Public: anyone can read this; nobody can change it.

deploy/azure: mode=vms|aks in main.bicep, HA servers carve-out, AKS germinate profile #139

Merged opened this view on origin

Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.

Merged

azure-aks-mode → main 10 commits

  • Merged. azure-aks-mode was merged into main.

Review

Sign in to review this change.
9 files changed +1637 −38
bin/germinate +44 −1
@@ -47,6 +47,26 @@ fi
4747 # daughter forks mitosis from when the seed's parent is gone or unreachable.
4848 # Overridable so an enterprise fleet can point at its own root forge.
4949 : "${MITOSIS_FORK_ROOT:=https://git.open-platform.sh/plat/mitosis}"
50 # CLUSTER_PROFILE — the GROUND the daughter grows on. k3s (default): bare VMs /
51 # k3d, node access assumed (registries.yaml mirror, ctr image side-load).
52 # aks: a managed Azure cluster deployed by deploy/azure/main.bicep -p mode=aks —
53 # no node access; ci-runner is built server-side with `az acr build`, the k3s
54 # node couplings (monitoring/grafana, node-fabric, registry-node-config,
55 # coredns rewrites) are trimmed by trim_gitops_aks, and Azure Monitor is the
56 # observability surface. Requires ACR=<login server> (from the bicep outputs).
57 : "${CLUSTER_PROFILE:=k3s}"
58 if [ "$CLUSTER_PROFILE" = aks ]; then
59 : "${ACR:?CLUSTER_PROFILE=aks requires ACR=<acr login server, e.g. mitosisabc.azurecr.io>}"
60 : "${AKS_RG:?CLUSTER_PROFILE=aks requires AKS_RG=<resource group of the mode=aks deployment>}"
61 command -v az >/dev/null 2>&1 || { echo "FATAL: CLUSTER_PROFILE=aks requires the az CLI (ci-runner builds via 'az acr build')"; exit 1; }
62 # The ACME solver swap (trim_gitops_aks 4d/4e) needs the bicep-provisioned
63 # identity + zone facts; derive them once here so the operator passes only RG.
64 : "${AKS_SUBSCRIPTION_ID:=$(az account show --query id -o tsv)}"
65 : "${AKS_DNS_ZONE:=$DOMAIN}"
66 : "${AKS_CERT_MANAGER_CLIENT_ID:=$(az identity show -g "$AKS_RG" -n mitosis-cert-manager --query clientId -o tsv 2>/dev/null || true)}"
67 [ -n "$AKS_CERT_MANAGER_CLIENT_ID" ] || echo "WARN: mitosis-cert-manager identity not found in $AKS_RG — LE solver stays on cloudflare and certs WILL fail"
68 export AKS_SUBSCRIPTION_ID AKS_DNS_ZONE AKS_CERT_MANAGER_CLIENT_ID AKS_RG
69 fi
5070 # BYO claude credentials (default empty) — sealed into plat-agents-secrets so a
5171 # daughter launched with one can build apps the instant it reconciles. Captured
5272 # into named locals so we can report (NOT echo) whether agents are credentialed,
@@ -124,6 +144,14 @@ GITOPS_DIR="$WORK/r-gitops"
124144 [ "$PROFILE" = lean ] && trim_gitops_lean "$GITOPS_DIR"
125145 strip_gitops_tenants "$GITOPS_DIR" # fresh daughter has no external orgs (any profile)
126146
147 # aks ground: strip the k3s node couplings + Grafana observability (Azure
148 # Monitor replaces it) and repoint the runner image at ACR. Runs AFTER the
149 # lean trim (it edits the already-trimmed tree) and BEFORE identity rewrite.
150 if [ "$CLUSTER_PROFILE" = aks ]; then
151 STEP="trim gitops for the aks ground"
152 trim_gitops_aks "$GITOPS_DIR" "$ACR"
153 fi
154
127155 # OLD-VINTAGE ONLY (dind seeds; current seeds ship no docker-daemon-config and
128156 # skip this whole block): pin dind's bridge MTU to the cluster overlay so image
129157 # builds' large downloads aren't silently dropped (k3d/flannel overlay = 1450;
@@ -732,7 +760,22 @@ CIRTAG=$(grep -ohE 'ci-runner:[0-9]+\.[0-9]+\.[0-9]+' \
732760 "$GITOPS_DIR"/clusters/local/platform/forgejo/*.yaml 2>/dev/null | head -1 | sed 's/.*://' || true)
733761 : "${CIRTAG:=0.1.0}"
734762 CIR_IMAGE="git.$DOMAIN/plat/ci-runner:$CIRTAG"
735 if [ ! -d "$WORK/r-ci-runner" ]; then
763 if [ "$CLUSTER_PROFILE" = aks ]; then
764 # Managed ground: no node runtime to side-load into. `az acr build` builds
765 # server-side in ACR (no local docker) and the ScaledJob already points at
766 # $ACR/plat/ci-runner (trim_gitops_aks rewrote it); kubelet pulls via the
767 # AcrPull role the bicep granted — the chicken-and-egg never exists here.
768 CIR_IMAGE="$ACR/plat/ci-runner:$CIRTAG"
769 if [ ! -d "$WORK/r-ci-runner" ]; then
770 echo " WARN: seed carries no plat/ci-runner — the isolated pool cannot start and"
771 echo " NO CI will run. Re-cut the seed from a mitosis that bundles it."
772 else
773 echo "==> building $CIR_IMAGE server-side (az acr build; managed ground has no node to side-load)"
774 az acr build --registry "${ACR%%.azurecr.io*}" --image "plat/ci-runner:$CIRTAG" "$WORK/r-ci-runner" >/dev/null \
775 && echo " $CIR_IMAGE built and pushed in ACR" \
776 || { echo "FATAL: az acr build failed — no runner image means no CI at all"; exit 1; }
777 fi
778 elif [ ! -d "$WORK/r-ci-runner" ]; then
736779 echo " WARN: seed carries no plat/ci-runner — the isolated pool cannot start and"
737780 echo " NO CI will run. Re-cut the seed from a mitosis that bundles it."
738781 elif ! command -v docker >/dev/null 2>&1; then
bin/lib.sh +346
@@ -351,6 +351,352 @@ verify_seed_bundles() {
351351 echo " seed integrity: every bundle matches its manifest sha256"
352352 }
353353
354 # trim_gitops_aks — adapt an (already lean-trimmed) gitops tree to a MANAGED
355 # Azure ground (deploy/azure/main.bicep -p mode=aks). Called by germinate when
356 # CLUSTER_PROFILE=aks, after trim_gitops_lean and before identity rewrite.
357 #
358 # What changes and why:
359 # 1. monitoring/monitoring-scrapes GO — Azure Monitor (managed Prometheus +
360 # Container Insights, enabled by the bicep) is the observability surface;
361 # Grafana and its OAuth/SSO surface do not exist on this ground.
362 # 2. node-fabric GOES — it exists to fix k3s flannel vxlan checksum offload;
363 # AKS (azure CNI overlay + cilium) has no such datapath.
364 # 3. registry-node-config GOES — the 127.0.0.1:31100 containerd realm-rewrite
365 # exists because k3s nodes can't resolve .local registries; AKS nodes pull
366 # git.<domain> over public DNS + trusted TLS, and infra images from ACR.
367 # 4. coredns GOES — the k3s coredns-custom rewrite kept *.domain in-cluster;
368 # on AKS the public zone resolves to the LB and hairpinning is fine.
369 # 5. The isolated runner's ScaledJob image is repointed at ACR (germinate
370 # builds it there with `az acr build`; kubelet pulls via AcrPull).
371 # 6. A Kyverno MUTATE policy (aks-image-redirect) rewrites explicit external
372 # registries (ghcr/quay/kyverno/k8s/forgejo/docker.io + the recipe's known
373 # docker.io shortnames) to ACR at Pod admission — one policy instead of
374 # per-chart values surgery. deploy/azure/mirror-images.sh MUST have run
375 # first or redirected pods ErrImagePull. kube-system is excluded (AKS
376 # managed addons are Microsoft's to pull from MCR).
377 # 7. restrict-image-registries allowlist learns the ACR + mcr.microsoft.com.
378 trim_gitops_aks() { # $1 = gitops clone $2 = ACR login server
379 local g="$1"
380 local acr="$2"
381 local P="$g/clusters/local/platform" fk="$g/clusters/local/flux-kustomizations.yaml"
382 local pk="$P/kustomization.yaml"
383
384 # 1-4) strip the k3s-ground components from the platform Kustomization + tree.
385 if [ -f "$pk" ]; then
386 awk -v strip="monitoring monitoring-scrapes node-fabric registry-node-config coredns" '
387 BEGIN { n = split(strip, a, " "); for (i = 1; i <= n; i++) drop[a[i]] = 1 }
388 /^[[:space:]]*-[[:space:]]/ {
389 item = $0
390 sub(/^[[:space:]]*-[[:space:]]*/, "", item)
391 sub(/\/.*$/, "", item)
392 sub(/[[:space:]]+$/, "", item)
393 if (item in drop) next
394 }
395 { print }
396 ' "$pk" > "$pk.trim" && mv "$pk.trim" "$pk"
397 fi
398 rm -rf "$P/monitoring" "$P/monitoring-scrapes" "$P/node-fabric" "$P/registry-node-config" "$P/coredns"
399
400 # monitoring ships as standalone Kustomization docs — chunk-drop any doc that
401 # NAMES monitoring (also catches monitoring-scrapes and dependsOn entries;
402 # anything depending on a stripped component cannot reconcile either).
403 if [ -f "$fk" ]; then
404 awk '
405 function flush() { if (buf != "" && buf !~ /name:[[:space:]]*monitoring/) printf "%s", buf; buf = "" }
406 /^---[[:space:]]*$/ { flush(); buf = $0 "\n"; next }
407 { buf = buf $0 "\n" }
408 END { flush() }
409 ' "$fk" > "$fk.trim" && mv "$fk.trim" "$fk"
410 fi
411
412 # Grafana-coupled surfaces outside monitoring/: the agents flow console rides
413 # grafana.<domain> + a forwardAuth against the Grafana session — no Grafana,
414 # no surface (plat-flow CLI drives the same API; an Entra-fronted host is the
415 # pilot follow-up). Presence-guarded: agents dir layouts vary by vintage.
416 local af agdir="$P/agents"
417 if [ -d "$agdir" ]; then
418 for af in $(grep -rIl 'grafana\.' "$agdir" 2>/dev/null || true); do
419 echo " trim(aks): dropping grafana-coupled $af"
420 local bn; bn=$(basename "$af")
421 rm -f "$af"
422 [ -f "$agdir/kustomization.yaml" ] && sed -i.bak "\|$bn|d" "$agdir/kustomization.yaml" && rm -f "$agdir/kustomization.yaml.bak"
423 done
424 fi
425
426 # 4b) control-plane nodeSelectors GO — the k3s recipe pins fail-closed control
427 # components (cert-manager, cnpg, mcp, kyverno, forgejo-pg) to the durable
428 # server node; AKS nodes carry no control-plane label, so the pin makes
429 # those pods permanently unschedulable (observed: first aks germination,
430 # cert-manager Pending → whole platform chain stalled). Drops the
431 # two-line `nodeSelector:` + control-plane pattern and bare control-plane
432 # lines wherever they appear under platform/.
433 local nsf
434 for nsf in $(grep -rIl 'node-role\.kubernetes\.io/control-plane' "$P" --include='*.yaml' 2>/dev/null || true); do
435 awk '
436 /^[[:space:]]*nodeSelector:[[:space:]]*$/ { hold=$0; held=1; next }
437 held && /node-role\.kubernetes\.io\/control-plane/ { held=0; next }
438 held { print hold; held=0 }
439 /node-role\.kubernetes\.io\/control-plane/ && !/^[[:space:]]*#/ { next }
440 { print }
441 END { if (held) print hold }
442 ' "$nsf" > "$nsf.aks" && mv "$nsf.aks" "$nsf"
443 echo " trim(aks): dropped control-plane pin in ${nsf#$g/}"
444 done
445 # …and the parents the strip can orphan: cert-manager's per-component keys
446 # (webhook/cainjector/startupapicheck) exist in the recipe ONLY to carry the
447 # pin — left empty they render null and nil-pointer the chart's templates
448 # (observed: first aks germination, UpgradeFailed on
449 # webhook-validating-webhook.yaml). POSIX classes: BSD sed has no \s.
450 local cmf="$P/cert-manager/helmrelease.yaml"
451 [ -f "$cmf" ] && sed -i.bak -E '/^[[:space:]]+(webhook|cainjector|startupapicheck):[[:space:]]*$/d' "$cmf" && rm -f "$cmf.bak"
452
453 # 4c) explicit local-path storage classes GO — AKS has no such class; a PVC
454 # naming it Pends forever and stalls the forgejo-pg handoff (observed:
455 # first aks germination, "Cluster is unrecoverable"). Dropping the line
456 # falls back to the cluster default (managed-csi — better: survives node
457 # loss). Covers CNPG `storageClass:` and PVC `storageClassName:` forms.
458 local spf
459 for spf in $(grep -rIlE 'storageClass(Name)?:[[:space:]]*local-path' "$P" --include='*.yaml' 2>/dev/null || true); do
460 sed -i.bak -E '/storageClass(Name)?:[[:space:]]*local-path/d' "$spf" && rm -f "$spf.bak"
461 echo " trim(aks): dropped local-path storage class in ${spf#$g/}"
462 done
463
464 # 4d) ACME solver: cloudflare → azureDNS via workload identity. The bicep
465 # ships a mitosis-cert-manager identity federated to the cert-manager
466 # ServiceAccount with DNS Zone Contributor on the platform zone; the
467 # germinate env supplies its clientID + subscription + resource group.
468 # Without this swap the wildcard cert challenges fail ("no Cloudflare
469 # credential") and CI cannot docker-login to git.<domain> over TLS.
470 local liss="$P/letsencrypt/clusterissuer.yaml"
471 if [ -f "$liss" ] && grep -q cloudflare "$liss"; then
472 if [ -n "${AKS_CERT_MANAGER_CLIENT_ID:-}" ] && [ -n "${AKS_SUBSCRIPTION_ID:-}" ] && [ -n "${AKS_RG:-}" ] && [ -n "${AKS_DNS_ZONE:-}" ]; then
473 python3 - "$liss" <<'PYEOF'
474 import os, sys
475 p = sys.argv[1]
476 s = open(p).read()
477 old = """ cloudflare:
478 apiTokenSecretRef:
479 name: cloudflare-dns-api
480 key: api-token"""
481 new = f""" # aks ground: Azure DNS via workload identity (no secret in-cluster)
482 azureDNS:
483 subscriptionID: {os.environ['AKS_SUBSCRIPTION_ID']}
484 resourceGroupName: {os.environ['AKS_RG']}
485 hostedZoneName: {os.environ['AKS_DNS_ZONE']}
486 environment: AzurePublicCloud
487 managedIdentity:
488 clientID: {os.environ['AKS_CERT_MANAGER_CLIENT_ID']}"""
489 if old in s:
490 open(p, 'w').write(s.replace(old, new))
491 print(" trim(aks): LE solver cloudflare -> azureDNS")
492 else:
493 print(" WARN trim(aks): cloudflare solver block shape unexpected — swap it by hand")
494 PYEOF
495 rm -f "$P/letsencrypt/cloudflare-dns-api.yaml"
496 [ -f "$P/letsencrypt/kustomization.yaml" ] && sed -i.bak '/cloudflare-dns-api/d' "$P/letsencrypt/kustomization.yaml" && rm -f "$P/letsencrypt/kustomization.yaml.bak"
497 else
498 echo " WARN trim(aks): AKS_CERT_MANAGER_CLIENT_ID/AKS_SUBSCRIPTION_ID/AKS_RG/AKS_DNS_ZONE unset — LE solver left on cloudflare (certs WILL fail)"
499 fi
500 fi
501
502 # 4e) cert-manager runs under that identity: annotate its ServiceAccount and
503 # opt pods into the workload-identity webhook.
504 if [ -f "$cmf" ] && [ -n "${AKS_CERT_MANAGER_CLIENT_ID:-}" ] && ! grep -q 'workload.identity' "$cmf"; then
505 python3 - "$cmf" <<'PYEOF'
506 import os, sys
507 p = sys.argv[1]
508 s = open(p).read()
509 anchor = " values:\n"
510 cid = os.environ['AKS_CERT_MANAGER_CLIENT_ID']
511 inject = f""" values:
512 # aks ground: federate with the mitosis-cert-manager identity (DNS-01).
513 serviceAccount:
514 annotations:
515 azure.workload.identity/client-id: {cid}
516 podLabels:
517 azure.workload.identity/use: "true"
518 """
519 if anchor in s:
520 open(p, 'w').write(s.replace(anchor, inject, 1))
521 print(" trim(aks): cert-manager wired to workload identity")
522 else:
523 print(" WARN trim(aks): cert-manager values anchor not found — wire workload identity by hand")
524 PYEOF
525 fi
526
527 # 4f) the git edge serves the PUBLIC (LE) certificate — AKS kubelets pull the
528 # plat images (mcp, agents, apps) from git.<domain> over public TLS and
529 # trust only public CAs; the plat-local cert x509-fails every pull
530 # (observed: second aks germination, mcp ImagePullBackOff at the final
531 # gate). The k3s ground keeps the internal cert (its nodes bypass via the
532 # containerd mirror).
533 local igf="$P/forgejo/ingress.yaml"
534 if [ -f "$igf" ] && grep -q 'secretName: wildcard-platform-tls' "$igf"; then
535 sed -i.bak 's/secretName: wildcard-platform-tls/secretName: wildcard-public-tls/' "$igf" && rm -f "$igf.bak"
536 echo " trim(aks): git edge cert -> wildcard-public-tls (kubelet pulls need a public CA)"
537 fi
538 # …and that certificate must come from LE PRODUCTION: the recipe pins the
539 # staging server (fine on k3s where the public cert is cosmetic), but staging
540 # certs are not publicly trusted, so kubelet pulls x509-fail exactly like the
541 # internal CA did (observed: second aks germination, "(STAGING) Artificial
542 # Amaranth" on the edge).
543 if [ -f "$liss" ]; then
544 sed -i.bak 's|https://acme-staging-v02.api.letsencrypt.org/directory|https://acme-v02.api.letsencrypt.org/directory|' "$liss" && rm -f "$liss.bak"
545 fi
546
547 # 4g) charts of k3s vintage HARDCODE storageClassName: local-path in PVC
548 # templates (agents chart, app templates) — no values override exists.
549 # A mutate policy strips it at admission so PVCs bind the cluster default
550 # (managed-csi). Remove once the charts parameterize storage.
551 if [ -d "$P/kyverno-policies" ]; then
552 cat > "$P/kyverno-policies/aks-storage-default.yaml" <<'EOF'
553 # aks ground only (written by trim_gitops_aks): strip hardcoded local-path
554 # from PVCs at admission; AKS binds the default class (managed-csi) instead.
555 apiVersion: kyverno.io/v1
556 kind: ClusterPolicy
557 metadata:
558 name: aks-storage-default
559 labels:
560 plat.sh/managed: "true"
561 spec:
562 background: false
563 rules:
564 - name: strip-local-path
565 match:
566 any:
567 - resources:
568 kinds: [PersistentVolumeClaim]
569 preconditions:
570 all:
571 - key: '{{ request.object.spec.storageClassName || '''' }}'
572 operator: Equals
573 value: local-path
574 mutate:
575 patchesJson6902: |-
576 - op: remove
577 path: /spec/storageClassName
578 EOF
579 if [ -f "$P/kyverno-policies/kustomization.yaml" ] && ! grep -q 'aks-storage-default' "$P/kyverno-policies/kustomization.yaml"; then
580 sed -i.bak '/^resources:/a\
581 - aks-storage-default.yaml' "$P/kyverno-policies/kustomization.yaml" && rm -f "$P/kyverno-policies/kustomization.yaml.bak"
582 fi
583 fi
584 # …but kyverno's webhooks skip SYSTEM namespaces (plat.sh/app selector), so
585 # the policy covers tenant apps only. The agents chart lives in a system
586 # namespace and hardcodes local-path too — strip it at Helm render time.
587 local ahf="$P/agents/helmrelease.yaml"
588 if [ -f "$ahf" ] && ! grep -q postRenderers "$ahf"; then
589 python3 - "$ahf" <<'PYEOF'
590 import sys
591 p = sys.argv[1]
592 s = open(p).read()
593 anchor = "spec:\n"
594 inject = """spec:
595 # aks ground: the chart hardcodes storageClassName local-path in its PVC
596 # templates and kyverno's webhooks skip system namespaces — strip the field
597 # at render time so PVCs bind the cluster default (managed-csi).
598 postRenderers:
599 - kustomize:
600 patches:
601 - target:
602 kind: PersistentVolumeClaim
603 patch: |
604 - op: remove
605 path: /spec/storageClassName
606 """
607 if anchor in s:
608 i = s.index(anchor)
609 open(p, 'w').write(s[:i] + inject + s[i+len(anchor):])
610 print(" trim(aks): agents postRenderer strips hardcoded local-path")
611 else:
612 print(" WARN trim(aks): agents helmrelease spec anchor not found — strip local-path by hand")
613 PYEOF
614 fi
615
616 # 5) runner image → ACR (domain-agnostic pattern: runs BEFORE identity rewrite,
617 # so the tree still carries the parent's domain).
618 local rf
619 for rf in $(grep -rIl 'plat/ci-runner:' "$P/forgejo" 2>/dev/null || true); do
620 sed -i.bak -E "s|git\.[^/\"']+/plat/ci-runner:|$acr/plat/ci-runner:|g" "$rf" && rm -f "$rf.bak"
621 echo " trim(aks): runner image -> $acr/plat/ci-runner (in $(basename "$rf"))"
622 done
623
624 # 6) admission-time ACR redirect for external images. One policy, no per-chart
625 # values surgery; idempotent (regex misses leave the image untouched).
626 local KP="$P/kyverno-policies"
627 if [ -d "$KP" ]; then
628 cat > "$KP/aks-image-redirect.yaml" <<'EOF'
629 # aks ground only (written by trim_gitops_aks): every explicit external
630 # registry the recipe pins is mirrored into ACR by deploy/azure/mirror-images.sh
631 # (same path, host swapped) — this policy makes pods pull the mirror without
632 # touching each chart's values. kube-system is Microsoft's (MCR-managed addons).
633 apiVersion: kyverno.io/v1
634 kind: ClusterPolicy
635 metadata:
636 name: aks-image-redirect
637 labels:
638 plat.sh/managed: "true"
639 spec:
640 background: false
641 rules:
642 - name: redirect-containers
643 match:
644 any:
645 - resources:
646 kinds: [Pod]
647 exclude:
648 any:
649 - resources:
650 namespaces: [kube-system, kube-node-lease, kube-public, gatekeeper-system]
651 mutate:
652 foreach:
653 - list: request.object.spec.containers
654 patchStrategicMerge:
655 spec:
656 containers:
657 - name: '{{ element.name }}'
658 image: '{{ regex_replace_all(''^(ghcr\.io|quay\.io|reg\.kyverno\.io|registry\.k8s\.io|code\.forgejo\.org|docker\.io)/(.*)$'', regex_replace_all(''^(traefik|alpine|caddy|busybox)((:|@).*)?$'', regex_replace_all(''^(minio|alpine)/(.*)$'', element.image, ''docker.io/$1/$2''), ''docker.io/library/$1$2''), ''__ACR__/$2'') }}'
659 - name: redirect-init-containers
660 match:
661 any:
662 - resources:
663 kinds: [Pod]
664 # preconditions guard: only pods that HAVE initContainers
665 preconditions:
666 all:
667 - key: '{{ request.object.spec.initContainers[] | length(@) }}'
668 operator: GreaterThan
669 value: 0
670 exclude:
671 any:
672 - resources:
673 namespaces: [kube-system, kube-node-lease, kube-public, gatekeeper-system]
674 mutate:
675 foreach:
676 - list: request.object.spec.initContainers
677 patchStrategicMerge:
678 spec:
679 initContainers:
680 - name: '{{ element.name }}'
681 image: '{{ regex_replace_all(''^(ghcr\.io|quay\.io|reg\.kyverno\.io|registry\.k8s\.io|code\.forgejo\.org|docker\.io)/(.*)$'', regex_replace_all(''^(traefik|alpine|caddy|busybox)((:|@).*)?$'', regex_replace_all(''^(minio|alpine)/(.*)$'', element.image, ''docker.io/$1/$2''), ''docker.io/library/$1$2''), ''__ACR__/$2'') }}'
682 EOF
683 sed -i.bak "s|__ACR__|$acr|g" "$KP/aks-image-redirect.yaml" && rm -f "$KP/aks-image-redirect.yaml.bak"
684 if [ -f "$KP/kustomization.yaml" ] && ! grep -q 'aks-image-redirect' "$KP/kustomization.yaml"; then
685 sed -i.bak '/^resources:/a\
686 - aks-image-redirect.yaml' "$KP/kustomization.yaml" && rm -f "$KP/kustomization.yaml.bak"
687 fi
688 # 7) the allowlist learns the mirror + Microsoft's registry (AKS addons).
689 if [ -f "$KP/restrict-image-registries.yaml" ] && ! grep -q "$acr" "$KP/restrict-image-registries.yaml"; then
690 sed -i.bak -E "0,/^([[:space:]]*)- \"?git\./s||\1- \"$acr/*\"\n\1- \"mcr.microsoft.com/*\"\n&|" \
691 "$KP/restrict-image-registries.yaml" && rm -f "$KP/restrict-image-registries.yaml.bak" \
692 || echo " WARN trim(aks): could not extend restrict-image-registries allowlist — add $acr by hand"
693 fi
694 fi
695
696 git -C "$g" add -A
697 git -C "$g" -c user.email=mitosis@plat -c user.name=mitosis commit -q -m "mitosis: aks ground (Azure Monitor replaces grafana; strip node-fabric/registry-node-config/coredns; runner + external images via ACR)" || true
698 }
699
354700 # publish_wiki — materialize the in-repo wiki/ source as the NATIVE Forgejo wiki
355701 # of <owner>/mitosis. The wiki is documentation-as-an-organ: it ships inside the
356702 # mitosis repo (reviewable, versioned with the code it documents), and every
deploy/README.md +2
@@ -46,6 +46,8 @@ official SSM parameter — no AMI table.
4646
4747 ## Azure (Bicep)
4848
49 Full guide, both modes (bare VMs or AKS): [`azure/README.md`](azure/README.md).
50
4951 ```sh
5052 az login # or a service principal
5153 az group create -n mitosis -l eastus
deploy/azure/README.md added +385
@@ -0,0 +1,385 @@
1 # Deploy Mitosis on Azure
2
3 This directory deploys a Mitosis platform on Microsoft Azure. One Bicep
4 template, `main.bicep`, does the deployment. The template has two modes. Set
5 the `mode` parameter to select one.
6
7 | Mode | What Azure runs | Who installs the platform | Select it when |
8 | --------------- | ------------------------------------------------------------------- | ----------------------------------------------- | ----------------------------------------------------------------- |
9 | `vms` (default) | 1 or 3 Ubuntu VMs with k3s | The leader VM, automatically, at its first boot | You want a sovereign platform on plain VMs. |
10 | `aks` | A managed AKS cluster with ACR, Azure DNS, Key Vault, Azure Monitor | You, from your machine, after the deployment | You want an Azure-managed cluster and Azure-native services only. |
11
12 The two modes install the same platform: Forgejo, Flux, MCP, CNPG, Kyverno,
13 KEDA, MinIO, and the app model. The guide for all clouds is
14 [`../README.md`](../README.md). Read it for the parts that are the same on
15 AWS, Azure, and DigitalOcean.
16
17 ## Words used in this document
18
19 | Word | Meaning |
20 | -------------- | --------------------------------------------------------------------------------------------------------------------------------------------- |
21 | Platform | The full Mitosis installation: git server, container registry, cluster services, and apps. |
22 | Germination | The procedure that installs the platform into an empty cluster. `bin/germinate` runs it. |
23 | Leader | The first VM in `vms` mode. It runs germination. |
24 | Sovereign key | The `age` key that encrypts the secrets of the platform. There is no other copy. If you lose it, you lose the secrets. |
25 | Resource group | The Azure container that holds all resources of one deployment. |
26 | Domain | The DNS name of the platform, for example `plat.example.com`. Services get names below it: `git.`, `mcp.`, `www.`, and one name for each app. |
27 | Seed | The archive `genesis/seed.tar.gz`. It holds the platform repositories that germination installs. |
28
29 ## Before you start
30
31 You need:
32
33 - An Azure subscription and the `az` CLI. Log in with `az login`.
34 - Permission to create resources in a resource group.
35 - For `vms` mode: an SSH public key, for example `~/.ssh/id_ed25519.pub`.
36 - For `aks` mode: `kubectl`, `flux`, `sops`, `age`, and `jq` on your machine,
37 and a seed.
38 - A domain that you control. In `vms` mode the domain is optional (see
39 [Zero-DNS](#zero-dns-vms-mode-only)). In `aks` mode the domain is required.
40 - Optional: one Claude credential, so the build crew can start work
41 immediately. Use `claudeCodeOauthToken` (subscription) or `anthropicApiKey`
42 (API billing). Do not set both.
43
44 Do these two steps one time on a new subscription:
45
46 1. Register the resource providers. Run
47 `az provider register -n <name> --wait` for each name the mode uses:
48 - `vms`: `Microsoft.Compute`, `Microsoft.Network`
49 - `aks`: `Microsoft.ContainerService`, `Microsoft.ContainerRegistry`,
50 `Microsoft.Network`, `Microsoft.OperationalInsights`,
51 `Microsoft.KeyVault`, `Microsoft.ManagedIdentity`
52 2. Check the vCPU quota with `az vm list-usage -l <region> -o table`. New
53 subscriptions get quota for the v6, v7, and B-series VM families only.
54 The default VM sizes in this template are in these families. `vms` mode
55 needs 4 free vCPUs for each VM. `aks` mode needs 8 free vCPUs.
56
57 CAUTION: `main.bicepparam` is in `.gitignore` because it can hold
58 credentials. Copy `main.bicepparam.example` and edit the copy. Do not commit
59 it.
60
61 ## Mode `vms` — sovereign k3s on VMs
62
63 ### Resources this mode creates
64
65 All resources go into the resource group that you name. Set `ha=true` to get
66 three VMs.
67
68 | Resource | Name | Details |
69 | -------------------------- | ------------------------------------------ | ------------------------------------------------------------------------------------ |
70 | Network security group | `mitosis-nodes` | Opens TCP 22 (SSH, from `allowedSshCidr`) and TCP 80, 443, 2222 (from the Internet). |
71 | Virtual network | `mitosis` | Address space `10.60.0.0/24`, one subnet `nodes`. |
72 | Public IP address (1 or 3) | `mitosis-leader`, `mitosis-server-1`, `-2` | Standard SKU, static. The IP address does not change when the VM restarts. |
73 | Network interface (1 or 3) | Same names as the public IPs | Static private IPs `10.60.0.10`, `.11`, `.12`. |
74 | Virtual machine (1 or 3) | Same names as the public IPs | Ubuntu 24.04 LTS, `Standard_D4as_v7` (4 vCPU, 16 GB), 80 GB StandardSSD OS disk. |
75
76 The VM does the rest at first boot. Cloud-init writes the registry recipe and
77 the node contract from [`../lib/`](../lib/). Then a systemd unit installs k3s,
78 clones Mitosis, and runs germination. Azure credentials never go to the VMs.
79
80 ### Steps
81
82 1. Create the resource group.
83
84 ```sh
85 az group create -n mitosis -l eastus
86 ```
87
88 2. Start the deployment. Replace the domain with yours.
89
90 ```sh
91 cd deploy/azure
92 az deployment group create -g mitosis -f main.bicep \
93 -p domain=plat.example.com \
94 -p adminSshPublicKey="$(cat ~/.ssh/id_ed25519.pub)"
95 ```
96
97 Options: add `-p ha=true` for three servers. Add
98 `-p vmSize=Standard_B4ms` for a cheaper, burstable VM. Add
99 `-p allowedSshCidr=<your-ip>/32` to limit SSH access.
100
101 3. Read the outputs. They show the IP addresses, the DNS record, and the next
102 commands.
103
104 ```sh
105 az deployment group show -g mitosis -n main --query properties.outputs
106 ```
107
108 4. Create one DNS record at your DNS provider. Use the `dnsRecord` output:
109 `*.plat.example.com A <leader-ip>`. In HA mode, put all three IPs in the
110 record. This one record covers `git.`, `mcp.`, `www.`, and all apps.
111
112 5. Watch germination. It takes approximately 30 to 45 minutes.
113
114 ```sh
115 ssh mitosis@<leader-ip> journalctl -u mitosis-node -f
116 ```
117
118 6. When germination is complete, read the platform card.
119
120 ```sh
121 ssh mitosis@<leader-ip> sudo cat /root/PLATFORM.md
122 ```
123
124 7. Copy the sovereign key off the VM immediately. The card shows how. The key
125 is at `/root/fork-<domain>.age` on the leader.
126
127 WARNING: Keep the sovereign key in a safe place. It is the only key to the
128 secrets of the platform. If you delete the VM and you do not have the key,
129 the secrets are lost.
130
131 The VM login user is `mitosis`, or the value of `adminUsername`. Azure VMs
132 have no root login. Git over SSH listens on port 2222.
133
134 ### Zero-DNS (`vms` mode only)
135
136 If you do not have a domain, set `-p domain=auto`. At first boot, the leader
137 reads its public IP and names the platform `<ip-with-dashes>.sslip.io`. This
138 name resolves without any DNS record. The platform is available at
139 `https://git.<that-name>` when germination is complete.
140
141 ### Add servers to a live platform
142
143 `main.bicep` with `ha=true` is for new deployments only. To add two servers
144 to a platform that already runs, use `servers.bicep`. It creates only the
145 joiner VMs (`mitosis-server-1`, `-2`) in the existing network.
146
147 1. Make sure the leader runs embedded etcd. On a single-node install, write
148 `cluster-init: true` to `/etc/rancher/k3s/config.yaml` on the leader and
149 restart k3s. k3s migrates from sqlite to etcd in place.
150 2. Read the join token from the leader:
151 `ssh mitosis@<leader-ip> sudo cat /var/lib/rancher/k3s/server/token`.
152 3. Deploy the joiners.
153
154 ```sh
155 az deployment group create -g mitosis -f servers.bicep \
156 -p domain=plat.example.com \
157 -p adminSshPublicKey="$(cat ~/.ssh/id_ed25519.pub)" \
158 -p k3sToken="<token-from-step-2>"
159 ```
160
161 4. Wait 5 to 10 minutes. Then check: `kubectl get nodes` shows 3 nodes
162 `Ready`.
163 5. Add the two new IPs to the `*.<domain>` A record.
164
165 NOTE: `k3sVersion` in `servers.bicep` must match the version the leader runs
166 (`k3s --version`).
167
168 ## Mode `aks` — Azure-managed cluster
169
170 In this mode Azure runs the cluster. There is no cloud-init and no node
171 contract. You run germination from your machine against the cluster
172 kubeconfig, after the deployment.
173
174 ### Resources this mode creates
175
176 | Resource | Name | Details |
177 | --------------------------------------- | --------------------------------------- | --------------------------------------------------------------------------------------------------------------------- |
178 | Log Analytics workspace | `mitosis-logs` | Container logs. Retention 30 days. |
179 | Container registry (ACR) | `mitosis<unique>` (or `acrName`) | Standard SKU. Holds the mirrored infra images and the CI runner image. |
180 | AKS cluster | `mitosis-aks` | 2 × `Standard_D4ds_v6` (4 vCPU, 16 GB), zones 2 and 3, Azure CNI overlay with Cilium, workload identity, OIDC issuer. |
181 | Role assignment | AcrPull on the ACR | Lets the cluster nodes pull images from the ACR. |
182 | Public DNS zone | `<domain>` | Created when `createDnsZone=true` (default). Delegate the domain to it. |
183 | Managed identity + federated credential | `mitosis-cert-manager` | Lets cert-manager answer DNS-01 challenges in the zone (DNS Zone Contributor). Gives public TLS certificates. |
184 | Key Vault | `mkv<unique>` (or `keyVaultName`) | Standard SKU, RBAC, soft delete on. Holds the sops key at pilot hardening. |
185 | Managed identity + federated credential | `mitosis-flux-sops` | Lets the Flux kustomize-controller decrypt sops secrets with a Key Vault key (Key Vault Crypto User). |
186 | Azure Monitor | Container Insights + managed Prometheus | Enabled on the cluster. Replaces the Grafana of the k3s recipe. |
187
188 Azure creates one more resource at germination: a Standard Load Balancer
189 with a public IP for the Traefik Service. It serves ports 80, 443, and 2222.
190 It lives in the node resource group (`MC_...`).
191
192 ### Steps
193
194 1. Create the resource group and start the deployment.
195
196 ```sh
197 az group create -n mitosis-aks -l eastus
198 cd deploy/azure
199 az deployment group create -g mitosis-aks -f main.bicep \
200 -p domain=plat.example.com -p mode=aks
201 ```
202
203 2. Read the outputs: `acrLoginServer`, `dnsNameServers`, `keyVaultName`,
204 `certManagerClientId`, `fluxSopsClientId`.
205
206 ```sh
207 az deployment group show -g mitosis-aks -n main --query properties.outputs
208 ```
209
210 3. Delegate the domain. At your registrar or DNS host, add NS records for
211 `plat.example.com` that point to the four `dnsNameServers`.
212
213 4. Mirror the infra images into the ACR. The copy runs on the Azure side and
214 takes approximately 2 minutes. Do this before germination. Pods that are
215 redirected to the ACR cannot start if the image is not there.
216
217 ```sh
218 deploy/azure/mirror-images.sh <acrLoginServer>
219 ```
220
221 5. Get the kubeconfig.
222
223 ```sh
224 az aks get-credentials -g mitosis-aks -n mitosis-aks
225 ```
226
227 6. Start germination from the repository root. Germination creates the
228 sovereign key on your machine. It needs the `az` CLI: it builds the CI
229 runner image with `az acr build` and reads the cert-manager identity from
230 the resource group.
231
232 ```sh
233 SEED=genesis/seed.tar.gz CLUSTER_PROFILE=aks \
234 ACR=<acrLoginServer> AKS_RG=mitosis-aks DOMAIN=plat.example.com \
235 FORK_KEY_ACK=1 bin/germinate
236 ```
237
238 7. Create the zone records as soon as the Traefik load balancer has a public
239 IP. This happens a few minutes after germination starts. Watch with
240 `kubectl get svc -n traefik traefik -w`.
241
242 ```sh
243 az network dns record-set a add-record -g mitosis-aks -z plat.example.com -n '*' -a <lb-ip> --ttl 60
244 az network dns record-set a add-record -g mitosis-aks -z plat.example.com -n '@' -a <lb-ip> --ttl 60
245 ```
246
247 CAUTION: The CI of the platform pushes to `git.<domain>` over public DNS.
248 Chart builds fail until these records exist. Use `--ttl 60` while you
249 make changes. A wrong cached record delays you by one full build.
250
251 8. Back up the sovereign key immediately.
252
253 9. Verify: run `scripts/coldstart-assert.sh`. Then build the first app from
254 `_app-template` and open `https://<app>.<domain>`. The certificate is
255 public.
256
257 ### Optional: Entra ID sign-in on Forgejo
258
259 Run this after germination. You need rights to create app registrations in
260 the tenant.
261
262 ```sh
263 KUBECONFIG=~/.kube/config deploy/azure/entra-forgejo-oidc.sh plat.example.com
264 ```
265
266 The script creates an Entra app registration, a client secret (2 years), and
267 an OpenID Connect sign-in source named `entra` in Forgejo. Users then see
268 "Sign in with entra" on `git.<domain>`. Local accounts continue to work.
269
270 ### What is different from `vms` mode
271
272 - Storage is the AKS default `managed-csi` (Azure Disk). Volumes survive node
273 loss. Germination removes the `local-path` storage class from the recipe.
274 A Kyverno policy removes it from tenant PVCs at admission.
275 - TLS is public. cert-manager gets certificates from Let's Encrypt
276 (production) with DNS-01 in the Azure DNS zone, through the
277 `mitosis-cert-manager` identity. The git edge serves this certificate, so
278 the cluster nodes can pull platform images over public TLS.
279 - Germination removes the control-plane `nodeSelector` pins of the k3s
280 recipe. AKS nodes do not have that label.
281 - The CI runner image is built on the Azure side with `az acr build`. There
282 is no image side-loading.
283 - A Kyverno policy, `aks-image-redirect`, points external registries to the
284 ACR at Pod admission. Nodes pull only from the ACR and from the platform
285 registry.
286 - Grafana, node-fabric, registry-node-config, and the CoreDNS rewrites are
287 removed at germination. Use **Insights** on the AKS resource for logs and
288 metrics.
289
290 ### Known limits (proof of concept)
291
292 - The Grafana checks in `coldstart-assert.sh` do not apply. Skip them.
293 - Images with an explicit registry, and the known docker.io short names
294 (traefik, alpine, caddy, busybox, minio), are redirected to the ACR. Other
295 docker.io short names are not.
296 - The flow console (agents operator UI) has no hostname in this mode. The
297 `kit/plat-flow` CLI works.
298 - CNPG clusters keep their `ghcr.io` image name in the spec. `aks-image-redirect`
299 changes only the Pod. CNPG then sees a difference and restarts the tenant
300 database. Fix: extend the policy to the CNPG `Cluster` resource.
301 - CNPG PodDisruptionBudgets block AKS node operations (scale, upgrade). Drain
302 the node by hand with `--disable-eviction` first.
303 - Some VM sizes attach few disks (`D4as_v7`: 12). Count the volumes before you
304 change the VM size.
305
306 The record of the first AKS platform, with costs, commands, and lessons, is
307 in [`aks-cost-experiment.md`](aks-cost-experiment.md).
308
309 ### Pilot hardening (before customer secrets)
310
311 1. sops → Key Vault: add an `azure_keyvault` recipient in `.sops.yaml`. Flux
312 decrypts through the `mitosis-flux-sops` identity. Then remove the age key
313 from the cluster. Keep the offline copy for emergencies.
314 2. Enable the agents module with the customer's Anthropic credential. Give
315 the flow console its own hostname and Entra authentication.
316 3. Mirror the app base images (bun, node, postgres, ...) into the ACR. Extend
317 `aks-image-redirect`. Then set `restrict-image-registries` to Enforce.
318 4. Private cluster, App Gateway + WAF, Azure Policy, as the customer requires.
319 5. Send CNPG backups to Azure Blob instead of MinIO.
320
321 ## Parameters
322
323 | Parameter | Mode | Default | Meaning |
324 | ----------------------------------------- | ----- | ---------------------- | --------------------------------------------------------------------- |
325 | `domain` | both | (required) | The domain of the platform. `auto` is allowed in `vms` mode only. |
326 | `mode` | both | `vms` | `vms` or `aks`. |
327 | `location` | both | Resource group region | Azure region for all resources. |
328 | `ha` | `vms` | `false` | `true` = three servers with embedded etcd. |
329 | `adminSshPublicKey` | `vms` | (required) | Your SSH public key. |
330 | `adminUsername` | `vms` | `mitosis` | VM login user. |
331 | `vmSize` | `vms` | `Standard_D4as_v7` | 4 vCPU / 8 GB is the minimum. |
332 | `diskGb` | `vms` | `80` | OS disk size. 80 GB is the minimum. |
333 | `allowedSshCidr` | `vms` | `*` | Source range for SSH. |
334 | `claudeCodeOauthToken`, `anthropicApiKey` | `vms` | empty | Set one. Goes to the leader only. In `aks` mode, set it at germination. |
335 | `k3sToken` | `vms` | new GUID | Pass your own to keep re-deployments from replacing the VMs. |
336 | `k3sVersion` | `vms` | `v1.31.5+k3s1` | k3s release. |
337 | `mitosisUrl`, `mitosisRef` | `vms` | `plat/mitosis`, `main` | Where the nodes clone Mitosis from. |
338 | `aksNodeCount` | `aks` | `2` | Nodes in the system pool. |
339 | `aksVmSize` | `aks` | `Standard_D4ds_v6` | Node size. |
340 | `aksZones` | `aks` | `['2', '3']` | Availability zones. `[]` = non-zonal. |
341 | `aksVersion` | `aks` | empty | Kubernetes version. Empty = the AKS default. |
342 | `createDnsZone` | `aks` | `true` | `false` = use a zone that you already have. |
343 | `acrName`, `keyVaultName` | `aks` | derived | Globally unique names. |
344
345 ## Outputs
346
347 | Output | Mode | Meaning |
348 | --------------------------------------------------------- | ----- | ------------------------------------------------------------ |
349 | `nodeIps`, `dnsRecord`, `nextSteps` | `vms` | The VM IPs, the one DNS record to create, the next commands. |
350 | `acrLoginServer`, `aksName`, `oidcIssuer` | `aks` | The ACR host, the cluster name, the OIDC issuer URL. |
351 | `dnsNameServers` | `aks` | The four name servers to delegate the domain to. |
352 | `keyVaultName`, `certManagerClientId`, `fluxSopsClientId` | `aks` | Client IDs for the sops and cert-manager identities. |
353
354 ## Cost (approximate, eastus, retail)
355
356 | Mode | Compute | Other |
357 | ------------------- | ------------------------------------- | ----------------------------------------------------- |
358 | `vms`, 1 VM | `Standard_D4as_v7` ≈ $125 / month | OS disk, public IP. |
359 | `vms`, 1 VM, budget | `Standard_B4ms` ≈ $60 / month | Burstable. Slower builds. |
360 | `vms`, HA | 3 × the single-VM price | |
361 | `aks`, default | 2 × `Standard_D4ds_v6` ≈ $364 / month | ACR, Standard Load Balancer, Log Analytics ingestion. |
362
363 ## Remove the deployment
364
365 ```sh
366 az group delete -n <resource-group>
367 ```
368
369 WARNING: The secrets of the platform are removed with the cluster. Keep the
370 sovereign key if you keep any backups.
371
372 NOTE (`aks` mode): the Key Vault has soft delete on. Purge it before you
373 create a new vault with the same name:
374 `az keyvault purge -n <keyVaultName>`.
375
376 ## Files in this directory
377
378 | File | Purpose |
379 | ------------------------- | ----------------------------------------------------------- |
380 | `main.bicep` | The deployment template. Both modes. |
381 | `main.bicepparam.example` | Example parameter file. Copy, edit, do not commit the copy. |
382 | `servers.bicep` | Add two k3s servers to a live `vms` platform. |
383 | `mirror-images.sh` | Copy the infra images into the ACR (`aks` mode). |
384 | `entra-forgejo-oidc.sh` | Add Entra ID sign-in to Forgejo (`aks` mode). |
385 | `aks-cost-experiment.md` | Record of the first AKS platform: costs, commands, lessons. |
deploy/azure/aks-cost-experiment.md added +268
@@ -0,0 +1,268 @@
1 # AKS cost-down experiment — mitosis-aks (git.aks.benjivers.com)
2
3 > **2026-08-18, end of day: TORN DOWN.** Ben ordered full teardown of both Azure
4 > mitosis platforms (RGs `mitosis` + `mitosis-aks`, cold, no data export).
5 > This file stays as the record of what the experiment found — the constraints,
6 > gotchas, and az commands below apply to any future re-germinate on Azure.
7
8 Started 2026-08-18. Goal: strip the AKS POC to one VM and reduce CPU/memory cost.
9 Every change below is logged with its exact rollback command. Steps are executed
10 top to bottom; status is updated as each one lands.
11
12 Cluster: `mitosis-aks` in RG `mitosis-aks`, eastus, AKS 1.35.6, Free tier.
13 kubectl context: `aks-mitosis`.
14
15 ---
16
17 ## Baseline (before any change) — captured 2026-08-18
18
19 - Node pool `system`: **2× Standard_D4ds_v6** (4 vCPU / 16 GB, ephemeral OS disk 80 GB),
20 zones [2,3], autoscaler OFF, mode System.
21 - Retail price D4ds_v6 eastus: **$0.249/hr** → 2 nodes ≈ **$364/mo** compute.
22 - Live usage: CPU ~370m + ~250m (6–9%); memory 6.2 GB + 5.6 GB (~50%/node).
23 - Requests: 2246m+2747m CPU (58%/71%), 3930Mi+5142Mi memory (33%/43%).
24 - Monitoring: Container Insights (omsagent) **enabled** → Log Analytics workspace
25 `/subscriptions/1d655bbf-4fc9-48dd-bf75-2d3f72bb64a1/resourceGroups/mitosis-aks/providers/Microsoft.OperationalInsights/workspaces/mitosis-logs`;
26 managed Prometheus (azureMonitorMetrics) **enabled**.
27 The ama-* pods request ~770m CPU / ~2.3 GB memory total, plus ingestion billing.
28 - Health: `https://git.aks.benjivers.com/api/healthz` → 200 (~0.16 s).
29 - All pod anti-affinity is `preferred` (kyverno, coredns, metrics-server,
30 konnectivity, cilium-operator) → nothing hard-blocks single-node scheduling.
31 - PDBs: all minAvailable=1, satisfiable on one node.
32 - Workloads present: forgejo (+ cnpg forgejo-pg), agents, mcp, minio, traefik,
33 sablier, registry-mirror, flux, kyverno, keda, cert-manager, cnpg-system,
34 backup cronjobs, tenant app `comed-fleet--www--pr-4` (www-pg cnpg cluster).
35
36 Single-node feasibility math (requests, daemonsets counted once):
37 - As-is: ~4.34 cores requested vs ~3.86 allocatable → **does not fit**.
38 - Minus monitoring (~770m): ~3.58 cores → fits. Memory ~5.4 GB requested → fits easily.
39
40 Bicep note: `deploy/azure/main.bicep` pins `aksVmSize = 'Standard_D4ds_v6'`,
41 `aksNodeCount` default (2, "POC floor"), and wires omsagent → `mitosis-logs`
42 (line ~336). A re-germinate will restore all of today's baseline unless the
43 bicep is updated to match whatever end-state we keep.
44
45 ---
46
47 ## Step 1 — disable Container Insights (omsagent addon)
48
49 Removes ama-logs daemonset + ama-logs-rs (~340m CPU / ~860Mi requests) and stops
50 Log Analytics ingestion billing.
51
52 ```sh
53 az aks disable-addons -g mitosis-aks -n mitosis-aks -a monitoring
54 ```
55
56 **Rollback:**
57 ```sh
58 az aks enable-addons -g mitosis-aks -n mitosis-aks -a monitoring \
59 --workspace-resource-id /subscriptions/1d655bbf-4fc9-48dd-bf75-2d3f72bb64a1/resourceGroups/mitosis-aks/providers/Microsoft.OperationalInsights/workspaces/mitosis-logs
60 ```
61 (The workspace `mitosis-logs` itself is NOT deleted by this step, so rollback is lossless
62 going forward; only the gap window has no logs.)
63
64 Status: **DONE 2026-08-18** — omsagent.enabled=false, all ama-logs pods gone.
65
66 ## Step 2 — disable managed Prometheus (azureMonitorMetrics)
67
68 Removes ama-metrics deployments + ama-metrics-node daemonset (~425m CPU / ~1.4Gi requests).
69 Grafana-on-Azure dashboards fed by this stop updating (in-cluster metrics-server is
70 unaffected; `kubectl top` keeps working).
71
72 ```sh
73 az aks update -g mitosis-aks -n mitosis-aks --disable-azure-monitor-metrics
74 ```
75
76 **Rollback:**
77 ```sh
78 az aks update -g mitosis-aks -n mitosis-aks --enable-azure-monitor-metrics
79 ```
80
81 Status: **DONE 2026-08-18** — metrics.enabled=false. Azure printed a warning that it
82 deletes `azmonitoring.coreos.com` custom resources (PodMonitor/ServiceMonitor CRs from
83 the managed addon); rollback recreates the defaults automatically.
84
85 ## Step 3 — scale node pool 2 → 1
86
87 ```sh
88 az aks nodepool scale -g mitosis-aks --cluster-name mitosis-aks -n system --node-count 1
89 ```
90
91 Expected: VMSS removes one node; pods reschedule onto the survivor. Duplicate
92 replicas (coredns, metrics-server, konnectivity, kyverno-admission ×3,
93 cilium-operator, azure-wi-webhook) co-locate — anti-affinity is preferred-only.
94 Brief blips while forgejo / traefik reschedule if they were on the removed node.
95
96 **Rollback:**
97 ```sh
98 az aks nodepool scale -g mitosis-aks --cluster-name mitosis-aks -n system --node-count 2
99 ```
100
101 **Verify after:** all pods Running (no Pending), `curl https://git.aks.benjivers.com/api/healthz`
102 → 200, `flux get kustomizations` all Ready, tenant app pods healthy.
103
104 Status: PENDING
105
106 ## Step 4 — cheaper VM size (EXECUTED 2026-08-18, landed on D4as_v7)
107
108 VM size cannot be changed in place; blue/green nodepool swap required:
109
110 | SKU | eastus $/hr | ~$/mo (1 node) | notes |
111 |---|---|---|---|
112 | Standard_D4ds_v6 (current) | 0.249 | 182 | local SSD → ephemeral OS disk (no disk cost) |
113 | Standard_D4as_v6 | 0.182 | 133 | AMD, no local disk → needs managed OS disk (+~$10/mo) |
114 | Standard_B4as_v2 | 0.150 | 110 | burstable 4vCPU/16GB; fine for this idle profile; managed OS disk |
115 | Standard_D2ds_v6 | 0.125 | 91 | 2vCPU/8GB — memory too small at current usage (~10GB) |
116
117 Procedure (when we do it):
118 ```sh
119 # add new System-mode pool
120 az aks nodepool add -g mitosis-aks --cluster-name mitosis-aks -n system2 \
121 --mode System --node-count 1 --node-vm-size Standard_B4as_v2
122 # drain + delete old pool
123 kubectl --context aks-mitosis cordon -l agentpool=system
124 kubectl --context aks-mitosis drain -l agentpool=system --ignore-daemonsets --delete-emptydir-data
125 az aks nodepool delete -g mitosis-aks --cluster-name mitosis-aks -n system
126 ```
127 **Rollback:** re-add a `system` pool with the original size
128 (`--node-vm-size Standard_D4ds_v6 --node-osdisk-type Ephemeral --node-osdisk-size 80 --zones 2 3`)
129 and delete `system2`. Use the `az rest` form below if the CLI crashes.
130
131 Status: **DONE 2026-08-18** — but NOT with B4as_v2. What actually happened:
132
133 - **Quota**: subscription has 0 vCPU quota for ALL B-v2 families (Basv2/Bsv2/Bpsv2)
134 in eastus → B4as_v2 impossible without a quota request. Old BSFamily (B4ms,
135 $0.166/hr) has 10 free vCPUs but B4ms sustained baseline is only ~0.9 cores —
136 platform idles at 1.5–1.8 cores → would throttle. Rejected.
137 - **Chosen**: Standard_D4as_v7 ($0.182/hr ≈ $133/mo), StandardDasv7Family had 6
138 free vCPUs. Full-performance, AMD, no local disk → Managed OS disk 64 GB.
139 - **az CLI bug**: `az aks nodepool add` crashes client-side
140 (`AttributeError: 'AgentPool' object has no attribute 'artifact_streaming_profile'`,
141 azure-cli SDK-model mismatch) even with --no-wait. Workaround — raw ARM call:
142 ```sh
143 az rest --method put --url "https://management.azure.com/subscriptions/1d655bbf-4fc9-48dd-bf75-2d3f72bb64a1/resourceGroups/mitosis-aks/providers/Microsoft.ContainerService/managedClusters/mitosis-aks/agentPools/system2?api-version=2025-05-01" \
144 --body '{"properties":{"mode":"System","count":1,"vmSize":"Standard_D4as_v7","osDiskSizeGB":64,"osDiskType":"Managed","osType":"Linux","availabilityZones":["2"]}}'
145 ```
146 (`az aks nodepool scale/show/list/delete` still work; only add is broken. Delete
147 old pool likewise done via `az rest --method delete`.)
148 - **Zone safety**: all PVs are StandardSSD_ZRS with multi-zone nodeAffinity, so
149 the new pool's zone didn't matter. Verified before the swap.
150 - Swap procedure used: create system2 → wait Ready → `kubectl cordon` +
151 `drain --disable-eviction` old node (PDB bypass again) → `az rest` DELETE old
152 `system` pool.
153 - **Transient after swap**: platform app images pull from git.aks.benjivers.com's
154 own Forgejo registry; while forgejo relocated nodes the registry 503'd →
155 ImagePullBackOff on agents/tenant pods for ~3 min, self-healed once forgejo was up.
156 Chicken-and-egg to remember for any single-node move.
157 - **NEW HARD CONSTRAINT — disk attach limit**: D4as_v7 attaches max **12** Azure
158 disks (old D4ds_v6: 16). Cluster already has **18** disk PVs; it only works
159 because hibernated PR apps keep their disks detached. At >12 concurrently-awake
160 disk consumers, pods will stick Pending "exceed max volume count". Mitigations:
161 move small volumes to azurefile (no attach limit), keep PR apps hibernated, or
162 go back to a Ddsv6-family SKU.
163
164 ## Step 5 (planned) — trim workload requests via gitops
165
166 Biggest non-system requests: agents 500m/1Gi, forgejo 250m/512Mi + pg 100m/512Mi,
167 traefik 200m/150Mi, kyverno-admission 3×100m (replicas → 1 saves 200m/256Mi).
168 These are Flux/Helm-managed — change in the gitops repo, not kubectl, or Flux
169 reverts it. Do after Steps 1–3 prove stable.
170
171 Status: NOT STARTED
172
173 ---
174
175 ## Execution log
176
177 - 2026-08-18: Step 1 done. `az aks disable-addons -a monitoring` → omsagent disabled, 0 ama pods left.
178 - 2026-08-18: Step 2 done. `--disable-azure-monitor-metrics` → managed Prometheus off.
179 Combined CPU requests fell 5.0 → 4.0 cores (1820m + 2167m across the 2 nodes).
180 - 2026-08-18: Step 3 attempt 1 **FAILED**: plain `az aks nodepool scale --node-count 1`
181 errored (pool provisioningState=Failed, node auto-uncordoned). Root cause: AKS drain
182 respects PDBs, and every CNPG cluster (forgejo-pg + each comed-fleet tenant pg) has
183 minAvailable=1 with 1 replica → 0 allowed disruptions → eviction blocked → timeout.
184 **Lesson: any future node op (scale/upgrade/resize) on this cluster needs a manual
185 force-drain first.**
186 - 2026-08-18: Fix applied:
187 `kubectl cordon aks-system-40081872-vmss000001`
188 `kubectl drain aks-system-40081872-vmss000001 --ignore-daemonsets --delete-emptydir-data --disable-eviction --grace-period=30 --timeout=180s`
189 (`--disable-eviction` deletes pods directly, bypassing PDBs; CNPG rebuilt each pg pod
190 on node 000000 within ~1 min. git.aks.benjivers.com healthz went 503 for ~1 min while
191 forgejo-pg restarted, back to 200 after.)
192 - 2026-08-18: `az aks nodepool delete-machines --machine-names ...vmss000001` REJECTED
193 (InvalidParameter: won't delete "all" system nodes — it ignores the cordoned node when
194 counting). Not usable here; plain scale-in works instead (VMSS default scale-in policy
195 removes the highest instance ID = the drained 000001).
196 - 2026-08-18: Step 3 attempt 2 — re-ran `az aks nodepool scale --node-count 1` after the
197 manual drain; in progress.
198 - Note: comed-fleet tenant apps (design/extract/invoices/m5bridge/www, prod + PRs) were
199 actively deploying during the scale-down — pg pod churn in those namespaces is a mix of
200 CNPG re-init and live fleet deploys, not all caused by us.
201 - 2026-08-18: Step 3 **DONE** — pool at 1 node (vmss000000), provisioningState Succeeded,
202 healthz 200. Forgejo took ~1 min of 503 during the pg force-drain, recovered on its own.
203 - 2026-08-18: **Post-scale incident**: node CPU requests hit 3827m (99%) and
204 comed-fleet--www--prod/www-pg-1 stuck FailedScheduling "Insufficient cpu" (CNPG kept
205 deleting/recreating the Pending pod — that's what the 0s-age pod churn was). Actual CPU
206 *usage* was only ~46%; this is request over-provisioning, not real load.
207 Quick fix applied (kubectl, NOT in git — see revert-risk note):
208 `kubectl scale deploy -n kyverno kyverno-admission-controller --replicas=1` (frees 200m)
209 `kubectl scale deploy -n kube-system metrics-server --replicas=1` (frees 156m)
210 → www-pg-1 scheduled and Running; requests down to 3727m (96%).
211 **Revert-risk**: kyverno replica count is Helm/Flux-managed — reverts on next
212 HelmRelease upgrade; metrics-server is an AKS managed addon — the addon-manager may
213 restore replicas=2 at any time. Make durable via gitops (Step 5) if single-node stays.
214 Rollback of the quick fix: same commands with `--replicas=3` / `--replicas=2`.
215 - Known-benign on 1 node: second cilium-operator replica Pending (hostPort conflict —
216 only one node to bind on). Clears itself if we ever scale back to 2.
217 - 2026-08-18: **Churn classification** — the constant comed-fleet pg pod recreation is NOT
218 from the scale-down: CNPG shows every tenant cluster in "Primary instance is being
219 restarted without a switchover" / UpgradingInstance loops, and those clusters are only
220 14–42 min old. The agent fleet was actively deploying/upgrading comed-fleet apps
221 throughout the experiment; each deploy rolls its pg. forgejo-pg: healthy.
222 Pre-existed the node change (seen in the very first pod snapshot).
223
224 ## End state (2026-08-18, after Step 4)
225
226 - **1× Standard_D4as_v7** (pool `system2`, zone 2, 64 GB managed OS disk), monitoring
227 addons off → **~$133/mo compute + ~$5/mo OS disk + $0 ingestion**
228 (baseline was ~$364/mo + Log Analytics ingestion → **~62% cut**).
229 - healthz 200, forgejo + core platform healthy, backups green (nightly dumps Complete,
230 weekly restore-drill scheduled).
231 - Node: CPU requests ~99% booked (usage far lower). Memory fine. Disk-attach ceiling
232 12 (18 PVs exist; OK only while PR apps stay hibernated).
233
234 ## Client-readiness findings (2026-08-18 probe)
235
236 1. **CRITICAL — CNPG restart loop on every tenant DB**: kyverno `aks-image-redirect`
237 mutates pod images to ACR, but CNPG Cluster spec still says ghcr.io → CNPG sees
238 spec≠pod, restarts primary forever ("Primary instance is being restarted without a
239 switchover" on all 9 comed-fleet clusters; tenant apps 503 with no ready DB).
240 forgejo-pg is immune (spec already ACR). Fix: extend aks-image-redirect to also
241 mutate `clusters.postgresql.cnpg.io` spec.imageName (keeps sovereignty policy
242 intact) — one-off `kubectl patch` of each Cluster's imageName to the ACR name
243 proves it (patch attempt was permission-blocked this session; needs Ben).
244 2. **Tenant TLS is the internal CA**: tenant ingresses use `wildcard-plat-tls`
245 (issuer plat-local-ca) → browsers reject; forgejo/mcp use the public LE
246 `wildcard-public-tls` (valid to 2026-11-16). Fix: tenant ingress template should
247 reference the public wildcard (reflect/copy it into tenant namespaces).
248 3. **CPU requests 99%** — no headroom for new tenants; needs Step 5 request trim.
249 4. **Disk attach ceiling** — see Step 4 note (12 < 18 PVs).
250 5. **CNPG PDBs block every AKS node op** (scale/upgrade) — manual
251 `drain --disable-eviction` required each time; platform-level fix desirable.
252 6. **No monitoring/alerting now** — we turned off both Azure layers; client-ready
253 needs at least cheap in-cluster alerting or re-enable metrics only.
254 7. **bicep drift** — main.bicep still pins 2×D4ds_v6 + omsagent; re-germinate
255 resurrects the $364/mo baseline. Update aksVmSize/aksNodeCount/omsagent.
256 8. az CLI `aks nodepool add` broken locally — use the az rest form (Step 4).
257
258 ## Full restore to baseline (if it all goes wrong)
259
260 ```sh
261 az aks nodepool scale -g mitosis-aks --cluster-name mitosis-aks -n system --node-count 2
262 az aks enable-addons -g mitosis-aks -n mitosis-aks -a monitoring \
263 --workspace-resource-id /subscriptions/1d655bbf-4fc9-48dd-bf75-2d3f72bb64a1/resourceGroups/mitosis-aks/providers/Microsoft.OperationalInsights/workspaces/mitosis-logs
264 az aks update -g mitosis-aks -n mitosis-aks --enable-azure-monitor-metrics
265 kubectl --context aks-mitosis scale deploy -n kyverno kyverno-admission-controller --replicas=3
266 kubectl --context aks-mitosis scale deploy -n kube-system metrics-server --replicas=2
267 ```
268 (order doesn't matter except: scale nodes FIRST if pods are stuck Pending)
deploy/azure/entra-forgejo-oidc.sh added +54
@@ -0,0 +1,54 @@
1 #!/usr/bin/env bash
2 # entra-forgejo-oidc.sh — wire Entra ID sign-in into a germinated aks-ground
3 # Forgejo. Run AFTER germination, by an operator with rights to create app
4 # registrations in the tenant (customer ops runs this in their own tenant).
5 #
6 # deploy/azure/entra-forgejo-oidc.sh <domain> [<display-name>]
7 #
8 # Creates (idempotently) an Entra app registration with the Forgejo OIDC
9 # redirect URI, a client secret (2y), and registers a generic OpenID Connect
10 # authentication source in Forgejo via the admin API. Users then sign in with
11 # "Entra ID" on git.<domain>; local accounts keep working (break-glass admin).
12 set -euo pipefail
13
14 DOMAIN="${1:?usage: entra-forgejo-oidc.sh <domain> [display-name]}"
15 NAME="${2:-mitosis-forgejo-$DOMAIN}"
16 REDIRECT="https://git.$DOMAIN/user/oauth2/entra/callback"
17 TENANT=$(az account show --query tenantId -o tsv)
18
19 APP_ID=$(az ad app list --display-name "$NAME" --query "[0].appId" -o tsv)
20 if [ -z "$APP_ID" ] || [ "$APP_ID" = "null" ]; then
21 echo "==> creating Entra app registration '$NAME'"
22 APP_ID=$(az ad app create --display-name "$NAME" \
23 --web-redirect-uris "$REDIRECT" \
24 --enable-id-token-issuance true \
25 --query appId -o tsv)
26 else
27 echo "==> reusing Entra app '$NAME' ($APP_ID); ensuring redirect URI"
28 az ad app update --id "$APP_ID" --web-redirect-uris "$REDIRECT"
29 fi
30
31 echo "==> minting client secret (2 years)"
32 SECRET=$(az ad app credential reset --id "$APP_ID" --display-name forgejo-oidc --years 2 --query password -o tsv)
33
34 # Forgejo admin API needs the platform admin credential — read it from the
35 # cluster the way the operator card documents (never echoed).
36 : "${KUBECONFIG:?export KUBECONFIG for the aks cluster first (az aks get-credentials)}"
37 ADMIN_PW=$(kubectl get secret -n forgejo forgejo-admin -o json | jq -r '.data.password' | base64 -d)
38
39 DISCOVERY="https://login.microsoftonline.com/$TENANT/v2.0/.well-known/openid-configuration"
40 echo "==> registering OIDC auth source 'entra' in Forgejo (discovery: tenant $TENANT)"
41 # The stable interface across Forgejo versions is the admin CLI inside the pod
42 # (the REST path for auth sources has moved between releases).
43 POD=$(kubectl get pod -n forgejo -l app=forgejo -o jsonpath='{.items[0].metadata.name}')
44 kubectl exec -n forgejo "$POD" -- forgejo admin auth add-oauth \
45 --name entra --provider openidConnect \
46 --key "$APP_ID" --secret "$SECRET" \
47 --auto-discover-url "$DISCOVERY" \
48 2>/dev/null \
49 || kubectl exec -n forgejo "$POD" -- forgejo admin auth update-oauth \
50 --id "$(kubectl exec -n forgejo "$POD" -- forgejo admin auth list 2>/dev/null | awk '/entra/{print $1; exit}')" \
51 --key "$APP_ID" --secret "$SECRET" --auto-discover-url "$DISCOVERY"
52
53 echo "==> done. 'Sign in with entra' is live on https://git.$DOMAIN"
54 : "$ADMIN_PW" # reserved: REST fallback for Forgejo versions without the auth CLI
deploy/azure/main.bicep +275 −37
@@ -1,62 +1,109 @@
1 // Mitosis on Azure — bare VMs, one deployment.
1 // Mitosis on Azure — one deployment, two grounds.
22 //
33 // az group create -n mitosis -l eastus
4 //
5 // VMs (sovereign k3s, the default):
46 // az deployment group create -g mitosis -f main.bicep \
57 // -p domain=plat.example.com -p adminSshPublicKey="$(cat ~/.ssh/id_ed25519.pub)"
68 //
7 // One VM by default; -p ha=true grows three k3s servers (embedded etcd).
8 // The leader germinates the platform; every node is born with the registry
9 // recipe and its own copy of the CI runner image. See ../README.md.
9 // AKS (Azure-pure: managed cluster + ACR + Azure DNS + Log Analytics + Key Vault):
10 // az deployment group create -g mitosis-aks -f main.bicep \
11 // -p domain=plat.example.com -p mode=aks
1012 //
11 // The node contract (env file + bootstrap script + unit + registry recipe) is
12 // defined ONCE in deploy/lib + recipe/ — this file renders the same template
13 // Terraform does, with replace() instead of templatefile().
13 // mode=vms: one VM by default; -p ha=true grows three k3s servers (embedded
14 // etcd). The leader germinates the platform; every node is born with the
15 // registry recipe and its own copy of the CI runner image. See ../README.md.
16 //
17 // mode=aks: a managed cluster the operator germinates AFTERWARDS with
18 // CLUSTER_PROFILE=aks bin/germinate (no cloud-init role — there are no nodes
19 // to bootstrap). Ships ACR (infra images + ci-runner via `az acr build`),
20 // a public Azure DNS zone for the domain (delegate NS at your registrar),
21 // Log Analytics + managed Prometheus (Azure Monitor replaces the k3s
22 // recipe's Grafana), Key Vault (sops key custody), and workload identities
23 // for cert-manager (DNS-01) and Flux (Key Vault sops decryption).
24 //
25 // The VM node contract (env file + bootstrap script + unit + registry recipe)
26 // is defined ONCE in deploy/lib + recipe/ — this file renders the same
27 // template Terraform does, with replace() instead of templatefile().
1428
1529 @description('The platform\'s sovereign domain (you own its DNS), e.g. plat.example.com.')
1630 param domain string
1731
18 @description('false = one VM. true = three k3s servers with embedded etcd (survives one node loss).')
32 @description('vms = self-managed k3s on bare VMs (sovereign default). aks = Azure-pure managed cluster (AKS + ACR + Azure DNS + Log Analytics + Key Vault); germinate with CLUSTER_PROFILE=aks after deploy.')
33 @allowed(['vms', 'aks'])
34 param mode string = 'vms'
35
36 // ── vms-mode parameters ──────────────────────────────────────────────────────
37
38 @description('vms: false = one VM. true = three k3s servers with embedded etcd (survives one node loss).')
1939 param ha bool = false
2040
21 @description('Your SSH public key (contents of ~/.ssh/id_ed25519.pub) — how you reach the VMs.')
22 param adminSshPublicKey string
41 @description('Your SSH public key (contents of ~/.ssh/id_ed25519.pub) — how you reach the VMs. REQUIRED for mode=vms; unused for mode=aks.')
42 param adminSshPublicKey string = ''
2343
2444 @description('VM login user (Azure images have no root login).')
2545 param adminUsername string = 'mitosis'
2646
27 @description('4 vCPU / 8 GB is the floor (germination builds several images). D4as_v7 = 4 vCPU / 16 GB and matches the quota fresh subscriptions actually get (v6/v7 + B-series; check az vm list-usage). B4ms is the burstable budget option; D4as_v5 for older subscriptions.')
47 @description('vms: 4 vCPU / 8 GB is the floor (germination builds several images). D4as_v7 = 4 vCPU / 16 GB and matches the quota fresh subscriptions actually get (v6/v7 + B-series; check az vm list-usage). B4ms is the burstable budget option; D4as_v5 for older subscriptions.')
2848 param vmSize string = 'Standard_D4as_v7'
2949
30 @description('OS disk size. Images + Postgres + the platform registry live here; 80 GB is a comfortable floor.')
50 @description('vms: OS disk size. Images + Postgres + the platform registry live here; 80 GB is a comfortable floor.')
3151 param diskGb int = 80
3252
33 @description('Source allowed to SSH (port 22). Tighten to your IP, e.g. 203.0.113.7/32.')
53 @description('vms: source allowed to SSH (port 22). Tighten to your IP, e.g. 203.0.113.7/32.')
3454 param allowedSshCidr string = '*'
3555
36 @description('Claude subscription credential for the build crew. Set exactly one of this or anthropicApiKey (or neither — the crew idles until you seal one later).')
56 @description('Claude subscription credential for the build crew. Set exactly one of this or anthropicApiKey (or neither — the crew idles until you seal one later). vms mode only; on aks the credential is sealed at germination.')
3757 @secure()
3858 param claudeCodeOauthToken string = ''
3959
40 @description('API-billing credential for the build crew (alternative to claudeCodeOauthToken).')
60 @description('API-billing credential for the build crew (alternative to claudeCodeOauthToken). vms mode only.')
4161 @secure()
4262 param anthropicApiKey string = ''
4363
44 @description('k3s cluster join token. Auto-generated per deployment — pass your own to keep re-deployments from replacing the VMs.')
64 @description('k3s cluster join token. Auto-generated per deployment — pass your own to keep re-deployments from replacing the VMs. vms mode only.')
4565 @secure()
4666 param k3sToken string = newGuid()
4767
48 @description('k3s release the nodes install — matches the version the platform is validated on.')
68 @description('k3s release the nodes install — matches the version the platform is validated on. vms mode only.')
4969 param k3sVersion string = 'v1.31.5+k3s1'
5070
51 @description('Where nodes clone mitosis from. Point at your own platform\'s mitosis repo to grow a child of YOUR platform.')
71 @description('Where nodes clone mitosis from. Point at your own platform\'s mitosis repo to grow a child of YOUR platform. vms mode only (aks germination clones on the operator host).')
5272 param mitosisUrl string = 'https://git.open-platform.sh/plat/mitosis'
5373
54 @description('Branch or tag of mitosis to deploy.')
74 @description('Branch or tag of mitosis to deploy. vms mode only.')
5575 param mitosisRef string = 'main'
5676
77 // ── aks-mode parameters ──────────────────────────────────────────────────────
78
79 @description('aks: node count for the (single, System-mode) pool. 2 is the POC floor; germination CI builds want the headroom.')
80 param aksNodeCount int = 2
81
82 @description('aks: node size. Pick a family with free regional quota (az vm list-usage); D4ds_v6 = 4 vCPU / 16 GB + temp disk.')
83 param aksVmSize string = 'Standard_D4ds_v6'
84
85 @description('aks: availability zones for the node pool. Empty array = non-zonal (regions without zones).')
86 param aksZones array = ['2', '3']
87
88 @description('aks: Kubernetes version. Empty = the AKS default for the region.')
89 param aksVersion string = ''
90
91 @description('aks: create a public Azure DNS zone for the domain (delegate NS records at your registrar). false = bring your own zone.')
92 param createDnsZone bool = true
93
94 @description('aks: ACR name (globally unique, alphanumeric). Default derives from the resource group.')
95 param acrName string = 'mitosis${uniqueString(resourceGroup().id)}'
96
97 @description('aks: Key Vault name (globally unique, 3-24 chars). Default derives from the resource group.')
98 param keyVaultName string = 'mkv${uniqueString(resourceGroup().id)}'
99
57100 param location string = resourceGroup().location
58101
59 var nodeCount = ha ? 3 : 1
102 // ═════════════════════════════════ vms mode ═════════════════════════════════
103
104 // nodeCount 0 in aks mode empties every vms-mode loop — same template, no drift
105 // for existing VM deployments (mode defaults to vms; resources render identically).
106 var nodeCount = mode == 'vms' ? (ha ? 3 : 1) : 0
60107 // 10.60/24 is clear of k3s's 10.42/16 (pods) + 10.43/16 (services). Static
61108 // private IPs (.10/.11/.12): every node knows the leader with no discovery.
62109 var subnetCidr = '10.60.0.0/24'
@@ -77,7 +124,7 @@ var cloudInitTmpl = loadTextContent('../lib/cloud-init.yaml.tmpl')
77124 var scriptB64 = base64(loadTextContent('../lib/mitosis-node.sh'))
78125 var unitB64 = base64(loadTextContent('../lib/mitosis-node.service'))
79126
80 resource nsg 'Microsoft.Network/networkSecurityGroups@2024-05-01' = {
127 resource nsg 'Microsoft.Network/networkSecurityGroups@2024-05-01' = if (mode == 'vms') {
81128 name: 'mitosis-nodes'
82129 location: location
83130 properties: {
@@ -114,7 +161,7 @@ resource nsg 'Microsoft.Network/networkSecurityGroups@2024-05-01' = {
114161 }
115162 }
116163
117 resource vnet 'Microsoft.Network/virtualNetworks@2024-05-01' = {
164 resource vnet 'Microsoft.Network/virtualNetworks@2024-05-01' = if (mode == 'vms') {
118165 name: 'mitosis'
119166 location: location
120167 properties: {
@@ -240,23 +287,214 @@ resource vm 'Microsoft.Compute/virtualMachines@2024-07-01' = [
240287 }
241288 ]
242289
290 // ═════════════════════════════════ aks mode ═════════════════════════════════
291 // A managed ground: no cloud-init, no node contract — the operator germinates
292 // against the kubeconfig (CLUSTER_PROFILE=aks). Everything a strict
293 // "Azure-pure" bar demands is provisioned here so germinate only consumes IDs.
294
295 resource logs 'Microsoft.OperationalInsights/workspaces@2023-09-01' = if (mode == 'aks') {
296 name: 'mitosis-logs'
297 location: location
298 properties: {
299 sku: { name: 'PerGB2018' }
300 retentionInDays: 30
301 }
302 }
303
304 resource acr 'Microsoft.ContainerRegistry/registries@2023-07-01' = if (mode == 'aks') {
305 name: acrName
306 location: location
307 sku: { name: 'Standard' }
308 properties: {
309 adminUserEnabled: false
310 }
311 }
312
313 resource aks 'Microsoft.ContainerService/managedClusters@2024-05-01' = if (mode == 'aks') {
314 name: 'mitosis-aks'
315 location: location
316 identity: { type: 'SystemAssigned' }
317 properties: {
318 dnsPrefix: 'mitosis'
319 kubernetesVersion: empty(aksVersion) ? null : aksVersion
320 agentPoolProfiles: [
321 {
322 name: 'system'
323 mode: 'System'
324 count: aksNodeCount
325 vmSize: aksVmSize
326 osType: 'Linux'
327 osDiskSizeGB: 80
328 availabilityZones: empty(aksZones) ? null : aksZones
329 }
330 ]
331 // Workload identity: cert-manager (DNS-01) and Flux (Key Vault sops)
332 // federate below against this issuer — no static credentials in-cluster.
333 oidcIssuerProfile: { enabled: true }
334 securityProfile: { workloadIdentity: { enabled: true } }
335 networkProfile: {
336 networkPlugin: 'azure'
337 networkPluginMode: 'overlay'
338 networkDataplane: 'cilium'
339 // Kyverno GENERATES per-tenant NetworkPolicies (plat-ns-network-governance);
340 // cilium is the overlay-compatible enforcer that actually honors them.
341 networkPolicy: 'cilium'
342 }
343 // Container Insights (logs) — the Azure-pure replacement for the k3s
344 // recipe's Grafana, together with managed Prometheus below.
345 addonProfiles: {
346 omsagent: {
347 enabled: true
348 config: {
349 logAnalyticsWorkspaceResourceID: logs.id
350 useAADAuth: 'true'
351 }
352 }
353 }
354 // Managed Prometheus (metrics into Azure Monitor workspace-less mode).
355 azureMonitorProfile: {
356 metrics: {
357 enabled: true
358 kubeStateMetrics: {}
359 }
360 }
361 }
362 }
363
364 // kubelet pulls infra images + ci-runner from ACR — AcrPull on the kubelet identity.
365 resource acrPull 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (mode == 'aks') {
366 name: guid(resourceGroup().id, 'acrpull', acrName)
367 scope: acr
368 properties: {
369 principalId: aks.properties.identityProfile.kubeletidentity.objectId
370 roleDefinitionId: subscriptionResourceId(
371 'Microsoft.Authorization/roleDefinitions',
372 '7f951dda-4ed3-4680-a7ca-43fe172d538d' // AcrPull
373 )
374 principalType: 'ServicePrincipal'
375 }
376 }
377
378 // Public zone for the platform domain — delegate its NS records at the
379 // registrar; cert-manager answers DNS-01 challenges here via workload identity.
380 resource dnsZone 'Microsoft.Network/dnsZones@2023-07-01-preview' = if (mode == 'aks' && createDnsZone) {
381 name: domain
382 location: 'global'
383 }
384
385 resource certManagerId 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = if (mode == 'aks') {
386 name: 'mitosis-cert-manager'
387 location: location
388 }
389
390 resource certManagerFed 'Microsoft.ManagedIdentity/userAssignedIdentities/federatedIdentityCredentials@2023-01-31' = if (mode == 'aks') {
391 parent: certManagerId
392 name: 'cert-manager'
393 properties: {
394 issuer: aks.properties.oidcIssuerProfile.issuerURL
395 subject: 'system:serviceaccount:cert-manager:cert-manager'
396 audiences: ['api://AzureADTokenExchange']
397 }
398 }
399
400 resource certManagerDns 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (mode == 'aks' && createDnsZone) {
401 name: guid(resourceGroup().id, 'certmgr-dns', domain)
402 scope: dnsZone
403 properties: {
404 principalId: certManagerId.properties.principalId
405 roleDefinitionId: subscriptionResourceId(
406 'Microsoft.Authorization/roleDefinitions',
407 'befefa01-2a29-4197-83a8-272ff33ce314' // DNS Zone Contributor
408 )
409 principalType: 'ServicePrincipal'
410 }
411 }
412
413 // Key Vault holds the sops KMS key at pilot hardening (sovereign custody the
414 // Azure way). Created now so the identity plumbing ships with the ground.
415 resource kv 'Microsoft.KeyVault/vaults@2023-07-01' = if (mode == 'aks') {
416 name: keyVaultName
417 location: location
418 properties: {
419 sku: { family: 'A', name: 'standard' }
420 tenantId: subscription().tenantId
421 enableRbacAuthorization: true
422 enableSoftDelete: true
423 }
424 }
425
426 resource fluxSopsId 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = if (mode == 'aks') {
427 name: 'mitosis-flux-sops'
428 location: location
429 }
430
431 resource fluxSopsFed 'Microsoft.ManagedIdentity/userAssignedIdentities/federatedIdentityCredentials@2023-01-31' = if (mode == 'aks') {
432 parent: fluxSopsId
433 name: 'kustomize-controller'
434 properties: {
435 issuer: aks.properties.oidcIssuerProfile.issuerURL
436 subject: 'system:serviceaccount:flux-system:kustomize-controller'
437 audiences: ['api://AzureADTokenExchange']
438 }
439 }
440
441 resource fluxSopsKv 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (mode == 'aks') {
442 name: guid(resourceGroup().id, 'flux-sops-kv', keyVaultName)
443 scope: kv
444 properties: {
445 principalId: fluxSopsId.properties.principalId
446 roleDefinitionId: subscriptionResourceId(
447 'Microsoft.Authorization/roleDefinitions',
448 '12338af0-0e69-4776-bea7-57ae8d297424' // Key Vault Crypto User
449 )
450 principalType: 'ServicePrincipal'
451 }
452 }
453
454 // ═════════════════════════════════ outputs ══════════════════════════════════
455
243456 // for-expressions may not nest inside join()/interpolation, and variable
244457 // loops can't touch runtime values — so the joined list is a ternary (ARM's
245458 // if() only evaluates the taken branch, so pip[1]/pip[2] resolve only in HA).
246 var allIps = ha
247 ? '${pip[0].properties.ipAddress}, ${pip[1].properties.ipAddress}, ${pip[2].properties.ipAddress}'
248 : pip[0].properties.ipAddress
459 var allIps = mode != 'vms'
460 ? ''
461 : ha
462 ? '${pip[0].properties.ipAddress}, ${pip[1].properties.ipAddress}, ${pip[2].properties.ipAddress}'
463 : pip[0].properties.ipAddress
464 var leaderIp = mode == 'vms' ? pip[0].properties.ipAddress : ''
249465
466 output mode string = mode
250467 output nodeIps array = [for i in range(0, nodeCount): pip[i].properties.ipAddress]
251 output dnsRecord string = '*.${domain} A ${allIps}'
252 output nextSteps string = join(
253 [
254 'Germination runs ~30-45 min on the leader. From here:'
255 ' watch it: ssh ${adminUsername}@${pip[0].properties.ipAddress} journalctl -u mitosis-node -f'
256 ' DNS: create *.${domain} A ${allIps}'
257 ' when done: ssh ${adminUsername}@${pip[0].properties.ipAddress} sudo cat /root/PLATFORM.md'
258 'The sovereign key + admin logins are on the leader — PLATFORM.md walks you through'
259 'copying the key somewhere safe (do it first).'
260 ],
261 '\n'
262 )
468 output dnsRecord string = mode == 'vms'
469 ? '*.${domain} A ${allIps}'
470 : 'delegate NS for ${domain} to the dnsNameServers output; apps resolve via the zone'
471 output acrLoginServer string = mode == 'aks' ? acr.properties.loginServer : ''
472 output aksName string = mode == 'aks' ? aks.name : ''
473 output oidcIssuer string = mode == 'aks' ? aks.properties.oidcIssuerProfile.issuerURL : ''
474 output dnsNameServers array = mode == 'aks' && createDnsZone ? dnsZone.properties.nameServers : []
475 output keyVaultName string = mode == 'aks' ? kv.name : ''
476 output certManagerClientId string = mode == 'aks' ? certManagerId.properties.clientId : ''
477 output fluxSopsClientId string = mode == 'aks' ? fluxSopsId.properties.clientId : ''
478 output nextSteps string = mode == 'vms'
479 ? join(
480 [
481 'Germination runs ~30-45 min on the leader. From here:'
482 ' watch it: ssh ${adminUsername}@${leaderIp} journalctl -u mitosis-node -f'
483 ' DNS: create *.${domain} A ${allIps}'
484 ' when done: ssh ${adminUsername}@${leaderIp} sudo cat /root/PLATFORM.md'
485 'The sovereign key + admin logins are on the leader — PLATFORM.md walks you through'
486 'copying the key somewhere safe (do it first).'
487 ],
488 '\n'
489 )
490 : join(
491 [
492 'AKS ground is up. From here:'
493 ' 1. delegate NS for ${domain} at your registrar (dnsNameServers output)'
494 ' 2. mirror infra images: deploy/azure/mirror-images.sh <acrLoginServer>'
495 ' 3. kubeconfig: az aks get-credentials -g <rg> -n mitosis-aks'
496 ' 4. germinate: CLUSTER_PROFILE=aks ACR=<acrLoginServer> DOMAIN=${domain} bin/germinate'
497 'The sovereign key is minted on YOUR machine during germination - back it up first thing.'
498 ],
499 '\n'
500 )
deploy/azure/mirror-images.sh added +70
@@ -0,0 +1,70 @@
1 #!/usr/bin/env bash
2 # mirror-images.sh — copy the platform's external infra images into ACR.
3 #
4 # deploy/azure/mirror-images.sh <acr-name-or-login-server>
5 #
6 # Part of the AKS ("Azure-pure") ground: nodes pull ONLY from ACR and from the
7 # platform's own registry (git.<domain>). This imports every third-party image
8 # the recipe pins so no pod ever reaches docker.io/ghcr/quay at runtime.
9 # `az acr import` copies server-side — nothing is pulled through this machine.
10 #
11 # Version pins mirror clusters/local/platform/*/helmrelease.yaml at the vintage
12 # this script last shipped; germinate's aks profile rewrites the HelmRelease
13 # values to $ACR/<same path>. Keep the two in sync when bumping charts.
14 # k3s-only images (rancher/klipper-lb, local-path-provisioner, mirrored-*) are
15 # deliberately absent — they have no AKS counterpart.
16 set -euo pipefail
17
18 ACR="${1:?usage: mirror-images.sh <acr-name-or-login-server>}"
19 ACR="${ACR%%.azurecr.io*}" # accept either form
20
21 IMAGES=(
22 # docker.io — core infra
23 "docker.io/library/traefik:v3.5"
24 "docker.io/library/alpine:3.20"
25 "docker.io/library/caddy:2-alpine"
26 "docker.io/alpine/git:2.43.0"
27 "quay.io/minio/minio:RELEASE.2025-02-07T23-21-09Z" # minio left docker.io; quay is canonical
28 "quay.io/minio/mc:RELEASE.2025-04-16T18-13-26Z"
29 # flux (installed by `flux install` — mirror for restrict-image-registries compliance)
30 "ghcr.io/fluxcd/helm-controller:v1.5.5"
31 "ghcr.io/fluxcd/kustomize-controller:v1.8.5"
32 "ghcr.io/fluxcd/source-controller:v1.8.5"
33 "ghcr.io/fluxcd/notification-controller:v1.8.4"
34 # keda
35 "ghcr.io/kedacore/keda:2.20.0"
36 "ghcr.io/kedacore/keda-metrics-apiserver:2.20.0"
37 "ghcr.io/kedacore/keda-admission-webhooks:2.20.0"
38 # cnpg
39 "ghcr.io/cloudnative-pg/cloudnative-pg:1.30.0"
40 "ghcr.io/cloudnative-pg/postgresql:18-system-trixie"
41 # app-template tenant DBs pin the patch tag (kyverno aks-image-redirect rewrites them to ACR)
42 "ghcr.io/cloudnative-pg/postgresql:18.4-system-trixie"
43 # sablier
44 "ghcr.io/sablierapp/sablier:1.16.1"
45 # cert-manager
46 "quay.io/jetstack/cert-manager-controller:v1.21.1"
47 "quay.io/jetstack/cert-manager-cainjector:v1.21.1"
48 "quay.io/jetstack/cert-manager-webhook:v1.21.1"
49 # kyverno
50 "reg.kyverno.io/kyverno/kyverno:v1.18.2"
51 "reg.kyverno.io/kyverno/kyverno-cli:v1.18.2"
52 "reg.kyverno.io/kyverno/background-controller:v1.18.2"
53 "reg.kyverno.io/kyverno/reports-controller:v1.18.2"
54 # forgejo (the product's own VCS+registry, pulled as an image at install)
55 "code.forgejo.org/forgejo/forgejo:15.0.5-rootless"
56 )
57
58 fail=0
59 for src in "${IMAGES[@]}"; do
60 # keep the source path under ACR verbatim (minus registry host) so germinate
61 # can rewrite registries by prefix only: ghcr.io/X:Y -> $ACR/X:Y
62 path="${src#*/}"
63 echo "==> $src -> $ACR/$path"
64 if ! az acr import --name "$ACR" --source "$src" --image "$path" --force >/dev/null 2>&1; then
65 echo " RETRY with explicit registry auth-free import"
66 az acr import --name "$ACR" --source "$src" --image "$path" --force || { echo " FAILED: $src"; fail=1; }
67 fi
68 done
69
70 [ "$fail" = 0 ] && echo "==> all images mirrored into $ACR.azurecr.io" || { echo "==> SOME IMPORTS FAILED — rerun; nothing partial is harmful"; exit 1; }
deploy/azure/servers.bicep added +193
@@ -0,0 +1,193 @@
1 // Mitosis on Azure — in-place HA expansion: add server nodes to a LIVE platform.
2 //
3 // az deployment group create -g mitosis -f servers.bicep \
4 // -p domain=plat.example.com -p adminSshPublicKey="$(cat ~/.ssh/id_ed25519.pub)" \
5 // -p k3sToken="$(ssh <leader> sudo cat /var/lib/rancher/k3s/server/token)"
6 //
7 // main.bicep's ha=true is for FRESH deployments only: re-running it against a
8 // live group would trip on the leader VM's immutable customData and REPLACE the
9 // NSG rule set (deleting any operator-added rules). This template creates ONLY
10 // the joiner nodes (mitosis-server-1/2, 10.60.0.11/.12), referencing the
11 // existing vnet/subnet and touching nothing else.
12 //
13 // Prerequisite: the leader must already run embedded etcd. On a sqlite
14 // single-node install, write `cluster-init: true` to /etc/rancher/k3s/config.yaml
15 // and restart k3s (k3s migrates sqlite -> etcd in place); THEN deploy this.
16 //
17 // The node contract (env file + bootstrap + unit + registry recipe) is the same
18 // deploy/lib + recipe/ template main.bicep renders — joiners run ROLE=server:
19 // join the leader, build their own ci-runner, and never germinate.
20
21 @description('The platform\'s sovereign domain, e.g. plat.example.com.')
22 param domain string
23
24 @description('Your SSH public key (contents of ~/.ssh/id_ed25519.pub).')
25 param adminSshPublicKey string
26
27 @description('The LIVE cluster\'s join token — read it from the leader: sudo cat /var/lib/rancher/k3s/server/token. (Not a fresh GUID: joiners must present the existing cluster\'s token.)')
28 @secure()
29 param k3sToken string
30
31 @description('VM login user (must match the leader\'s).')
32 param adminUsername string = 'mitosis'
33
34 @description('Match the leader\'s size unless capacity-planning says otherwise.')
35 param vmSize string = 'Standard_D4as_v7'
36
37 @description('OS disk size, GB.')
38 param diskGb int = 80
39
40 @description('k3s release — MUST match the version the leader runs (k3s --version).')
41 param k3sVersion string = 'v1.31.5+k3s1'
42
43 @description('Where joiners clone mitosis from (ci-runner bootstrap). Point at the live platform\'s own mirror for sovereignty.')
44 param mitosisUrl string = 'https://git.open-platform.sh/plat/mitosis'
45
46 @description('Branch or tag of mitosis.')
47 param mitosisRef string = 'main'
48
49 @description('Availability zone per joiner (index 0 -> mitosis-server-1, 1 -> mitosis-server-2). Empty string = non-zonal, matching a non-zonal leader.')
50 param zones array = ['2', '3']
51
52 @description('The leader\'s static private IP on the nodes subnet.')
53 param leaderIp string = '10.60.0.10'
54
55 param location string = resourceGroup().location
56
57 // Joiners are indexed 1..2 — .11/.12 next to the leader's .10 (main.bicep contract).
58 var serverIndexes = [1, 2]
59
60 resource vnet 'Microsoft.Network/virtualNetworks@2024-05-01' existing = {
61 name: 'mitosis'
62 }
63
64 resource subnet 'Microsoft.Network/virtualNetworks/subnets@2024-05-01' existing = {
65 parent: vnet
66 name: 'nodes'
67 }
68
69 var nodeEnv = [
70 for i in serverIndexes: join(
71 [
72 '# /etc/mitosis/node.env — written at provision time; read by mitosis-node.sh'
73 'CLOUD=\'azure\''
74 'ROLE=\'server\''
75 'HA=\'true\''
76 'DOMAIN=\'${domain}\''
77 'LEADER_IP=\'${leaderIp}\''
78 'K3S_TOKEN=\'${k3sToken}\''
79 'K3S_VERSION=\'${k3sVersion}\''
80 'MITOSIS_URL=\'${mitosisUrl}\''
81 'MITOSIS_REF=\'${mitosisRef}\''
82 'SSH_USER=\'${adminUsername}\''
83 'CLAUDE_CODE_OAUTH_TOKEN=\'\''
84 'ANTHROPIC_API_KEY=\'\''
85 ],
86 '\n'
87 )
88 ]
89
90 var registries = replace(
91 loadTextContent('../../recipe/registries.yaml.tmpl'),
92 '__DOMAIN__',
93 domain
94 )
95 var cloudInitTmpl = loadTextContent('../lib/cloud-init.yaml.tmpl')
96 var cloudInit = [
97 for (idx, j) in serverIndexes: replace(
98 replace(
99 replace(
100 replace(cloudInitTmpl, '\${registries_b64}', base64(registries)),
101 '\${node_env_b64}',
102 base64(nodeEnv[j])
103 ),
104 '\${script_b64}',
105 base64(loadTextContent('../lib/mitosis-node.sh'))
106 ),
107 '\${unit_b64}',
108 base64(loadTextContent('../lib/mitosis-node.service'))
109 )
110 ]
111
112 resource pip 'Microsoft.Network/publicIPAddresses@2024-05-01' = [
113 for i in serverIndexes: {
114 name: 'mitosis-server-${i}'
115 location: location
116 sku: { name: 'Standard' }
117 properties: { publicIPAllocationMethod: 'Static' }
118 }
119 ]
120
121 resource nic 'Microsoft.Network/networkInterfaces@2024-05-01' = [
122 for (idx, j) in serverIndexes: {
123 name: 'mitosis-server-${idx}'
124 location: location
125 properties: {
126 ipConfigurations: [
127 {
128 name: 'primary'
129 properties: {
130 subnet: { id: subnet.id }
131 privateIPAllocationMethod: 'Static'
132 privateIPAddress: '10.60.0.${10 + idx}'
133 publicIPAddress: { id: pip[j].id }
134 }
135 }
136 ]
137 }
138 }
139 ]
140
141 resource vm 'Microsoft.Compute/virtualMachines@2024-07-01' = [
142 for (idx, j) in serverIndexes: {
143 name: 'mitosis-server-${idx}'
144 location: location
145 zones: empty(zones[j]) ? null : [zones[j]]
146 properties: {
147 hardwareProfile: { vmSize: vmSize }
148 storageProfile: {
149 imageReference: {
150 publisher: 'Canonical'
151 offer: 'ubuntu-24_04-lts'
152 sku: 'server'
153 version: 'latest'
154 }
155 osDisk: {
156 createOption: 'FromImage'
157 diskSizeGB: diskGb
158 managedDisk: { storageAccountType: 'StandardSSD_LRS' }
159 }
160 }
161 osProfile: {
162 computerName: 'mitosis-server-${idx}'
163 adminUsername: adminUsername
164 customData: base64(cloudInit[j])
165 linuxConfiguration: {
166 disablePasswordAuthentication: true
167 ssh: {
168 publicKeys: [
169 {
170 path: '/home/${adminUsername}/.ssh/authorized_keys'
171 keyData: adminSshPublicKey
172 }
173 ]
174 }
175 }
176 }
177 networkProfile: {
178 networkInterfaces: [{ id: nic[j].id }]
179 }
180 }
181 }
182 ]
183
184 output serverIps array = [for (idx, j) in serverIndexes: pip[j].properties.ipAddress]
185 output nextSteps string = join(
186 [
187 'Joiners boot, wait for the leader apiserver, and join etcd (~5-10 min).'
188 ' watch: ssh ${adminUsername}@<serverIp> journalctl -u mitosis-node -f'
189 ' verify: kubectl get nodes (3 Ready, roles control-plane,etcd,master)'
190 ' DNS: add each server IP to the *.${domain} record set (round-robin edge).'
191 ],
192 '\n'
193 )

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs