Public: anyone can read this; nobody can change it.
DOM-2 [NO-GO / DO NOT MERGE]: germinate on Cilium CNI — evidence it breaks first-try germination #59
feat/dom-2-cilium-fqdn-egress → main 1 commit
1 file
changed +39
−0
bin/up +39
| @@ -31,6 +31,9 @@ done | ||
| 31 | 31 | : "${DOMAIN:=plat.local}" # platform identity; mirror host is git.$DOMAIN |
| 32 | 32 | : "${CLUSTER:=plat}" # k3d cluster name -> kube context k3d-$CLUSTER |
| 33 | 33 | : "${K3S_IMAGE:=rancher/k3s:v1.31.5-k3s1}" # pin a modern k3s: config_path/certs.d era |
| 34 | : "${CILIUM_VERSION:=1.17.17}" # DOM-2: CNI pinned. Cilium replaces flannel+kube-router; | |
| 35 | # it enforces standard NetworkPolicy AND adds the | |
| 36 | # CiliumNetworkPolicy toFQDNs egress the mcp overlay renders. | |
| 34 | 37 | : "${HTTP_PORT:=80}" # host port -> traefik :80 (set PORTS=none to skip) |
| 35 | 38 | : "${HTTPS_PORT:=443}" # host port -> traefik :443 (so git.$DOMAIN works via /etc/hosts) |
| 36 | 39 | : "${SEED:=}" |
| @@ -97,10 +100,46 @@ EOF | ||
| 97 | 100 | --image "$K3S_IMAGE" \ |
| 98 | 101 | --registry-config "$RECIPE" \ |
| 99 | 102 | --k3s-arg '--disable=traefik@server:0' \ |
| 103 | --k3s-arg '--flannel-backend=none@server:0' \ | |
| 104 | --k3s-arg '--disable-network-policy@server:0' \ | |
| 105 | --k3s-arg '--disable-kube-proxy@server:0' \ | |
| 100 | 106 | "${PORT_ARGS[@]}" \ |
| 101 | 107 | --wait || { echo "up: cluster create failed. If host ports ${HTTP_PORT}/${HTTPS_PORT} are already in use, retry with HTTP_PORT=8080 HTTPS_PORT=8443 nix run .#up (or PORTS=none)." >&2; exit 1; } |
| 102 | 108 | fi |
| 103 | 109 | kubectl config use-context "k3d-$CLUSTER" >/dev/null |
| 110 | # DOM-2: the cluster was created with NO CNI (--flannel-backend=none), NO | |
| 111 | # kube-router netpol controller (--disable-network-policy), and NO kube-proxy | |
| 112 | # (--disable-kube-proxy), so nodes stay NotReady until a CNI lands. Install | |
| 113 | # Cilium now, BEFORE germinate, so every platform pod schedules onto the Cilium | |
| 114 | # dataplane. Cilium enforces standard networking.k8s.io/v1 NetworkPolicy | |
| 115 | # natively (the platform default-deny + allow rules keep working) and adds | |
| 116 | # CiliumNetworkPolicy toFQDNs egress, which the mcp overlay renderer emits | |
| 117 | # per-app from `allowedHosts` (domain-defined egress). | |
| 118 | # | |
| 119 | # kubeProxyReplacement=true is NOT optional here: the platform's in-cluster | |
| 120 | # registry mirror is reached by containerd via a hostPort (127.0.0.1:31100, the | |
| 121 | # registry-node-config DaemonSet), and Cilium only programs hostPort when it | |
| 122 | # replaces kube-proxy. Without it, mcp/agents (and every in-cluster-built app | |
| 123 | # image) fail ImagePull and the platform never finishes germinating. Cilium's | |
| 124 | # agent must reach the apiserver directly (there is no kube-proxy to program the | |
| 125 | # kubernetes ClusterIP yet), so point it at the node's InternalIP:6443 — an | |
| 126 | # address reachable from both the host-networked agent and the pod-networked | |
| 127 | # operator. | |
| 128 | K8S_SVC_HOST="$(kubectl get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}')" | |
| 129 | echo "==> installing Cilium ${CILIUM_VERSION} as the CNI (kube-proxy-free; apiserver ${K8S_SVC_HOST}:6443)…" | |
| 130 | helm repo add cilium https://helm.cilium.io/ >/dev/null 2>&1 || true | |
| 131 | helm repo update cilium >/dev/null 2>&1 || true | |
| 132 | helm upgrade --install cilium cilium/cilium \ | |
| 133 | --version "$CILIUM_VERSION" \ | |
| 134 | --namespace kube-system \ | |
| 135 | --set operator.replicas=1 \ | |
| 136 | --set kubeProxyReplacement=true \ | |
| 137 | --set k8sServiceHost="$K8S_SVC_HOST" \ | |
| 138 | --set k8sServicePort=6443 \ | |
| 139 | --wait --timeout 6m \ | |
| 140 | || { echo "up: Cilium install failed — the platform needs a CNI. Aborting." >&2; exit 1; } | |
| 141 | kubectl -n kube-system rollout status ds/cilium --timeout=6m | |
| 142 | kubectl wait --for=condition=Ready nodes --all --timeout=6m | |
| 104 | 143 | else |
| 105 | 144 | cat >&2 <<EOF |
| 106 | 145 | up: k3d not found, so I can't create a local cluster for you. |
new above
offline