674da39
@@ -127,6 +127,7 @@
EOF
      --network "k3d-$CLUSTER" \
      --registry-config "$RECIPE" \
      --k3s-arg '--disable=traefik@server:0' \
      --k3s-arg '--cluster-init@server:0' \
      "${PORT_ARGS[@]}" \
      "${_wait_arg[@]}" || { echo "up: cluster create failed. If host ports ${HTTP_PORT}/${HTTPS_PORT} are already in use, retry with HTTP_PORT=8080 HTTPS_PORT=8443 nix run .#up (or PORTS=none)." >&2; exit 1; }
  fi
@@ -142,7 +142,10 @@
ARGS=(--disable=traefik --write-kubeconfig-mode 644 --tls-san "$DOMAIN" --tls-sa
[ -n "$PUBLIC_IP" ] && ARGS+=(--tls-san "$PUBLIC_IP")
[ -n "$PRIVATE_IP" ] && ARGS+=(--node-ip "$PRIVATE_IP" --tls-san "$PRIVATE_IP")
case "$ROLE" in
  leader) [ "$HA" = true ] && ARGS+=(--cluster-init) ;;
  # Embedded etcd always — single node included. k3s's default kine/SQLite
  # datastore hit its wall on the mother platform at ~200 CNPG clusters (57 GB
  # WAL, API saturated, 2026-08-26); etcd is the datastore a platform grows on.
  leader) ARGS+=(--cluster-init) ;;
  server)
    echo "==> waiting for the leader apiserver at $LEADER_IP…"
    for i in $(seq 1 180); do
@@ -0,0 +1,36 @@
# Datastore: embedded etcd, not SQLite
k3s defaults to kine over SQLite. A platform outgrows it: the mother platform
saturated its API server at ~200 CNPG clusters (each Postgres instance renews a
lease every ~5 s → 40–60 writes/s; the SQLite WAL reached 57 GB; controllers
lost leader election, Flux stopped rolling, CI froze — 2026-08-26). Every new
platform therefore starts with `--cluster-init` (embedded etcd): `bin/up` for
k3d, `deploy/lib/mitosis-node.sh` for VMs, HA or not.
## Migrating a live platform (k3d)
k3s migrates SQLite → etcd itself when started with `cluster-init` against an
existing datastore. On the docker host:
1. Back up: `cp state.db state.db.pre-etcd` inside
   `<k3s volume>/server/db/` (checkpoint the WAL first if it is large:
   `sqlite3 state.db "PRAGMA wal_checkpoint(TRUNCATE);"` — a throwaway
   `alpine` container with `sqlite` mounted on the volume works when the host
   has no sqlite3).
2. Append to the server container's `/etc/rancher/k3s/config.yaml`:
   ```yaml
   cluster-init: true
   etcd-arg:
     - "quota-backend-bytes=8589934592"
   ```
3. Restart the server container. `docker restart` may stall ("did not receive
   an exit event"); `docker start <container>` then brings it up. Watch
   `docker logs` for "Managed etcd cluster initializing". The old file is kept
   as `state.db.migrated`.
4. Verify `kubectl get nodes` (all Ready), API errors gone, load down. Delete
   the two SQLite copies after a day.
Rollback (never needed so far): restore `config.yaml`, delete `server/db/etcd`,
rename `state.db.migrated` → `state.db`, start the container.
Bare k3s VMs: same `config.yaml` edit, then `systemctl restart k3s`.
		@@ -127,6 +127,7 @@ EOF
127	127	--network "k3d-$CLUSTER" \
128	128	--registry-config "$RECIPE" \
129	129	--k3s-arg '--disable=traefik@server:0' \
	130	--k3s-arg '--cluster-init@server:0' \
130	131	"${PORT_ARGS[@]}" \
131	132	"${_wait_arg[@]}" \|\| { echo "up: cluster create failed. If host ports ${HTTP_PORT}/${HTTPS_PORT} are already in use, retry with HTTP_PORT=8080 HTTPS_PORT=8443 nix run .#up (or PORTS=none)." >&2; exit 1; }
132	133	fi
		@@ -142,7 +142,10 @@ ARGS=(--disable=traefik --write-kubeconfig-mode 644 --tls-san "$DOMAIN" --tls-sa
142	142	[ -n "$PUBLIC_IP" ] && ARGS+=(--tls-san "$PUBLIC_IP")
143	143	[ -n "$PRIVATE_IP" ] && ARGS+=(--node-ip "$PRIVATE_IP" --tls-san "$PRIVATE_IP")
144	144	case "$ROLE" in
145		leader) [ "$HA" = true ] && ARGS+=(--cluster-init) ;;
	145	# Embedded etcd always — single node included. k3s's default kine/SQLite
	146	# datastore hit its wall on the mother platform at ~200 CNPG clusters (57 GB
	147	# WAL, API saturated, 2026-08-26); etcd is the datastore a platform grows on.
	148	leader) ARGS+=(--cluster-init) ;;
146	149	server)
147	150	echo "==> waiting for the leader apiserver at $LEADER_IP…"
148	151	for i in $(seq 1 180); do
		@@ -0,0 +1,36 @@
	1	# Datastore: embedded etcd, not SQLite
	2
	3	k3s defaults to kine over SQLite. A platform outgrows it: the mother platform
	4	saturated its API server at ~200 CNPG clusters (each Postgres instance renews a
	5	lease every ~5 s → 40–60 writes/s; the SQLite WAL reached 57 GB; controllers
	6	lost leader election, Flux stopped rolling, CI froze — 2026-08-26). Every new
	7	platform therefore starts with `--cluster-init` (embedded etcd): `bin/up` for
	8	k3d, `deploy/lib/mitosis-node.sh` for VMs, HA or not.
	9
	10	## Migrating a live platform (k3d)
	11
	12	k3s migrates SQLite → etcd itself when started with `cluster-init` against an
	13	existing datastore. On the docker host:
	14
	15	1. Back up: `cp state.db state.db.pre-etcd` inside
	16	`<k3s volume>/server/db/` (checkpoint the WAL first if it is large:
	17	`sqlite3 state.db "PRAGMA wal_checkpoint(TRUNCATE);"` — a throwaway
	18	`alpine` container with `sqlite` mounted on the volume works when the host
	19	has no sqlite3).
	20	2. Append to the server container's `/etc/rancher/k3s/config.yaml`:
	21	```yaml
	22	cluster-init: true
	23	etcd-arg:
	24	- "quota-backend-bytes=8589934592"
	25	```
	26	3. Restart the server container. `docker restart` may stall ("did not receive
	27	an exit event"); `docker start <container>` then brings it up. Watch
	28	`docker logs` for "Managed etcd cluster initializing". The old file is kept
	29	as `state.db.migrated`.
	30	4. Verify `kubectl get nodes` (all Ready), API errors gone, load down. Delete
	31	the two SQLite copies after a day.
	32
	33	Rollback (never needed so far): restore `config.yaml`, delete `server/db/etcd`,
	34	rename `state.db.migrated` → `state.db`, start the container.
	35
	36	Bare k3s VMs: same `config.yaml` edit, then `systemctl restart k3s`.