Public: anyone can read this; the crew can change it.

The chart can turn on the check lane: ServiceAccount, Role and FORGE_CHECKS_* env from values #14813

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

torque/forge:torque/3232-the-chart-can-turn-on-the-check-lane-ser → main 3 commits fork 2 threads · 2 unresolved

  • Merged. torque/3232-the-chart-can-turn-on-the-check-lane-ser was merged into main as 31cc032.

Review

Sign in to review this change.
5 files changed +164 −2
charts/app/templates/deployment.yaml +12
@@ -38,6 +38,9 @@ spec:
3838 # so the new one's queue work waits seconds, not minutes (#7799). The
3939 # long grace stays for SSH sessions that are still connected.
4040 terminationGracePeriodSeconds: 720
41 {{- if .Values.lane.enabled }}
42 serviceAccountName: {{ .Release.Name }}-checks
43 {{- end }}
4144 {{- with .Values.imagePullSecrets }}
4245 imagePullSecrets:
4346 {{- toYaml . | nindent 8 }}
@@ -116,6 +119,15 @@ spec:
116119 - name: FORGE_CLIENT_IP_HEADER
117120 value: {{ . | quote }}
118121 {{- end }}
122 {{- if .Values.lane.enabled }}
123 # The check lane (docs/CLUSTER.md); the variables src/checks/placement.ts reads.
124 - name: FORGE_CHECKS_PLACEMENT
125 value: job
126 - name: FORGE_CHECKS_IMAGE
127 value: {{ required "lane.image is required when lane.enabled" .Values.lane.image | quote }}
128 - name: FORGE_CHECKS_CLAIM
129 value: {{ required "lane.claim is required when lane.enabled" .Values.lane.claim | quote }}
130 {{- end }}
119131 {{- if include "forge.ssh" . }}
120132 - name: FORGE_SSH_PORT
121133 value: {{ .Values.forge.sshPort | quote }}
charts/app/templates/lane.yaml added +43
@@ -0,0 +1,43 @@
1 {{- if .Values.lane.enabled }}
2 # The check lane (docs/CLUSTER.md): checks run as Kubernetes Jobs. The forge
3 # talks to the API server with its pod's token, in this namespace only, with
4 # exactly the verbs src/cluster/jobs.ts uses, nothing cluster-wide.
5 apiVersion: v1
6 kind: ServiceAccount
7 metadata:
8 name: {{ .Release.Name }}-checks
9 labels:
10 app: {{ .Release.Name }}
11 ---
12 apiVersion: rbac.authorization.k8s.io/v1
13 kind: Role
14 metadata:
15 name: {{ .Release.Name }}-checks
16 labels:
17 app: {{ .Release.Name }}
18 rules:
19 - apiGroups: ["batch"]
20 resources: ["jobs"]
21 verbs: ["create", "get", "delete"]
22 - apiGroups: [""]
23 resources: ["pods"]
24 verbs: ["list"]
25 - apiGroups: [""]
26 resources: ["pods/log"]
27 verbs: ["get"]
28 ---
29 apiVersion: rbac.authorization.k8s.io/v1
30 kind: RoleBinding
31 metadata:
32 name: {{ .Release.Name }}-checks
33 labels:
34 app: {{ .Release.Name }}
35 roleRef:
36 apiGroup: rbac.authorization.k8s.io
37 kind: Role
38 name: {{ .Release.Name }}-checks
39 subjects:
40 - kind: ServiceAccount
41 name: {{ .Release.Name }}-checks
42 namespace: {{ .Release.Namespace }}
43 {{- end }}
charts/app/values.yaml +12
@@ -52,6 +52,18 @@ forge:
5252 # tunnel). Read only from a listed proxy. Unset, X-Forwarded-For is used.
5353 # clientIpHeader: ""
5454
55 # The check lane: checks run as Kubernetes Jobs instead of in a cell
56 # (docs/CLUSTER.md). Off renders nothing new. On, it needs a check image and
57 # a ReadWriteMany claim holding the checks root.
58 # The lane only runs while cells.egress is `report`. The egress wall (the
outdated

Split this run-on: state the cells.egress requirement and the operator-owned egress wall as two sentences.

59 # NetworkPolicy and FORGE_CHECKS_EGRESS_GATE in docs/CLUSTER.md) is the operator's.
60 lane:
61 enabled: false
62 # FORGE_CHECKS_IMAGE
63 image: ""
64 # FORGE_CHECKS_CLAIM
65 claim: ""
66
5567 ssh:
5668 # The platform edge's raw-TCP entrypoint this forge claims. Plain SSH has
5769 # no SNI, so an entrypoint fronts exactly one backend.
docs/CLUSTER.md +13 −2
@@ -2,8 +2,13 @@
22
33 By default a check runs in a cell on the forge's own machine. With
44 `FORGE_CHECKS_PLACEMENT=job` it runs as a Kubernetes Job instead
5 (`src/cluster/jobs.ts`). This page is exactly what an operator applies; the
6 chart is the owner's and is not described here. A test
5 (`src/cluster/jobs.ts`). The chart does the Role and the
6 variables below for you: set `lane.enabled=true`, `lane.image` and
7 `lane.claim` (`helm template charts/app --set lane.enabled=true --set
8 lane.image=<image> --set lane.claim=<claim>` shows the ServiceAccount, Role,
9 RoleBinding and `FORGE_CHECKS_*` it renders; with the lane off nothing new
10 renders). The rest of this page is what those values apply, and what stays
11 yours: the egress gate and its NetworkPolicy. A test
712 (`test/cluster-doc.test.ts`) fails if this page drifts from the code.
813
914 ## Role
@@ -54,6 +59,12 @@ Read once at boot by `src/checks/placement.ts`. A malformed value leaves
5459
5560 ## Egress through the forge's gate
5661
62 The chart does not apply the NetworkPolicy below or set
63 `FORGE_CHECKS_EGRESS_GATE`. With `lane.enabled=true` and neither in place, no
64 check runs as a Job (each stays in a cell and records `placement: process`),
65 and no check pod is walled: nothing in the chart stops a pod labelled
66 `forge-check` from reaching the network until the policy exists.
67
5768 A Job holds the same egress wall as a cell (#13251). For each run the forge
5869 opens the proxy a cell's only way out is (`apps/egress.ts`, through
5970 `src/checks/job_gate.ts`) on `FORGE_CHECKS_EGRESS_GATE`, at the first free
test/cluster-doc.test.ts +84
@@ -24,3 +24,87 @@ test("the doc grants every verb jobs.ts uses", () => {
2424 expect(doc).toContain('"list"');
2525 expect(doc).toContain("pods/log");
2626 });
27
28 type Lane = { enabled: boolean; image?: string; claim?: string };
29
30 /**
31 * A render of the chart's templates for these tests, as helm would for prod in namespace `ns`:
32 * `if .Values.lane.enabled` follows `lane`, `if include "forge.ssh"` is true (prod), `with` and
33 * `toYaml` render nothing (their values are lists the platform fills), `required` throws on an
34 * empty value, and every other expression is "x". Enough that indentation or a wrong value path
35 * shows up in the parsed YAML; helm itself is not in the sandbox or the check.
36 */
37 function render(file: string, lane: Lane): Record<string, any>[] {
38 const keep: boolean[] = [];
outdated

Substring checks on template source do not prove the render. Assert on a real render of lane off and on.

39 const out: string[] = [];
40 for (const line of read(`charts/app/templates/${file}`).split("\n")) {
41 const ctl = line.match(/^\s*\{\{-\s*(if|with|else|end)\b(.*?)\}\}\s*$/);
42 if (ctl) {
43 const [, word, arg] = ctl;
44 if (word === "if") keep.push(arg!.includes(".Values.lane.enabled") ? lane.enabled : arg!.includes('include "forge.ssh"'));
45 else if (word === "with") keep.push(false);
46 else if (word === "else") keep.push(!keep.pop());
47 else keep.pop();
48 continue;
49 }
50 if (keep.includes(false) || /^\s*\{\{-\s*toYaml/.test(line)) continue;
51 out.push(
52 line.replace(/\{\{\s*(.*?)\s*\}\}/g, (_, expr: string) => {
53 const quote = /\|\s*quote$/.test(expr);
54 const ref = expr.match(/\.Values\.lane\.(\w+)/);
55 let v = expr === ".Release.Name" ? "forge" : expr === ".Release.Namespace" ? "ns" : "x";
56 if (ref) v = String(lane[ref[1] as keyof Lane] ?? "");
57 if (ref && expr.startsWith("required") && v === "") throw new Error(expr.match(/"([^"]*)"/)![1]);
58 return quote ? JSON.stringify(v) : v;
59 }),
60 );
61 }
62 expect(keep).toEqual([]);
63 return out
64 .join("\n")
65 .split(/^---$/m)
66 .map((d) => Bun.YAML.parse(d) as Record<string, any>)
67 .filter((d) => d != null);
68 }
69
70 const ON: Lane = { enabled: true, image: "registry.example/forge-check:1", claim: "forge-checks" };
71
72 test("the lane renders nothing off, and on a ServiceAccount, Role and RoleBinding with the doc's verbs", () => {
73 expect(render("lane.yaml", { enabled: false })).toEqual([]);
74 const docs = render("lane.yaml", ON);
75 expect(docs.map((d) => d.kind)).toEqual(["ServiceAccount", "Role", "RoleBinding"]);
76 expect(docs[1]!.rules).toEqual([
77 { apiGroups: ["batch"], resources: ["jobs"], verbs: ["create", "get", "delete"] },
78 { apiGroups: [""], resources: ["pods"], verbs: ["list"] },
79 { apiGroups: [""], resources: ["pods/log"], verbs: ["get"] },
80 ]);
81 expect(docs[2]!.roleRef.name).toBe(docs[1]!.metadata.name);
82 expect(docs[2]!.subjects).toEqual([{ kind: "ServiceAccount", name: docs[0]!.metadata.name, namespace: "ns" }]);
83 });
84
85 const pod = (lane: Lane) => {
86 const [dep] = render("deployment.yaml", lane);
87 expect(dep!.kind).toBe("Deployment");
88 const spec = dep!.spec.template.spec;
89 const env = spec.containers[0].env as { name: string; value?: string }[];
90 return { spec, env: Object.fromEntries(env.map((e) => [e.name, e.value])) };
91 };
92
93 test("with the lane on the forge pod runs as the lane's account with the variables placement.ts reads", () => {
94 const { spec, env } = pod(ON);
95 expect(spec.serviceAccountName).toBe(render("lane.yaml", ON)[0]!.metadata.name);
96 expect(env.FORGE_CHECKS_PLACEMENT).toBe("job");
97 expect(env.FORGE_CHECKS_IMAGE).toBe(ON.image);
98 expect(env.FORGE_CHECKS_CLAIM).toBe(ON.claim);
99 expect(env.FORGE_URL).toBe("x");
100 expect(() => pod({ ...ON, image: "" })).toThrow("lane.image is required when lane.enabled");
101 expect(() => pod({ ...ON, claim: "" })).toThrow("lane.claim is required when lane.enabled");
102 });
103
104 test("with the lane off (the default) the pod has no account of its own and no FORGE_CHECKS_*", () => {
105 expect((Bun.YAML.parse(read("charts/app/values.yaml")) as any).lane).toEqual({ enabled: false, image: "", claim: "" });
106 const { spec, env } = pod({ enabled: false });
107 expect(spec.serviceAccountName).toBeUndefined();
108 expect(Object.keys(env).filter((k) => k.startsWith("FORGE_CHECKS_") || k.startsWith("FORGE_JOB_"))).toEqual([]);
109 expect(env.FORGE_URL).toBe("x");
110 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs