Public: anyone can read this; the crew can change it.
While cells.egress is enforce, a check never runs as a cluster Job: it runs in a cell and says why #3191
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/3190-while-cells-egress-is-enforce-a-check-ne → main 2 commits fork
4 files
changed +52
−3
docs/CLUSTER.md +10 −1
| @@ -51,6 +51,15 @@ Read once at boot by `src/checks/placement.ts`. A malformed value leaves | ||
| 51 | 51 | | `FORGE_CHECKS_NODE_SELECTOR` | `pool=checks,disk=ssd` | `key=value,key=value`; optional | |
| 52 | 52 | | `FORGE_CHECKS_TOLERATIONS` | `[{"key":"dedicated","operator":"Equal","value":"checks","effect":"NoSchedule"}]` | JSON list, at most 20; optional | |
| 53 | 53 | |
| 54 | ## Only while egress is reported | |
| 55 | ||
| 56 | The lane applies only while `cells.egress` is `report`. Under `enforce` (the | |
| 57 | default) a Job cannot hold egress, so every check stays in a cell: the run | |
| 58 | records `placement: process` and its log says | |
| 59 | `placement: job skipped, egress is enforced and a Job cannot hold it`. The | |
| 60 | setting is read on every run, so an admin's change takes effect on the next | |
| 61 | check. | |
| 62 | ||
| 54 | 63 | ## Label for a NetworkPolicy |
| 55 | 64 | |
| 56 | 65 | The forge does not enforce egress on a Job. Every check pod carries the label |
| @@ -65,7 +74,7 @@ podSelector: | ||
| 65 | 74 | ## Confirming the lane is on |
| 66 | 75 | |
| 67 | 76 | 1. Push to a pull, or re-run its check. |
| 68 | 2. The run's page says "ran as a cluster job on <node>"; a `process` run says | |
| 77 | 2. With `cells.egress` at `report`, the run's page says "ran as a cluster job on <node>"; a `process` run says | |
| 69 | 78 | nothing. |
| 70 | 79 | 3. In the cluster, `kubectl get jobs,pods -l forge-check=true` shows the Job |
| 71 | 80 | while it runs (it is deleted when the run ends). |
src/checks/service.ts +8 −2
| @@ -766,9 +766,15 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle { | ||
| 766 | 766 | // The check is repository code: it runs in a cell of its own |
| 767 | 767 | // (`apps/contain.ts`), its checkout and a home its only places to write, |
| 768 | 768 | // the forge, the registry and what it declared its only way out. |
| 769 | const job = placement.mode === "job" ? placement : undefined; | |
| 769 | // A Job cannot hold egress (#1428), so while it is enforced the check | |
| 770 | // stays in a cell. Read per run: an admin can flip it at runtime. | |
| 771 | const fellBack = placement.mode === "job" && egressSetting(app.db) === "enforce"; | |
| 772 | const placed: Placement = fellBack ? { mode: "process" } : placement; | |
| 773 | const job = placed.mode === "job" ? placed : undefined; | |
| 770 | 774 | if (x.runId !== undefined) |
| 771 | noteRunProgress(app.db, x.runId, { placement: placement.mode }); | |
| 775 | noteRunProgress(app.db, x.runId, { placement: placed.mode }); | |
| 776 | if (fellBack) | |
| 777 | write("placement: job skipped, egress is enforced and a Job cannot hold it"); | |
| 772 | 778 | // A Job is its own boundary: no cell here, the cluster holds it. |
| 773 | 779 | const cells = job === undefined ? await cellsAvailable() : undefined; |
| 774 | 780 | if (cells !== undefined) { |
test/checks-job-vanish.test.ts +3
| @@ -10,6 +10,7 @@ import { createApp, type App } from "../src/app.ts"; | ||
| 10 | 10 | import { commitToBranch, createRepo } from "../src/actions/repos.ts"; |
| 11 | 11 | import { startChecks, type ChecksHandle } from "../src/checks/service.ts"; |
| 12 | 12 | import { VANISHED_SUMMARY, listRuns } from "../src/core/runs.ts"; |
| 13 | import { setCapacity } from "../src/core/capacity.ts"; | |
| 13 | 14 | import { createUser } from "../src/core/users.ts"; |
| 14 | 15 | import type { Repo } from "../src/core/repos.ts"; |
| 15 | 16 | import { serve, type Servers } from "../src/http/server.ts"; |
| @@ -45,6 +46,8 @@ describe("a vanished check Job", () => { | ||
| 45 | 46 | beforeAll(async () => { |
| 46 | 47 | dataDir = await mkdtemp(join(tmpdir(), "forge-vanish-")); |
| 47 | 48 | app = createApp({ dataDir, port: 0 }); |
| 49 | // Jobs run only while egress is reported (#3190). | |
| 50 | setCapacity(app.db, "cells.egress", "report", "test"); | |
| 48 | 51 | servers = await serve(app); |
| 49 | 52 | const fetch = async (_u: string, init: RequestInit) => |
| 50 | 53 | init.method === "GET" ? new Response("{}", { status: 404 }) : Response.json({}); |
test/checks-placement.test.ts +31
| @@ -19,6 +19,7 @@ import { listRuns, type Run } from "../src/core/runs.ts"; | ||
| 19 | 19 | import { createUser, type User } from "../src/core/users.ts"; |
| 20 | 20 | import type { Repo } from "../src/core/repos.ts"; |
| 21 | 21 | import { serve, type Servers } from "../src/http/server.ts"; |
| 22 | import { setCapacity } from "../src/core/capacity.ts"; | |
| 22 | 23 | import { readRunLog } from "../src/runs/log.ts"; |
| 23 | 24 | |
| 24 | 25 | const checkJson = (run: string): string => `{"check": {"run": ${JSON.stringify(run)}, "install": ""}}\n`; |
| @@ -148,6 +149,10 @@ describe("checks placed", () => { | ||
| 148 | 149 | }), |
| 149 | 150 | ); |
| 150 | 151 | afterAll(down); |
| 152 | // Egress is enforced by default, which keeps checks in cells. | |
| 153 | beforeAll(() => { | |
| 154 | setCapacity(app.db, "cells.egress", "report", alice.id); | |
| 155 | }); | |
| 151 | 156 | |
| 152 | 157 | test("a check runs as a Job: log in the run, node and placement recorded", async () => { |
| 153 | 158 | const run = await settled(await push("exit 0")); |
| @@ -209,6 +214,32 @@ describe("checks placed", () => { | ||
| 209 | 214 | }); |
| 210 | 215 | }); |
| 211 | 216 | |
| 217 | describe("job under enforce", () => { | |
| 218 | beforeAll(() => boot({ mode: "job", image: "oven/bun:1", claim: "checks-root", nodeSelector: {}, tolerations: [] })); | |
| 219 | afterAll(down); | |
| 220 | ||
| 221 | test("never calls the cluster: the check runs in a cell and says why", async () => { | |
| 222 | calls.length = 0; | |
| 223 | const run = await settled(await push("echo in a cell")); | |
| 224 | expect(run.state).toBe("ok"); | |
| 225 | expect(run.placement).toBe("process"); | |
| 226 | expect(run.node).toBe(""); | |
| 227 | expect(calls).toHaveLength(0); | |
| 228 | const log = readRunLog(dataDir, run.id).lines.join("\n"); | |
| 229 | expect(log).toContain("placement: job skipped, egress is enforced and a Job cannot hold it"); | |
| 230 | expect(log).toContain("in a cell"); | |
| 231 | const page = await (await globalThis.fetch(`${servers.url}/alice/lib/runs/${run.id}`)).text(); | |
| 232 | expect(page).not.toContain("cluster job"); | |
| 233 | }); | |
| 234 | ||
| 235 | test("is read per run: flipping to report sends the next check to a Job", async () => { | |
| 236 | setCapacity(app.db, "cells.egress", "report", alice.id); | |
| 237 | const run = await settled(await push("echo flipped")); | |
| 238 | expect(run.placement).toBe("job"); | |
| 239 | expect(calls.some((c) => c.method === "POST")).toBe(true); | |
| 240 | }); | |
| 241 | }); | |
| 242 | ||
| 212 | 243 | describe("process", () => { |
| 213 | 244 | beforeAll(() => boot(undefined)); |
| 214 | 245 | afterAll(down); |
new above
offline