Public: anyone can read this; the crew can change it.
checks.placement: job runs a check as a cluster Job and records node and placement; process unchanged #1863
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/1815-checks-placement-job-runs-a-check-as-a-c → main 4 commits fork 6 threads · 6 unresolved
15 files
changed +606
−36
docs/APPS.md +18
| @@ -358,6 +358,23 @@ A `check` that is present and wrong fails with the reason, so a commit is | ||
| 358 | 358 | never silently unchecked. A check under way when the forge restarts is |
| 359 | 359 | queued again, and the forge that carries on runs it from the start. |
| 360 | 360 | |
| 361 | **Placement.** `FORGE_CHECKS_PLACEMENT=process` (the default) runs a check | |
| 362 | in a cell on the forge's machine. `job` runs install and check as one | |
| 363 | Kubernetes Job (`src/cluster/jobs.ts`) instead: `FORGE_CHECKS_IMAGE` is its | |
| 364 | image, `FORGE_CHECKS_CLAIM` a ReadWriteMany claim holding the forge's checks | |
| 365 | root (mounted at the same path, where the checkout is exported), | |
| 366 | `FORGE_CHECKS_NODE_SELECTOR` (`key=value,…`) and `FORGE_CHECKS_TOLERATIONS` | |
| 367 | (a JSON list) say where it may land. Only `CI` and `FORGE_*` variables cross | |
| 368 | to the Job. Its log is written into the run's log when it ends, and the run | |
| 369 | records `placement` and the `node` its pod ran on. A malformed setting leaves | |
| 370 | `process` in place. No cell wraps a Job. The pod mounts only its own | |
| 371 | checkout (`subPath` = the run id), mounts no service-account token, runs | |
| 372 | non-root without privilege escalation, and carries the check's limits as | |
| 373 | `resources.limits`. **Egress is not enforced by the forge on a Job**: the | |
| 374 | run is recorded as uncontained ("egress not enforced on a cluster job"), and | |
| 375 | the pod carries the label `forge-check=true` so an operator's NetworkPolicy | |
| 376 | can select it and hold what `egress` declares. | |
| 377 | ||
| 361 | 378 | Checks share `FORGE_CHECKS_CONCURRENCY` slots (default `1`), and when one |
| 362 | 379 | frees, it goes to the most recently requested check waiting, not the one |
| 363 | 380 | queued longest — a fresh push is never stuck behind pulls that queued hours |
| @@ -1079,6 +1096,7 @@ a link to the milestone (`docs/PLANNING.md`); its progress never does. | ||
| 1079 | 1096 | | `FORGE_PREVIEW_IDLE_DAYS` | `3` | days a pull may sit untouched before its preview sleeps; `0` = never | |
| 1080 | 1097 | | `FORGE_CREW_CONCURRENCY` | `2` | agent runs at once (live: Settings → Capacity) | |
| 1081 | 1098 | | `FORGE_CHECKS_CONCURRENCY` | `1` | checks at once (live: Settings → Capacity) | |
| 1099 | | `FORGE_CHECKS_PLACEMENT` | `process` | `job` runs a check's commands as a cluster Job (see Placement above) | | |
| 1082 | 1100 | | `FORGE_CHECKS_DRAIN_S` | `300` | seconds a roll lets running checks finish first | |
| 1083 | 1101 | | `FORGE_CHECK_SHARDS` | the machine's cores | test shards per check (live: Settings → Capacity) | |
| 1084 | 1102 | | `FORGE_CELLS_EGRESS` | `enforce` | `report` lets undeclared egress through on 80/443 (live: Settings → Capacity) | |
scripts/gate.ts +1
| @@ -50,6 +50,7 @@ export const GATE: readonly string[] = [ | ||
| 50 | 50 | "test/chaos.test.ts", |
| 51 | 51 | "test/flow-stage.test.ts", |
| 52 | 52 | "test/cluster-jobs.test.ts", |
| 53 | "test/checks-placement.test.ts", | |
| 53 | 54 | "test/fuzz-found.test.ts", |
| 54 | 55 | "test/fuzz-parsers.test.ts", |
| 55 | 56 | "test/attach-look.test.ts", |
src/actions/runs.ts +1
| @@ -42,6 +42,7 @@ export function runPayload(run: Run): Record<string, unknown> { | ||
| 42 | 42 | summary: run.summary, |
| 43 | 43 | costUsd: run.costUsd, |
| 44 | 44 | turns: run.turns, |
| 45 | ...(run.placement === "job" ? { placement: run.placement, node: run.node } : {}), | |
| 45 | 46 | ...(run.issueId === null ? {} : { issueId: run.issueId }), |
| 46 | 47 | }; |
| 47 | 48 | } |
src/checks/job_step.ts added +61
| @@ -0,0 +1,61 @@ | ||
| 1 | /** | |
| 2 | * A check's steps as one cluster Job (`checks.placement: job`): install and | |
| 3 | * check run in order in the Job's container, over the checks root the claim | |
| 4 | * holds. The Job's log is written into the run's log and its node recorded; | |
| 5 | * the answer has `runStep`'s shape, so the caller is the same for both. | |
| 6 | */ | |
| 7 | import { runJob, type JobOptions, type JobSpec } from "../cluster/jobs.ts"; | |
| 8 | import { err, ok, type Result } from "../kernel/result.ts"; | |
| 9 | import type { Placement } from "./placement.ts"; | |
| 10 | ||
| 11 | /** What a NetworkPolicy selects a check's pod by (docs/APPS.md, Placement). */ | |
| 12 | export const JOB_LABELS = { "forge-check": "true" }; | |
| 13 | ||
| 14 | export async function runJobStep( | |
| 15 | placement: Extract<Placement, { mode: "job" }>, | |
| 16 | input: { | |
| 17 | runId: string; | |
| 18 | dir: string; | |
| 19 | commands: string[]; | |
| 20 | env: Record<string, string>; | |
| 21 | log: (line: string) => void; | |
| 22 | signal: AbortSignal; | |
| 23 | timeoutMs: number; | |
| 24 | limits: { memoryMiB: number; cpus: number; diskMiB: number }; | |
| 25 | /** The Job's own output, apart from the echoed commands. */ | |
| 26 | output: string[]; | |
| 27 | onNode: (node: string) => void; | |
| 28 | options?: JobOptions; | |
| 29 | }, | |
| 30 | ): Promise<Result<void, string>> { | |
| 31 | // Only the check's own variables cross to the cluster, never the forge's. | |
| 32 | const env = Object.fromEntries(Object.entries(input.env).filter(([k]) => k === "CI" || k.startsWith("FORGE_"))); | |
| 33 | const spec: JobSpec = { | |
| 34 | name: `check-${input.runId.toLowerCase().replace(/[^a-z0-9]/g, "-")}`.slice(0, 53).replace(/-+$/, ""), | |
| 35 | image: placement.image, | |
| 36 | command: ["/bin/sh", "-c", input.commands.join(" && ")], | |
outdated
High: the claim holds the whole checks root and is mounted at this run's dir, which is also the working directory. The check runs over every concurrent checkout (cross-repo and private read/write) instead of its own. Use |
||
| 37 | env, | |
| 38 | nodeSelector: placement.nodeSelector, | |
| 39 | tolerations: placement.tolerations, | |
| 40 | // Only this run's own checkout, never the root the claim holds. | |
| 41 | volume: { claim: placement.claim, mountPath: input.dir, subPath: input.runId }, | |
| 42 | limits: input.limits, | |
| 43 | labels: JOB_LABELS, | |
| 44 | }; | |
| 45 | for (const c of input.commands) input.log(`$ ${c}`); | |
| 46 | let result; | |
| 47 | try { | |
| 48 | result = await runJob(spec, { timeoutMs: input.timeoutMs, signal: input.signal, ...input.options }); | |
| 49 | } catch (error) { | |
| 50 | if (input.signal.aborted) return err("check cancelled"); | |
| 51 | return err(`job could not run: ${error instanceof Error ? error.message : String(error)}`); | |
| 52 | } | |
| 53 | if (result.node) input.onNode(result.node); | |
| 54 | for (const line of result.lines) { | |
| 55 | input.output.push(line); | |
| 56 | input.log(line); | |
| 57 | } | |
| 58 | if (result.timedOut) return err(`check took longer than ${Math.round(input.timeoutMs / 1000)}s`); | |
| 59 | if (result.exitCode === 0) return ok(); | |
| 60 | return err(`check exited with code ${result.exitCode ?? "unknown"}`); | |
| 61 | } |
src/checks/placement.ts added +90
| @@ -0,0 +1,90 @@ | ||
| 1 | /** | |
| 2 | * Where a check's commands run: `process` (the default, a cell on the | |
| 3 | * forge's own machine) or `job` (a Kubernetes Job, through | |
| 4 | * `cluster/jobs.ts`). Read from the forge's environment once, validated at | |
| 5 | * this boundary; anything malformed leaves the default in place. | |
| 6 | * FORGE_CHECKS_PLACEMENT process | job | |
| 7 | * FORGE_CHECKS_IMAGE the Job's image | |
| 8 | * FORGE_CHECKS_CLAIM a ReadWriteMany claim holding the checks root | |
| 9 | * FORGE_CHECKS_NODE_SELECTOR key=value,key=value | |
| 10 | * FORGE_CHECKS_TOLERATIONS a JSON list of tolerations | |
| 11 | */ | |
| 12 | import type { Toleration } from "../cluster/jobs.ts"; | |
| 13 | ||
| 14 | export type Placement = | |
| 15 | | { readonly mode: "process" } | |
| 16 | | { | |
| 17 | readonly mode: "job"; | |
| 18 | readonly image: string; | |
| 19 | readonly claim: string; | |
| 20 | readonly nodeSelector: Record<string, string>; | |
| 21 | readonly tolerations: Toleration[]; | |
| 22 | }; | |
| 23 | ||
| 24 | const LABEL = /^[A-Za-z0-9]([-A-Za-z0-9_./]{0,61}[A-Za-z0-9])?$/; | |
| 25 | const EFFECTS = ["NoSchedule", "PreferNoSchedule", "NoExecute"]; | |
| 26 | ||
| 27 | export function parseSelector(text: string): Record<string, string> | undefined { | |
| 28 | const out: Record<string, string> = {}; | |
| 29 | for (const pair of text.split(",").filter((p) => p.trim() !== "")) { | |
| 30 | const at = pair.indexOf("="); | |
| 31 | const key = pair.slice(0, at).trim(); | |
| 32 | const value = pair.slice(at + 1).trim(); | |
| 33 | if (at < 1 || !LABEL.test(key) || !(value === "" || LABEL.test(value))) return undefined; | |
| 34 | out[key] = value; | |
| 35 | } | |
| 36 | return out; | |
| 37 | } | |
| 38 | ||
| 39 | export function parseTolerations(text: string): Toleration[] | undefined { | |
| 40 | let list: unknown; | |
| 41 | try { | |
| 42 | list = JSON.parse(text); | |
| 43 | } catch { | |
| 44 | return undefined; | |
| 45 | } | |
| 46 | if (!Array.isArray(list) || list.length > 20) return undefined; | |
| 47 | const out: Toleration[] = []; | |
| 48 | for (const t of list as Record<string, unknown>[]) { | |
| 49 | if (typeof t !== "object" || t === null) return undefined; | |
| 50 | const str = (k: string) => (typeof t[k] === "string" ? (t[k] as string) : undefined); | |
| 51 | const key = str("key"); | |
| 52 | const operator = str("operator"); | |
| 53 | const value = str("value"); | |
| 54 | const effect = str("effect"); | |
| 55 | if (operator !== undefined && operator !== "Exists" && operator !== "Equal") return undefined; | |
| 56 | if (effect !== undefined && !EFFECTS.includes(effect)) return undefined; | |
| 57 | if ((key !== undefined && !LABEL.test(key)) || (value !== undefined && !(value === "" || LABEL.test(value)))) | |
| 58 | return undefined; | |
| 59 | out.push({ | |
| 60 | ...(key === undefined ? {} : { key }), | |
| 61 | ...(operator === undefined ? {} : { operator }), | |
| 62 | ...(value === undefined ? {} : { value }), | |
| 63 | ...(effect === undefined ? {} : { effect: effect as NonNullable<Toleration["effect"]> }), | |
| 64 | }); | |
| 65 | } | |
| 66 | return out; | |
| 67 | } | |
| 68 | ||
| 69 | export function placementFromEnv(env: Record<string, string | undefined> = process.env): Placement { | |
| 70 | if (env["FORGE_CHECKS_PLACEMENT"]?.trim() !== "job") return { mode: "process" }; | |
| 71 | const image = env["FORGE_CHECKS_IMAGE"]?.trim() ?? ""; | |
| 72 | const claim = env["FORGE_CHECKS_CLAIM"]?.trim() ?? ""; | |
| 73 | const nodeSelector = parseSelector(env["FORGE_CHECKS_NODE_SELECTOR"] ?? ""); | |
| 74 | const tolerations = parseTolerations(env["FORGE_CHECKS_TOLERATIONS"]?.trim() || "[]"); | |
| 75 | // A job placement that cannot be built is not guessed at. | |
| 76 | if (image === "" || /\s/.test(image) || claim === "" || !nodeSelector || !tolerations) return { mode: "process" }; | |
| 77 | return { mode: "job", image, claim, nodeSelector, tolerations }; | |
| 78 | } | |
| 79 | ||
| 80 | /** | |
| 81 | * Where a run ran, in plain words, for a person to read. Nothing for the | |
| 82 | * default `process` or a run that predates the record: only the unusual is | |
| 83 | * said. The node comes from the cluster, so it is cut to printable ASCII | |
| 84 | * here as well as escaped by whatever shows it. | |
| 85 | */ | |
| 86 | export function placementWords(run: { placement: string; node: string }): string { | |
| 87 | if (run.placement !== "job") return ""; | |
| 88 | const node = run.node.replace(/[^\x21-\x7e]/g, "").slice(0, 253); | |
| 89 | return node === "" ? "ran as a cluster job" : `ran as a cluster job on ${node}`; | |
| 90 | } |
src/checks/service.ts +75 −30
| @@ -71,6 +71,10 @@ import { | ||
| 71 | 71 | type CellSpec, |
| 72 | 72 | } from "../apps/contain.ts"; |
| 73 | 73 | import { summariseCheckFailure } from "./summary.ts"; |
| 74 | import type { Result } from "../kernel/result.ts"; | |
| 75 | import { placementFromEnv, type Placement } from "./placement.ts"; | |
| 76 | import { runJobStep } from "./job_step.ts"; | |
| 77 | import type { JobOptions } from "../cluster/jobs.ts"; | |
| 74 | 78 | import type { CheckDetail } from "../core/runs.ts"; |
| 75 | 79 | |
| 76 | 80 | const log = createLog("checks"); |
| @@ -93,6 +97,13 @@ export interface ChecksOptions { | ||
| 93 | 97 | * @default 0 |
| 94 | 98 | */ |
| 95 | 99 | readonly drainMs?: number | undefined; |
| 100 | /** | |
| 101 | * Where a check's commands run (`checks/placement.ts`). @default from the | |
| 102 | * environment: `process`. | |
| 103 | */ | |
| 104 | readonly placement?: Placement | undefined; | |
| 105 | /** The kube client a `job` placement speaks through; tests stand one in. */ | |
| 106 | readonly job?: JobOptions | undefined; | |
| 96 | 107 | } |
| 97 | 108 | |
| 98 | 109 | export interface ChecksHandle { |
| @@ -198,6 +209,7 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle { | ||
| 198 | 209 | const capacity = (): number => |
| 199 | 210 | capacityCount(app.db, "checks.concurrency", options.concurrency ?? 1); |
| 200 | 211 | const gate = new Gate(capacity()); |
| 212 | const placement = options.placement ?? placementFromEnv(); | |
| 201 | 213 | /** Full gates on land in flight: not slots in the queue, but stopped with it. */ |
| 202 | 214 | const gates = new Set<AbortController>(); |
| 203 | 215 | const inFlight = new Map< |
| @@ -657,15 +669,29 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle { | ||
| 657 | 669 | // The check is repository code: it runs in a cell of its own |
| 658 | 670 | // (`apps/contain.ts`), its checkout and a home its only places to write, |
| 659 | 671 | // the forge, the registry and what it declared its only way out. |
| 660 | const cells = await cellsAvailable(); | |
| 672 | const job = placement.mode === "job" ? placement : undefined; | |
| 661 | 673 | if (x.runId !== undefined) |
| 662 | noteRunProgress(app.db, x.runId, { | |
| 663 | sandbox: { enforced: cells.ok, note: cells.ok ? "" : cells.reason }, | |
| 664 | }); | |
| 665 | if (!cells.ok) | |
| 666 | write(`uncontained here: ${cells.reason}; the check runs unrestricted`); | |
| 674 | noteRunProgress(app.db, x.runId, { placement: placement.mode }); | |
| 675 | // A Job is its own boundary: no cell here, the cluster holds it. | |
The cell enforced egress ( |
||
| 676 | const cells = job === undefined ? await cellsAvailable() : undefined; | |
| 677 | if (cells !== undefined) { | |
| 678 | if (x.runId !== undefined) | |
| 679 | noteRunProgress(app.db, x.runId, { | |
| 680 | sandbox: { enforced: cells.ok, note: cells.ok ? "" : cells.reason }, | |
| 681 | }); | |
| 682 | if (!cells.ok) | |
| 683 | write(`uncontained here: ${cells.reason}; the check runs unrestricted`); | |
| 684 | } else { | |
| 685 | // The cluster holds a Job, but not as a cell does: the egress a | |
| 686 | // check declared is not enforced here, and the run says so. | |
| 687 | if (x.runId !== undefined) | |
| 688 | noteRunProgress(app.db, x.runId, { | |
| 689 | sandbox: { enforced: false, note: "egress not enforced on a cluster job" }, | |
| 690 | }); | |
| 691 | write("placement: job"); | |
| 692 | } | |
| 667 | 693 | const home = `${dir}.home`; |
| 668 | const cell: CellSpec | undefined = cells.ok | |
| 694 | const cell: CellSpec | undefined = cells?.ok | |
| 669 | 695 | ? { |
| 670 | 696 | dir, |
| 671 | 697 | writable: [], |
| @@ -701,36 +727,55 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle { | ||
| 701 | 727 | // plain processes here — as they do on a runner without bwrap. |
| 702 | 728 | FORGE_CELLS: "off", |
| 703 | 729 | }); |
| 704 | if (manifest.install !== null) { | |
| 705 | const installed = await runStep("install", manifest.install, { | |
| 706 | cwd: dir, | |
| 730 | let ran: Result<void, string>; | |
| 731 | const checkLines: string[] = []; | |
| 732 | if (job !== undefined) { | |
| 733 | ran = await runJobStep(job, { | |
| 734 | runId: run.id, | |
| 735 | dir, | |
| 736 | commands: manifest.install === null ? [command] : [manifest.install, command], | |
| 707 | 737 | env, |
| 708 | 738 | log: write, |
| 739 | output: checkLines, | |
| 740 | signal, | |
| 741 | timeoutMs: manifest.timeout * 1000, | |
| 742 | limits: resolveLimits("step", manifest.limits), | |
| 743 | onNode: (node) => { | |
| 744 | if (x.runId !== undefined) noteRunProgress(app.db, x.runId, { node }); | |
| 745 | }, | |
| 746 | ...(options.job === undefined ? {} : { options: options.job }), | |
| 747 | }); | |
| 748 | } else { | |
| 749 | if (manifest.install !== null) { | |
| 750 | const installed = await runStep("install", manifest.install, { | |
| 751 | cwd: dir, | |
| 752 | env, | |
| 753 | log: write, | |
| 754 | signal, | |
| 755 | timeoutMs: manifest.timeout * 1000, | |
| 756 | cell, | |
| 757 | }); | |
| 758 | if (!installed.ok) return { ok: false, error: installed.error }; | |
| 759 | } | |
| 760 | // The check step's own lines are kept beside what `write` already | |
| 761 | // logs, so a failure can be summarised once here — never by reading | |
| 762 | // the log back, on this run or any later render of it. `runStep` | |
| 763 | // echoes the command as its first line before any output of the | |
| 764 | // command's own; that echo is not output to summarise from. | |
| 765 | let sawEcho = false; | |
| 766 | ran = await runStep("check", command, { | |
| 767 | cwd: dir, | |
| 768 | env, | |
| 769 | log: (line) => { | |
| 770 | if (sawEcho) checkLines.push(line); | |
| 771 | sawEcho = true; | |
| 772 | write(line); | |
| 773 | }, | |
| 709 | 774 | signal, |
| 710 | 775 | timeoutMs: manifest.timeout * 1000, |
| 711 | 776 | cell, |
| 712 | 777 | }); |
| 713 | if (!installed.ok) return { ok: false, error: installed.error }; | |
| 714 | 778 | } |
| 715 | // The check step's own lines are kept beside what `write` already | |
| 716 | // logs, so a failure can be summarised once here — never by reading | |
| 717 | // the log back, on this run or any later render of it. `runStep` | |
| 718 | // echoes the command as its first line before any output of the | |
| 719 | // command's own; that echo is not output to summarise from. | |
| 720 | const checkLines: string[] = []; | |
| 721 | let sawEcho = false; | |
| 722 | const ran = await runStep("check", command, { | |
| 723 | cwd: dir, | |
| 724 | env, | |
| 725 | log: (line) => { | |
| 726 | if (sawEcho) checkLines.push(line); | |
| 727 | sawEcho = true; | |
| 728 | write(line); | |
| 729 | }, | |
| 730 | signal, | |
| 731 | timeoutMs: manifest.timeout * 1000, | |
| 732 | cell, | |
| 733 | }); | |
| 734 | 779 | if (!ran.ok) { |
| 735 | 780 | const { summary, detail } = summariseCheckFailure( |
| 736 | 781 | checkLines, |
src/cluster/jobs.ts +65 −4
| @@ -26,6 +26,12 @@ export type JobSpec = { | ||
| 26 | 26 | env?: Record<string, string>; |
| 27 | 27 | nodeSelector?: Record<string, string>; |
| 28 | 28 | tolerations?: Toleration[]; |
| 29 | /** A claim mounted at `mountPath`, which is also the container's working directory. */ | |
| 30 | volume?: { claim: string; mountPath: string; subPath: string }; | |
| 31 | /** Resource ceilings for the container. */ | |
| 32 | limits?: { memoryMiB: number; cpus: number; diskMiB: number }; | |
| 33 | /** Labels on the pod, for a NetworkPolicy to select it by. */ | |
| 34 | labels?: Record<string, string>; | |
| 29 | 35 | }; |
| 30 | 36 | |
| 31 | 37 | export type JobResult = { |
| @@ -33,6 +39,8 @@ export type JobResult = { | ||
| 33 | 39 | exitCode: number | null; |
| 34 | 40 | lines: string[]; |
| 35 | 41 | timedOut: boolean; |
| 42 | /** The node the Job's pod ran on; absent when it never reached one. */ | |
| 43 | node?: string; | |
| 36 | 44 | }; |
| 37 | 45 | |
| 38 | 46 | export type Credentials = { server: string; token: string; ca?: string; namespace: string }; |
| @@ -47,6 +55,8 @@ export type JobOptions = { | ||
| 47 | 55 | sleep?: (ms: number) => Promise<void>; |
| 48 | 56 | /** The most log bytes read; the rest of the log is left unread. */ |
| 49 | 57 | maxLogBytes?: number; |
| 58 | /** Aborted: the Job is deleted and the call throws. */ | |
| 59 | signal?: AbortSignal; | |
| 50 | 60 | }; |
| 51 | 61 | |
| 52 | 62 | /** The service account the pod runs as; throws when there is none. */ |
| @@ -73,6 +83,23 @@ export function manifest(spec: JobSpec): object { | ||
| 73 | 83 | if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name) || typeof value !== "string") throw new Error(`bad env ${name}`); |
| 74 | 84 | return { name, value }; |
| 75 | 85 | }); |
| 86 | const vol = spec.volume; | |
| 87 | if ( | |
| 88 | vol && | |
| 89 | (!NAME.test(vol.claim) || | |
| 90 | !/^\/[\w./-]+$/.test(vol.mountPath) || | |
| 91 | vol.mountPath.includes("..") || | |
| 92 | !/^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/.test(vol.subPath)) | |
| 93 | ) | |
| 94 | throw new Error("bad volume"); | |
| 95 | const lim = spec.limits; | |
| 96 | if (lim && ![lim.memoryMiB, lim.cpus, lim.diskMiB].every((n) => Number.isFinite(n) && n > 0)) | |
| 97 | throw new Error("bad limits"); | |
| 98 | const labels: Record<string, string> = {}; | |
outdated
High: this pod runs repository code with the namespace's service-account token auto-mounted, no resources and no securityContext. Add |
||
| 99 | for (const [k, v] of Object.entries(spec.labels ?? {})) { | |
| 100 | if (!/^[A-Za-z0-9][-A-Za-z0-9_.\/]{0,62}$/.test(k) || !/^[A-Za-z0-9]?[-A-Za-z0-9_.]{0,62}$/.test(v)) throw new Error(`bad label ${k}`); | |
| 101 | labels[k] = v; | |
| 102 | } | |
| 76 | 103 | return { |
| 77 | 104 | apiVersion: "batch/v1", |
| 78 | 105 | kind: "Job", |
| @@ -81,12 +108,38 @@ export function manifest(spec: JobSpec): object { | ||
| 81 | 108 | backoffLimit: 0, |
| 82 | 109 | ttlSecondsAfterFinished: 600, |
| 83 | 110 | template: { |
| 84 | metadata: { labels: { "job-name": spec.name } }, | |
| 111 | metadata: { labels: { ...labels, "job-name": spec.name } }, | |
| 85 | 112 | spec: { |
| 86 | 113 | restartPolicy: "Never", |
| 114 | // Repository code gets no kube credential and no privilege. | |
| 115 | automountServiceAccountToken: false, | |
| 116 | securityContext: { runAsNonRoot: true }, | |
| 87 | 117 | nodeSelector: spec.nodeSelector ?? {}, |
| 88 | 118 | tolerations: spec.tolerations ?? [], |
| 89 | containers: [{ name: "job", image: spec.image, command: spec.command, env }], | |
| 119 | containers: [ | |
| 120 | { | |
| 121 | name: "job", | |
| 122 | image: spec.image, | |
| 123 | command: spec.command, | |
| 124 | env, | |
| 125 | securityContext: { allowPrivilegeEscalation: false, capabilities: { drop: ["ALL"] } }, | |
| 126 | ...(lim | |
| 127 | ? { | |
| 128 | resources: { | |
| 129 | limits: { | |
| 130 | memory: `${Math.ceil(lim.memoryMiB)}Mi`, | |
| 131 | cpu: String(lim.cpus), | |
| 132 | "ephemeral-storage": `${Math.ceil(lim.diskMiB)}Mi`, | |
| 133 | }, | |
| 134 | }, | |
| 135 | } | |
| 136 | : {}), | |
| 137 | ...(vol | |
| 138 | ? { workingDir: vol.mountPath, volumeMounts: [{ name: "work", mountPath: vol.mountPath, subPath: vol.subPath }] } | |
| 139 | : {}), | |
| 140 | }, | |
| 141 | ], | |
| 142 | ...(vol ? { volumes: [{ name: "work", persistentVolumeClaim: { claimName: vol.claim } }] } : {}), | |
| 90 | 143 | }, |
| 91 | 144 | }, |
| 92 | 145 | }, |
| @@ -97,6 +150,7 @@ export function manifest(spec: JobSpec): object { | ||
| 97 | 150 | type Job = { status?: { succeeded?: number; failed?: number } }; |
| 98 | 151 | type Pod = { |
| 99 | 152 | metadata?: { name?: string }; |
| 153 | spec?: { nodeName?: string }; | |
| 100 | 154 | status?: { containerStatuses?: { state?: { terminated?: { exitCode?: number } } }[] }; |
| 101 | 155 | }; |
| 102 | 156 | type PodList = { items?: Pod[] }; |
| @@ -152,16 +206,20 @@ export async function runJob(spec: JobSpec, opts: JobOptions = {}): Promise<JobR | ||
| 152 | 206 | const deadline = now() + timeoutMs; |
| 153 | 207 | let done: "succeeded" | "failed" | null = null; |
| 154 | 208 | while (!done) { |
| 209 | if (opts.signal?.aborted) throw new Error("job cancelled"); | |
| 155 | 210 | const job = await json<Job>("GET", jobPath); |
| 156 | 211 | if ((job.status?.succeeded ?? 0) > 0) done = "succeeded"; |
| 157 | 212 | else if ((job.status?.failed ?? 0) > 0) done = "failed"; |
| 158 | else if (now() >= deadline) return { exitCode: null, lines: await logLines(), timedOut: true }; | |
| 213 | else if (now() >= deadline) { | |
| 214 | const pod = await findPod().catch(() => undefined); | |
| 215 | return { exitCode: null, lines: await logLines(pod?.metadata?.name), timedOut: true, ...nodeOf(pod) }; | |
| 216 | } | |
| 159 | 217 | else await sleep(pollMs); |
| 160 | 218 | } |
| 161 | 219 | const pod = await findPod(); |
| 162 | 220 | const code: unknown = pod?.status?.containerStatuses?.[0]?.state?.terminated?.exitCode; |
| 163 | 221 | const exitCode = typeof code === "number" ? code : done === "succeeded" ? 0 : null; |
| 164 | return { exitCode, lines: await logLines(pod?.metadata?.name), timedOut: false }; | |
| 222 | return { exitCode, lines: await logLines(pod?.metadata?.name), timedOut: false, ...nodeOf(pod) }; | |
| 165 | 223 | } finally { |
| 166 | 224 | if (created) { |
| 167 | 225 | try { |
| @@ -191,6 +249,9 @@ export async function runJob(spec: JobSpec, opts: JobOptions = {}): Promise<JobR | ||
| 191 | 249 | } |
| 192 | 250 | } |
| 193 | 251 | |
| 252 | const nodeOf = (pod?: Pod): { node?: string } => | |
| 253 | typeof pod?.spec?.nodeName === "string" && pod.spec.nodeName ? { node: pod.spec.nodeName } : {}; | |
| 254 | ||
| 194 | 255 | /** Read a body chunk by chunk, stopping at `max` bytes; the rest is never held. */ |
| 195 | 256 | async function readCapped(body: ReadableStream<Uint8Array>, max: number): Promise<string> { |
| 196 | 257 | const reader = body.getReader(); |
src/core/runs.ts +16 −1
| @@ -76,6 +76,10 @@ export interface Run { | ||
| 76 | 76 | readonly sandboxEnforced: boolean | null; |
| 77 | 77 | /** Why it was not, when it was not. */ |
| 78 | 78 | readonly sandboxNote: string; |
| 79 | /** Where a check ran: `process`, `job`, or empty when unrecorded. */ | |
| 80 | readonly placement: string; | |
| 81 | /** The node a `job` check's pod ran on; empty otherwise. */ | |
| 82 | readonly node: string; | |
| 79 | 83 | /** A failed check's own detail, when its output was recognised; null otherwise. */ |
| 80 | 84 | readonly checkDetail: CheckDetail | null; |
| 81 | 85 | /** The variant group this run belongs to; null for an ordinary run. */ |
| @@ -108,6 +112,8 @@ interface RunRow { | ||
| 108 | 112 | sandbox_enforced: number | null; |
| 109 | 113 | sandbox_note: string; |
| 110 | 114 | check_detail: string; |
| 115 | placement: string; | |
| 116 | node: string; | |
| 111 | 117 | variant_group: string | null; |
| 112 | 118 | created_at: string; |
| 113 | 119 | started_at: string | null; |
| @@ -136,6 +142,8 @@ const toRun = (r: RunRow): Run => ({ | ||
| 136 | 142 | sandboxEnforced: |
| 137 | 143 | r.sandbox_enforced === null ? null : r.sandbox_enforced !== 0, |
| 138 | 144 | sandboxNote: r.sandbox_note, |
| 145 | placement: r.placement, | |
| 146 | node: r.node, | |
| 139 | 147 | checkDetail: r.check_detail ? (JSON.parse(r.check_detail) as CheckDetail) : null, |
| 140 | 148 | variantGroup: r.variant_group, |
| 141 | 149 | createdAt: r.created_at, |
| @@ -342,13 +350,18 @@ export function noteRunProgress( | ||
| 342 | 350 | subscriptionId?: string | undefined; |
| 343 | 351 | /** Whether the shell is held to the sandbox, and why not when it is not. */ |
| 344 | 352 | sandbox?: { enforced: boolean; note: string } | undefined; |
| 353 | /** Where a check ran, and the node when a cluster Job did. */ | |
| 354 | placement?: string | undefined; | |
| 355 | node?: string | undefined; | |
| 345 | 356 | }, |
| 346 | 357 | ): void { |
| 347 | 358 | db.query( |
| 348 | 359 | `UPDATE run SET session_id = COALESCE(?2, session_id), cost_usd = COALESCE(?3, cost_usd), |
| 349 | 360 | branch = COALESCE(?4, branch), subscription_id = COALESCE(?5, subscription_id), |
| 350 | 361 | sandbox_enforced = COALESCE(?6, sandbox_enforced), |
| 351 | sandbox_note = COALESCE(?7, sandbox_note) | |
| 362 | sandbox_note = COALESCE(?7, sandbox_note), | |
| 363 | placement = COALESCE(?8, placement), | |
| 364 | node = COALESCE(?9, node) | |
| 352 | 365 | WHERE id = ?1`, |
| 353 | 366 | ).run( |
| 354 | 367 | runId, |
| @@ -358,6 +371,8 @@ export function noteRunProgress( | ||
| 358 | 371 | progress.subscriptionId ?? null, |
| 359 | 372 | progress.sandbox === undefined ? null : progress.sandbox.enforced ? 1 : 0, |
| 360 | 373 | progress.sandbox?.note.slice(0, 300) ?? null, |
| 374 | progress.placement?.slice(0, 20) ?? null, | |
| 375 | progress.node?.slice(0, 253) ?? null, | |
| 361 | 376 | ); |
| 362 | 377 | } |
| 363 | 378 |
src/core/schema.ts +6
| @@ -1804,6 +1804,12 @@ export const migrations: readonly string[] = [ | ||
| 1804 | 1804 | at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) |
| 1805 | 1805 | ) WITHOUT ROWID; |
| 1806 | 1806 | `, |
| 1807 | // 91 — where a check ran (#1815): `process` (the forge's own cell) or `job` | |
| 1808 | // (a cluster Job), and the node a Job's pod landed on. Empty: not recorded. | |
| 1809 | ` | |
| 1810 | ALTER TABLE run ADD COLUMN placement TEXT NOT NULL DEFAULT ''; | |
| 1811 | ALTER TABLE run ADD COLUMN node TEXT NOT NULL DEFAULT ''; | |
| 1812 | `, | |
| 1807 | 1813 | ]; |
| 1808 | 1814 | |
| 1809 | 1815 | /** |
src/http/mcp.ts +2
| @@ -153,6 +153,7 @@ import { manifestFor } from "../actions/here.ts"; | ||
| 153 | 153 | import { clientBucket } from "./peer.ts"; |
| 154 | 154 | import { fileReport } from "../actions/reports.ts"; |
| 155 | 155 | import { BODY_MAX, REQUEST_MAX, TITLE_MAX } from "../core/reports.ts"; |
| 156 | import { placementWords } from "../checks/placement.ts"; | |
| 156 | 157 | import { everydayOf } from "../core/everyday.ts"; |
| 157 | 158 | import { nextStepsFor } from "../core/next-steps.ts"; |
| 158 | 159 | |
| @@ -755,6 +756,7 @@ const TOOLS: readonly Tool[] = [ | ||
| 755 | 756 | since: r.startedAt ?? r.createdAt, |
| 756 | 757 | // What a run costs is the owner's fact (`canSeeOwnerFacts`). |
| 757 | 758 | costUsd: owned.has(r.repoId) ? r.costUsd : 0, |
| 759 | ...(placementWords(r) === "" ? {} : { ran: placementWords(r) }), | |
Nit: |
||
| 758 | 760 | }; |
| 759 | 761 | }), |
| 760 | 762 | ]); |
src/tui/screens/runlist.ts +2
| @@ -11,6 +11,7 @@ | ||
| 11 | 11 | * the summary that says what failed. |
| 12 | 12 | */ |
| 13 | 13 | |
| 14 | import { placementWords } from "../../checks/placement.ts"; | |
| 14 | 15 | import { getIssuesByIds, type Issue } from "../../core/issues.ts"; |
| 15 | 16 | import type { Repo } from "../../core/repos.ts"; |
| 16 | 17 | import type { Run, RunState } from "../../core/runs.ts"; |
| @@ -108,6 +109,7 @@ export function runRows( | ||
| 108 | 109 | meta: metaOf( |
| 109 | 110 | run.commitOid === "" ? "" : run.commitOid.slice(0, 7), |
| 110 | 111 | costText(run.costUsd), |
| 112 | placementWords(run), | |
| 111 | 113 | relTime(runWhen(run)), |
| 112 | 114 | ), |
| 113 | 115 | }; |
src/web/studio/run.ts +2 −1
| @@ -27,7 +27,7 @@ import { page } from "../layout.ts"; | ||
| 27 | 27 | import { btn, sha } from "../ui.ts"; |
| 28 | 28 | import { previewCard } from "../pages/apps.ts"; |
| 29 | 29 | import { costText, durationText, runTitle, runTokens } from "../pages/runs.ts"; |
| 30 | import { sandboxLine } from "./sandbox.ts"; | |
| 30 | import { placementLine, sandboxLine } from "./sandbox.ts"; | |
| 31 | 31 | import { logBox, runVoice, RUN_TRANSCRIPT_CAP } from "./log.ts"; |
| 32 | 32 | import { changeHref, tellHref } from "./nouns.ts"; |
| 33 | 33 | import { unnamedRun, voiceFor, type Actor } from "./voice.ts"; |
| @@ -176,6 +176,7 @@ export async function runView( | ||
| 176 | 176 | </p>`, |
| 177 | 177 | )} |
| 178 | 178 | ${sandboxLine(run)} |
| 179 | ${placementLine(run)} | |
| 179 | 180 | ${when( |
| 180 | 181 | held.length > 0, |
| 181 | 182 | () => |
src/web/studio/sandbox.ts +7
| @@ -6,8 +6,15 @@ | ||
| 6 | 6 | */ |
| 7 | 7 | |
| 8 | 8 | import type { Run } from "../../core/runs.ts"; |
| 9 | import { placementWords } from "../../checks/placement.ts"; | |
| 9 | 10 | import { html, type Html } from "../html.ts"; |
| 10 | 11 | |
| 12 | /** Where a check ran, when that was a cluster job; nothing for the default. */ | |
| 13 | export function placementLine(run: Run): Html | null { | |
| 14 | const words = placementWords(run); | |
| 15 | return words === "" ? null : html`<p class="muted small" id="run-placement">${words[0]!.toUpperCase() + words.slice(1)}.</p>`; | |
| 16 | } | |
| 17 | ||
| 11 | 18 | export function sandboxLine(run: Run): Html | null { |
| 12 | 19 | if (run.sandboxEnforced === null) return null; |
| 13 | 20 | const agent = run.kind === "agent"; |
test/checks-placement.test.ts added +222
| @@ -0,0 +1,222 @@ | ||
| 1 | /** | |
| 2 | * `checks.placement` (#1815): `process` is today's check; `job` runs it as a | |
| 3 | * cluster Job through a stand-in kube client, streams its log into the run | |
| 4 | * and records the node and the placement. | |
| 5 | */ | |
| 6 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 7 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 8 | import { tmpdir } from "node:os"; | |
| 9 | import { join } from "node:path"; | |
| 10 | import { createApp, type App } from "../src/app.ts"; | |
| 11 | import { commitToBranch, createRepo } from "../src/actions/repos.ts"; | |
| 12 | import { parseSelector, parseTolerations, placementFromEnv } from "../src/checks/placement.ts"; | |
| 13 | import { startChecks, type ChecksHandle } from "../src/checks/service.ts"; | |
| 14 | import { runRows } from "../src/tui/screens/runlist.ts"; | |
| 15 | import { placementLine } from "../src/web/studio/sandbox.ts"; | |
| 16 | import { runPayload } from "../src/actions/runs.ts"; | |
| 17 | import { placementWords } from "../src/checks/placement.ts"; | |
| 18 | import { listRuns, type Run } from "../src/core/runs.ts"; | |
| 19 | import { createUser, type User } from "../src/core/users.ts"; | |
| 20 | import type { Repo } from "../src/core/repos.ts"; | |
| 21 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 22 | import { readRunLog } from "../src/runs/log.ts"; | |
| 23 | ||
| 24 | const checkJson = (run: string): string => `{"check": {"run": ${JSON.stringify(run)}, "install": ""}}\n`; | |
| 25 | ||
| 26 | describe("placement from the environment", () => { | |
| 27 | const job = { | |
| 28 | FORGE_CHECKS_PLACEMENT: "job", | |
| 29 | FORGE_CHECKS_IMAGE: "oven/bun:1", | |
| 30 | FORGE_CHECKS_CLAIM: "checks-root", | |
| 31 | }; | |
| 32 | test("process is the default, and what is malformed leaves it", () => { | |
| 33 | expect(placementFromEnv({})).toEqual({ mode: "process" }); | |
| 34 | expect(placementFromEnv({ FORGE_CHECKS_PLACEMENT: "process" })).toEqual({ mode: "process" }); | |
| 35 | expect(placementFromEnv({ FORGE_CHECKS_PLACEMENT: "job" })).toEqual({ mode: "process" }); | |
| 36 | expect(placementFromEnv({ ...job, FORGE_CHECKS_NODE_SELECTOR: "bad key=x" })).toEqual({ mode: "process" }); | |
| 37 | expect(placementFromEnv({ ...job, FORGE_CHECKS_TOLERATIONS: "[{\"effect\":\"Sometimes\"}]" })).toEqual({ mode: "process" }); | |
| 38 | }); | |
| 39 | test("job carries its node selector and tolerations", () => { | |
| 40 | expect( | |
| 41 | placementFromEnv({ | |
| 42 | ...job, | |
| 43 | FORGE_CHECKS_NODE_SELECTOR: "pool=ci, arch=arm64", | |
| 44 | FORGE_CHECKS_TOLERATIONS: '[{"key":"ci","operator":"Exists","effect":"NoSchedule"}]', | |
| 45 | }), | |
| 46 | ).toEqual({ | |
| 47 | mode: "job", | |
| 48 | image: "oven/bun:1", | |
| 49 | claim: "checks-root", | |
| 50 | nodeSelector: { pool: "ci", arch: "arm64" }, | |
| 51 | tolerations: [{ key: "ci", operator: "Exists", effect: "NoSchedule" }], | |
| 52 | }); | |
| 53 | expect(parseSelector("a")).toBeUndefined(); | |
| 54 | expect(parseTolerations("{}")).toBeUndefined(); | |
| 55 | }); | |
| 56 | }); | |
| 57 | ||
| 58 | describe("checks placed", () => { | |
| 59 | let dataDir: string; | |
| 60 | let app: App; | |
| 61 | let servers: Servers; | |
| 62 | let checks: ChecksHandle; | |
| 63 | let alice: User; | |
| 64 | let repo: Repo; | |
| 65 | const calls: { method: string; url: string; body?: string }[] = []; | |
| 66 | let exit = 0; | |
| 67 | ||
| 68 | // A stand-in kube API: the Job succeeds at once, its pod on node n7. | |
| 69 | const fetch = async (url: string, init: RequestInit) => { | |
| 70 | calls.push({ method: String(init.method), url, ...(typeof init.body === "string" ? { body: init.body } : {}) }); | |
| 71 | const path = new URL(url).pathname; | |
| 72 | if (init.method === "POST" || init.method === "DELETE") return Response.json({}); | |
| 73 | if (path.endsWith("/log")) return new Response("from the job\nsecond line\n"); | |
| 74 | if (path.endsWith("/pods")) | |
| 75 | return Response.json({ | |
| 76 | items: [ | |
| 77 | { | |
| 78 | metadata: { name: "p1" }, | |
| 79 | spec: { nodeName: "n7" }, | |
| 80 | status: { containerStatuses: [{ state: { terminated: { exitCode: exit } } }] }, | |
| 81 | }, | |
| 82 | ], | |
| 83 | }); | |
| 84 | return Response.json({ status: exit === 0 ? { succeeded: 1 } : { failed: 1 } }); | |
| 85 | }; | |
| 86 | ||
| 87 | async function settled(oid: string): Promise<Run> { | |
| 88 | const deadline = Date.now() + 30_000; | |
| 89 | while (Date.now() < deadline) { | |
| 90 | const run = listRuns(app.db, repo.id, { kind: "check", limit: 50 }).find((r) => r.commitOid === oid); | |
| 91 | if (run && (run.state === "ok" || run.state === "failed")) return run; | |
| 92 | await Bun.sleep(50); | |
| 93 | } | |
| 94 | throw new Error("check did not settle"); | |
| 95 | } | |
| 96 | ||
| 97 | async function boot(placement: Parameters<typeof startChecks>[1]["placement"]): Promise<void> { | |
| 98 | dataDir = await mkdtemp(join(tmpdir(), "forge-placement-")); | |
| 99 | app = createApp({ dataDir, port: 0 }); | |
| 100 | servers = await serve(app); | |
| 101 | checks = startChecks(app, { | |
| 102 | forgeUrl: servers.url, | |
| 103 | concurrency: 1, | |
| 104 | placement, | |
| 105 | job: { | |
| 106 | fetch: fetch as never, | |
| 107 | credentials: { server: "https://k", token: "tok", namespace: "forge" }, | |
| 108 | pollMs: 1, | |
| 109 | }, | |
| 110 | }); | |
| 111 | const a = await createUser(app.db, { | |
| 112 | trusted: true, | |
| 113 | username: "alice", | |
| 114 | email: "[email protected]", | |
| 115 | password: "a-good-passphrase", | |
| 116 | adminIfFirst: true, | |
| 117 | }); | |
| 118 | if (!a.ok) throw new Error(a.error); | |
| 119 | alice = a.value; | |
| 120 | const made = await createRepo(app, alice, { name: "lib", visibility: "public" }); | |
| 121 | if (!made.ok) throw new Error(String(made.error)); | |
| 122 | repo = made.value; | |
| 123 | } | |
| 124 | async function down(): Promise<void> { | |
| 125 | await checks.stop(); | |
| 126 | servers.stop(); | |
| 127 | app.close(); | |
| 128 | await rm(dataDir, { recursive: true, force: true }); | |
| 129 | } | |
| 130 | const push = async (run: string) => { | |
| 131 | const c = await commitToBranch(app, alice, repo, { | |
| 132 | message: "Declare a check", | |
| 133 | files: [{ path: "forge.json", content: checkJson(run) }], | |
| 134 | }); | |
| 135 | if (!c.ok) throw new Error(JSON.stringify(c.error)); | |
| 136 | return c.value.commitOid; | |
| 137 | }; | |
| 138 | ||
| 139 | describe("job", () => { | |
| 140 | beforeAll(() => | |
| 141 | boot({ | |
| 142 | mode: "job", | |
| 143 | image: "oven/bun:1", | |
| 144 | claim: "checks-root", | |
| 145 | nodeSelector: { pool: "ci" }, | |
| 146 | tolerations: [{ key: "ci", operator: "Exists" }], | |
| 147 | }), | |
| 148 | ); | |
| 149 | afterAll(down); | |
| 150 | ||
| 151 | test("a check runs as a Job: log in the run, node and placement recorded", async () => { | |
| 152 | const run = await settled(await push("exit 0")); | |
| 153 | expect(run.state).toBe("ok"); | |
| 154 | expect(run.placement).toBe("job"); | |
| 155 | expect(run.node).toBe("n7"); | |
| 156 | const log = readRunLog(dataDir, run.id).lines.join("\n"); | |
| 157 | expect(log).toContain("from the job"); | |
| 158 | expect(log).toContain("$ exit 0"); | |
| 159 | const made = JSON.parse(calls.find((c) => c.method === "POST")!.body!); | |
| 160 | const pod = made.spec.template.spec; | |
| 161 | expect(pod.nodeSelector).toEqual({ pool: "ci" }); | |
| 162 | expect(pod.tolerations).toEqual([{ key: "ci", operator: "Exists" }]); | |
| 163 | expect(pod.volumes[0].persistentVolumeClaim.claimName).toBe("checks-root"); | |
Nit: this assertion is the same as the line above it. Drop one. |
||
| 164 | expect(pod.volumes[0].persistentVolumeClaim.claimName).toBe("checks-root"); | |
| 165 | expect(pod.containers[0].volumeMounts[0].subPath).toBe(run.id); | |
| 166 | expect(pod.automountServiceAccountToken).toBe(false); | |
| 167 | expect(pod.containers[0].resources.limits.memory).toMatch(/Mi$/); | |
| 168 | expect(made.spec.template.metadata.labels["forge-check"]).toBe("true"); | |
| 169 | expect(run.sandboxEnforced).toBe(false); | |
| 170 | expect(run.sandboxNote).toBe("egress not enforced on a cluster job"); | |
| 171 | expect(pod.containers[0].image).toBe("oven/bun:1"); | |
| 172 | expect(pod.containers[0].command.at(-1)).toBe("exit 0"); | |
| 173 | expect(calls.some((c) => c.method === "DELETE")).toBe(true); | |
| 174 | }); | |
| 175 | ||
| 176 | test("the page, the terminal and the events say where a job ran; a process run says nothing", async () => { | |
| 177 | const run = await settled(await push("exit 0 # page")); | |
| 178 | expect(run.placement).toBe("job"); | |
| 179 | const page = await (await globalThis.fetch(`${servers.url}/alice/lib/runs/${run.id}`)).text(); | |
| 180 | expect(page).toContain("Ran as a cluster job on n7."); | |
| 181 | expect(runRows([run], new Map())[0]!.meta).toContain("ran as a cluster job on n7"); | |
| 182 | expect(runPayload(run)).toMatchObject({ placement: "job", node: "n7" }); | |
| 183 | const plain = { ...run, placement: "process", node: "" }; | |
| 184 | expect(placementLine(plain)).toBeNull(); | |
| 185 | expect(runRows([plain], new Map())[0]!.meta).not.toContain("cluster"); | |
| 186 | expect(runPayload(plain)).not.toHaveProperty("node"); | |
| 187 | }); | |
| 188 | ||
| 189 | test("a node name from the cluster is escaped on the page and clean in the terminal", () => { | |
| 190 | const odd = { placement: "job", node: "<b>n\u001b[31m7" }; | |
| 191 | expect(placementWords(odd)).toBe("ran as a cluster job on <b>n[31m7"); | |
| 192 | expect(String(placementLine({ ...odd } as never))).toContain("<b>"); | |
| 193 | }); | |
| 194 | ||
| 195 | test("a failing Job fails the check, still on its node", async () => { | |
| 196 | exit = 2; | |
| 197 | try { | |
| 198 | const run = await settled(await push("exit 2")); | |
| 199 | expect(run.state).toBe("failed"); | |
| 200 | expect(run.summary).toContain("second line"); | |
| 201 | expect(run.node).toBe("n7"); | |
| 202 | } finally { | |
| 203 | exit = 0; | |
| 204 | } | |
| 205 | }); | |
| 206 | }); | |
| 207 | ||
| 208 | describe("process", () => { | |
| 209 | beforeAll(() => boot(undefined)); | |
| 210 | afterAll(down); | |
| 211 | ||
| 212 | test("the default runs here, calls no cluster and records process", async () => { | |
| 213 | calls.length = 0; | |
| 214 | const run = await settled(await push("echo here")); | |
| 215 | expect(run.state).toBe("ok"); | |
| 216 | expect(run.placement).toBe("process"); | |
| 217 | expect(run.node).toBe(""); | |
| 218 | expect(readRunLog(dataDir, run.id).lines.join("\n")).toContain("here"); | |
| 219 | expect(calls).toHaveLength(0); | |
| 220 | }); | |
| 221 | }); | |
| 222 | }); |
test/cluster-jobs.test.ts +38
| @@ -86,3 +86,41 @@ test("a log larger than the cap comes back truncated, by bytes, and is asked for | ||
| 86 | 86 | expect(r.lines).toEqual(["ééééé"]); |
| 87 | 87 | expect(k.calls.some((c) => c.url.endsWith("/log?limitBytes=10"))).toBe(true); |
| 88 | 88 | }); |
| 89 | ||
| 90 | test("the pod's node is reported, and a volume is mounted where the container works", async () => { | |
| 91 | const k = kube([{ succeeded: 1 }]); | |
| 92 | const base = k.fetch; | |
| 93 | const fetch = async (url: string, init: any) => | |
| 94 | new URL(url).pathname.endsWith("/pods") | |
| 95 | ? Response.json({ items: [{ metadata: { name: "p1" }, spec: { nodeName: "n3" } }] }) | |
| 96 | : base(url, init); | |
| 97 | const r = await runJob({ ...spec, volume: { claim: "root", mountPath: "/data/checks/x", subPath: "x" } }, { fetch, credentials, ...clock() }); | |
| 98 | expect(r.node).toBe("n3"); | |
| 99 | const pod = JSON.parse(k.calls[0]!.body!).spec.template.spec; | |
| 100 | expect(pod.containers[0].workingDir).toBe("/data/checks/x"); | |
| 101 | expect(pod.volumes).toEqual([{ name: "work", persistentVolumeClaim: { claimName: "root" } }]); | |
| 102 | expect(pod.containers[0].volumeMounts).toEqual([{ name: "work", mountPath: "/data/checks/x", subPath: "x" }]); | |
| 103 | expect(() => manifest({ ...spec, volume: { claim: "root", mountPath: "/a/b", subPath: "../other" } })).toThrow("bad volume"); | |
| 104 | expect(() => manifest({ ...spec, volume: { claim: "root", mountPath: "/a/b", subPath: "a/b" } })).toThrow("bad volume"); | |
| 105 | expect(() => manifest({ ...spec, volume: { claim: "root", mountPath: "/a/../b", subPath: "x" } })).toThrow("bad volume"); | |
| 106 | }); | |
| 107 | ||
| 108 | test("an aborted call deletes the Job and throws", async () => { | |
| 109 | const k = kube([{ active: 1 }]); | |
| 110 | const ac = new AbortController(); | |
| 111 | ac.abort(); | |
| 112 | await expect(runJob(spec, { fetch: k.fetch, credentials, signal: ac.signal, ...clock() })).rejects.toThrow("cancelled"); | |
| 113 | expect(deleted(k.calls)).toBe(1); | |
| 114 | }); | |
| 115 | ||
| 116 | test("the pod holds no service-account token, no privilege, and the declared limits", () => { | |
| 117 | const pod = (manifest({ ...spec, limits: { memoryMiB: 2048, cpus: 2, diskMiB: 1024 }, labels: { "forge-check": "true" } }) as any).spec.template; | |
| 118 | expect(pod.spec.automountServiceAccountToken).toBe(false); | |
| 119 | expect(pod.spec.securityContext).toEqual({ runAsNonRoot: true }); | |
| 120 | const c = pod.spec.containers[0]; | |
| 121 | expect(c.securityContext).toEqual({ allowPrivilegeEscalation: false, capabilities: { drop: ["ALL"] } }); | |
| 122 | expect(c.resources.limits).toEqual({ memory: "2048Mi", cpu: "2", "ephemeral-storage": "1024Mi" }); | |
| 123 | expect(pod.metadata.labels).toEqual({ "forge-check": "true", "job-name": "build-1" }); | |
| 124 | expect(() => manifest({ ...spec, limits: { memoryMiB: 0, cpus: 1, diskMiB: 1 } })).toThrow("bad limits"); | |
| 125 | expect(() => manifest({ ...spec, labels: { "a b": "c" } })).toThrow("bad label"); | |
| 126 | }); |
new above
offline
Nit:
runAsNonRoot: truemakes a pod fail to start when its image runs as root, and common images likeoven/bun:1do (the tests use it as the example). Say here thatFORGE_CHECKS_IMAGEmust run as a non-root user who can write to the claim. Otherwise the first real setup fails at pod start with no hint why.