Public: anyone can read this; the crew can change it.
The release proof boots a 1-CPU, 512 MiB forge, runs the first-hour journey and fails when memory at rest is over budget #12579
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/12532-the-release-proof-boots-a-1-cpu-512-mib → main 11 commits fork 1 thread · 1 unresolved
14 files
changed +599
−7
docs/features/footprint.md +9
| @@ -24,6 +24,15 @@ Where to read it: | ||
| 24 | 24 | |
| 25 | 25 | The release proof reads the footprint once the new version answers `/healthz`, before the smoke checks load it, and keeps it in `release/proof.json` (`footprint`). `/system` shows this process's last reading, else the proof's. |
| 26 | 26 | |
| 27 | ## The small forge in the release proof (#12532) | |
| 28 | ||
| 29 | Once the smoke checks and the stranger's journey pass, the proof boots a second forge from the same checkout, held by the kernel to 1 CPU and 512 MiB (a cell cgroup of its own: `memory.max` 512 MiB, `cpu.max` one core, swap off), on an empty scratch directory under `<data>/release/small`, and walks its first hour (`src/release/small_forge.ts`): `/readyz`, the home page, a first admin (a code minted into its database, redeemed at `POST /api/v1/first-admin`), a repository, a push of a commit whose `forge.json` declares `check.run: "true"`, and that commit's check reaching `ok`. Its `VmRSS` is read at rest (3 s after ready, before any request) and after the walk, with the cgroup's `memory.peak` and OOM kills. | |
| 30 | ||
| 31 | - A reading at rest over `BUDGET.restBytes` fails the proof, naming the number: "the 1 CPU / 512 MiB forge held 300.0 MiB at rest; the budget is 256.0 MiB". An OOM kill or a step that does not finish fails it too, retried sooner like a slow answer. | |
| 32 | - It runs only where it cannot hurt the forge beside it: the pod must have room for 512 MiB beyond the forge's own use and the reserve (`cellMemoryCeilingMiB`, #12519), and the kernel must hold the cap. Anywhere else it is skipped and the reading says why; a skip does not fail the proof. | |
| 33 | - The scratch forge's environment is named in `smallEnv`: no token or secret of this forge's, no terminal, no apps or crew, loopback only. The token git pushes with travels in git's environment, never its arguments. Its directory and cgroup are removed after. | |
| 34 | - `/system` shows it in "What the forge costs": a row "On 1 CPU, 512 MiB" with memory at rest, after, and peak, or why it was not measured; over budget at rest becomes the section's verdict. The terminal's System dialog says the same words on a line of its own, `On 1 CPU, 512 MiB: …`, after the verdict and before `off:`. It is kept in `release/proof.json` as `small`. | |
| 35 | ||
| 27 | 36 | Reading the disk figure walks the whole data directory (`treeBytes`), once per release proof and never on a page view; on a big data directory that walk can take seconds. It is asynchronous, so it does not hold the event loop. |
| 28 | 37 | |
| 29 | 38 | This is the measurement step. The work that cuts the footprint is tracked in its own issues, each blocking #12092: the cell ceiling on a small pod (#12346), reading the pod's limit in `/system` and health (#12347), a launcher-free start (#12348), lazy-loading what a small forge never uses (#12349), an ARM image (#12350), and the serve/work split only on machines with cores (#12351). |
scripts/gate.ts +1
| @@ -56,6 +56,7 @@ export const SLOW: readonly Exclusion[] = [ | ||
| 56 | 56 | { file: "test/account-ceiling-reserve.test.ts", reason: "runs two stand-in agent runs together against a daily spend ceiling; 11.4 s a test on the gate box", issue: 1242 }, |
| 57 | 57 | { file: "test/api-pulls-timing.test.ts", reason: "seeds 130 open pulls with git repositories to time the pulls API; 7.0 s a test on the gate box", issue: 1242 }, |
| 58 | 58 | { file: "test/apps-adversarial-teardown.test.ts", reason: "races a live preview's deploy against its close and waits out the window; 3.3 s a test on the gate box", issue: 1242 }, |
| 59 | { file: "test/small-forge.test.ts", reason: "boots a whole second forge from the checkout and walks home, first admin, push and check on it; 12 s a test in the sandbox", issue: 12532 }, | |
| 59 | 60 | { file: "test/guide-journey.test.ts", reason: "runs the first-app guide end to end: signup, clone, push and a real deploy; 5.7 s a test on the gate box", issue: 12069 }, |
| 60 | 61 | { file: "test/apps-crew.test.ts", reason: "runs the crew against a real app, a restart and a push that cannot boot; 3.0 s a test on the gate box", issue: 1242 }, |
| 61 | 62 | { file: "test/apps-measure.test.ts", reason: "sends 601 reports through the app proxy to reach the rate limit; 3.7 s a test on the gate box", issue: 1242 }, |
src/main.ts +4
| @@ -103,6 +103,7 @@ import { running } from "./kernel/version.ts"; | ||
| 103 | 103 | import { record as recordDegraded } from "./kernel/degraded_history.ts"; |
| 104 | 104 | import { vitals } from "./kernel/vitals.ts"; |
| 105 | 105 | import { markReady, readFootprint, keepFootprint } from "./system/footprint.ts"; |
| 106 | import { walkSmallForge } from "./release/small_forge.ts"; | |
| 106 | 107 | import { proveRunning } from "./release/prove.ts"; |
| 107 | 108 | import { walkJourney } from "./release/journey.ts"; |
| 108 | 109 | import { removeForge } from "./spawn/manage.ts"; |
| @@ -892,6 +893,9 @@ proveRunning({ | ||
| 892 | 893 | token: process.env["FORGE_TOKEN"], |
| 893 | 894 | journey: walkJourney, |
| 894 | 895 | footprint: async () => keepFootprint(await readFootprint(app.config.dataDir)), |
| 896 | // A 1 CPU / 512 MiB forge beside this one, walked through its first hour (#12532). | |
| 897 | small: () => walkSmallForge({ dataDir: app.config.dataDir }), | |
| 898 | keepFootprint, | |
| 895 | 899 | proberKey: proberKey(app.hookSecret), |
| 896 | 900 | // The probe forge is removed here, in-process, by what `POST /forges/:name/remove` runs: |
| 897 | 901 | // production has no FORGE_TOKEN to ask itself with (#12573). The journey refuses any name |
src/release/prove.ts +38 −3
| @@ -28,7 +28,8 @@ import { UNSTAMPED } from "../kernel/version.ts"; | ||
| 28 | 28 | import { ownProbeHeaders } from "../kernel/probe.ts"; |
| 29 | 29 | import type { Log } from "../kernel/log.ts"; |
| 30 | 30 | import { during } from "../perf/jobs.ts"; |
| 31 | import type { Footprint } from "../system/footprint.ts"; | |
| 31 | import { BUDGET, withSmall, type Footprint } from "../system/footprint.ts"; | |
| 32 | import { SMALL, smallFailure, type SmallReading } from "./small_reading.ts"; | |
| 32 | 33 | |
| 33 | 34 | /** How long the public URL has to answer as the new version. */ |
| 34 | 35 | export const GIVE_UP_MS = 10 * 60_000; |
| @@ -82,6 +83,11 @@ export interface Proof { | ||
| 82 | 83 | readonly journey?: { readonly readyMs: number | null; readonly firstActivityMs: number | null; readonly screens: readonly string[] }; |
| 83 | 84 | /** What the forge cost at rest when this version first answered, before any load (#12092). */ |
| 84 | 85 | readonly footprint?: Footprint; |
| 86 | /** | |
| 87 | * The 1 CPU / 512 MiB forge booted beside this one and walked through its first hour | |
| 88 | * (#12532). | |
| 89 | */ | |
| 90 | readonly small?: SmallReading; | |
| 85 | 91 | /** A person asked for it to be proven again now: the next tick does, whatever the backoff says. */ |
| 86 | 92 | readonly retryNow?: boolean; |
| 87 | 93 | /** How many walks of the stranger's journey it took (1 when it passed or failed at once). */ |
| @@ -338,6 +344,13 @@ export interface ProveOptions { | ||
| 338 | 344 | readonly journey?: typeof walkJourney; |
| 339 | 345 | /** Reads the footprint; the proof records it as soon as the version answers, before any load. */ |
| 340 | 346 | readonly footprint?: () => Promise<Footprint>; |
| 347 | /** | |
| 348 | * Boots a 1 CPU / 512 MiB forge and walks its first hour (`release/small_forge.ts`), once the | |
| 349 | * smoke checks and the journey pass; a reading at rest over the budget fails the proof. | |
| 350 | */ | |
| 351 | readonly small?: () => Promise<SmallReading>; | |
| 352 | /** Hears the footprint once the small forge's reading is in it, for /system. */ | |
| 353 | readonly keepFootprint?: (f: Footprint) => void; | |
| 341 | 354 | /** |
| 342 | 355 | * This boot's prober key (`release/prober.ts`), handed to the journey so the forge's rate |
| 343 | 356 | * limits know it. |
| @@ -375,6 +388,7 @@ export async function proveRelease(o: ProveOptions): Promise<Proof | undefined> | ||
| 375 | 388 | const doFetch = ((url, init) => baseFetch(url, o.signal ? { ...init, signal: init?.signal ? AbortSignal.any([init.signal, o.signal]) : o.signal } : init)) as typeof fetch; |
| 376 | 389 | const failedBefore = last?.version === o.version && !last.ok ? last : undefined; |
| 377 | 390 | let measured: Footprint | undefined; |
| 391 | let small: SmallReading | undefined; | |
| 378 | 392 | const finish = (failure?: string, slow = false, gaveUp = false, walked?: JourneyResult, tries?: number, leftover: readonly string[] = []): Proof => { |
| 379 | 393 | if (o.signal?.aborted) throw new Error("the proof was aborted as hung"); |
| 380 | 394 | const at = new Date(now()).toISOString(); |
| @@ -383,6 +397,7 @@ export async function proveRelease(o: ProveOptions): Promise<Proof | undefined> | ||
| 383 | 397 | at, |
| 384 | 398 | ok: failure === undefined, |
| 385 | 399 | ...(measured === undefined ? {} : { footprint: measured }), |
| 400 | ...(small === undefined ? {} : { small }), | |
| 386 | 401 | ...(walked === undefined ? {} : { journey: { readyMs: walked.readyMs, firstActivityMs: walked.firstActivityMs, screens: walked.screens } }), |
| 387 | 402 | ...(tries === undefined ? {} : { tries }), |
| 388 | 403 | ...(walked === undefined ? {} : { removed: walked.removed, ...(walked.removed ? {} : { removeReason: walked.removeReason ?? "no reason was given" }) }), |
| @@ -424,8 +439,24 @@ export async function proveRelease(o: ProveOptions): Promise<Proof | undefined> | ||
| 424 | 439 | anonymousBoard: true, |
| 425 | 440 | heavy: { version: o.version, probe: ownProbeHeaders(o.dataDir), ...o.heavy }, ...(o.appHost ? { appHost: o.appHost } : {}), |
| 426 | 441 | }); |
| 427 | if (failures[0] !== undefined || o.journey === undefined) | |
| 428 | return finish(failures[0] === undefined ? undefined : checkLine(failures[0]), failures[0]?.slow === true || failures[0]?.empty === true || failures[0]?.hung === true); | |
| 442 | // Last, once all else passed: the small forge. Over budget at rest is the release's to answer | |
| 443 | // for; a step that did not finish is retried sooner, like a slow answer. | |
| 444 | const proveSmall = async (): Promise<{ failure?: string; slow: boolean }> => { | |
| 445 | if (o.small === undefined) return { slow: false }; | |
| 446 | small = await o.small().catch((error): SmallReading => ({ at: new Date(now()).toISOString(), ...SMALL, ran: false, why: `it could not start: ${String(error).slice(0, 200)}`, steps: [], restBytes: null, afterBytes: null, peakBytes: null, oomKills: 0 })); | |
| 447 | if (measured !== undefined) { | |
| 448 | measured = withSmall(measured, small); | |
| 449 | o.keepFootprint?.(measured); | |
| 450 | } | |
| 451 | const failure = smallFailure(small); | |
| 452 | const over = small.restBytes !== null && small.restBytes > BUDGET.restBytes; | |
| 453 | return failure === undefined ? { slow: false } : { failure, slow: !over }; | |
| 454 | }; | |
| 455 | if (failures[0] !== undefined) return finish(checkLine(failures[0]), failures[0]?.slow === true || failures[0]?.empty === true || failures[0]?.hung === true); | |
| 456 | if (o.journey === undefined) { | |
| 457 | const s = await proveSmall(); | |
| 458 | return finish(s.failure, s.slow); | |
| 459 | } | |
| 429 | 460 | // The stranger's journey, the last check: a failed walk is a failed proof, retried on `retryAfter` like any other (one card, paged once). |
| 430 | 461 | // A restart wakes every app: the first walk waits for the host to settle, and a failure is |
| 431 | 462 | // tried again a few minutes later (twice at most) before it is recorded. |
| @@ -446,6 +477,10 @@ export async function proveRelease(o: ProveOptions): Promise<Proof | undefined> | ||
| 446 | 477 | await sleep(o.giveUpMs === undefined ? JOURNEY_RETRY_MS : Math.min(JOURNEY_RETRY_MS, o.giveUpMs)); |
| 447 | 478 | } |
| 448 | 479 | const w = walked!; |
| 480 | if (w.ok) { | |
| 481 | const s = await proveSmall(); | |
| 482 | if (s.failure !== undefined) return finish(s.failure, s.slow, false, w, tries, leftover); | |
| 483 | } | |
| 449 | 484 | return finish(w.ok ? undefined : `a stranger's journey failed: ${w.failure ?? "it did not finish"}`, false, false, w, tries, leftover); |
| 450 | 485 | } catch (error) { |
| 451 | 486 | return finish(`the smoke checks could not run: ${String(error)}`); |
src/release/small_forge.ts added +263
| @@ -0,0 +1,263 @@ | ||
| 1 | /** | |
| 2 | * The small forge (#12532, the Done line of #12092): once per release the | |
| 3 | * proof boots a second forge from this same checkout, held by the kernel to | |
| 4 | * 1 CPU and 512 MiB (a cell cgroup of its own, `apps/cgroup.ts`), on an | |
| 5 | * empty scratch data directory, and walks the first hour on it: the home | |
| 6 | * page, a first admin, a repository, a push, and the check that push runs. | |
| 7 | * Its resident memory is read at rest (ready, before any request) and again | |
| 8 | * after the walk; a reading at rest over `BUDGET.restBytes` is a failed | |
| 9 | * proof that names the number. | |
| 10 | * | |
| 11 | * It never risks the forge it runs beside: it starts only when the pod has | |
| 12 | * room for a whole 512 MiB forge beyond this one's own use and the reserve | |
| 13 | * (`cellMemoryCeilingMiB`, #12519), and only where the kernel can hold the | |
| 14 | * cap. Anywhere else it is skipped, and the reading says why: a forge that | |
| 15 | * cannot be held to 512 MiB proves nothing about 512 MiB. | |
| 16 | * | |
| 17 | * The scratch forge gets a short, named environment, never this forge's: | |
| 18 | * no token, no secret, no terminal, no apps or crew; it listens on loopback | |
| 19 | * only and its directory is removed when the walk ends. | |
| 20 | */ | |
| 21 | ||
| 22 | import { Database } from "bun:sqlite"; | |
| 23 | import { mkdirSync, readFileSync, rmSync } from "node:fs"; | |
| 24 | import { join, resolve } from "node:path"; | |
| 25 | import { cellMemoryCeilingMiB } from "../core/capacity.ts"; | |
| 26 | import { mintFirstAdmin } from "../core/first_admin.ts"; | |
| 27 | import { cgroupHost, enterArgv, openCellCgroup, type CellCgroup } from "../apps/cgroup.ts"; | |
| 28 | import { memoryLimitBytes } from "../system/memory_limit.ts"; | |
| 29 | import { SMALL, type SmallReading, type SmallStep } from "./small_reading.ts"; | |
| 30 | ||
| 31 | const MIB = 1024 * 1024; | |
| 32 | ||
| 33 | /** How long the small forge has to answer `/readyz`, ms; #12092 measured 5.8 s on this size. */ | |
| 34 | export const BOOT_MS = 60_000; | |
| 35 | /** How long its check has to reach a verdict, ms. */ | |
| 36 | export const CHECK_MS = 120_000; | |
| 37 | /** How long the forge rests after ready before its memory is read, ms. */ | |
| 38 | export const REST_MS = 3_000; | |
| 39 | ||
| 40 | /** `VmRSS` from a `/proc/<pid>/status`, bytes; null when absent. */ | |
| 41 | export function vmRss(status: string | undefined): number | null { | |
| 42 | const m = /^VmRSS:\s+(\d+)\s+kB$/m.exec(status ?? ""); | |
| 43 | return m ? Number(m[1]) * 1024 : null; | |
| 44 | } | |
| 45 | ||
| 46 | /** A one-number cgroup file (`memory.peak`), bytes; null when absent or not a number. */ | |
| 47 | export function cgroupNumber(text: string | undefined): number | null { | |
| 48 | const n = Number((text ?? "").trim()); | |
| 49 | return text !== undefined && text.trim() !== "" && Number.isFinite(n) && n >= 0 ? n : null; | |
| 50 | } | |
| 51 | ||
| 52 | /** | |
| 53 | * Whether this pod can hold a 512 MiB forge beside this one without the OOM killer | |
| 54 | * choosing between them: the ceiling a cell may have (#12519) is at least its size. | |
| 55 | */ | |
| 56 | export function roomFor(podBytes: number, forgeBytes: number, memoryMiB: number = SMALL.memoryMiB): string | undefined { | |
| 57 | const ceiling = cellMemoryCeilingMiB(podBytes / MIB, forgeBytes / MIB); | |
| 58 | return ceiling >= memoryMiB ? undefined : `this pod has room for ${ceiling} MiB beside the forge, under the ${memoryMiB} MiB a small forge needs`; | |
| 59 | } | |
| 60 | ||
| 61 | /** The environment the small forge boots with: named here, nothing inherited but PATH. */ | |
| 62 | export function smallEnv(dataDir: string, port: number, path: string | undefined, testSocket?: string): Record<string, string> { | |
| 63 | return { | |
| 64 | // A sandbox that cannot bind a unix socket says so to this forge's tests; the small forge | |
| 65 | // hears the same. | |
| 66 | ...(testSocket === undefined || testSocket === "" ? {} : { FORGE_TEST_SOCKET: testSocket }), | |
| 67 | PATH: path ?? "/usr/local/bin:/usr/bin:/bin", | |
| 68 | HOME: dataDir, | |
| 69 | FORGE_DATA: dataDir, | |
| 70 | FORGE_APP_DATA: "", | |
| 71 | PORT: String(port), | |
| 72 | FORGE_HOST: "127.0.0.1", | |
| 73 | FORGE_URL: `http://127.0.0.1:${port}`, | |
| 74 | FORGE_SSH_PORT: "0", | |
| 75 | FORGE_APPS: "0", | |
| 76 | FORGE_SIGNUP: "invite", | |
| 77 | // Its own check runs in the cell it stands in: no nested cgroup of its own. | |
| 78 | FORGE_CELL_CGROUP: "off", | |
| 79 | FORGE_CHECKS_CONCURRENCY: "1", | |
| 80 | GIT_TERMINAL_PROMPT: "0", | |
| 81 | }; | |
| 82 | } | |
| 83 | ||
| 84 | /** What `walkSmallForge` is handed; each part a test's to replace. */ | |
| 85 | export interface SmallOptions { | |
| 86 | /** | |
| 87 | * This forge's data directory: the scratch forge lives under `<it>/release/small` while it | |
| 88 | * runs. | |
| 89 | */ | |
| 90 | readonly dataDir: string; | |
| 91 | /** The forge's entry, `src/main.ts` of this checkout. */ | |
| 92 | readonly entry?: string; | |
| 93 | /** Opens the cell the small forge runs in; undefined where the kernel cannot hold it. */ | |
| 94 | readonly openCell?: () => CellCgroup | undefined; | |
| 95 | /** Why there is no cell, when `openCell` gives none. */ | |
| 96 | readonly noCellWhy?: () => string; | |
| 97 | /** The pod's memory and this forge's resident size, bytes. */ | |
| 98 | readonly podBytes?: number; | |
| 99 | readonly forgeBytes?: number; | |
| 100 | readonly now?: () => number; | |
| 101 | readonly bootMs?: number; | |
| 102 | readonly checkMs?: number; | |
| 103 | readonly restMs?: number; | |
| 104 | } | |
| 105 | ||
| 106 | const read = (path: string): string | undefined => { | |
| 107 | try { | |
| 108 | return readFileSync(path, "utf8"); | |
| 109 | } catch { | |
| 110 | return undefined; | |
| 111 | } | |
| 112 | }; | |
| 113 | ||
| 114 | /** A loopback port free now. */ | |
| 115 | function freePort(): number { | |
| 116 | const s = Bun.listen({ hostname: "127.0.0.1", port: 0, socket: { data() {} } }); | |
| 117 | const port = s.port; | |
| 118 | s.stop(true); | |
| 119 | return port; | |
| 120 | } | |
| 121 | ||
| 122 | async function run(argv: string[], cwd: string, env: Record<string, string>): Promise<{ code: number; err: string }> { | |
| 123 | const p = Bun.spawn(argv, { cwd, env, stdout: "ignore", stderr: "pipe" }); | |
| 124 | const err = await new Response(p.stderr).text(); | |
| 125 | return { code: await p.exited, err: err.trim().slice(-300) }; | |
| 126 | } | |
| 127 | ||
| 128 | /** | |
| 129 | * Boots the small forge, walks its first hour and reads what it cost. Never throws: a step | |
| 130 | * that fails is a step in the reading, and the forge and its directory are gone after. | |
| 131 | */ | |
| 132 | export async function walkSmallForge(o: SmallOptions): Promise<SmallReading> { | |
| 133 | const now = o.now ?? Date.now; | |
| 134 | const at = new Date(now()).toISOString(); | |
| 135 | const base = { at, cpus: SMALL.cpus, memoryMiB: SMALL.memoryMiB }; | |
| 136 | const skipped = (why: string): SmallReading => ({ ...base, ran: false, why, steps: [], restBytes: null, afterBytes: null, peakBytes: null, oomKills: 0 }); | |
| 137 | const room = roomFor(o.podBytes ?? memoryLimitBytes(), o.forgeBytes ?? process.memoryUsage().rss); | |
| 138 | if (room !== undefined) return skipped(room); | |
| 139 | const scratch = join(o.dataDir, "release", "small"); | |
| 140 | rmSync(scratch, { recursive: true, force: true }); | |
| 141 | const data = join(scratch, "data"); | |
| 142 | const work = join(scratch, "work"); | |
| 143 | mkdirSync(data, { recursive: true }); | |
| 144 | mkdirSync(work, { recursive: true }); | |
| 145 | const port = freePort(); | |
| 146 | // Opened last before the spawn, so nothing between can leave it standing. | |
| 147 | const cell = (o.openCell ?? (() => openCellCgroup(SMALL.memoryMiB, cgroupHost(), { weight: 10, cpus: SMALL.cpus })))(); | |
| 148 | if (cell === undefined) { | |
| 149 | rmSync(scratch, { recursive: true, force: true }); | |
| 150 | const host = cgroupHost(); | |
| 151 | return skipped(`the kernel cannot hold a forge to ${SMALL.memoryMiB} MiB here: ${o.noCellWhy?.() ?? (host.ok ? "the cell's cgroup did not open" : host.reason)}`); | |
| 152 | } | |
| 153 | const url = `http://127.0.0.1:${port}`; | |
| 154 | const env = smallEnv(data, port, process.env.PATH, process.env["FORGE_TEST_SOCKET"]); | |
| 155 | const entry = o.entry ?? resolve(import.meta.dir, "../main.ts"); | |
| 156 | const steps: SmallStep[] = []; | |
| 157 | let restBytes: number | null = null; | |
| 158 | let afterBytes: number | null = null; | |
| 159 | const proc = Bun.spawn(enterArgv(join(cell.dir, "cgroup.procs"), [process.execPath, entry]), { cwd: resolve(import.meta.dir, "../.."), env, stdout: "ignore", stderr: "ignore" }); | |
| 160 | const rss = (): number | null => vmRss(read(`/proc/${proc.pid}/status`)); | |
| 161 | const step = async (name: SmallStep["name"], fn: () => Promise<string | undefined>): Promise<boolean> => { | |
| 162 | const started = now(); | |
| 163 | let why: string | undefined; | |
| 164 | try { | |
| 165 | why = await fn(); | |
| 166 | } catch (error) { | |
| 167 | why = String(error).slice(0, 300); | |
| 168 | } | |
| 169 | steps.push({ name, ms: now() - started, ok: why === undefined, ...(why === undefined ? {} : { why }) }); | |
| 170 | return why === undefined; | |
| 171 | }; | |
| 172 | const ask = (path: string, init: RequestInit = {}): Promise<Response> => fetch(`${url}${path}`, { redirect: "manual", ...init, signal: AbortSignal.timeout(10_000) }); | |
| 173 | try { | |
| 174 | const walked = | |
| 175 | (await step("boot", async () => { | |
| 176 | const until = now() + (o.bootMs ?? BOOT_MS); | |
| 177 | for (;;) { | |
| 178 | if (proc.exitCode !== null) return `it exited with ${proc.exitCode} before it was ready`; | |
| 179 | try { | |
| 180 | const res = await ask("/readyz"); | |
| 181 | await res.body?.cancel(); | |
| 182 | if (res.status === 200) break; | |
| 183 | } catch {} | |
| 184 | if (now() > until) return `/readyz did not answer 200 in ${Math.round((o.bootMs ?? BOOT_MS) / 1000)} s`; | |
| 185 | await Bun.sleep(250); | |
| 186 | } | |
| 187 | await Bun.sleep(o.restMs ?? REST_MS); | |
| 188 | restBytes = rss(); | |
| 189 | return undefined; | |
| 190 | })) && | |
| 191 | (await step("home", async () => { | |
| 192 | const res = await ask("/"); | |
| 193 | const body = await res.text(); | |
| 194 | return res.status === 200 && body.includes("<html") ? undefined : `/ answered ${res.status}`; | |
| 195 | })); | |
| 196 | let token = ""; | |
| 197 | const authed = walked && (await step("first admin", async () => { | |
| 198 | const db = new Database(join(data, "forge.db")); | |
| 199 | let code: string | undefined; | |
| 200 | try { | |
| 201 | code = mintFirstAdmin(db)?.code; | |
| 202 | } finally { | |
| 203 | db.close(); | |
| 204 | } | |
| 205 | if (code === undefined) return "no first-admin code could be minted"; | |
| 206 | const res = await ask("/api/v1/first-admin", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ code, username: "prover", email: "[email protected]", scope: "admin", expiresIn: 3600 }) }); | |
| 207 | const body = (await res.json().catch(() => ({}))) as { token?: unknown }; | |
| 208 | if (res.status !== 201 || typeof body.token !== "string") return `/api/v1/first-admin answered ${res.status}`; | |
| 209 | token = body.token; | |
| 210 | return undefined; | |
| 211 | })); | |
| 212 | const bearer = { authorization: `Bearer ${token}`, "content-type": "application/json" }; | |
| 213 | const made = authed && (await step("repository", async () => { | |
| 214 | const res = await ask("/api/v1/repos", { method: "POST", headers: bearer, body: JSON.stringify({ name: "first-hour" }) }); | |
| 215 | await res.body?.cancel(); | |
| 216 | return res.status === 201 ? undefined : `/api/v1/repos answered ${res.status}`; | |
| 217 | })); | |
| 218 | let oid = ""; | |
| 219 | // The token reaches git in its environment, never in an argument a process list shows. | |
| 220 | const gitEnv = { | |
| 221 | ...env, | |
| 222 | GIT_AUTHOR_NAME: "prover", GIT_AUTHOR_EMAIL: "[email protected]", GIT_COMMITTER_NAME: "prover", GIT_COMMITTER_EMAIL: "[email protected]", | |
| 223 | GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: "/dev/null", | |
| 224 | GIT_CONFIG_COUNT: "1", GIT_CONFIG_KEY_0: "http.extraHeader", GIT_CONFIG_VALUE_0: `Authorization: Basic ${btoa(`prover:${token}`)}`, | |
| 225 | }; | |
| 226 | const pushed = made && (await step("push", async () => { | |
| 227 | await Bun.write(join(work, "forge.json"), `${JSON.stringify({ check: { run: "true", timeout: 60 } })}\n`); | |
| 228 | await Bun.write(join(work, "README.md"), "# first hour\n"); | |
| 229 | for (const argv of [["git", "init", "-q", "-b", "main"], ["git", "add", "."], ["git", "commit", "-q", "-m", "first hour"], ["git", "push", "-q", `${url}/prover/first-hour.git`, "main"]]) { | |
| 230 | const r = await run(argv, work, gitEnv); | |
| 231 | if (r.code !== 0) return `${argv.slice(0, 2).join(" ")} exited ${r.code}: ${r.err}`; | |
| 232 | } | |
| 233 | const head = Bun.spawnSync(["git", "rev-parse", "HEAD"], { cwd: work, env: gitEnv, stdout: "pipe", stderr: "ignore" }); | |
| 234 | oid = head.stdout.toString().trim(); | |
| 235 | return /^[0-9a-f]{40}$/.test(oid) ? undefined : "the pushed commit could not be read"; | |
| 236 | })); | |
| 237 | if (pushed) | |
| 238 | await step("check", async () => { | |
| 239 | const until = now() + (o.checkMs ?? CHECK_MS); | |
| 240 | let state = "none"; | |
| 241 | for (;;) { | |
| 242 | const res = await ask(`/api/v1/repos/prover/first-hour/commits/${oid}/check`, { headers: bearer }); | |
| 243 | const body = (await res.json().catch(() => ({}))) as { state?: unknown; summary?: unknown }; | |
| 244 | state = typeof body.state === "string" ? body.state : `status ${res.status}`; | |
| 245 | if (state === "ok") return undefined; | |
| 246 | if (state === "failed" || state === "superseded") return `the check ended ${state}: ${String(body.summary ?? "").slice(0, 200)}`; | |
| 247 | if (now() > until) return `the check was still ${state} after ${Math.round((o.checkMs ?? CHECK_MS) / 1000)} s`; | |
| 248 | await Bun.sleep(500); | |
| 249 | } | |
| 250 | }); | |
| 251 | afterBytes = rss(); | |
| 252 | } finally { | |
| 253 | proc.kill("SIGTERM"); | |
| 254 | const gone = await Promise.race([proc.exited.then(() => true), Bun.sleep(10_000).then(() => false)]); | |
| 255 | if (!gone) proc.kill("SIGKILL"); | |
| 256 | await proc.exited; | |
| 257 | } | |
| 258 | const peakBytes = cgroupNumber(read(join(cell.dir, "memory.peak"))); | |
| 259 | const oomKills = cell.oomKills(); | |
| 260 | cell.remove(); | |
| 261 | rmSync(scratch, { recursive: true, force: true }); | |
| 262 | return { ...base, ran: true, steps, restBytes, afterBytes, peakBytes, oomKills }; | |
| 263 | } |
src/release/small_reading.ts added +58
| @@ -0,0 +1,58 @@ | ||
| 1 | /** | |
| 2 | * What the small forge cost (#12532), as plain data: the machine it is held to, | |
| 3 | * the reading the release proof keeps, and the one line that fails the proof. | |
| 4 | * Kept apart from `small_forge.ts` so the proof, the footprint and the Workers | |
| 5 | * that reach them read a reading without importing the walk that takes it. | |
| 6 | */ | |
| 7 | ||
| 8 | import { BUDGET } from "../system/footprint.ts"; | |
| 9 | ||
| 10 | const MIB = 1024 * 1024; | |
| 11 | ||
| 12 | /** The machine the small forge is held to. */ | |
| 13 | export const SMALL = { cpus: 1, memoryMiB: 512 } as const; | |
| 14 | ||
| 15 | /** One step of the walk, how long it took, and what went wrong when it did. */ | |
| 16 | export interface SmallStep { | |
| 17 | readonly name: "boot" | "home" | "first admin" | "repository" | "push" | "check"; | |
| 18 | readonly ms: number; | |
| 19 | readonly ok: boolean; | |
| 20 | readonly why?: string; | |
| 21 | } | |
| 22 | ||
| 23 | /** What the small forge cost; every number measured on it, none on this forge. */ | |
| 24 | export interface SmallReading { | |
| 25 | readonly at: string; | |
| 26 | readonly cpus: number; | |
| 27 | readonly memoryMiB: number; | |
| 28 | /** False when it was not run; `why` says why. */ | |
| 29 | readonly ran: boolean; | |
| 30 | readonly why?: string; | |
| 31 | readonly steps: readonly SmallStep[]; | |
| 32 | /** Resident memory of its process at rest: ready, before any request. */ | |
| 33 | readonly restBytes: number | null; | |
| 34 | /** Resident memory after the walk. */ | |
| 35 | readonly afterBytes: number | null; | |
| 36 | /** The most its cgroup held (`memory.peak`), its children counted. */ | |
| 37 | readonly peakBytes: number | null; | |
| 38 | /** How many times the kernel OOM-killed in it. */ | |
| 39 | readonly oomKills: number; | |
| 40 | } | |
| 41 | ||
| 42 | const mib = (n: number): string => `${(n / MIB).toFixed(1)} MiB`; | |
| 43 | ||
| 44 | /** | |
| 45 | * Why the reading fails the proof, in one line that names the number; undefined when it | |
| 46 | * passes or did not run. Over budget at rest is checked first: it is the number the | |
| 47 | * release is held to. | |
| 48 | */ | |
| 49 | export function smallFailure(r: SmallReading, restBudget: number = BUDGET.restBytes): string | undefined { | |
| 50 | if (!r.ran) return undefined; | |
| 51 | const machine = `the ${r.cpus} CPU / ${r.memoryMiB} MiB forge`; | |
| 52 | if (r.restBytes !== null && r.restBytes > restBudget) return `${machine} held ${mib(r.restBytes)} at rest; the budget is ${mib(restBudget)}`; | |
| 53 | if (r.oomKills > 0) return `${machine} was OOM-killed ${r.oomKills === 1 ? "once" : `${r.oomKills} times`} during its first hour`; | |
| 54 | const bad = r.steps.find((s) => !s.ok); | |
| 55 | if (bad !== undefined) return `${machine} failed at ${bad.name}: ${bad.why ?? "it did not finish"}`; | |
| 56 | if (r.restBytes === null) return `${machine}'s memory at rest could not be read`; | |
| 57 | return undefined; | |
| 58 | } |
src/system/footprint.ts +27
| @@ -9,6 +9,7 @@ import { arch } from "node:os"; | ||
| 9 | 9 | import { machineLimits, type MachineLimits } from "../core/capacity.ts"; |
| 10 | 10 | import { findChromium } from "../looks/capture.ts"; |
| 11 | 11 | import { treeBytes } from "./largest_dirs.ts"; |
| 12 | import type { SmallReading } from "../release/small_reading.ts"; | |
| 12 | 13 | import { largePageCache } from "./cache_share.ts"; |
| 13 | 14 | |
| 14 | 15 | const MIB = 1024 * 1024; |
| @@ -49,6 +50,16 @@ export interface Footprint { | ||
| 49 | 50 | readonly diskBytes: number | null; |
| 50 | 51 | readonly optional: readonly Optional[]; |
| 51 | 52 | readonly withinBudget: boolean; |
| 53 | /** The 1 CPU / 512 MiB forge the release proof booted and walked (#12532); unset until it has. */ | |
| 54 | readonly small?: SmallReading; | |
| 55 | } | |
| 56 | ||
| 57 | /** Whether the small forge's memory at rest is over the budget; false when it did not run. */ | |
| 58 | const smallOver = (s: SmallReading | undefined): boolean => s !== undefined && s.ran && s.restBytes !== null && s.restBytes > BUDGET.restBytes; | |
| 59 | ||
| 60 | /** The reading with the small forge's in it, its verdict counted. */ | |
| 61 | export function withSmall(f: Footprint, small: SmallReading): Footprint { | |
| 62 | return { ...f, small, withinBudget: f.withinBudget && !smallOver(small) }; | |
| 52 | 63 | } |
| 53 | 64 | |
| 54 | 65 | /** Start to the listener answering, in ms, once `markReady` has said so; null before. */ |
| @@ -127,6 +138,7 @@ const seconds = (ms: number): string => `${(ms / 1000).toFixed(1)} s`; | ||
| 127 | 138 | export function footprintVerdict(f: Footprint): string { |
| 128 | 139 | const memory = `the forge is using ${mib(f.rssBytes)}; it should stay under ${mib(BUDGET.restBytes)}.`; |
| 129 | 140 | if (f.rssBytes > BUDGET.restBytes) return `Over budget: ${memory}`; |
| 141 | if (smallOver(f.small)) return `Over budget: a forge held to ${f.small!.cpus} CPU and ${f.small!.memoryMiB} MiB used ${mib(f.small!.restBytes!)} at rest; it should stay under ${mib(BUDGET.restBytes)}.`; | |
| 130 | 142 | if (f.readyMs === null) return "Over budget: the forge has not said its first page is up yet."; |
| 131 | 143 | if (f.readyMs > BUDGET.readyMs) return `Over budget: the first page came ${seconds(f.readyMs)} after start; it should come within ${seconds(BUDGET.readyMs)}.`; |
| 132 | 144 | return `Within budget: ${memory}`; |
| @@ -147,5 +159,20 @@ export function footprintFacts(f: Footprint): Fact[] { | ||
| 147 | 159 | { label: "First page", value: f.readyMs === null ? "not up yet" : `${seconds(f.readyMs)} after start; budget ${seconds(BUDGET.readyMs)}` }, |
| 148 | 160 | { label: "Data on disk", value: f.diskBytes === null ? "not read" : `${mib(f.diskBytes)}; an empty forge's budget ${mib(BUDGET.emptyDiskBytes)}` }, |
| 149 | 161 | { label: "Machine", value: `${f.arch}, ${f.cpus} CPU, ${mib(f.machineBytes)}${f.limitedBy === "cgroup" ? ", limited by cgroup" : ""}` }, |
| 162 | ...(f.small === undefined ? [] : [{ label: `On ${f.small.cpus} CPU, ${f.small.memoryMiB} MiB`, value: smallWords(f.small) }]), | |
| 150 | 163 | ]; |
| 151 | 164 | } |
| 165 | ||
| 166 | /** | |
| 167 | * The small forge's reading in words: memory at rest and after its first hour, or why it | |
| 168 | * did not run. | |
| 169 | */ | |
| 170 | export function smallWords(s: SmallReading): string { | |
| 171 | if (!s.ran) return `not measured: ${s.why ?? "it did not run"}`; | |
| 172 | const bad = s.steps.find((x) => !x.ok); | |
| 173 | const rest = s.restBytes === null ? "memory at rest not read" : `${mib(s.restBytes)} at rest`; | |
| 174 | const after = s.afterBytes === null ? "" : `, ${mib(s.afterBytes)} after a push and a check`; | |
| 175 | const peak = s.peakBytes === null ? "" : `, peak ${mib(s.peakBytes)}`; | |
| 176 | const oom = s.oomKills > 0 ? `; OOM-killed ${s.oomKills === 1 ? "once" : `${s.oomKills} times`}` : ""; | |
| 177 | return `${rest}${after}${peak}; budget ${mib(BUDGET.restBytes)} at rest${oom}${bad === undefined ? "" : `; stopped at ${bad.name}: ${bad.why ?? "it did not finish"}`}`; | |
| 178 | } |
src/tui/screens/system.ts +5 −3
| @@ -51,7 +51,7 @@ import { dialog } from "../ui/widgets.ts"; | ||
| 51 | 51 | import { Scroll, ago, confirmScreen, drawLines, hitScroll, later, pickerScreen } from "./common.ts"; |
| 52 | 52 | import { largestDirs, type LargestDirs } from "../../system/largest_dirs.ts"; |
| 53 | 53 | import { dbSize, dbSizeNeed, type DbSize } from "../../system/db_size.ts"; |
| 54 | import { footprintVerdict, lastFootprint } from "../../system/footprint.ts"; | |
| 54 | import { footprintVerdict, lastFootprint, smallWords } from "../../system/footprint.ts"; | |
| 55 | 55 | import { lastProof } from "../../release/prove.ts"; |
| 56 | 56 | import type { PaletteEntry } from "./entries.ts"; |
| 57 | 57 | import { shapeWords } from "../../engine/mode.ts"; |
| @@ -191,7 +191,8 @@ export function largestLines(session: Session, found: LargestDirs | null, inner | ||
| 191 | 191 | |
| 192 | 192 | /** |
| 193 | 193 | * What the forge costs, as /system's "What the forge costs" says it on first view |
| 194 | * (#12092): the verdict in words, then what is off. A few lines, since the System | |
| 194 | * (#12092): the verdict in words, the small forge's reading when the release proof | |
| 195 | * took one (#12532), then what is off. A few lines, since the System | |
| 195 | 196 | * dialog does not scroll; the measured facts and why each part is off are behind |
| 196 | 197 | * the page's disclosure. Empty for anyone the page refuses. |
| 197 | 198 | */ |
| @@ -200,7 +201,8 @@ export function footprintLines(session: Session, f = lastFootprint() ?? lastProo | ||
| 200 | 201 | const head = "What the forge costs"; |
| 201 | 202 | if (f === null) return [head, "not read yet; the release proof reads it once this version answers"]; |
| 202 | 203 | const off = f.optional.filter((o) => !o.on).map((o) => o.name); |
| 203 | return [head, ...wrap(footprintVerdict(f), inner), ...(off.length === 0 ? [] : [truncate(`off: ${off.join(", ")}`, inner)])]; | |
| 204 | const small = f.small === undefined ? [] : wrap(`On ${f.small.cpus} CPU, ${f.small.memoryMiB} MiB: ${smallWords(f.small)}`, inner); | |
| 205 | return [head, ...wrap(footprintVerdict(f), inner), ...small, ...(off.length === 0 ? [] : [truncate(`off: ${off.join(", ")}`, inner)])]; | |
| 204 | 206 | } |
| 205 | 207 | |
| 206 | 208 | /** |
test/footprint.test.ts +22 −1
| @@ -2,7 +2,8 @@ import { afterEach, expect, test } from "bun:test"; | ||
| 2 | 2 | import { mkdtempSync, writeFileSync } from "node:fs"; |
| 3 | 3 | import { join } from "node:path"; |
| 4 | 4 | import { tmpdir } from "node:os"; |
| 5 | import { BUDGET, clearReadyMark, footprintFacts, footprintVerdict, markReady, optionals, readFootprint, type Footprint } from "../src/system/footprint.ts"; | |
| 5 | import { BUDGET, clearReadyMark, footprintFacts, footprintVerdict, markReady, optionals, readFootprint, smallWords, withSmall, type Footprint } from "../src/system/footprint.ts"; | |
| 6 | import type { SmallReading } from "../src/release/small_reading.ts"; | |
| 6 | 7 | import { footprintSection, regions } from "../src/web/pages/system.ts"; |
| 7 | 8 | import { cgroupLimits, machineLimits } from "../src/core/capacity.ts"; |
| 8 | 9 | import { buildSystem } from "../src/system/view.ts"; |
| @@ -97,6 +98,26 @@ test("the release proof records the footprint it is handed once the version answ | ||
| 97 | 98 | expect(lastProof(dir)?.footprint?.rssBytes).toBe(123 * MIB); |
| 98 | 99 | }); |
| 99 | 100 | |
| 101 | // #12532: the small forge's reading shows in /system. | |
| 102 | const small = (over: Partial<SmallReading> = {}): SmallReading => ({ at: "2026-11-14T00:00:00Z", cpus: 1, memoryMiB: 512, ran: true, steps: [], restBytes: 170 * MIB, afterBytes: 382 * MIB, peakBytes: 509 * MIB, oomKills: 0, ...over }); | |
| 103 | ||
| 104 | test("the small forge's reading is a fact on /system, and over budget at rest is the verdict", () => { | |
| 105 | const within = withSmall(reading(), small()); | |
| 106 | expect(within.withinBudget).toBe(true); | |
| 107 | const html = String(footprintSection(within)); | |
| 108 | expect(html).toContain("<dt>On 1 CPU, 512 MiB</dt><dd>170 MiB at rest, 382 MiB after a push and a check, peak 509 MiB; budget 256 MiB at rest</dd>"); | |
| 109 | const over = withSmall(reading(), small({ restBytes: 300 * MIB })); | |
| 110 | expect(over.withinBudget).toBe(false); | |
| 111 | expect(footprintVerdict(over)).toBe("Over budget: a forge held to 1 CPU and 512 MiB used 300 MiB at rest; it should stay under 256 MiB."); | |
| 112 | expect(String(footprintSection(over))).toContain('data-within="false"'); | |
| 113 | }); | |
| 114 | ||
| 115 | test("a small forge that did not run or stopped says so in words", () => { | |
| 116 | expect(smallWords(small({ ran: false, why: "no cgroup here", restBytes: null }))).toBe("not measured: no cgroup here"); | |
| 117 | expect(smallWords(small({ afterBytes: null, peakBytes: null, oomKills: 2, steps: [{ name: "check", ms: 1, ok: false, why: "it ended failed" }] }))).toBe("170 MiB at rest; budget 256 MiB at rest; OOM-killed 2 times; stopped at check: it ended failed"); | |
| 118 | expect(withSmall(reading(), small({ ran: false, restBytes: 900 * MIB })).withinBudget).toBe(true); | |
| 119 | }); | |
| 120 | ||
| 100 | 121 | // ---- the cgroup's limits, not the host's (#12518) ---- |
| 101 | 122 | |
| 102 | 123 | const GIB = 1024 * MIB; |
test/release-prove.test.ts +52
| @@ -13,6 +13,8 @@ import { GIVE_UP_MS, HANG_MS, JOURNEY_TRIES, JOURNEY_RETRY_MS, SETTLE_MS, markPr | ||
| 13 | 13 | import { journeyLine, plain } from "../src/release/words.ts"; |
| 14 | 14 | import type { JourneyResult } from "../src/release/journey.ts"; |
| 15 | 15 | import { owned } from "../src/crew/incidents.ts"; |
| 16 | import type { SmallReading } from "../src/release/small_reading.ts"; | |
| 17 | import type { Footprint } from "../src/system/footprint.ts"; | |
| 16 | 18 | |
| 17 | 19 | const CSS = '<html><link rel="stylesheet" href="/a.css"></html>'; |
| 18 | 20 | |
| @@ -615,3 +617,53 @@ describe("a hung proof is asked again sooner (#9482)", () => { | ||
| 615 | 617 | } |
| 616 | 618 | }); |
| 617 | 619 | }); |
| 620 | ||
| 621 | // #12532: the 1 CPU / 512 MiB forge's reading is the proof's last step. | |
| 622 | describe("the small forge in the proof", () => { | |
| 623 | let dir: string; | |
| 624 | beforeEach(() => { | |
| 625 | dir = mkdtempSync(join(tmpdir(), "forge-prove-small-")); | |
| 626 | }); | |
| 627 | afterEach(() => rmSync(dir, { recursive: true, force: true })); | |
| 628 | const MIB = 1024 * 1024; | |
| 629 | const steps = ["boot", "home", "first admin", "repository", "push", "check"].map((name) => ({ name, ms: 10, ok: true })) as SmallReading["steps"]; | |
| 630 | const reading = (over: Partial<SmallReading> = {}): SmallReading => ({ at: "2026-11-14T00:00:00Z", cpus: 1, memoryMiB: 512, ran: true, steps, restBytes: 170 * MIB, afterBytes: 380 * MIB, peakBytes: 509 * MIB, oomKills: 0, ...over }); | |
| 631 | const prove = (version: string, small: () => Promise<SmallReading>, extra: Partial<ProveOptions> = {}) => | |
| 632 | proveRelease({ dataDir: dir, version, url: "https://forge.test", fetch: site(version).fake, sleep: async () => {}, heavy: { watchMs: 30, pollMs: 5 }, small, ...extra }); | |
| 633 | ||
| 634 | test("a reading at rest over the budget fails the proof, naming the number", async () => { | |
| 635 | const p = await prove("3.0.0", async () => reading({ restBytes: 300 * MIB })); | |
| 636 | expect(p?.ok).toBe(false); | |
| 637 | expect(p?.failure).toBe("the 1 CPU / 512 MiB forge held 300.0 MiB at rest; the budget is 256.0 MiB"); | |
| 638 | // A budget broken is the release's, not a moment's load: not retried as slow. | |
| 639 | expect(p?.slow).toBeUndefined(); | |
| 640 | expect(lastProof(dir)?.small?.restBytes).toBe(300 * MIB); | |
| 641 | }); | |
| 642 | ||
| 643 | test("a reading under the budget passes and is kept with the proof and in the footprint", async () => { | |
| 644 | const kept: Footprint[] = []; | |
| 645 | const fp = { at: "x", arch: "x64", cpus: 4, machineBytes: 2048 * MIB, rssBytes: 200 * MIB, heapBytes: 1, ageSeconds: 1, readyMs: 100, diskBytes: 1, optional: [], withinBudget: true } satisfies Footprint; | |
| 646 | const p = await prove("3.0.1", async () => reading(), { footprint: async () => fp, keepFootprint: (f) => void kept.push(f) }); | |
| 647 | expect(p?.ok).toBe(true); | |
| 648 | expect(p?.small?.restBytes).toBe(170 * MIB); | |
| 649 | expect(p?.footprint?.small?.afterBytes).toBe(380 * MIB); | |
| 650 | expect(kept[0]?.small?.ran).toBe(true); | |
| 651 | }); | |
| 652 | ||
| 653 | test("a step that did not finish fails the proof as a slow one; a skip does not fail it", async () => { | |
| 654 | const stopped = await prove("3.0.2", async () => reading({ steps: [{ name: "boot", ms: 60_000, ok: false, why: "/readyz did not answer 200 in 60 s" }], restBytes: null })); | |
| 655 | expect(stopped?.failure).toBe("the 1 CPU / 512 MiB forge failed at boot: /readyz did not answer 200 in 60 s"); | |
| 656 | expect(stopped?.slow).toBe(true); | |
| 657 | const skipped = await prove("3.0.3", async () => reading({ ran: false, why: "no cgroup here", steps: [], restBytes: null })); | |
| 658 | expect(skipped?.ok).toBe(true); | |
| 659 | expect(skipped?.small?.why).toBe("no cgroup here"); | |
| 660 | }); | |
| 661 | ||
| 662 | test("the small forge runs only once the smoke checks pass", async () => { | |
| 663 | let ran = 0; | |
| 664 | const fake = site("3.0.4", { "/": () => new Response("down", { status: 500 }) }).fake; | |
| 665 | const p = await prove("3.0.4", async () => (ran++, reading()), { fetch: fake }); | |
| 666 | expect(p?.ok).toBe(false); | |
| 667 | expect(ran).toBe(0); | |
| 668 | }); | |
| 669 | }); |
test/small-forge.test.ts added +106
| @@ -0,0 +1,106 @@ | ||
| 1 | /** | |
| 2 | * The release proof's small forge (#12532): the verdict names the number, the | |
| 3 | * pod's room and the kernel's cap decide whether it runs at all, it boots | |
| 4 | * with a named environment that carries none of this forge's secrets, and a | |
| 5 | * real boot of this checkout walks home, first admin, repository, push and | |
| 6 | * check, reading its memory at rest. | |
| 7 | */ | |
| 8 | ||
| 9 | import { afterAll, describe, expect, test } from "bun:test"; | |
| 10 | import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; | |
| 11 | import { tmpdir } from "node:os"; | |
| 12 | import { join } from "node:path"; | |
| 13 | import { cgroupNumber, roomFor, smallEnv, vmRss, walkSmallForge } from "../src/release/small_forge.ts"; | |
| 14 | import { smallFailure, type SmallReading } from "../src/release/small_reading.ts"; | |
| 15 | import type { CellCgroup } from "../src/apps/cgroup.ts"; | |
| 16 | ||
| 17 | const MIB = 1024 * 1024; | |
| 18 | const ok = ["boot", "home", "first admin", "repository", "push", "check"].map((name) => ({ name, ms: 1, ok: true })) as SmallReading["steps"]; | |
| 19 | const reading = (over: Partial<SmallReading> = {}): SmallReading => ({ at: "x", cpus: 1, memoryMiB: 512, ran: true, steps: ok, restBytes: 170 * MIB, afterBytes: 380 * MIB, peakBytes: null, oomKills: 0, ...over }); | |
| 20 | const scratch = mkdtempSync(join(tmpdir(), "small-forge-")); | |
| 21 | afterAll(() => rmSync(scratch, { recursive: true, force: true })); | |
| 22 | ||
| 23 | describe("smallFailure", () => { | |
| 24 | test("over the budget at rest names the number; at the budget passes", () => { | |
| 25 | expect(smallFailure(reading({ restBytes: 300 * MIB }))).toBe("the 1 CPU / 512 MiB forge held 300.0 MiB at rest; the budget is 256.0 MiB"); | |
| 26 | expect(smallFailure(reading({ restBytes: 256 * MIB }))).toBeUndefined(); | |
| 27 | expect(smallFailure(reading({ restBytes: 200 * MIB }), 100 * MIB)).toBe("the 1 CPU / 512 MiB forge held 200.0 MiB at rest; the budget is 100.0 MiB"); | |
| 28 | }); | |
| 29 | ||
| 30 | test("an OOM kill, a step that stopped and an unread reading each fail; a skip does not", () => { | |
| 31 | expect(smallFailure(reading({ oomKills: 1 }))).toBe("the 1 CPU / 512 MiB forge was OOM-killed once during its first hour"); | |
| 32 | expect(smallFailure(reading({ steps: [{ name: "push", ms: 1, ok: false, why: "git push exited 128" }] }))).toBe("the 1 CPU / 512 MiB forge failed at push: git push exited 128"); | |
| 33 | expect(smallFailure(reading({ restBytes: null }))).toBe("the 1 CPU / 512 MiB forge's memory at rest could not be read"); | |
| 34 | expect(smallFailure(reading({ ran: false, restBytes: 900 * MIB }))).toBeUndefined(); | |
| 35 | }); | |
| 36 | }); | |
| 37 | ||
| 38 | test("VmRSS and memory.peak are read in bytes", () => { | |
| 39 | expect(vmRss("Name:\tbun\nVmRSS:\t 174080 kB\nVmSwap:\t0 kB\n")).toBe(170 * MIB); | |
| 40 | expect(vmRss(undefined)).toBeNull(); | |
| 41 | expect(cgroupNumber("533725184\n")).toBe(533725184); | |
| 42 | expect(cgroupNumber("")).toBeNull(); | |
| 43 | expect(cgroupNumber("max")).toBeNull(); | |
| 44 | }); | |
| 45 | ||
| 46 | test("a 512 MiB forge starts only where the pod has room for it beside this one (#12519)", () => { | |
| 47 | // 2048 MiB pod, forge at 400: min(1536, 2048 − 400 − 64) = 1536 ≥ 512. | |
| 48 | expect(roomFor(2048 * MIB, 400 * MIB)).toBeUndefined(); | |
| 49 | // 1024 MiB pod, forge at 400: min(768, 560) = 560 ≥ 512. | |
| 50 | expect(roomFor(1024 * MIB, 400 * MIB)).toBeUndefined(); | |
| 51 | // 512 MiB pod, forge at 170: min(384, 278) = 278. | |
| 52 | expect(roomFor(512 * MIB, 170 * MIB)).toBe("this pod has room for 278 MiB beside the forge, under the 512 MiB a small forge needs"); | |
| 53 | }); | |
| 54 | ||
| 55 | test("the small forge's environment is named, carrying no secret of this forge's", () => { | |
| 56 | const env = smallEnv("/d", 4321, "/bin", undefined); | |
| 57 | expect(env.FORGE_DATA).toBe("/d"); | |
| 58 | expect(env.PORT).toBe("4321"); | |
| 59 | expect(env.FORGE_URL).toBe("http://127.0.0.1:4321"); | |
| 60 | expect(env.FORGE_SSH_PORT).toBe("0"); | |
| 61 | expect(env.FORGE_APPS).toBe("0"); | |
| 62 | expect(Object.keys(env).some((k) => /TOKEN|SECRET|KEY|PASSPHRASE/.test(k))).toBe(false); | |
| 63 | expect(smallEnv("/d", 1, "/bin", "port").FORGE_TEST_SOCKET).toBe("port"); | |
| 64 | }); | |
| 65 | ||
| 66 | describe("walkSmallForge", () => { | |
| 67 | test("no room in the pod skips it, saying why, before any cell opens", async () => { | |
| 68 | let opened = 0; | |
| 69 | const r = await walkSmallForge({ dataDir: scratch, podBytes: 512 * MIB, forgeBytes: 170 * MIB, openCell: () => (opened++, undefined) }); | |
| 70 | expect(r.ran).toBe(false); | |
| 71 | expect(r.why).toContain("room for 278 MiB"); | |
| 72 | expect(opened).toBe(0); | |
| 73 | }); | |
| 74 | ||
| 75 | test("where the kernel cannot hold the cap it is skipped, never run unheld", async () => { | |
| 76 | const r = await walkSmallForge({ dataDir: scratch, podBytes: 4096 * MIB, forgeBytes: 170 * MIB, openCell: () => undefined, noCellWhy: () => "FORGE_CELL_CGROUP=off" }); | |
| 77 | expect(r).toMatchObject({ ran: false, why: "the kernel cannot hold a forge to 512 MiB here: FORGE_CELL_CGROUP=off", restBytes: null }); | |
| 78 | }); | |
| 79 | ||
| 80 | test( | |
| 81 | "a real boot of this checkout walks home, first admin, repository, push and check, and reads its memory", | |
| 82 | async () => { | |
| 83 | // The cgroup is a plain directory: the cell's argv writes its pid to cgroup.procs as it | |
| 84 | // would to the kernel's. | |
| 85 | const cellDir = join(scratch, "cell"); | |
| 86 | mkdirSync(cellDir, { recursive: true }); | |
| 87 | writeFileSync(join(cellDir, "cgroup.procs"), ""); | |
| 88 | writeFileSync(join(cellDir, "memory.peak"), `${400 * MIB}\n`); | |
| 89 | let removed = false; | |
| 90 | const cell: CellCgroup = { dir: cellDir, maxBytes: 512 * MIB, oomKills: () => 0, cpuCapped: true, ioCapped: false, remove: () => void (removed = true) }; | |
| 91 | const data = join(scratch, "forge"); | |
| 92 | const r = await walkSmallForge({ dataDir: data, podBytes: 4096 * MIB, forgeBytes: 170 * MIB, openCell: () => cell, restMs: 200 }); | |
| 93 | expect(r.steps.filter((s) => !s.ok)).toEqual([]); | |
| 94 | expect(r.steps.map((s) => s.name)).toEqual(["boot", "home", "first admin", "repository", "push", "check"]); | |
| 95 | expect(r.ran).toBe(true); | |
| 96 | expect(r.restBytes).toBeGreaterThan(10 * MIB); | |
| 97 | expect(r.afterBytes).toBeGreaterThan(10 * MIB); | |
| 98 | expect(r.peakBytes).toBe(400 * MIB); | |
| 99 | // It entered the cell, and left nothing behind. | |
| 100 | expect(Number(await Bun.file(join(cellDir, "cgroup.procs")).text())).toBeGreaterThan(0); | |
| 101 | expect(removed).toBe(true); | |
| 102 | expect(existsSync(join(data, "release", "small"))).toBe(false); | |
| 103 | }, | |
| 104 | 120_000, | |
| 105 | ); | |
| 106 | }); |
test/spawn-pipes.test.ts +1
| @@ -41,6 +41,7 @@ const PIPING: Record<string, { sites: string[]; reason: string }> = { | ||
| 41 | 41 | "src/release/devtools.ts": { sites: ["stderr"], reason: "stderr read for the DevTools address, then drained in a background loop; stdout ignored" }, |
| 42 | 42 | "src/release/launch.ts": { sites: ["stderr"], reason: "hear() reads stderr to its end for the release's life, passing every chunk through to ours and keeping the tail until ready; stdout inherited" }, |
| 43 | 43 | "src/release/boot.ts": { sites: ["stderr"], reason: "stderr read to its end in a background loop, awaited before the reason is read; stdout ignored" }, |
| 44 | "src/release/small_forge.ts": { sites: ["stderr", "stdout"], reason: "run() reads each git stderr to its end via Response.text() before the exit is awaited, stdout ignored; spawnSync buffers rev-parse stdout, stderr ignored" }, | |
| 44 | 45 | "src/snapshot/boot.ts": { sites: ["stderr"], reason: "stderr read to its end in a background loop, awaited before the exit is read for a lost port; stdout ignored" }, |
| 45 | 46 | "src/release/checkout.ts": { sites: ["stdout+stderr"], reason: "stdout and stderr both read via Response.text()" }, |
| 46 | 47 | "src/checks/install.ts": { sites: ["stdout+stderr"], reason: "prefillInstallCache reads stdout and stderr to the end via Response.text() while the exit is awaited" }, |
test/spawn-sites.test.ts +2
| @@ -79,6 +79,8 @@ const SITES: Record<string, string> = { | ||
| 79 | 79 | "bun (process.execPath) on src/main.ts in the release tree", |
| 80 | 80 | "src/release/swap.ts": |
| 81 | 81 | "bun (process.execPath) on the release entry file", |
| 82 | "src/release/small_forge.ts": | |
| 83 | "bun (process.execPath) on src/main.ts inside a cgroup cell, and git with fixed argv in a scratch directory it just made, the token in the environment only", | |
| 82 | 84 | "src/snapshot/boot.ts": |
| 83 | 85 | "bun (process.execPath) on src/main.ts in the snapshot tree", |
| 84 | 86 | "src/spawn/archive.ts": |
test/tui-system.test.ts +11
| @@ -22,6 +22,7 @@ import type { SystemView } from "../src/system/view.ts"; | ||
| 22 | 22 | import { changeCapacity } from "../src/actions/capacity.ts"; |
| 23 | 23 | import { EGRESS_REPORTED } from "../src/system/egress_line.ts"; |
| 24 | 24 | import { keepFootprint } from "../src/system/footprint.ts"; |
| 25 | import type { SmallReading } from "../src/release/small_reading.ts"; | |
| 25 | 26 | import { headlessTerminal } from "../src/tui/term/terminal.ts"; |
| 26 | 27 | import { askingOf } from "../src/perf/asking.ts"; |
| 27 | 28 | import { quietLogs } from "./helpers/globals.ts"; |
| @@ -455,6 +456,16 @@ describe("tui system footprint (#12092)", () => { | ||
| 455 | 456 | expect(footprintLines(new Session(app, identity(carol)), null)[1]).toContain("not read yet"); |
| 456 | 457 | }); |
| 457 | 458 | |
| 459 | test("the small forge's reading, or why it was not taken, is a line of its own (#12532)", () => { | |
| 460 | const small: SmallReading = { at: fp.at, cpus: 1, memoryMiB: 512, ran: true, steps: [], restBytes: 200 * MiB, afterBytes: 230 * MiB, peakBytes: 260 * MiB, oomKills: 0 }; | |
| 461 | const text = (s: SmallReading): string => footprintLines(new Session(app, identity(carol)), { ...fp, small: s }).join(" ").replace(/\s+/g, " "); | |
| 462 | expect(text(small)).toContain("On 1 CPU, 512 MiB: 200 MiB at rest, 230 MiB after a push and a check, peak 260 MiB; budget 256 MiB at rest"); | |
| 463 | expect(text({ ...small, ran: false, why: "the kernel cannot hold a forge to 512 MiB here", restBytes: null, afterBytes: null, peakBytes: null })).toContain("On 1 CPU, 512 MiB: not measured: the kernel cannot hold a forge to 512 MiB here"); | |
| 464 | expect(footprintLines(new Session(app, identity(carol)), { ...fp, small }).at(-1)).toBe("off: Chromium (looks)"); | |
| 465 | expect(footprintLines(new Session(app, identity(carol)), fp).join(" ")).not.toContain("On 1 CPU"); | |
| 466 | expect(footprintLines(new Session(app, identity(bob)), { ...fp, small })).toEqual([]); | |
| 467 | }); | |
| 468 | ||
| 458 | 469 | test("the System dialog carries the verdict and the off line", async () => { |
| 459 | 470 | keepFootprint(fp); |
| 460 | 471 | const terminal = headlessTerminal(120, 60); |
new above
offline
This row reaches the page only. Add the same words to the terminal's
footprintLines(src/tui/screens/system.ts:198) so a terminal user sees the small forge's reading or its skip reason, and test it.