Public: anyone can read this; the crew can change it.

A build that hits its CPU, memory or time limit says which on the app page in one line #10608

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/10527-a-build-that-hits-its-cpu-memory-or-time → main 3 commits fork 3 threads · 3 unresolved

  • Merged. scout/10527-a-build-that-hits-its-cpu-memory-or-time was merged into main as c6bd5db.

Review

Sign in to review this change.
7 files changed +216 −10
docs/APPS.md +9
@@ -584,6 +584,15 @@ are capped by the machine (`FORGE_CELL_MEMORY_MAX_MIB`, default 4096; the
584584 number of cores; `FORGE_CELL_DISK_MAX_MIB`, default 16384). A cell's `/tmp`
585585 holds at most 256 MiB.
586586
587 An install or test step is also bounded in time: 15 minutes by default.
588 `app.limits.time` (seconds, 1–3600) can lower it, and is held to 30 minutes. A
589 build stopped by its time, memory or cores says which in one line, in the
590 deploy log and as the deploy's error: "the build used its 15 minutes", "the
591 build ran out of its 2 GiB", "the build used more than its 2 cores".
592 `app.limits.memory` and `app.limits.cpus` hold the running app only: an
593 install or test step keeps the step's own memory and cores, so a 128 MiB app
594 still has room to install.
595
587596 Memory is held twice. The watch reads every cell's resident memory once a
588597 second and stops one over its limit. Where the forge holds a delegated
589598 cgroup v2 (`apps/cgroup.ts`), the kernel holds it as well: each cell runs
src/apps/contain.ts +39 −1
@@ -55,6 +55,8 @@ export interface Limits {
5555 readonly cpus: number;
5656 /** Bytes on disk under the cell's writable directories, MiB. */
5757 readonly diskMiB: number;
58 /** Wall-clock time of a build step (install, test), ms; unset for an app, which runs on. */
59 readonly timeMs?: number | undefined;
5860 }
5961
6062 /** What `forge.json` may ask for (`app.limits`, `check.limits`). */
@@ -63,15 +65,19 @@ export type DeclaredLimits = Partial<Limits>;
6365 /** A long-lived app, and a step (install, test, check) that ends. */
6466 export const DEFAULT_LIMITS: Record<"app" | "step", Limits> = {
6567 app: { memoryMiB: 512, cpus: 1, diskMiB: 1024 },
66 step: { memoryMiB: 2048, cpus: 2, diskMiB: 4096 },
68 step: { memoryMiB: 2048, cpus: 2, diskMiB: 4096, timeMs: 15 * 60_000 },
6769 };
6870
71 /** The most time a build step may be given, whatever forge.json declares. */
72 export const STEP_TIME_MAX_MS = 30 * 60_000;
73
6974 /** The most any declaration gets on this machine. */
7075 export function maxLimits(podFile = POD_MEMORY_FILE): Limits {
7176 return {
7277 memoryMiB: memoryCeiling(process.env["FORGE_CELL_MEMORY_MAX_MIB"], podFile),
7378 cpus: availableParallelism(),
7479 diskMiB: positive(process.env["FORGE_CELL_DISK_MAX_MIB"]) ?? 16384,
80 timeMs: STEP_TIME_MAX_MS,
7581 };
7682 }
7783
@@ -167,9 +173,41 @@ export function resolveLimits(
167173 ceilingFrom: memoryCeilingSource(process.env["FORGE_CELL_MEMORY_MAX_MIB"], podFile),
168174 cpus: Math.min(declared?.cpus ?? base.cpus, max.cpus),
169175 diskMiB: Math.min(declared?.diskMiB ?? base.diskMiB, max.diskMiB),
176 ...(base.timeMs === undefined
177 ? {}
178 : { timeMs: Math.min(declared?.timeMs ?? base.timeMs, max.timeMs ?? STEP_TIME_MAX_MS) }),
170179 };
171180 }
172181
182 const plural = (n: number, unit: string): string => `${n} ${unit}${n === 1 ? "" : "s"}`;
183
184 /** A duration in the words a person reads: "15 minutes", "1 second". */
185 export function durationWords(ms: number): string {
186 const seconds = Math.max(1, Math.round(ms / 1000));
187 return seconds >= 60 && seconds % 60 === 0 ? plural(seconds / 60, "minute") : plural(seconds, "second");
188 }
189
190 /** MiB as "2 GiB" where it is whole, else "1536 MiB". */
191 export function sizeWords(mib: number): string {
192 return mib >= 1024 && mib % 1024 === 0 ? `${mib / 1024} GiB` : `${mib} MiB`;
193 }
194
195 /** The one line for a build stopped by its time limit. */
196 export function timeLine(limits: Limits): string {
197 return `the build used its ${durationWords(limits.timeMs ?? DEFAULT_LIMITS.step.timeMs ?? 0)}`;
198 }
199
200 /**
201 * The one line for a build the cell stopped, from the watch's reason: which
202 * limit, and what it was. Undefined for a reason that is not a limit.
203 */
204 export function breachLine(why: string, limits: Limits): string | undefined {
205 if (/memory/.test(why)) return `the build ran out of its ${sizeWords(limits.memoryMiB)}`;
206 if (/cores/.test(why)) return `the build used more than its ${plural(limits.cpus, "core")}`;
207 if (/disk/.test(why)) return `the build ran out of its ${sizeWords(limits.diskMiB)} of disk`;
208 return undefined;
209 }
210
173211 /** Always reachable from inside a cell: the npm registry, for install steps. */
174212 export const STEP_HOSTS = ["registry.npmjs.org"];
175213
src/apps/manifest.ts +8 −3
@@ -285,6 +285,7 @@ const LIMIT_FIELDS = {
285285 memory: { key: "memoryMiB", min: 64, max: 65536, unit: "MiB" },
286286 cpus: { key: "cpus", min: 0.1, max: 64, unit: "cores" },
287287 disk: { key: "diskMiB", min: 64, max: 1048576, unit: "MiB" },
288 time: { key: "timeMs", min: 1, max: 3600, unit: "seconds a build step may take" },

time is shared by check.limits, where nothing uses it. Reject it for check, or honour it.

288289 } as const;
289290
290291 /** `app.limits` / `check.limits`: `{ memory, cpus, disk }`, each optional. */
@@ -296,18 +297,22 @@ function limitsField(
296297 if (value === undefined) return ok({});
297298 if (!isObject(value))
298299 return invalid(`forge.json: ${name}.limits must be an object`);
299 const out: { memoryMiB?: number; cpus?: number; diskMiB?: number } = {};
300 const out: { memoryMiB?: number; cpus?: number; diskMiB?: number; timeMs?: number } = {};
300301 for (const [field, raw] of Object.entries(value)) {
301302 const rule = LIMIT_FIELDS[field as keyof typeof LIMIT_FIELDS];
302303 if (rule === undefined)
303304 return invalid(
304 `forge.json: ${name}.limits takes memory, cpus and disk, not ${field}`,
305 `forge.json: ${name}.limits takes memory, cpus, disk and time, not ${field}`,
306 );
307 if (field === "time" && name === "check")
308 return invalid(
309 "forge.json: check.limits.time is not read: a check's time is the forge's own; app.limits.time lowers the build steps",
305310 );
306311 if (typeof raw !== "number" || !(raw >= rule.min && raw <= rule.max))
307312 return invalid(
308313 `forge.json: ${name}.limits.${field} must be a number of ${rule.unit}, ${rule.min}–${rule.max}`,
309314 );
310 out[rule.key] = raw;
315 out[rule.key] = rule.key === "timeMs" ? raw * 1000 : raw;
311316 }
312317 return ok(out);
313318 }
src/apps/process.ts +29 −6
@@ -28,6 +28,9 @@ import {
2828 killTree,
2929 openCell,
3030 resolveLimits,
31 breachLine,
32 timeLine,
33 type Limits,
3134 signalTree,
3235 STEP_HOSTS,
3336 type Cell,
@@ -321,6 +324,11 @@ export async function runStep(
321324 timeoutMs?: number;
322325 /** Run inside a cell (`apps/contain.ts`) instead of as a plain child. */
323326 cell?: CellSpec | undefined;
327 /**
328 * Set for a build step (install, test): its limits. The time is the
329 * step's bound, and a stop by any limit is one line saying which.
330 */
331 build?: Limits | undefined;
324332 },
325333 ): Promise<Result<void, string>> {
326334 options.log(`$ ${command}`);
@@ -357,14 +365,18 @@ export async function runStep(
357365 if (recent.length > TAIL_LINES) recent.shift();
358366 options.log(line);
359367 };
360 const timeoutMs = options.timeoutMs ?? STEP_TIMEOUT_MS;
368 const timeoutMs = options.timeoutMs ?? options.build?.timeMs ?? STEP_TIMEOUT_MS;
361369 let why: string | undefined;
362370 // A killed process's pipes are not guaranteed to close on their own
363371 // (see `pump`); this stops the reads directly the same moment the kill
364372 // is sent, so a step in flight never outlives its own cancellation.
365373 const stopReading = new AbortController();
366374 const timer = setTimeout(() => {
367 why = `${name} took longer than ${Math.round(timeoutMs / 1000)}s`;
375 why =
376 options.build === undefined
377 ? `${name} took longer than ${Math.round(timeoutMs / 1000)}s`
378 : timeLine({ ...options.build, timeMs: timeoutMs });
379 if (options.build !== undefined) options.log(why);
368380 kill();
369381 stopReading.abort();
370382 }, timeoutMs);
@@ -376,7 +388,7 @@ export async function runStep(
376388 if (options.signal.aborted) onAbort();
377389 else options.signal.addEventListener("abort", onAbort, { once: true });
378390 cell?.watch(proc.pid, (breach) => {
379 why = `${name} stopped: it ${breach}`;
391 why = (options.build && breachLine(breach, options.build)) || `${name} stopped: it ${breach}`;
380392 killTree(proc.pid);
381393 stopReading.abort();
382394 });
@@ -389,7 +401,8 @@ export async function runStep(
389401 if (why !== undefined) return err(why);
390402 // The kernel's OOM kill at the cell's cap is a breach no watch tick saw.
391403 const kernel = code === 0 ? undefined : cell?.breach();
392 if (kernel !== undefined) return err(`${name} stopped: it ${kernel}`);
404 if (kernel !== undefined)
405 return err((options.build && breachLine(kernel, options.build)) || `${name} stopped: it ${kernel}`);
393406 // Ended by a signal this run never sent: no verdict of the step's own.
394407 const killer = code === 0 ? undefined : (proc.signalCode ?? SIGNAL_CODES[code]);
395408 if (killer !== undefined) return err(`${name} was killed by ${killer}`);
@@ -405,6 +418,15 @@ export async function runStep(
405418 }
406419 }
407420
421 /** What a build step of this deploy is held to: the defaults, with forge.json's `limits.time` able to lower the time. */
422 export function stepLimits(input: Pick<StartInput, "manifest">): Limits {
423 // A build keeps the step's own memory and cores: `app.limits.memory` and
424 // `app.limits.cpus` hold the running app, and a 128 MiB web app still
425 // needs room for `bun install`. Only `app.limits.time` reaches a build,
426 // and only to lower it.
427 return resolveLimits("step", { timeMs: input.manifest.limits?.timeMs });
428 }
429
408430 /** The cell a step of this deploy runs in: the checkout and the home, the registry and what the app declared. */
409431 export function stepCell(input: StartInput, use?: ToolchainUse): CellSpec | undefined {
410432 const base = input.cell;
@@ -417,7 +439,7 @@ export function stepCell(input: StartInput, use?: ToolchainUse): CellSpec | unde
417439 hide: base.hide,
418440 egress: [...STEP_HOSTS, ...(input.manifest.egress ?? [])],
419441 reach: [input.forgeUrl],
420 limits: resolveLimits("step", undefined),
442 limits: stepLimits(input),
421443 onUndeclared: base.onUndeclared,
422444 allowed: base.allowed,
423445 onRefused: base.onRefused,
@@ -489,7 +511,7 @@ async function install(
489511 }
490512 const plan = installPlan(command, input, env, filled, use);
491513 const step = (): Promise<Result<void, string>> =>
492 runStep("install", plan.command, { cwd: input.dir, env: plan.env, log: input.log, signal: input.signal, cell: plan.cell });
514 runStep("install", plan.command, { cwd: input.dir, env: plan.env, log: input.log, signal: input.signal, cell: plan.cell, build: stepLimits(input) });
493515 if (plan.lock !== undefined) {
494516 const held = await withLock(plan.lock, step, { signal: input.signal });
495517 return held.ok ? held.value : held;
@@ -550,6 +572,7 @@ export async function startProcess(
550572 log,
551573 signal: testAbort.signal,
552574 cell: stepCell(input, use),
575 build: stepLimits(input),
553576 });
554577
555578 // The port is chosen, then released, then bound by the app: another
test/apps-proxy-unavailable.test.ts +2
@@ -109,6 +109,8 @@ describe("unavailable page", () => {
109109 const failed = await page(`pull-${open}`, { state: "failed", error: `boom token=${secret}` }).text();
110110 expect(failed).toContain("boom");
111111 expect(failed).not.toContain(secret);
112 const limit = await page(`pull-${open}`, { state: "failed", error: "the build used its 15 minutes" }).text();
113 expect(limit).toContain("the build used its 15 minutes");
112114 const running = await page(`pull-${open}`, { state: "running", port: 1 }).text();
113115 expect(running).toContain("It is running.");
114116 });
test/apps.test.ts +23
@@ -780,6 +780,29 @@ exec bun -e "Bun.serve({port:Number(process.env.PORT),hostname:'127.0.0.1',fetch
780780 notForgePaths(await page.text());
781781 }, 60_000);
782782
783 test("a deploy whose install runs past limits.time says so in one line on the app page", async () => {
784 // Invariant: a build stopped by its time limit fails the deploy with that sentence, in the summary, the status and the app page (#10527).
785 const { repo, record, production: run } = await deployApp(leaky, "slow-build", [
786 {
787 path: "forge.json",
788 content: JSON.stringify({
789 app: { run: "sh run.sh", install: "sleep 30", health: "/", ready: 3, limits: { time: 1 } },
790 }),
791 },
792 { path: "run.sh", content: "exit 1\n" },
793 ]);
794 expect(run.state).toBe("failed");
795 const sentence = "the build used its 1 second";
796 expect(run.summary ?? "").toContain(sentence);
797 expect(leaky.runtime.envStatus(record, "production")?.error ?? "").toContain(sentence);
798 const cookie = await signedInCookie(leaky.servers);
799 const page = await fetch(`${leaky.servers.url}/${repo.ownerName}/${repo.name}`, {
800 headers: { cookie: `${cookie}; forge_mode=studio` },
801 });
802 expect(page.status).toBe(200);
803 expect(await page.text()).toContain(sentence);
804 }, 60_000);
805
783806 /**
784807 * Issue #254: the terminal command has a `login@host` in it, which an
785808 * email-scanning edge on the forge's own domain rewrites into a `mailto:`
test/build-limits.test.ts added +106
@@ -0,0 +1,106 @@
1 /**
2 * A build stopped by its time, memory or CPU limit says which in one line,
3 * and forge.json `limits.time` can lower the time, never past the ceiling.
4 */
5
6 import { describe, expect, test } from "bun:test";
7 import { mkdtemp, rm } from "node:fs/promises";
8 import { tmpdir } from "node:os";
9 import { join } from "node:path";
10 import { breachLine, cellsAvailable, resolveLimits, STEP_TIME_MAX_MS, type CellSpec } from "../src/apps/contain.ts";
11 import type { AppManifest } from "../src/apps/manifest.ts";
12 import { parseCheck, parseManifest } from "../src/apps/manifest.ts";
13 import { childEnv, runStep, startProcess, stepLimits } from "../src/apps/process.ts";
14
15 const run = (command: string, build: ReturnType<typeof resolveLimits>, cell?: CellSpec, dir = tmpdir()) =>
16 runStep("install", command, {
17 cwd: dir,
18 env: childEnv({}),
19 log: () => {},
20 signal: new AbortController().signal,
21 cell,
22 build,
23 });
24
25 const cells = await cellsAvailable();
26
27 describe("build limits", () => {
28 test("the default is 15 minutes and forge.json can lower it, not raise it past the ceiling", () => {
29 expect(resolveLimits("step", undefined).timeMs).toBe(15 * 60_000);
30 expect(resolveLimits("step", { timeMs: 60_000 }).timeMs).toBe(60_000);
31 expect(resolveLimits("step", { timeMs: 10 * 3_600_000 }).timeMs).toBe(STEP_TIME_MAX_MS);
32 expect(resolveLimits("app", undefined).timeMs).toBeUndefined();
33 const m = parseManifest(JSON.stringify({ app: { run: "bun x.ts", limits: { time: 90 } } }), undefined);
34 expect(m.ok && stepLimits({ manifest: m.value }).timeMs).toBe(90_000);
35 expect(parseManifest(JSON.stringify({ app: { run: "bun x.ts", limits: { time: 0 } } }), undefined).ok).toBe(false);
36 const chk = parseCheck(JSON.stringify({ check: { run: "bun test", limits: { time: 60 } } }), undefined);
37 expect(chk.ok).toBe(false);
38 });
outdated

Drop the redundant 'longer' run. It costs a second and asserts nothing new.

39
40 test("the app's running memory and cores do not reach its build", () => {
41 const of = (limits: object) => {
42 const m = parseManifest(JSON.stringify({ app: { run: "bun x.ts", limits } }), undefined);
43 if (!m.ok) throw new Error("manifest");
44 return stepLimits({ manifest: m.value });
45 };
46 const step = resolveLimits("step", undefined);
47 expect(of({ memory: 128, cpus: 0.5 })).toMatchObject({ memoryMiB: step.memoryMiB, cpus: step.cpus });
48 expect(of({ memory: 8192, cpus: 8 })).toMatchObject({ memoryMiB: step.memoryMiB, cpus: step.cpus });
49 });
50
51 test("a sleeping install past its time fails with the time sentence", async () => {
52 const result = await run("sleep 30", resolveLimits("step", { timeMs: 1000 }));
53 expect(result).toEqual({ ok: false, error: "the build used its 1 second" });
54 }, 20_000);
55
56 test("a deploy whose install runs past limits.time fails with the sentence", async () => {
57 const dir = await mkdtemp(join(tmpdir(), "build-limits-"));
58 try {
59 const started = await startProcess({
60 dir,
61 dataDir: dir,
62 manifest: { run: "sleep 1", install: "sleep 30", health: "/", ready: 8, limits: { timeMs: 1000 } } as AppManifest,
63 env: "production",
64 url: "http://app.test",
65 commitOid: "0".repeat(40),
66 forgeUrl: "http://forge.test",
67 identityKey: "key",
68 log: () => {},
69 signal: new AbortController().signal,
70 });
71 expect(started.ok).toBe(false);
72 if (!started.ok) expect(started.error.message).toContain("the build used its 1 second");
73 } finally {
74 await rm(dir, { recursive: true, force: true });
75 }
76 }, 30_000);
77
78 test("the sentences for memory and cores", () => {
79 const limits = resolveLimits("step", undefined);
80 expect(breachLine("used 2100 MiB of memory, over its 2048 MiB limit (the default)", limits)).toBe(
81 "the build ran out of its 2 GiB",
82 );
83 expect(breachLine("reached its 2048 MiB memory cap, and the kernel killed it", limits)).toBe(
84 "the build ran out of its 2 GiB",
85 );
86 expect(breachLine("used 2.5 cores for 5s, over its limit of 2", limits)).toBe("the build used more than its 2 cores");
87 expect(breachLine("something else", limits)).toBeUndefined();
88 });
89
90 test.skipIf(!cells.ok)("an install over its memory is stopped with the memory sentence (needs a cell)", async () => {
91 const dir = await mkdtemp(join(tmpdir(), "build-limits-"));
92 try {
93 const limits = { ...resolveLimits("step", undefined), memoryMiB: 128 };
94 const spec: CellSpec = { dir, writable: [], home: join(dir, ".home"), hide: [], egress: [], reach: [], limits };
95 const result = await run(
96 `bun -e "const a = []; for (;;) { a.push(new Uint8Array(16 << 20).fill(1)); await Bun.sleep(10); }"`,
97 limits,
98 spec,
99 dir,
100 );
101 expect(result).toEqual({ ok: false, error: "the build ran out of its 128 MiB" });
102 } finally {
103 await rm(dir, { recursive: true, force: true });
104 }
105 }, 30_000);
106 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs