Public: anyone can read this; the crew can change it.

The restore drill's cost ceiling is a setting with its reason beside it, and a drill past it stops and says so #15177

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/15168-the-restore-drill-s-cost-ceiling-is-a-se → main 1 commit fork

  • Merged. mason/15168-the-restore-drill-s-cost-ceiling-is-a-se was merged into main as dd73caf.

Review

Sign in to review this change.
7 files changed +116 −18
docs/features/backup-truth.md +1 −1

Diff not shown — a line 2,497 characters wide (limit 2,000). +1 −1

View file
src/settings/limits.ts +22
@@ -21,6 +21,7 @@ import {
2121 setCapacity,
2222 type CapacityKey,
2323 } from "../core/capacity.ts";
24 import { totalmem } from "node:os";
2425 import { memoryLimitBytes } from "../system/memory_limit.ts";
2526 import { err, ok, type Result } from "../kernel/result.ts";
2627 import { DEPLOY_PULL_KEEP_DAYS } from "../system/retention_deploys.ts";
@@ -56,6 +57,7 @@ export type LimitKey =
5657 | "spawn.timers_max"
5758 | "forges.wakes_per_forge_hour"
5859 | "forge.memory_floor_mb"
60 | "drill.memory_mb"
5961 | "guest.storage_mb"
6062 | "guest.fork_storage_mb"
6163 | "guest.cooldown_minutes"
@@ -93,6 +95,18 @@ function capacityLimit(key: CapacityKey, unit: string, reason: string): LimitDef
9395 };
9496 }
9597
98 /**
99 * The restore drill's memory ceiling when nobody has set `drill.memory_mb`, MiB. The 2026-11-14
100 * drill of the first chunked snapshot held 9 to 13 GiB, so a fixed small number would be a drill
101 * that kills itself: 60% of the machine's memory, between 3 and 16 GiB, or
102 * `FORGE_DRILL_MEMORY_MIB` when that is a whole number of at least 512.
103 */
104 export function drillMemoryDefaultMiB(env: Record<string, string | undefined> = process.env, machine: number = totalmem()): number {
105 const set = env["FORGE_DRILL_MEMORY_MIB"] ?? "";
106 if (/^[0-9]{3,6}$/.test(set) && Number(set) >= 512) return Number(set);
107 return Math.floor(Math.min(16 * 1024, Math.max(3 * 1024, (machine * 0.6) / 1024 ** 2)));
108 }
109
96110 export const LIMITS: Record<LimitKey, LimitDef> = {
97111 "crew.concurrency": capacityLimit(
98112 "crew.concurrency",
@@ -320,6 +334,14 @@ export const LIMITS: Record<LimitKey, LimitDef> = {
320334 max: 1_048_576,
321335 reason: "Hosted forges and apps are each held to their own memory, but together they can fill the machine; this much stays with the forge, so they are slowed or stopped before it is. The default is a quarter of the machine's memory, 2 GiB at most; 0 keeps none.",
322336 },
337 "drill.memory_mb": {
338 label: "Memory a restore drill may hold",
339 unit: "MiB",
340 default: drillMemoryDefaultMiB(),
341 min: 512,
342 max: 1_048_576,
343 reason: "A restore drill restores and boots a whole forge in its own process on this machine; past this much resident memory it is stopped and kept red with memory named as the cause, so a drill never takes the machine. The default is 60% of the machine's memory, kept between 3 and 16 GiB, or FORGE_DRILL_MEMORY_MIB.",
344 },
323345 "guest.storage_mb": {
324346 label: "Storage one guest may hold in their forks",
325347 unit: "MB",
src/snapshot/drill_cost.ts +6 −2
@@ -7,10 +7,14 @@
77 * strings out.
88 */
99
10 import { LIMITS } from "../settings/limits.ts";
1011 import { formatBytes, formatSpan, type DrillResult } from "./drill.ts";
1112
12 /** How the limit is chosen (`drillMemoryLimit` in drill_job.ts), in the words the page uses. */
13 export const LIMIT_RULE = "60% of the machine's memory, kept between 3 and 16 GiB, or FORGE_DRILL_MEMORY_MIB";
13 /**
14 * Where the limit comes from, in the words the page uses: the `drill.memory_mb` setting on
15 * `/admin/limits`, by its label, and the default it has when nobody set it.
16 */
17 export const LIMIT_RULE = `the “${LIMITS["drill.memory_mb"].label}” limit on /admin/limits; by default 60% of the machine's memory, kept between 3 and 16 GiB, or FORGE_DRILL_MEMORY_MIB`;
1418
1519 /** Share of the limit, whole percent, rounded up so a drill at 99.6% never reads as having room. */
1620 const share = (peak: number, limit: number): number => (limit > 0 ? Math.ceil((peak / limit) * 100) : 0);
src/snapshot/drill_job.ts +9 −14
@@ -10,7 +10,7 @@
1010 * and removes every scratch directory, since no drill is running at boot.
1111 *
1212 * The child's memory is bounded: `--smol`, and its resident set is read every
13 * few seconds and the child is killed, the drill red, above `DRILL_MEMORY_BYTES`.
13 * few seconds and the child is killed, the drill red, above its limit (`drill.memory_mb`).
1414 * Every drill run here keeps its `cost` (elapsed, peak resident, limit; #13253),
1515 * and a child that ends without a result is a red drill whose reason names the
1616 * cause (memory, time, or how it exited), never only a log line.
@@ -20,26 +20,21 @@ import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync
2020 import { totalmem } from "node:os";
2121 import { join } from "node:path";
2222 import { during } from "../perf/jobs.ts";
23 import { drillMemoryDefaultMiB } from "../settings/limits.ts";
2324 import { describeProgress, DRILL_MAX_MS, type DrillCost, type DrillResult } from "./drill.ts";
2425 import type { ChildCost, DrillJob, DrillLine } from "./drill_child.ts";
2526 import type { Progress } from "./restore.ts";
2627
27 const GIB = 1024 ** 3;
28
2928 /**
30 * The most resident memory the drill's process may hold. The 2026-11-14 drill of the first chunked
31 * snapshot held 9 to 13 GiB in the serving process, so a fixed small number would be a drill that
32 * kills itself: the limit is 60% of the machine's memory, between 3 and 16 GiB, or
33 * `FORGE_DRILL_MEMORY_MIB` when that is a whole number of at least 512. It is stated on the Backups
34 * page beside the last drill's measured peak (`drill_cost.ts`), so each drill says how near it
35 * came.
29 * The most resident memory the drill's process may hold when the caller names none, bytes: the
30 * `drill.memory_mb` default (`drillMemoryDefaultMiB` in settings/limits.ts). The serving forge
31 * passes the setting's value at each drill, so a change on `/admin/limits` governs the next one.
32 * It is stated on the Backups page beside the last drill's measured peak (`drill_cost.ts`), so
33 * each drill says how near it came.
3634 */
3735 export function drillMemoryLimit(env: Record<string, string | undefined> = process.env, machine: number = totalmem()): number {
38 const set = env["FORGE_DRILL_MEMORY_MIB"] ?? "";
39 if (/^[0-9]{3,6}$/.test(set) && Number(set) >= 512) return Number(set) * 1024 ** 2;
40 return Math.min(16 * GIB, Math.max(3 * GIB, Math.floor(machine * 0.6)));
36 return drillMemoryDefaultMiB(env, machine) * 1024 ** 2;
4137 }
42 export const DRILL_MEMORY_BYTES = drillMemoryLimit();
4338 const WATCH_MS = 3_000;
4439 const CHILD = join(import.meta.dir, "drill_child.ts");
4540
@@ -162,7 +157,7 @@ export async function runDrillInChild(
162157 log: DrillLog,
163158 options: { readonly limit?: number; readonly watchMs?: number; readonly script?: string } = {},
164159 ): Promise<DrillResult> {
165 const limit = options.limit ?? DRILL_MEMORY_BYTES;
160 const limit = options.limit ?? drillMemoryLimit();
166161 const id = drillId();
167162 const at = new Date().toISOString();
168163 let state: DrillState = { id, startedAt: at, phase: "starting", done: 0, total: 0, bytes: 0, rss: 0, limit };
src/snapshot/service.ts +10
@@ -40,6 +40,7 @@ import { noteOffBox, recordSnapshotOutcome, SNAPSHOT_RETRY_MS } from "./outcome.
4040 import { remakeOf, s3FromEnv, type Store } from "./store.ts";
4141 import { verifySnapshot } from "./verify.ts";
4242 import { during } from "../perf/jobs.ts";
43 import { limitValue } from "../settings/limits.ts";
4344
4445 const log = createLog("snapshots");
4546
@@ -172,6 +173,9 @@ export function startSnapshots(
172173 readonly drillWaitMs?: number | undefined;
173174 /** Test seam: the drill itself. */
174175 readonly drill?: ((deps: DrillDeps) => Promise<DrillResult>) | undefined;
176 /** Test seams: the script the drill's own process runs, and how often its memory is read. */
177 readonly drillScript?: string | undefined;
178 readonly drillWatchMs?: number | undefined;
175179 /** Test seam: the outside drill (`kit.ts`). */
176180 readonly outsideDrill?: ((kit: Kit, passphrase: string) => Promise<OutsideDrill>) | undefined;
177181 /** Test seams: how soon after boot an owed drill runs, and when a red one is owed again. */
@@ -506,6 +510,12 @@ export function startSnapshots(
506510 drillNow = st;
507511 },
508512 (message, fields) => log.info(message, fields),
513 // The ceiling is read at each drill, so a change on /admin/limits governs the next.
514 {
515 limit: limitValue(app.db, "drill.memory_mb") * 1024 ** 2,
516 ...(options.drillScript === undefined ? {} : { script: options.drillScript }),
517 ...(options.drillWatchMs === undefined ? {} : { watchMs: options.drillWatchMs }),
518 },
509519 ));
510520 const result = await run(deps).catch(
511521 (error): DrillResult => ({
test/backups-drill-cost.test.ts +1 −1
@@ -66,7 +66,7 @@ test("a measured drill: took, peak with its share, and the limit with its rule,
6666 expect(rows).toEqual([
6767 "Took: 1 min 2 s",
6868 "Peak memory: 412 MiB, 14% of the limit",
69 "Limit: 3 GiB: 60% of the machine&#39;s memory, kept between 3 and 16 GiB, or FORGE_DRILL_MEMORY_MIB",
69 "Limit: 3 GiB: the “Memory a restore drill may hold” limit on /admin/limits; by default 60% of the machine&#39;s memory, kept between 3 and 16 GiB, or FORGE_DRILL_MEMORY_MIB",
7070 ]);
7171 // The rule is said once on the page.
7272 expect(page.split("between 3 and 16 GiB").length - 1).toBe(1);
test/drill-cost-setting.test.ts added +67
@@ -0,0 +1,67 @@
1 /**
2 * The restore drill's memory ceiling is a limit on /admin/limits with its reason beside it
3 * (#15168): the serving forge reads `drill.memory_mb` at each drill, and a drill past it ends red
4 * naming memory as the cause, with the ceiling it ran under kept in its cost.
5 */
6 import { afterEach, beforeEach, expect, test } from "bun:test";
7 import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
8 import { tmpdir } from "node:os";
9 import { join } from "node:path";
10 import { createApp } from "../src/app.ts";
11 import { LIMIT_RULE } from "../src/snapshot/drill_cost.ts";
12 import { drillMemoryLimit } from "../src/snapshot/drill_job.ts";
13 import { lastDrill } from "../src/snapshot/drills.ts";
14 import { startSnapshots } from "../src/snapshot/service.ts";
15 import { dirStore } from "../src/snapshot/store.ts";
16 import { drillMemoryDefaultMiB, LIMITS, limitValue, setLimit } from "../src/settings/limits.ts";
17 import { createUser, getUserByName } from "../src/core/users.ts";
18
19 const MiB = 1024 ** 2;
20 let root = "";
21 beforeEach(() => {
22 root = mkdtempSync(join(tmpdir(), "forge-drill-cost-setting-"));
23 });
24 afterEach(() => rmSync(root, { recursive: true, force: true }));
25
26 test("the drill's memory ceiling is a limit with its reason, defaulting to the machine rule", () => {
27 const def = LIMITS["drill.memory_mb"];
28 expect(def.unit).toBe("MiB");
29 expect(def.min).toBe(512);
30 expect(def.reason).toContain("memory named as the cause");
31 expect(def.default).toBe(drillMemoryDefaultMiB());
32 expect(drillMemoryLimit()).toBe(def.default * MiB);
33 expect(drillMemoryDefaultMiB({ FORGE_DRILL_MEMORY_MIB: "4096" }, 64 * 1024 * MiB)).toBe(4096);
34 expect(drillMemoryDefaultMiB({}, 64 * 1024 * MiB)).toBe(16 * 1024);
35 // The Backups page's Limit row names the setting by its label.
36 expect(LIMIT_RULE).toContain(def.label);
37 expect(LIMIT_RULE).toContain("/admin/limits");
38 });
39
40 test("a drill past the set ceiling ends red naming memory, and keeps the ceiling it ran under", async () => {
41 const app = createApp({ dataDir: join(root, "data"), port: 0 });
42 // A child that holds well past 512 MiB and then stays: killed at a reading, never a result.
43 const script = join(root, "holds.ts");
44 writeFileSync(
45 script,
46 "await Bun.stdin.text(); const held = new Uint8Array(700 * 1024 ** 2).fill(1); setInterval(() => held[0]++, 60_000);",
47 );
48 const svc = startSnapshots(app, dirStore(join(root, "store")), { version: "t", nightly: false, drillScript: script, drillWatchMs: 50 });
49 try {
50 const made = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
51 if (!made.ok) throw new Error(made.error);
52 const admin = getUserByName(app.db, "alice")!;
53 expect(setLimit(app.db, "drill.memory_mb", 512, admin.id).ok).toBe(true);
54 expect(limitValue(app.db, "drill.memory_mb")).toBe(512);
55 const result = await svc.drill();
56 expect(result.verdict).toBe("red");
57 expect(result.failedStep).toBe("restore");
58 expect(result.reason).toStartWith("memory: the drill process held ");
59 expect(result.reason).toContain("over its 512 MiB limit");
60 expect(result.cost?.limitBytes).toBe(512 * MiB);
61 // Recorded as the board reads it.
62 expect(lastDrill(app.db)?.reason).toContain("over its 512 MiB limit");
63 } finally {
64 await svc.stop();
65 app.close();
66 }
67 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs