Public: anyone can read this; the crew can change it.
The restore drill's cost ceiling is a setting with its reason beside it, and a drill past it stops and says so #15177
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/15168-the-restore-drill-s-cost-ceiling-is-a-se → main 1 commit fork
7 files
changed +116
−18
docs/features/backup-truth.md +1 −1
Diff not shown — a line 2,497 characters wide (limit 2,000). +1 −1
View filesrc/settings/limits.ts +22
| @@ -21,6 +21,7 @@ import { | ||
| 21 | 21 | setCapacity, |
| 22 | 22 | type CapacityKey, |
| 23 | 23 | } from "../core/capacity.ts"; |
| 24 | import { totalmem } from "node:os"; | |
| 24 | 25 | import { memoryLimitBytes } from "../system/memory_limit.ts"; |
| 25 | 26 | import { err, ok, type Result } from "../kernel/result.ts"; |
| 26 | 27 | import { DEPLOY_PULL_KEEP_DAYS } from "../system/retention_deploys.ts"; |
| @@ -56,6 +57,7 @@ export type LimitKey = | ||
| 56 | 57 | | "spawn.timers_max" |
| 57 | 58 | | "forges.wakes_per_forge_hour" |
| 58 | 59 | | "forge.memory_floor_mb" |
| 60 | | "drill.memory_mb" | |
| 59 | 61 | | "guest.storage_mb" |
| 60 | 62 | | "guest.fork_storage_mb" |
| 61 | 63 | | "guest.cooldown_minutes" |
| @@ -93,6 +95,18 @@ function capacityLimit(key: CapacityKey, unit: string, reason: string): LimitDef | ||
| 93 | 95 | }; |
| 94 | 96 | } |
| 95 | 97 | |
| 98 | /** | |
| 99 | * The restore drill's memory ceiling when nobody has set `drill.memory_mb`, MiB. The 2026-11-14 | |
| 100 | * drill of the first chunked snapshot held 9 to 13 GiB, so a fixed small number would be a drill | |
| 101 | * that kills itself: 60% of the machine's memory, between 3 and 16 GiB, or | |
| 102 | * `FORGE_DRILL_MEMORY_MIB` when that is a whole number of at least 512. | |
| 103 | */ | |
| 104 | export function drillMemoryDefaultMiB(env: Record<string, string | undefined> = process.env, machine: number = totalmem()): number { | |
| 105 | const set = env["FORGE_DRILL_MEMORY_MIB"] ?? ""; | |
| 106 | if (/^[0-9]{3,6}$/.test(set) && Number(set) >= 512) return Number(set); | |
| 107 | return Math.floor(Math.min(16 * 1024, Math.max(3 * 1024, (machine * 0.6) / 1024 ** 2))); | |
| 108 | } | |
| 109 | ||
| 96 | 110 | export const LIMITS: Record<LimitKey, LimitDef> = { |
| 97 | 111 | "crew.concurrency": capacityLimit( |
| 98 | 112 | "crew.concurrency", |
| @@ -320,6 +334,14 @@ export const LIMITS: Record<LimitKey, LimitDef> = { | ||
| 320 | 334 | max: 1_048_576, |
| 321 | 335 | reason: "Hosted forges and apps are each held to their own memory, but together they can fill the machine; this much stays with the forge, so they are slowed or stopped before it is. The default is a quarter of the machine's memory, 2 GiB at most; 0 keeps none.", |
| 322 | 336 | }, |
| 337 | "drill.memory_mb": { | |
| 338 | label: "Memory a restore drill may hold", | |
| 339 | unit: "MiB", | |
| 340 | default: drillMemoryDefaultMiB(), | |
| 341 | min: 512, | |
| 342 | max: 1_048_576, | |
| 343 | reason: "A restore drill restores and boots a whole forge in its own process on this machine; past this much resident memory it is stopped and kept red with memory named as the cause, so a drill never takes the machine. The default is 60% of the machine's memory, kept between 3 and 16 GiB, or FORGE_DRILL_MEMORY_MIB.", | |
| 344 | }, | |
| 323 | 345 | "guest.storage_mb": { |
| 324 | 346 | label: "Storage one guest may hold in their forks", |
| 325 | 347 | unit: "MB", |
src/snapshot/drill_cost.ts +6 −2
| @@ -7,10 +7,14 @@ | ||
| 7 | 7 | * strings out. |
| 8 | 8 | */ |
| 9 | 9 | |
| 10 | import { LIMITS } from "../settings/limits.ts"; | |
| 10 | 11 | import { formatBytes, formatSpan, type DrillResult } from "./drill.ts"; |
| 11 | 12 | |
| 12 | /** How the limit is chosen (`drillMemoryLimit` in drill_job.ts), in the words the page uses. */ | |
| 13 | export const LIMIT_RULE = "60% of the machine's memory, kept between 3 and 16 GiB, or FORGE_DRILL_MEMORY_MIB"; | |
| 13 | /** | |
| 14 | * Where the limit comes from, in the words the page uses: the `drill.memory_mb` setting on | |
| 15 | * `/admin/limits`, by its label, and the default it has when nobody set it. | |
| 16 | */ | |
| 17 | export const LIMIT_RULE = `the “${LIMITS["drill.memory_mb"].label}” limit on /admin/limits; by default 60% of the machine's memory, kept between 3 and 16 GiB, or FORGE_DRILL_MEMORY_MIB`; | |
| 14 | 18 | |
| 15 | 19 | /** Share of the limit, whole percent, rounded up so a drill at 99.6% never reads as having room. */ |
| 16 | 20 | const share = (peak: number, limit: number): number => (limit > 0 ? Math.ceil((peak / limit) * 100) : 0); |
src/snapshot/drill_job.ts +9 −14
| @@ -10,7 +10,7 @@ | ||
| 10 | 10 | * and removes every scratch directory, since no drill is running at boot. |
| 11 | 11 | * |
| 12 | 12 | * The child's memory is bounded: `--smol`, and its resident set is read every |
| 13 | * few seconds and the child is killed, the drill red, above `DRILL_MEMORY_BYTES`. | |
| 13 | * few seconds and the child is killed, the drill red, above its limit (`drill.memory_mb`). | |
| 14 | 14 | * Every drill run here keeps its `cost` (elapsed, peak resident, limit; #13253), |
| 15 | 15 | * and a child that ends without a result is a red drill whose reason names the |
| 16 | 16 | * cause (memory, time, or how it exited), never only a log line. |
| @@ -20,26 +20,21 @@ import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync | ||
| 20 | 20 | import { totalmem } from "node:os"; |
| 21 | 21 | import { join } from "node:path"; |
| 22 | 22 | import { during } from "../perf/jobs.ts"; |
| 23 | import { drillMemoryDefaultMiB } from "../settings/limits.ts"; | |
| 23 | 24 | import { describeProgress, DRILL_MAX_MS, type DrillCost, type DrillResult } from "./drill.ts"; |
| 24 | 25 | import type { ChildCost, DrillJob, DrillLine } from "./drill_child.ts"; |
| 25 | 26 | import type { Progress } from "./restore.ts"; |
| 26 | 27 | |
| 27 | const GIB = 1024 ** 3; | |
| 28 | ||
| 29 | 28 | /** |
| 30 | * The most resident memory the drill's process may hold. The 2026-11-14 drill of the first chunked | |
| 31 | * snapshot held 9 to 13 GiB in the serving process, so a fixed small number would be a drill that | |
| 32 | * kills itself: the limit is 60% of the machine's memory, between 3 and 16 GiB, or | |
| 33 | * `FORGE_DRILL_MEMORY_MIB` when that is a whole number of at least 512. It is stated on the Backups | |
| 34 | * page beside the last drill's measured peak (`drill_cost.ts`), so each drill says how near it | |
| 35 | * came. | |
| 29 | * The most resident memory the drill's process may hold when the caller names none, bytes: the | |
| 30 | * `drill.memory_mb` default (`drillMemoryDefaultMiB` in settings/limits.ts). The serving forge | |
| 31 | * passes the setting's value at each drill, so a change on `/admin/limits` governs the next one. | |
| 32 | * It is stated on the Backups page beside the last drill's measured peak (`drill_cost.ts`), so | |
| 33 | * each drill says how near it came. | |
| 36 | 34 | */ |
| 37 | 35 | export function drillMemoryLimit(env: Record<string, string | undefined> = process.env, machine: number = totalmem()): number { |
| 38 | const set = env["FORGE_DRILL_MEMORY_MIB"] ?? ""; | |
| 39 | if (/^[0-9]{3,6}$/.test(set) && Number(set) >= 512) return Number(set) * 1024 ** 2; | |
| 40 | return Math.min(16 * GIB, Math.max(3 * GIB, Math.floor(machine * 0.6))); | |
| 36 | return drillMemoryDefaultMiB(env, machine) * 1024 ** 2; | |
| 41 | 37 | } |
| 42 | export const DRILL_MEMORY_BYTES = drillMemoryLimit(); | |
| 43 | 38 | const WATCH_MS = 3_000; |
| 44 | 39 | const CHILD = join(import.meta.dir, "drill_child.ts"); |
| 45 | 40 | |
| @@ -162,7 +157,7 @@ export async function runDrillInChild( | ||
| 162 | 157 | log: DrillLog, |
| 163 | 158 | options: { readonly limit?: number; readonly watchMs?: number; readonly script?: string } = {}, |
| 164 | 159 | ): Promise<DrillResult> { |
| 165 | const limit = options.limit ?? DRILL_MEMORY_BYTES; | |
| 160 | const limit = options.limit ?? drillMemoryLimit(); | |
| 166 | 161 | const id = drillId(); |
| 167 | 162 | const at = new Date().toISOString(); |
| 168 | 163 | let state: DrillState = { id, startedAt: at, phase: "starting", done: 0, total: 0, bytes: 0, rss: 0, limit }; |
src/snapshot/service.ts +10
| @@ -40,6 +40,7 @@ import { noteOffBox, recordSnapshotOutcome, SNAPSHOT_RETRY_MS } from "./outcome. | ||
| 40 | 40 | import { remakeOf, s3FromEnv, type Store } from "./store.ts"; |
| 41 | 41 | import { verifySnapshot } from "./verify.ts"; |
| 42 | 42 | import { during } from "../perf/jobs.ts"; |
| 43 | import { limitValue } from "../settings/limits.ts"; | |
| 43 | 44 | |
| 44 | 45 | const log = createLog("snapshots"); |
| 45 | 46 | |
| @@ -172,6 +173,9 @@ export function startSnapshots( | ||
| 172 | 173 | readonly drillWaitMs?: number | undefined; |
| 173 | 174 | /** Test seam: the drill itself. */ |
| 174 | 175 | readonly drill?: ((deps: DrillDeps) => Promise<DrillResult>) | undefined; |
| 176 | /** Test seams: the script the drill's own process runs, and how often its memory is read. */ | |
| 177 | readonly drillScript?: string | undefined; | |
| 178 | readonly drillWatchMs?: number | undefined; | |
| 175 | 179 | /** Test seam: the outside drill (`kit.ts`). */ |
| 176 | 180 | readonly outsideDrill?: ((kit: Kit, passphrase: string) => Promise<OutsideDrill>) | undefined; |
| 177 | 181 | /** Test seams: how soon after boot an owed drill runs, and when a red one is owed again. */ |
| @@ -506,6 +510,12 @@ export function startSnapshots( | ||
| 506 | 510 | drillNow = st; |
| 507 | 511 | }, |
| 508 | 512 | (message, fields) => log.info(message, fields), |
| 513 | // The ceiling is read at each drill, so a change on /admin/limits governs the next. | |
| 514 | { | |
| 515 | limit: limitValue(app.db, "drill.memory_mb") * 1024 ** 2, | |
| 516 | ...(options.drillScript === undefined ? {} : { script: options.drillScript }), | |
| 517 | ...(options.drillWatchMs === undefined ? {} : { watchMs: options.drillWatchMs }), | |
| 518 | }, | |
| 509 | 519 | )); |
| 510 | 520 | const result = await run(deps).catch( |
| 511 | 521 | (error): DrillResult => ({ |
test/backups-drill-cost.test.ts +1 −1
| @@ -66,7 +66,7 @@ test("a measured drill: took, peak with its share, and the limit with its rule, | ||
| 66 | 66 | expect(rows).toEqual([ |
| 67 | 67 | "Took: 1 min 2 s", |
| 68 | 68 | "Peak memory: 412 MiB, 14% of the limit", |
| 69 | "Limit: 3 GiB: 60% of the machine's memory, kept between 3 and 16 GiB, or FORGE_DRILL_MEMORY_MIB", | |
| 69 | "Limit: 3 GiB: the “Memory a restore drill may hold” limit on /admin/limits; by default 60% of the machine's memory, kept between 3 and 16 GiB, or FORGE_DRILL_MEMORY_MIB", | |
| 70 | 70 | ]); |
| 71 | 71 | // The rule is said once on the page. |
| 72 | 72 | expect(page.split("between 3 and 16 GiB").length - 1).toBe(1); |
test/drill-cost-setting.test.ts added +67
| @@ -0,0 +1,67 @@ | ||
| 1 | /** | |
| 2 | * The restore drill's memory ceiling is a limit on /admin/limits with its reason beside it | |
| 3 | * (#15168): the serving forge reads `drill.memory_mb` at each drill, and a drill past it ends red | |
| 4 | * naming memory as the cause, with the ceiling it ran under kept in its cost. | |
| 5 | */ | |
| 6 | import { afterEach, beforeEach, expect, test } from "bun:test"; | |
| 7 | import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; | |
| 8 | import { tmpdir } from "node:os"; | |
| 9 | import { join } from "node:path"; | |
| 10 | import { createApp } from "../src/app.ts"; | |
| 11 | import { LIMIT_RULE } from "../src/snapshot/drill_cost.ts"; | |
| 12 | import { drillMemoryLimit } from "../src/snapshot/drill_job.ts"; | |
| 13 | import { lastDrill } from "../src/snapshot/drills.ts"; | |
| 14 | import { startSnapshots } from "../src/snapshot/service.ts"; | |
| 15 | import { dirStore } from "../src/snapshot/store.ts"; | |
| 16 | import { drillMemoryDefaultMiB, LIMITS, limitValue, setLimit } from "../src/settings/limits.ts"; | |
| 17 | import { createUser, getUserByName } from "../src/core/users.ts"; | |
| 18 | ||
| 19 | const MiB = 1024 ** 2; | |
| 20 | let root = ""; | |
| 21 | beforeEach(() => { | |
| 22 | root = mkdtempSync(join(tmpdir(), "forge-drill-cost-setting-")); | |
| 23 | }); | |
| 24 | afterEach(() => rmSync(root, { recursive: true, force: true })); | |
| 25 | ||
| 26 | test("the drill's memory ceiling is a limit with its reason, defaulting to the machine rule", () => { | |
| 27 | const def = LIMITS["drill.memory_mb"]; | |
| 28 | expect(def.unit).toBe("MiB"); | |
| 29 | expect(def.min).toBe(512); | |
| 30 | expect(def.reason).toContain("memory named as the cause"); | |
| 31 | expect(def.default).toBe(drillMemoryDefaultMiB()); | |
| 32 | expect(drillMemoryLimit()).toBe(def.default * MiB); | |
| 33 | expect(drillMemoryDefaultMiB({ FORGE_DRILL_MEMORY_MIB: "4096" }, 64 * 1024 * MiB)).toBe(4096); | |
| 34 | expect(drillMemoryDefaultMiB({}, 64 * 1024 * MiB)).toBe(16 * 1024); | |
| 35 | // The Backups page's Limit row names the setting by its label. | |
| 36 | expect(LIMIT_RULE).toContain(def.label); | |
| 37 | expect(LIMIT_RULE).toContain("/admin/limits"); | |
| 38 | }); | |
| 39 | ||
| 40 | test("a drill past the set ceiling ends red naming memory, and keeps the ceiling it ran under", async () => { | |
| 41 | const app = createApp({ dataDir: join(root, "data"), port: 0 }); | |
| 42 | // A child that holds well past 512 MiB and then stays: killed at a reading, never a result. | |
| 43 | const script = join(root, "holds.ts"); | |
| 44 | writeFileSync( | |
| 45 | script, | |
| 46 | "await Bun.stdin.text(); const held = new Uint8Array(700 * 1024 ** 2).fill(1); setInterval(() => held[0]++, 60_000);", | |
| 47 | ); | |
| 48 | const svc = startSnapshots(app, dirStore(join(root, "store")), { version: "t", nightly: false, drillScript: script, drillWatchMs: 50 }); | |
| 49 | try { | |
| 50 | const made = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true }); | |
| 51 | if (!made.ok) throw new Error(made.error); | |
| 52 | const admin = getUserByName(app.db, "alice")!; | |
| 53 | expect(setLimit(app.db, "drill.memory_mb", 512, admin.id).ok).toBe(true); | |
| 54 | expect(limitValue(app.db, "drill.memory_mb")).toBe(512); | |
| 55 | const result = await svc.drill(); | |
| 56 | expect(result.verdict).toBe("red"); | |
| 57 | expect(result.failedStep).toBe("restore"); | |
| 58 | expect(result.reason).toStartWith("memory: the drill process held "); | |
| 59 | expect(result.reason).toContain("over its 512 MiB limit"); | |
| 60 | expect(result.cost?.limitBytes).toBe(512 * MiB); | |
| 61 | // Recorded as the board reads it. | |
| 62 | expect(lastDrill(app.db)?.reason).toContain("over its 512 MiB limit"); | |
| 63 | } finally { | |
| 64 | await svc.stop(); | |
| 65 | app.close(); | |
| 66 | } | |
| 67 | }); |
new above
offline