Public: anyone can read this; the crew can change it.

The Backups page shows the last restore drill's duration and peak memory beside the limit it states #13284

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

torque/forge:torque/13253-the-backups-page-shows-the-last-restore → main 5 commits fork 2 threads · 2 unresolved

  • Merged. torque/13253-the-backups-page-shows-the-last-restore was merged into main as 895c932.

Review

Sign in to review this change.
14 files changed +444 −37
docs/features/backup-truth.md +1 −1

Diff not shown — a line 2,313 characters wide (limit 2,000). +1 −1

View file
src/actions/backups_now.ts +6 −2
@@ -11,6 +11,7 @@ import type { App } from "../app.ts";
1111 import type { User } from "../core/users.ts";
1212 import { err, ok, type Result } from "../kernel/result.ts";
1313 import type { DrillStep } from "../snapshot/drill.ts";
14 import { costLine } from "../snapshot/drill_cost.ts";
1415 import { lastDrill } from "../snapshot/drills.ts";
1516 import { snapshotsOf } from "../snapshot/service.ts";
1617 import { fileSize } from "../web/ui.ts";
@@ -146,7 +147,8 @@ export function drillStatus(app: App): string | undefined {
146147
147148 /**
148149 * Wait for the running drill, telling `tick` where it stands each `everyMs`, and answer its
149 * verdict in the words `drillNow` uses; the terminal's door to a drill that is a job (#12592).
150 * verdict in the words `drillNow` uses, then its cost (`costLine`); the terminal's door to a drill
151 * that is a job (#12592).
150152 */
151153 export async function followDrill(app: App, tick: (says: string) => void, everyMs = 250): Promise<string> {
152154 for (let says = drillStatus(app); says !== undefined; says = drillStatus(app)) {
@@ -160,7 +162,9 @@ export async function followDrill(app: App, tick: (says: string) => void, everyM
160162 d = undefined; // the forge closed while the drill ran
161163 }
162164 if (d === undefined) return "The drill ended without a verdict.";
163 return d.verdict === "green" ? "The drill restored green." : `The drill failed at ${d.failedStep}.`;
165 // Its process's cost beside its limit, as the Backups page and MCP say it (#13253).
166 const said = d.verdict === "green" ? "The drill restored green." : `The drill failed at ${d.failedStep}.`;
167 return `${said} Its process ${costLine(d)}.`;
164168 }
165169
166170 /**
src/http/mcp/tools/restore_drill.ts +11 −4
@@ -1,4 +1,5 @@
11 import { scopeAllows } from "../../../core/tokens.ts";
2 import { costLine } from "../../../snapshot/drill_cost.ts";
23 import { lastDrill } from "../../../snapshot/drills.ts";
34 import { snapshotsOf, type SnapshotsHandle } from "../../../snapshot/service.ts";
45 import { good, bad, type Tool, mayBoards } from "../shared.ts";
@@ -10,7 +11,7 @@ const phaseWords = (st: ReturnType<SnapshotsHandle["startDrill"]>["state"]): Rec
1011
1112 export const tool: Tool = {
1213 name: "restore_drill",
13 doc: "`action` start (default) answers at once `{ started, id, phase, bytes, ... }`; one drill runs at a time, a second ask says `started: false` and where the running one is; `status` answers the running drill's phase and bytes, or the newest verdict `{ verdict, failedStep?, says, at, from?, offSite? }` (red, not an error, when it failed); it runs in its own process; a forge admin's, with an admin token",
14 doc: "`action` start (default) answers at once `{ started, id, phase, bytes, ... }`; one drill runs at a time, a second ask says `started: false` and where the running one is; `status` answers the running drill's phase and bytes, or the newest verdict `{ verdict, failedStep?, says, at, from?, offSite?, cost, elapsedMs?, peakMiB?, limitMiB? }` (red, not an error, when it failed); it runs in its own process; a forge admin's, with an admin token",
1415 description: "Start a restore drill of the newest snapshot, or read how it is going.",
1516 inputSchema: { type: "object", properties: { action: { type: "string", enum: ["start", "status"] } } },
1617 writes: true,
@@ -37,11 +38,17 @@ export const tool: Tool = {
3738 const d = lastDrill(ctx.app.db);
3839 if (d === undefined) return good({ verdict: "none", says: "No drill has run." });
3940 // Where it restored from (the off-site lane when the forge has one, #11805), never a key.
40 const from = d.from === undefined ? {} : { from: d.from, offSite: d.offSite === true };
41 const extra = {
42 ...(d.from === undefined ? {} : { from: d.from, offSite: d.offSite === true }),
43 // The drill process's elapsed time and peak memory beside its limit, as the Backups page says
44 // it (#13253).
45 cost: costLine(d),
46 ...(d.cost === undefined ? {} : { elapsedMs: d.cost.elapsedMs, peakMiB: Math.round(d.cost.peakBytes / 1024 ** 2), limitMiB: Math.round(d.cost.limitBytes / 1024 ** 2) }),
47 };
4148 const answer =
4249 d.verdict === "green"
43 ? good({ verdict: "green", at: d.at, says: "The drill restored green.", ...from })
44 : good({ verdict: "red", at: d.at, failedStep: d.failedStep, says: `The drill failed at ${d.failedStep}${d.reason ? `: ${d.reason}` : ""}.`, ...from });
50 ? good({ verdict: "green", at: d.at, says: "The drill restored green.", ...extra })
51 : good({ verdict: "red", at: d.at, failedStep: d.failedStep, says: `The drill failed at ${d.failedStep}${d.reason ? `: ${d.reason}` : ""}.`, ...extra });
4552 return { ...answer, card: { at: d.at } };
4653 },
4754 };
src/snapshot/drill.ts +34 −5
@@ -102,14 +102,20 @@ const PART_WORD: Record<PhaseTime["phase"], string> = {
102102 boot: "start and migrate",
103103 };
104104
105 /** The drill's time by part, longest first: download, decompress, clone, open, migrate. Parts that share a word are added. */
105 /**
106 * The drill's time by part, longest first: download, decompress, clone, open, migrate. Parts that
107 * share a word are added.
108 */
106109 export function timeByPart(phases: readonly PhaseTime[]): { part: string; ms: number }[] {
107110 const sums = new Map<string, number>();
108111 for (const p of phases) sums.set(PART_WORD[p.phase], (sums.get(PART_WORD[p.phase]) ?? 0) + p.ms);
109112 return [...sums].map(([part, ms]) => ({ part, ms })).sort((a, b) => b.ms - a.ms);
110113 }
111114
112 /** "download about 8 min, decompress 40 s, open 2 s; download was the slowest"; empty when nothing was timed. */
115 /**
116 * "download about 8 min, decompress 40 s, open 2 s; download was the slowest"; empty when nothing
117 * was timed.
118 */
113119 export function describeTimes(phases: readonly PhaseTime[]): string {
114120 const parts = timeByPart(phases).filter((p) => p.part !== "key");
115121 if (parts.length === 0) return "";
@@ -119,7 +125,10 @@ export function describeTimes(phases: readonly PhaseTime[]): string {
119125
120126 /** One restore phase and the milliseconds it took. */
121127 export interface PhaseTime {
122 /** A restore phase, then `open` (the database read) and `boot` (the restored forge starting and migrating). */
128 /**
129 * A restore phase, then `open` (the database read) and `boot` (the restored forge starting and
130 * migrating).
131 */
123132 readonly phase: Progress["phase"] | "open" | "boot";
124133 readonly ms: number;
125134 }
@@ -160,15 +169,35 @@ export interface DrillResult {
160169 readonly budgetMs: number;
161170 /** Why it is red, for the person reading; never a value. */
162171 readonly reason: string | null;
163 /** The store it restored from (`Store.where`, never a key); absent on drills kept before #11805. */
172 /**
173 * The store it restored from (`Store.where`, never a key); absent on drills kept before #11805.
174 */
164175 readonly from?: string | undefined;
165176 /** Whether that store is off the forge's machine (`isOffBox`). */
166177 readonly offSite?: boolean | undefined;
167178 /** The manifest format of the snapshot drilled (1 before chunks); absent on older drills. */
168179 readonly format?: number | undefined;
180 /**
181 * What the drill's own process cost (#13253); absent on drills kept before, or run in the serving
182 * process.
183 */
184 readonly cost?: DrillCost | undefined;
169185 }
170186
171 /** The restored forge's front door: `/` answers below 400 (a redirect to sign in is an answer), or this throws. */
187 /** The drill process's wall time and peak resident memory, and the memory limit it ran under. */
188 export interface DrillCost {
189 /** From spawn to exit, ms. */
190 readonly elapsedMs: number;
191 /** Peak resident set of the drill process, bytes (getrusage `ru_maxrss`). */
192 readonly peakBytes: number;
193 /** The resident limit it was killed above, bytes (`drillMemoryLimit`). */
194 readonly limitBytes: number;
195 }
196
197 /**
198 * The restored forge's front door: `/` answers below 400 (a redirect to sign in is an answer), or
199 * this throws.
200 */
172201 export async function answersRoot(base: string): Promise<void> {
173202 const res = await fetch(`${base}/`, { redirect: "manual", signal: AbortSignal.timeout(5_000) });
174203 await res.body?.cancel();
src/snapshot/drill_child.ts +22 −3
@@ -6,7 +6,10 @@
66 *
77 * One JSON job on stdin ({ dataDir, store, passphrase }), so no secret is on a
88 * command line or in the environment. One JSON object per line on stdout:
9 * `{ progress }` as the restore moves, then `{ result }`.
9 * `{ progress }` as the restore moves, then `{ result, cost }`: `cost` is this
10 * process's elapsed time and peak resident memory (#13253), read once per
11 * progress report (`process.memoryUsage.rss()`, one syscall) and at the end
12 * from getrusage, which counts every peak between the reports.
1013 */
1114
1215 import { runDrill, type DrillResult } from "./drill.ts";
@@ -20,13 +23,28 @@ export interface DrillJob {
2023 readonly passphrase?: string | undefined;
2124 }
2225
23 export type DrillLine = { readonly progress: Progress } | { readonly result: DrillResult };
26 /** The child's own cost; the parent adds the limit it ran under. */
27 export interface ChildCost {
28 readonly elapsedMs: number;
29 readonly peakBytes: number;
30 }
31
32 export type DrillLine = { readonly progress: Progress } | { readonly result: DrillResult; readonly cost?: ChildCost };
33
34 /**
35 * Peak resident bytes so far: the larger of the readings taken and getrusage's
36 * `maxRSS`, which Node and Bun give in KiB for the own process.
37 */
38 export function peakSoFar(readings: number): number {
39 return Math.max(readings, process.resourceUsage().maxRSS * 1024);
40 }
2441
2542 const say = (line: DrillLine): void => void process.stdout.write(`${JSON.stringify(line)}\n`);
2643
2744 if (import.meta.main) {
2845 const job = JSON.parse(await Bun.stdin.text()) as DrillJob;
2946 let last = 0;
47 let peak = 0;
3048 const result = await runDrill({
3149 store: storeFrom(job.store),
3250 dataDir: job.dataDir,
@@ -38,9 +56,10 @@ if (import.meta.main) {
3856 last = now;
3957 // Buffers of finished reads are given back at once, not at the collector's leisure.
4058 Bun.gc(false);
59 peak = Math.max(peak, process.memoryUsage.rss());
4160 say({ progress });
4261 },
4362 });
44 say({ result });
63 say({ result, cost: { elapsedMs: Math.round(process.uptime() * 1_000), peakBytes: peakSoFar(peak) } });
4564 process.exit(0);
4665 }
src/snapshot/drill_cost.ts added +45
@@ -0,0 +1,45 @@
1 /**
2 * What a restore drill's own process cost beside the limit it ran under
3 * (#13253), so each drill proves or disproves the limit. The Backups page shows
4 * it as `dl` rows (`costFacts`); the MCP `restore_drill` status and the
5 * terminal's drill screens say it in one line (`costLine`). How the limit is
6 * chosen (`LIMIT_RULE`) is said once, in the Limit row. Plain data in,
7 * strings out.
8 */
9
10 import { formatBytes, formatSpan, type DrillResult } from "./drill.ts";
11
outdated

Principle 1: this rule text is repeated on every drill line and in the MCP answer, so the page explains itself twice. Keep it in one place.

12 /** How the limit is chosen (`drillMemoryLimit` in drill_job.ts), in the words the page uses. */
13 export const LIMIT_RULE = "60% of the machine's memory, kept between 3 and 16 GiB, or FORGE_DRILL_MEMORY_MIB";
14
15 /** Share of the limit, whole percent, rounded up so a drill at 99.6% never reads as having room. */
16 const share = (peak: number, limit: number): number => (limit > 0 ? Math.ceil((peak / limit) * 100) : 0);
17
18 /**
19 * "took 1 min 2 s, peak 412 MiB of its 3 GiB limit (14%)"; for a drill that kept no cost, says so.
20 */
21 export function costLine(d: Pick<DrillResult, "cost">): string {
22 const c = d.cost;
23 if (c === undefined) return "elapsed time and peak memory not recorded for this drill";
24 return `took ${formatSpan(c.elapsedMs)}, peak ${formatBytes(c.peakBytes)} of its ${formatBytes(c.limitBytes)} limit (${share(c.peakBytes, c.limitBytes)}%)`;
25 }
26
27 /** One fact of the drill's cost: its label and its value, as a `dl` row says it. */
28 export interface CostFact {
29 readonly label: string;
30 readonly value: string;
31 }
32
33 /**
34 * Took, peak (with its share of the limit) and the limit; one row saying none was kept, for an
35 * older drill.
36 */
37 export function costFacts(d: Pick<DrillResult, "cost">): CostFact[] {
38 const c = d.cost;
39 if (c === undefined) return [{ label: "Drill cost", value: "elapsed time and peak memory not recorded for this drill" }];
40 return [
41 { label: "Took", value: formatSpan(c.elapsedMs) },
42 { label: "Peak memory", value: `${formatBytes(c.peakBytes)}, ${share(c.peakBytes, c.limitBytes)}% of the limit` },
43 { label: "Limit", value: `${formatBytes(c.limitBytes)}: ${LIMIT_RULE}` },
44 ];
45 }
src/snapshot/drill_job.ts +49 −10
@@ -11,14 +11,17 @@
1111 *
1212 * The child's memory is bounded: `--smol`, and its resident set is read every
1313 * few seconds and the child is killed, the drill red, above `DRILL_MEMORY_BYTES`.
14 * Every drill run here keeps its `cost` (elapsed, peak resident, limit; #13253),
15 * and a child that ends without a result is a red drill whose reason names the
16 * cause (memory, time, or how it exited), never only a log line.
1417 */
1518
1619 import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
1720 import { totalmem } from "node:os";
1821 import { join } from "node:path";
1922 import { during } from "../perf/jobs.ts";
20 import { describeProgress, DRILL_MAX_MS, type DrillResult } from "./drill.ts";
21 import type { DrillJob, DrillLine } from "./drill_child.ts";
23 import { describeProgress, DRILL_MAX_MS, type DrillCost, type DrillResult } from "./drill.ts";
24 import type { ChildCost, DrillJob, DrillLine } from "./drill_child.ts";
2225 import type { Progress } from "./restore.ts";
2326
2427 const GIB = 1024 ** 3;
@@ -28,8 +31,8 @@ const GIB = 1024 ** 3;
2831 * snapshot held 9 to 13 GiB in the serving process, so a fixed small number would be a drill that
2932 * kills itself: the limit is 60% of the machine's memory, between 3 and 16 GiB, or
3033 * `FORGE_DRILL_MEMORY_MIB` when that is a whole number of at least 512. It is stated on the Backups
31 * page. Not yet measured on a real-size store: the child reads the bound and logs what it held
32 * (`restore drill working`), so the first real drill tunes this number.
34 * page beside the last drill's measured peak (`drill_cost.ts`), so each drill says how near it
35 * came.
3336 */
3437 export function drillMemoryLimit(env: Record<string, string | undefined> = process.env, machine: number = totalmem()): number {
3538 const set = env["FORGE_DRILL_MEMORY_MIB"] ?? "";
@@ -117,10 +120,30 @@ export function cutDrill(cut: { id: string; at: string }): DrillResult {
117120 };
118121 }
119122
123 /**
124 * The red result of a drill whose process gave none: the cause is the reason, and its cost is kept.
125 */
126 export function deadDrill(at: string, cause: string, cost: DrillCost): DrillResult {
127 return {
128 at,
129 snapshotId: null,
130 ms: cost.elapsedMs,
131 counts: { repositories: 0, issues: 0, users: 0, pulls: 0, apps: 0 },
132 health: "unreached",
133 verdict: "red",
134 failedStep: "restore",
135 phases: [],
136 budgetMs: 0,
137 reason: cause,
138 cost,
139 };
140 }
141
120142 /**
121143 * Run the drill in its own process. `watch` hears each state change (at most
122144 * about one a second, and one per memory reading); the promise answers with the
123 * drill's result, or rejects with why the process gave none. The scratch is
145 * drill's result, carrying its `cost`; when the process gave none (killed over
146 * its memory or time, or it died), a red result naming why. The scratch is
124147 * removed whatever the end. `options` are test seams: the limit, how often memory is read, and the
125148 * script the child runs.
126149 */
@@ -141,6 +164,8 @@ export async function runDrillInChild(
141164 mkdirSync(parent(job.dataDir), { recursive: true });
142165 writeFileSync(marker(job.dataDir), JSON.stringify({ id, at }));
143166 log("restore drill started", { id, limitMiB: Math.round(limit / 1024 ** 2) });
167 const began = performance.now();
168 let peak = 0;
144169 // Its own session, so the end of the drill ends the restored forge it booted too.
145170 const child = Bun.spawn([process.execPath, "--smol", options.script ?? CHILD], {
146171 stdin: "pipe",
@@ -162,9 +187,10 @@ export async function runDrillInChild(
162187 let lastLog = 0;
163188 const watcher = setInterval(() => void during("drill-watch", () => {
164189 const rss = residentBytes(child.pid);
190 peak = Math.max(peak, rss);
165191 set({ rss });
166192 if (rss > limit) {
167 killed = `the drill process held ${Math.round(rss / 1024 ** 2)} MiB, over its ${Math.round(limit / 1024 ** 2)} MiB limit`;
193 killed = `memory: the drill process held ${Math.round(rss / 1024 ** 2)} MiB, over its ${Math.round(limit / 1024 ** 2)} MiB limit`;
168194 stop();
169195 }
170196 if (Date.now() - lastLog >= 15_000) {
@@ -173,10 +199,11 @@ export async function runDrillInChild(
173199 }
174200 }), options.watchMs ?? WATCH_MS);
175201 const deadline = setTimeout(() => {
176 killed = "the drill process outlived every budget";
202 killed = "time: the drill process outlived every budget";
177203 stop();
178204 }, DRILL_MAX_MS + 5 * 60_000);
179205 let result: DrillResult | undefined;
206 let childCost: ChildCost | undefined;
180207 let stderr = "";
181208 const reading = (async () => {
182209 let buf = "";
@@ -190,7 +217,10 @@ export async function runDrillInChild(
190217 try {
191218 const v = JSON.parse(line) as DrillLine;
192219 if ("progress" in v) set({ phase: v.progress.phase, done: v.progress.done, total: v.progress.total, bytes: v.progress.bytes });
193 else if ("result" in v) result = v.result;
220 else if ("result" in v) {
221 result = v.result;
222 childCost = v.cost;
223 }
194224 } catch {
195225 // not ours
196226 }
@@ -211,13 +241,22 @@ export async function runDrillInChild(
211241 // Whatever the end, nothing of the drill stays on disk.
212242 sweepDrills(job.dataDir);
213243 }
244 // getrusage of the reaped child counts every peak, even one between readings or at a kill.
245 const reaped = child.resourceUsage()?.maxRSS ?? 0;
246 const cost: DrillCost = {
247 elapsedMs: childCost?.elapsedMs ?? Math.round(performance.now() - began),
248 peakBytes: Math.max(peak, reaped, childCost?.peakBytes ?? 0),
249 limitBytes: limit,
250 };
251 log("restore drill cost", { id, elapsedMs: cost.elapsedMs, peakMiB: Math.round(cost.peakBytes / 1024 ** 2), limitMiB: Math.round(limit / 1024 ** 2) });
214252 if (result !== undefined) {
215253 log("restore drill ended", { id, verdict: result.verdict });
216 return result;
254 return { ...result, cost };
217255 }
218256 const why = killed ?? `the drill process exited with ${child.exitCode ?? "no code"}${scrubbed(stderr) ? `: ${scrubbed(stderr)}` : ""}`;
219257 const where = state.phase === "starting" ? "" : ` in ${describeProgress({ phase: state.phase as Progress["phase"], done: state.done, total: state.total, bytes: state.bytes })}`;
220 throw new Error(`${why}${where}`);
258 log("restore drill ended", { id, verdict: "red", cause: why });
259 return deadDrill(at, `${why}${where}`, cost);
221260 }
222261
223262 /** A marker is left only while a drill runs; true when one is. */
src/snapshot/drills.ts +30 −3
@@ -5,7 +5,7 @@
55 */
66
77 import type { Database } from "bun:sqlite";
8 import type { DrillResult, DrillStep, PhaseTime } from "./drill.ts";
8 import type { DrillCost, DrillResult, DrillStep, PhaseTime } from "./drill.ts";
99
1010 interface Row {
1111 at: string;
@@ -24,7 +24,10 @@ interface Row {
2424
2525 const PHASES = new Set(["manifest", "files", "unpack", "bundles", "key", "open", "boot"]);
2626
27 /** Where a kept drill restored from; empty when it was kept before that was recorded, or the JSON is not ours. */
27 /**
28 * Where a kept drill restored from; empty when it was kept before that was recorded, or the JSON is
29 * not ours.
30 */
2831 export function parseFrom(raw: string | null): { from?: string; offSite?: boolean; format?: number } {
2932 try {
3033 const v = JSON.parse(raw ?? "null") as { from?: unknown; offSite?: unknown; format?: unknown } | null;
@@ -36,7 +39,27 @@ export function parseFrom(raw: string | null): { from?: string; offSite?: boolea
3639 }
3740 }
3841
39 /** The stored JSON as phases; anything that is not a list of known phases with whole ms reads as none. */
42 /**
43 * A kept drill's cost (#13253); absent when it kept none, or any of it is not a whole, finite,
44 * non-negative number.
45 */
46 export function parseCost(raw: string | null): { cost?: DrillCost } {
47 try {
48 const c = (JSON.parse(raw ?? "null") as { cost?: unknown } | null)?.cost as Record<string, unknown> | undefined;
49 if (typeof c !== "object" || c === null) return {};
50 const whole = (v: unknown): v is number => typeof v === "number" && Number.isSafeInteger(v) && v >= 0;
51 const { elapsedMs, peakBytes, limitBytes } = c;
52 if (!whole(elapsedMs) || !whole(peakBytes) || !whole(limitBytes)) return {};
53 return { cost: { elapsedMs, peakBytes, limitBytes } };
54 } catch {
55 return {};
56 }
57 }
58
59 /**
60 * The stored JSON as phases; anything that is not a list of known phases with whole ms reads as
61 * none.
62 */
4063 export function parsePhases(raw: string | null): { phases: PhaseTime[]; budgetMs: number } {
4164 const none = { phases: [], budgetMs: 0 };
4265 if (raw === null) return none;
@@ -79,6 +102,9 @@ export function recordDrill(db: Database, result: DrillResult): void {
79102 budgetMs: result.budgetMs,
80103 ...(result.from === undefined ? {} : { from: result.from, offSite: result.offSite === true }),
81104 ...(result.format === undefined ? {} : { format: result.format }),
105 ...(result.cost === undefined
106 ? {}
107 : { cost: { elapsedMs: Math.round(result.cost.elapsedMs), peakBytes: Math.round(result.cost.peakBytes), limitBytes: Math.round(result.cost.limitBytes) } }),
82108 }),
83109 );
84110 }
@@ -119,5 +145,6 @@ function asResult(row: Row | null): DrillResult | undefined {
119145 phases: kept.phases,
120146 budgetMs: kept.budgetMs,
121147 ...parseFrom(row.phases),
148 ...parseCost(row.phases),
122149 };
123150 }
src/web/pages/backups.ts +10 −2
@@ -25,6 +25,7 @@ import { mirrorSealedSet, mirrorNeed, MIRROR_SEALED } from "../../snapshot/mirro
2525 import { resolveSnapshots, sealedSet } from "../../snapshot/destination.ts";
2626 import { keyOf } from "../../snapshot/snapshot.ts";
2727 import { formatBytes, formatMinutes, formatSpan, timeByPart, type DrillResult } from "../../snapshot/drill.ts";
28 import { costFacts } from "../../snapshot/drill_cost.ts";
2829 import { lastDrill } from "../../snapshot/drills.ts";
2930 import { backupVerdict } from "../../snapshot/bucket.ts";
3031 import { bodiesOf, type Store } from "../../snapshot/store.ts";
@@ -124,6 +125,11 @@ function drillTiming(d: DrillResult): string {
124125 function drillLine(ctx: Ctx): Html {
125126 const d = lastDrill(ctx.app.db);
126127 const timing = d === undefined ? "" : drillTiming(d);
128 // The drill process's cost beside its limit, a fact a row (#13253).
129 const cost =
outdated

Design principles 1 and 7: a one-sentence paragraph carrying three facts plus the limit rule. Render took / peak / limit as dl rows, and say the rule once.

130 d === undefined
131 ? ""
132 : html`<dl class="status-rows" id="drill-cost">${costFacts(d).map((f) => html`<div class="status-row"><dt>${f.label}</dt><dd>${f.value}</dd></div>`)}</dl>`;
127133 const line =
128134 d === undefined
129135 ? html`<p class="break">No restore drill has run yet.</p>`
@@ -150,7 +156,7 @@ function drillLine(ctx: Ctx): Html {
150156 : html`<p class="break">${badge("Running", { tone: "attn", icon: "clock" })} ${says}${"bytes" in running
151157 ? html` <span class="muted small">${formatBytes(running.bytes)} read; its process holds ${formatBytes(running.rss)} of at most ${formatBytes(running.limit)}.</span>`
152158 : ""}</p>`;
153 return html`${line}${now}
159 return html`${line}${cost}${now}
154160 <form method="post" action="/settings/backups/drill" class="mt">
155161 ${btn("Run a drill", { type: "submit", icon: "clock" })}
156162 </form>`;
@@ -495,7 +501,9 @@ function destinationCard(ctx: Ctx): Html {
495501 </section>`;
496502 }
497503
498 /** The mirror: its age and reachability from the last pass, and the write-only form that seals it. */
504 /**
505 * The mirror: its age and reachability from the last pass, and the write-only form that seals it.
506 */
499507 function mirrorCard(ctx: Ctx): Html {
500508 const svc = snapshotsOf(ctx.app);
501509 const state = svc?.mirrored();
test/backups-drill-cost.test.ts added +78
@@ -0,0 +1,78 @@
1 /**
2 * The Backups page says the last drill's elapsed time and peak memory beside its limit, as `dl`
3 * rows (#13253).
4 */
5 import { afterAll, beforeAll, expect, test } from "bun:test";
6 import { mkdtempSync, rmSync } from "node:fs";
7 import { tmpdir } from "node:os";
8 import { join } from "node:path";
9 import { createApp, type App } from "../src/app.ts";
10 import { createUser } from "../src/core/users.ts";
11 import { serve, type Servers } from "../src/http/server.ts";
12 import type { DrillResult } from "../src/snapshot/drill.ts";
13 import { recordDrill } from "../src/snapshot/drills.ts";
14 import { startSnapshots, type SnapshotsHandle } from "../src/snapshot/service.ts";
15 import { dirStore } from "../src/snapshot/store.ts";
16 import { quietLogs } from "./helpers/globals.ts";
17
18 quietLogs();
19
20 let root = "";
21 let app: App;
22 let svc: SnapshotsHandle;
23 let servers: Servers;
24 let cookie = "";
25
26 const MiB = 1024 ** 2;
27 const drill: DrillResult = {
28 at: "2026-11-15T08:10:00.000Z", snapshotId: "s", ms: 60_000, counts: { repositories: 1, issues: 0, users: 1, pulls: 0, apps: 0 },
29 health: "ok", verdict: "green", failedStep: null, phases: [], budgetMs: 600_000, reason: null,
30 };
31
32 beforeAll(async () => {
33 root = mkdtempSync(join(tmpdir(), "forge-backups-cost-"));
34 app = createApp({ dataDir: root, port: 0 });
35 const u = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
36 if (!u.ok) throw new Error(u.error);
37 svc = startSnapshots(app, dirStore(join(root, "store")), { version: "t", nightly: false });
38 servers = await serve(app);
39 const r = await fetch(`${servers.url}/signin`, {
40 method: "POST",
41 headers: { origin: servers.url },
42 body: new URLSearchParams({ username: "alice", password: "a-good-passphrase" }),
43 redirect: "manual",
44 });
45 cookie = (r.headers.get("set-cookie") ?? "").split(";")[0]!;
46 });
47
48 afterAll(async () => {
49 servers.stop();
50 await svc.stop();
51 app.close();
52 rmSync(root, { recursive: true, force: true });
53 });
54
55 const costShown = async (): Promise<{ rows: string[]; page: string }> => {
56 const page = await (await fetch(`${servers.url}/settings/backups`, { headers: { cookie } })).text();
57 const dl = /<dl class="status-rows" id="drill-cost">([\s\S]*?)<\/dl>/.exec(page)?.[1] ?? "";
58 const rows = [...dl.matchAll(/<dt>([^<]*)<\/dt><dd>([^<]*)<\/dd>/g)].map((m) => `${m[1]}: ${m[2]}`);
59 return { rows, page };
60 };
61
62 test("a measured drill: took, peak with its share, and the limit with its rule, a fact a row", async () => {
63 recordDrill(app.db, { ...drill, cost: { elapsedMs: 62_000, peakBytes: 412 * MiB, limitBytes: 3072 * MiB } });
64 const { rows, page } = await costShown();
65 // 412 / 3072 = 13.4%, rounded up to 14%; the apostrophe is escaped on the way out.
66 expect(rows).toEqual([
67 "Took: 1 min 2 s",
68 "Peak memory: 412 MiB, 14% of the limit",
69 "Limit: 3 GiB: 60% of the machine&#39;s memory, kept between 3 and 16 GiB, or FORGE_DRILL_MEMORY_MIB",
70 ]);
71 // The rule is said once on the page.
72 expect(page.split("between 3 and 16 GiB").length - 1).toBe(1);
73 });
74
75 test("a drill kept before the cost was measured says so in one row", async () => {
76 recordDrill(app.db, drill);
77 expect((await costShown()).rows).toEqual(["Drill cost: elapsed time and peak memory not recorded for this drill"]);
78 });
test/drill-job.test.ts +124 −7
@@ -6,8 +6,10 @@ import { join } from "node:path";
66 import { createApp } from "../src/app.ts";
77 import { judge, type LaneRead } from "../src/snapshot/bucket.ts";
88 import type { DrillResult } from "../src/snapshot/drill.ts";
9 import type { DrillLine } from "../src/snapshot/drill_child.ts";
910 import { cutDrill, drillMemoryLimit, markerLeft, residentBytes, runDrillInChild, scrubbed, sweepDrills } from "../src/snapshot/drill_job.ts";
10 import { lastDrill } from "../src/snapshot/drills.ts";
11 import { costFacts, costLine, LIMIT_RULE } from "../src/snapshot/drill_cost.ts";
12 import { lastDrill, parseCost, recordDrill } from "../src/snapshot/drills.ts";
1113 import { startSnapshots } from "../src/snapshot/service.ts";
1214 import { dirStore } from "../src/snapshot/store.ts";
1315
@@ -43,25 +45,140 @@ test("a drill with an empty store ends red in its own process, leaving no direct
4345 expect(result.reason).toContain("no snapshot");
4446 expect(existsSync(join(root, "drill", "running.json"))).toBe(false);
4547 expect(existsSync(join(root, "drill")) ? readdirSync(join(root, "drill")).join(" ") : "").toBe("");
48 // A real drill in the real child, measured: a Bun process holds tens of MiB at the least.
49 expect(result.cost!.elapsedMs).toBeGreaterThan(0);
50 expect(result.cost!.peakBytes).toBeGreaterThan(16 * 1024 ** 2);
51 expect(result.cost!.peakBytes).toBeLessThan(result.cost!.limitBytes);
4652 });
4753
48 test("a drill over its memory limit is killed and says so", async () => {
54 test("the drill's own process ends with its result and its own elapsed time and peak memory", async () => {
55 // drill_child.ts run as the job runs it: one JSON job on stdin, a real runDrill on an empty
56 // store.
57 const child = Bun.spawn([process.execPath, "--smol", join(import.meta.dir, "..", "src", "snapshot", "drill_child.ts")], {
58 stdin: "pipe",
59 stdout: "pipe",
60 stderr: "ignore",
61 env: { PATH: process.env["PATH"] ?? "" },
62 });
63 child.stdin.write(JSON.stringify({ dataDir: join(root, "child"), store: { kind: "dir", root: join(root, "store") } }));
64 await child.stdin.end();
65 const out = await new Response(child.stdout).text();
66 expect(await child.exited).toBe(0);
67 const last = JSON.parse(out.trim().split("\n").at(-1)!) as DrillLine;
68 if (!("result" in last)) throw new Error(`no result line: ${out}`);
69 expect(last.result.verdict).toBe("red");
70 expect(last.result.reason).toContain("no snapshot");
71 // process.uptime in ms, and getrusage's maxRSS in bytes: whole, and of a real Bun process.
72 expect(Number.isSafeInteger(last.cost?.elapsedMs)).toBe(true);
73 expect(last.cost!.elapsedMs).toBeGreaterThan(0);
74 expect(last.cost!.elapsedMs).toBeLessThan(60_000);
75 expect(last.cost!.peakBytes).toBeGreaterThan(16 * 1024 ** 2);
76 expect(last.cost!.peakBytes % 1024).toBe(0);
77 });
78
79 test("a drill over its memory limit is a red drill naming memory as the cause, with what it held", async () => {
4980 // A child that holds still, a limit of one byte and a quick reading: killed at the first reading.
5081 const script = join(root, "idle.ts");
5182 writeFileSync(script, "await Bun.stdin.text(); await Bun.sleep(30_000);"); // real-timer: a real child process is the thing under test
52 const err = await runDrillInChild(
83 const result = await runDrillInChild(
5384 { dataDir: root, store: { kind: "dir", root: join(root, "store") } },
5485 () => {},
5586 () => {},
5687 { limit: 1, watchMs: 50, script },
57 ).then(
58 () => "no error",
59 (e) => String(e),
6088 );
61 expect(err).toContain("over its 0 MiB limit");
89 expect(result.verdict).toBe("red");
90 expect(result.failedStep).toBe("restore");
91 expect(result.reason).toStartWith("memory: the drill process held ");
92 expect(result.reason).toContain("over its 0 MiB limit");
93 // The kill is measured too: getrusage of the reaped child, and the time to the kill.
94 expect(result.cost?.limitBytes).toBe(1);
95 expect(result.cost!.peakBytes).toBeGreaterThan(1024 ** 2);
96 expect(result.cost!.elapsedMs).toBeGreaterThan(0);
97 expect(result.cost!.elapsedMs).toBeLessThan(30_000);
6298 expect(markerLeft(root)).toBe(false);
6399 });
64100
101 test("a child that dies without a result is a red drill saying how it exited, never a silent end", async () => {
102 const script = join(root, "dies.ts");
103 writeFileSync(script, "await Bun.stdin.text(); process.exit(7);");
104 const result = await runDrillInChild({ dataDir: root, store: { kind: "dir", root: join(root, "store") } }, () => {}, () => {}, { script });
105 expect(result.verdict).toBe("red");
106 expect(result.reason).toContain("the drill process exited with 7");
107 expect(result.cost?.peakBytes).toBeGreaterThan(0);
108 });
109
110 test("a drill's own process reads back its elapsed time and peak memory beside its limit", async () => {
111 // The child speaks the real protocol (drill_child.ts): a progress line, then its result and
112 // cost, the peak read by the real `peakSoFar`. A full restore and boot in the child takes
113 // seconds; what is under test is the cost crossing the process boundary and being kept.
114 const script = join(root, "speaks.ts");
115 const child = join(import.meta.dir, "..", "src", "snapshot", "drill_child.ts");
116 const drilled: DrillResult = {
117 at: "2026-11-15T08:00:00.000Z", snapshotId: "s-1", ms: 5, counts: { repositories: 1, issues: 0, users: 1, pulls: 0, apps: 0 },
118 health: "ok", verdict: "green", failedStep: null, phases: [], budgetMs: 0, reason: null,
119 };
120 writeFileSync(
121 script,
122 `import { peakSoFar } from ${JSON.stringify(child)};
123 await Bun.stdin.text();
124 const held = new Uint8Array(32 * 1024 ** 2).fill(1);
125 console.log(JSON.stringify({ progress: { phase: "download", done: 1, total: 1, bytes: held.length } }));
126 console.log(JSON.stringify({ result: ${JSON.stringify(drilled)}, cost: { elapsedMs: 1234, peakBytes: peakSoFar(0) } }));
127 process.exit(0);`,
128 );
129 const limit = 3 * 1024 ** 3;
130 const logged: string[] = [];
131 const phases: string[] = [];
132 const result = await runDrillInChild(
133 { dataDir: join(root, "drill-data"), store: { kind: "dir", root: join(root, "store") } },
134 (s) => phases.push(s.phase),
135 (m) => logged.push(m),
136 { limit, script },
137 );
138 expect(result.verdict).toBe("green");
139 expect(result.counts.repositories).toBe(1);
140 expect(phases).toContain("download");
141 const cost = result.cost!;
142 expect(cost.limitBytes).toBe(limit);
143 // The child's own elapsed time is the one kept; its peak includes the 32 MiB it held.
144 expect(cost.elapsedMs).toBe(1234);
145 expect(cost.peakBytes).toBeGreaterThan(32 * 1024 ** 2);
146 expect(cost.peakBytes).toBeLessThan(limit);
147 expect(logged).toContain("restore drill cost");
148 expect(markerLeft(join(root, "drill-data"))).toBe(false);
149 // Kept and read back whole, the same numbers.
150 const app = createApp({ dataDir: join(root, "kept"), port: 0 });
151 try {
152 recordDrill(app.db, result);
153 expect(lastDrill(app.db)?.cost).toEqual(cost);
154 } finally {
155 app.close();
156 }
157 });
158
159 test("the cost line says elapsed, peak, the limit and its share; the facts say the rule once", () => {
160 const MiB = 1024 ** 2;
161 const d = { cost: { elapsedMs: 62_000, peakBytes: 412 * MiB, limitBytes: 3072 * MiB } };
162 // 412 MiB of 3 GiB = 13.41%, rounded up to 14%.
163 expect(costLine(d)).toBe("took 1 min 2 s, peak 412 MiB of its 3 GiB limit (14%)");
164 expect(costLine({})).toBe("elapsed time and peak memory not recorded for this drill");
165 expect(costFacts(d)).toEqual([
166 { label: "Took", value: "1 min 2 s" },
167 { label: "Peak memory", value: "412 MiB, 14% of the limit" },
168 { label: "Limit", value: `3 GiB: ${LIMIT_RULE}` },
169 ]);
170 expect(LIMIT_RULE).toContain("between 3 and 16 GiB");
171 expect(costFacts({})).toEqual([{ label: "Drill cost", value: "elapsed time and peak memory not recorded for this drill" }]);
172 });
173
174 test("a kept cost that is not three whole numbers reads as none", () => {
175 expect(parseCost(JSON.stringify({ cost: { elapsedMs: 1, peakBytes: 2, limitBytes: 3 } }))).toEqual({ cost: { elapsedMs: 1, peakBytes: 2, limitBytes: 3 } });
176 for (const bad of [{ elapsedMs: -1, peakBytes: 2, limitBytes: 3 }, { elapsedMs: 1, peakBytes: "2", limitBytes: 3 }, { elapsedMs: 1.5, peakBytes: 2, limitBytes: 3 }, null, "x"])
177 expect(parseCost(JSON.stringify({ cost: bad }))).toEqual({});
178 expect(parseCost("not json")).toEqual({});
179 expect(parseCost(null)).toEqual({});
180 });
181
65182 test("the limit follows the machine, within bounds, and an override of at least 512 MiB", () => {
66183 const GiB = 1024 ** 3;
67184 expect(drillMemoryLimit({}, 2 * GiB)).toBe(3 * GiB);
test/mcp-restore-drill.test.ts +6
@@ -64,6 +64,7 @@ describe("MCP restore_drill", () => {
6464 phases: [],
6565 budgetMs: 0,
6666 reason: green ? null : "a sealed value does not open",
67 cost: { elapsedMs: 62_000, peakBytes: 412 * 1024 ** 2, limitBytes: 3072 * 1024 ** 2 },
6768 };
6869 },
6970 });
@@ -118,6 +119,11 @@ describe("MCP restore_drill", () => {
118119 expect(verdict.says).toBe("The drill restored green.");
119120 expect(lastDrill(app.db)?.snapshotId).toBe("snap-1");
120121 });
122 test("status prints the drill's elapsed time and peak memory in one line beside its limit (#13253)", async () => {
123 const { verdict } = await drillAndRead(token("alice"));
124 expect(verdict.cost).toBe("took 1 min 2 s, peak 412 MiB of its 3 GiB limit (14%)");
125 expect([verdict.elapsedMs, verdict.peakMiB, verdict.limitMiB]).toEqual([62_000, 412, 3072]);
126 });
121127 test("a second ask while one runs says so and does not start another", async () => {
122128 const t = token("alice");
123129 let open!: () => void;
test/tui-drill.test.ts +15
@@ -127,6 +127,21 @@ test("a restore drill says its outcome in words and leaves the domain row alone"
127127 h.app.stop();
128128 });
129129
130 test("the verdict says the drill process's cost beside its limit (#13253)", async () => {
131 await handle?.stop();
132 handle = startSnapshots(forge, dirStore(join(dir, "store")), {
133 version: "0",
134 nightly: false,
135 drill: async () => ({ ...result("green"), cost: { elapsedMs: 62_000, peakBytes: 412 * 1024 ** 2, limitBytes: 3 * 1024 ** 3 } }),
136 });
137 const h = await open(alice);
138 await h.press("d");
139 await h.press("left");
140 const text = await h.pressUntil("enter", "peak 412 MiB");
141 expect(text).toContain("The drill restored green. Its process took 1 min 2 s, peak 412 MiB of its 3 GiB limit (14%).");
142 h.app.stop();
143 });
144
130145 test("declining runs nothing", async () => {
131146 const h = await open(alice);
132147 const before = runs;
test/tui-resilience-drill.test.ts +13
@@ -131,6 +131,19 @@ describe("a restore drill from the terminal's Resilience dialog", () => {
131131 h.stop();
132132 });
133133
134 test("the verdict says the drill process's cost beside its limit, as the Backups page does (#13253)", async () => {
135 verdict = { ...GREEN, cost: { elapsedMs: 62_000, peakBytes: 412 * 1024 ** 2, limitBytes: 3 * 1024 ** 3 } };
136 const h = boot("alice");
137 await h.open();
138 await h.press("r", "y");
139 await h.waitFor("peak 412 MiB");
140 // The dialog wraps the line; its words, border and padding removed, read as one.
141 const said = h.text().replace(/[│╭╮╰╯─]/g, " ").replace(/\s+/g, " ");
142 expect(said).toContain("The drill restored green. Its process took 1 min 2 s, peak 412 MiB of its 3 GiB limit (14%).");
143 verdict = GREEN;
144 h.stop();
145 });
146
134147 test("r then n, or esc, runs nothing", async () => {
135148 const h = boot("alice");
136149 await h.open();

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs