Public: anyone can read this; the crew can change it.
The Backups page shows the last restore drill's duration and peak memory beside the limit it states #13284
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/13253-the-backups-page-shows-the-last-restore → main 5 commits fork 2 threads · 2 unresolved
14 files
changed +444
−37
docs/features/backup-truth.md +1 −1
Diff not shown — a line 2,313 characters wide (limit 2,000). +1 −1
View filesrc/actions/backups_now.ts +6 −2
| @@ -11,6 +11,7 @@ import type { App } from "../app.ts"; | ||
| 11 | 11 | import type { User } from "../core/users.ts"; |
| 12 | 12 | import { err, ok, type Result } from "../kernel/result.ts"; |
| 13 | 13 | import type { DrillStep } from "../snapshot/drill.ts"; |
| 14 | import { costLine } from "../snapshot/drill_cost.ts"; | |
| 14 | 15 | import { lastDrill } from "../snapshot/drills.ts"; |
| 15 | 16 | import { snapshotsOf } from "../snapshot/service.ts"; |
| 16 | 17 | import { fileSize } from "../web/ui.ts"; |
| @@ -146,7 +147,8 @@ export function drillStatus(app: App): string | undefined { | ||
| 146 | 147 | |
| 147 | 148 | /** |
| 148 | 149 | * Wait for the running drill, telling `tick` where it stands each `everyMs`, and answer its |
| 149 | * verdict in the words `drillNow` uses; the terminal's door to a drill that is a job (#12592). | |
| 150 | * verdict in the words `drillNow` uses, then its cost (`costLine`); the terminal's door to a drill | |
| 151 | * that is a job (#12592). | |
| 150 | 152 | */ |
| 151 | 153 | export async function followDrill(app: App, tick: (says: string) => void, everyMs = 250): Promise<string> { |
| 152 | 154 | for (let says = drillStatus(app); says !== undefined; says = drillStatus(app)) { |
| @@ -160,7 +162,9 @@ export async function followDrill(app: App, tick: (says: string) => void, everyM | ||
| 160 | 162 | d = undefined; // the forge closed while the drill ran |
| 161 | 163 | } |
| 162 | 164 | if (d === undefined) return "The drill ended without a verdict."; |
| 163 | return d.verdict === "green" ? "The drill restored green." : `The drill failed at ${d.failedStep}.`; | |
| 165 | // Its process's cost beside its limit, as the Backups page and MCP say it (#13253). | |
| 166 | const said = d.verdict === "green" ? "The drill restored green." : `The drill failed at ${d.failedStep}.`; | |
| 167 | return `${said} Its process ${costLine(d)}.`; | |
| 164 | 168 | } |
| 165 | 169 | |
| 166 | 170 | /** |
src/http/mcp/tools/restore_drill.ts +11 −4
| @@ -1,4 +1,5 @@ | ||
| 1 | 1 | import { scopeAllows } from "../../../core/tokens.ts"; |
| 2 | import { costLine } from "../../../snapshot/drill_cost.ts"; | |
| 2 | 3 | import { lastDrill } from "../../../snapshot/drills.ts"; |
| 3 | 4 | import { snapshotsOf, type SnapshotsHandle } from "../../../snapshot/service.ts"; |
| 4 | 5 | import { good, bad, type Tool, mayBoards } from "../shared.ts"; |
| @@ -10,7 +11,7 @@ const phaseWords = (st: ReturnType<SnapshotsHandle["startDrill"]>["state"]): Rec | ||
| 10 | 11 | |
| 11 | 12 | export const tool: Tool = { |
| 12 | 13 | name: "restore_drill", |
| 13 | doc: "`action` start (default) answers at once `{ started, id, phase, bytes, ... }`; one drill runs at a time, a second ask says `started: false` and where the running one is; `status` answers the running drill's phase and bytes, or the newest verdict `{ verdict, failedStep?, says, at, from?, offSite? }` (red, not an error, when it failed); it runs in its own process; a forge admin's, with an admin token", | |
| 14 | doc: "`action` start (default) answers at once `{ started, id, phase, bytes, ... }`; one drill runs at a time, a second ask says `started: false` and where the running one is; `status` answers the running drill's phase and bytes, or the newest verdict `{ verdict, failedStep?, says, at, from?, offSite?, cost, elapsedMs?, peakMiB?, limitMiB? }` (red, not an error, when it failed); it runs in its own process; a forge admin's, with an admin token", | |
| 14 | 15 | description: "Start a restore drill of the newest snapshot, or read how it is going.", |
| 15 | 16 | inputSchema: { type: "object", properties: { action: { type: "string", enum: ["start", "status"] } } }, |
| 16 | 17 | writes: true, |
| @@ -37,11 +38,17 @@ export const tool: Tool = { | ||
| 37 | 38 | const d = lastDrill(ctx.app.db); |
| 38 | 39 | if (d === undefined) return good({ verdict: "none", says: "No drill has run." }); |
| 39 | 40 | // Where it restored from (the off-site lane when the forge has one, #11805), never a key. |
| 40 | const from = d.from === undefined ? {} : { from: d.from, offSite: d.offSite === true }; | |
| 41 | const extra = { | |
| 42 | ...(d.from === undefined ? {} : { from: d.from, offSite: d.offSite === true }), | |
| 43 | // The drill process's elapsed time and peak memory beside its limit, as the Backups page says | |
| 44 | // it (#13253). | |
| 45 | cost: costLine(d), | |
| 46 | ...(d.cost === undefined ? {} : { elapsedMs: d.cost.elapsedMs, peakMiB: Math.round(d.cost.peakBytes / 1024 ** 2), limitMiB: Math.round(d.cost.limitBytes / 1024 ** 2) }), | |
| 47 | }; | |
| 41 | 48 | const answer = |
| 42 | 49 | d.verdict === "green" |
| 43 | ? good({ verdict: "green", at: d.at, says: "The drill restored green.", ...from }) | |
| 44 | : good({ verdict: "red", at: d.at, failedStep: d.failedStep, says: `The drill failed at ${d.failedStep}${d.reason ? `: ${d.reason}` : ""}.`, ...from }); | |
| 50 | ? good({ verdict: "green", at: d.at, says: "The drill restored green.", ...extra }) | |
| 51 | : good({ verdict: "red", at: d.at, failedStep: d.failedStep, says: `The drill failed at ${d.failedStep}${d.reason ? `: ${d.reason}` : ""}.`, ...extra }); | |
| 45 | 52 | return { ...answer, card: { at: d.at } }; |
| 46 | 53 | }, |
| 47 | 54 | }; |
src/snapshot/drill.ts +34 −5
| @@ -102,14 +102,20 @@ const PART_WORD: Record<PhaseTime["phase"], string> = { | ||
| 102 | 102 | boot: "start and migrate", |
| 103 | 103 | }; |
| 104 | 104 | |
| 105 | /** The drill's time by part, longest first: download, decompress, clone, open, migrate. Parts that share a word are added. */ | |
| 105 | /** | |
| 106 | * The drill's time by part, longest first: download, decompress, clone, open, migrate. Parts that | |
| 107 | * share a word are added. | |
| 108 | */ | |
| 106 | 109 | export function timeByPart(phases: readonly PhaseTime[]): { part: string; ms: number }[] { |
| 107 | 110 | const sums = new Map<string, number>(); |
| 108 | 111 | for (const p of phases) sums.set(PART_WORD[p.phase], (sums.get(PART_WORD[p.phase]) ?? 0) + p.ms); |
| 109 | 112 | return [...sums].map(([part, ms]) => ({ part, ms })).sort((a, b) => b.ms - a.ms); |
| 110 | 113 | } |
| 111 | 114 | |
| 112 | /** "download about 8 min, decompress 40 s, open 2 s; download was the slowest"; empty when nothing was timed. */ | |
| 115 | /** | |
| 116 | * "download about 8 min, decompress 40 s, open 2 s; download was the slowest"; empty when nothing | |
| 117 | * was timed. | |
| 118 | */ | |
| 113 | 119 | export function describeTimes(phases: readonly PhaseTime[]): string { |
| 114 | 120 | const parts = timeByPart(phases).filter((p) => p.part !== "key"); |
| 115 | 121 | if (parts.length === 0) return ""; |
| @@ -119,7 +125,10 @@ export function describeTimes(phases: readonly PhaseTime[]): string { | ||
| 119 | 125 | |
| 120 | 126 | /** One restore phase and the milliseconds it took. */ |
| 121 | 127 | export interface PhaseTime { |
| 122 | /** A restore phase, then `open` (the database read) and `boot` (the restored forge starting and migrating). */ | |
| 128 | /** | |
| 129 | * A restore phase, then `open` (the database read) and `boot` (the restored forge starting and | |
| 130 | * migrating). | |
| 131 | */ | |
| 123 | 132 | readonly phase: Progress["phase"] | "open" | "boot"; |
| 124 | 133 | readonly ms: number; |
| 125 | 134 | } |
| @@ -160,15 +169,35 @@ export interface DrillResult { | ||
| 160 | 169 | readonly budgetMs: number; |
| 161 | 170 | /** Why it is red, for the person reading; never a value. */ |
| 162 | 171 | readonly reason: string | null; |
| 163 | /** The store it restored from (`Store.where`, never a key); absent on drills kept before #11805. */ | |
| 172 | /** | |
| 173 | * The store it restored from (`Store.where`, never a key); absent on drills kept before #11805. | |
| 174 | */ | |
| 164 | 175 | readonly from?: string | undefined; |
| 165 | 176 | /** Whether that store is off the forge's machine (`isOffBox`). */ |
| 166 | 177 | readonly offSite?: boolean | undefined; |
| 167 | 178 | /** The manifest format of the snapshot drilled (1 before chunks); absent on older drills. */ |
| 168 | 179 | readonly format?: number | undefined; |
| 180 | /** | |
| 181 | * What the drill's own process cost (#13253); absent on drills kept before, or run in the serving | |
| 182 | * process. | |
| 183 | */ | |
| 184 | readonly cost?: DrillCost | undefined; | |
| 169 | 185 | } |
| 170 | 186 | |
| 171 | /** The restored forge's front door: `/` answers below 400 (a redirect to sign in is an answer), or this throws. */ | |
| 187 | /** The drill process's wall time and peak resident memory, and the memory limit it ran under. */ | |
| 188 | export interface DrillCost { | |
| 189 | /** From spawn to exit, ms. */ | |
| 190 | readonly elapsedMs: number; | |
| 191 | /** Peak resident set of the drill process, bytes (getrusage `ru_maxrss`). */ | |
| 192 | readonly peakBytes: number; | |
| 193 | /** The resident limit it was killed above, bytes (`drillMemoryLimit`). */ | |
| 194 | readonly limitBytes: number; | |
| 195 | } | |
| 196 | ||
| 197 | /** | |
| 198 | * The restored forge's front door: `/` answers below 400 (a redirect to sign in is an answer), or | |
| 199 | * this throws. | |
| 200 | */ | |
| 172 | 201 | export async function answersRoot(base: string): Promise<void> { |
| 173 | 202 | const res = await fetch(`${base}/`, { redirect: "manual", signal: AbortSignal.timeout(5_000) }); |
| 174 | 203 | await res.body?.cancel(); |
src/snapshot/drill_child.ts +22 −3
| @@ -6,7 +6,10 @@ | ||
| 6 | 6 | * |
| 7 | 7 | * One JSON job on stdin ({ dataDir, store, passphrase }), so no secret is on a |
| 8 | 8 | * command line or in the environment. One JSON object per line on stdout: |
| 9 | * `{ progress }` as the restore moves, then `{ result }`. | |
| 9 | * `{ progress }` as the restore moves, then `{ result, cost }`: `cost` is this | |
| 10 | * process's elapsed time and peak resident memory (#13253), read once per | |
| 11 | * progress report (`process.memoryUsage.rss()`, one syscall) and at the end | |
| 12 | * from getrusage, which counts every peak between the reports. | |
| 10 | 13 | */ |
| 11 | 14 | |
| 12 | 15 | import { runDrill, type DrillResult } from "./drill.ts"; |
| @@ -20,13 +23,28 @@ export interface DrillJob { | ||
| 20 | 23 | readonly passphrase?: string | undefined; |
| 21 | 24 | } |
| 22 | 25 | |
| 23 | export type DrillLine = { readonly progress: Progress } | { readonly result: DrillResult }; | |
| 26 | /** The child's own cost; the parent adds the limit it ran under. */ | |
| 27 | export interface ChildCost { | |
| 28 | readonly elapsedMs: number; | |
| 29 | readonly peakBytes: number; | |
| 30 | } | |
| 31 | ||
| 32 | export type DrillLine = { readonly progress: Progress } | { readonly result: DrillResult; readonly cost?: ChildCost }; | |
| 33 | ||
| 34 | /** | |
| 35 | * Peak resident bytes so far: the larger of the readings taken and getrusage's | |
| 36 | * `maxRSS`, which Node and Bun give in KiB for the own process. | |
| 37 | */ | |
| 38 | export function peakSoFar(readings: number): number { | |
| 39 | return Math.max(readings, process.resourceUsage().maxRSS * 1024); | |
| 40 | } | |
| 24 | 41 | |
| 25 | 42 | const say = (line: DrillLine): void => void process.stdout.write(`${JSON.stringify(line)}\n`); |
| 26 | 43 | |
| 27 | 44 | if (import.meta.main) { |
| 28 | 45 | const job = JSON.parse(await Bun.stdin.text()) as DrillJob; |
| 29 | 46 | let last = 0; |
| 47 | let peak = 0; | |
| 30 | 48 | const result = await runDrill({ |
| 31 | 49 | store: storeFrom(job.store), |
| 32 | 50 | dataDir: job.dataDir, |
| @@ -38,9 +56,10 @@ if (import.meta.main) { | ||
| 38 | 56 | last = now; |
| 39 | 57 | // Buffers of finished reads are given back at once, not at the collector's leisure. |
| 40 | 58 | Bun.gc(false); |
| 59 | peak = Math.max(peak, process.memoryUsage.rss()); | |
| 41 | 60 | say({ progress }); |
| 42 | 61 | }, |
| 43 | 62 | }); |
| 44 | say({ result }); | |
| 63 | say({ result, cost: { elapsedMs: Math.round(process.uptime() * 1_000), peakBytes: peakSoFar(peak) } }); | |
| 45 | 64 | process.exit(0); |
| 46 | 65 | } |
src/snapshot/drill_cost.ts added +45
| @@ -0,0 +1,45 @@ | ||
| 1 | /** | |
| 2 | * What a restore drill's own process cost beside the limit it ran under | |
| 3 | * (#13253), so each drill proves or disproves the limit. The Backups page shows | |
| 4 | * it as `dl` rows (`costFacts`); the MCP `restore_drill` status and the | |
| 5 | * terminal's drill screens say it in one line (`costLine`). How the limit is | |
| 6 | * chosen (`LIMIT_RULE`) is said once, in the Limit row. Plain data in, | |
| 7 | * strings out. | |
| 8 | */ | |
| 9 | ||
| 10 | import { formatBytes, formatSpan, type DrillResult } from "./drill.ts"; | |
| 11 | ||
| 12 | /** How the limit is chosen (`drillMemoryLimit` in drill_job.ts), in the words the page uses. */ | |
| 13 | export const LIMIT_RULE = "60% of the machine's memory, kept between 3 and 16 GiB, or FORGE_DRILL_MEMORY_MIB"; | |
| 14 | ||
| 15 | /** Share of the limit, whole percent, rounded up so a drill at 99.6% never reads as having room. */ | |
| 16 | const share = (peak: number, limit: number): number => (limit > 0 ? Math.ceil((peak / limit) * 100) : 0); | |
| 17 | ||
| 18 | /** | |
| 19 | * "took 1 min 2 s, peak 412 MiB of its 3 GiB limit (14%)"; for a drill that kept no cost, says so. | |
| 20 | */ | |
| 21 | export function costLine(d: Pick<DrillResult, "cost">): string { | |
| 22 | const c = d.cost; | |
| 23 | if (c === undefined) return "elapsed time and peak memory not recorded for this drill"; | |
| 24 | return `took ${formatSpan(c.elapsedMs)}, peak ${formatBytes(c.peakBytes)} of its ${formatBytes(c.limitBytes)} limit (${share(c.peakBytes, c.limitBytes)}%)`; | |
| 25 | } | |
| 26 | ||
| 27 | /** One fact of the drill's cost: its label and its value, as a `dl` row says it. */ | |
| 28 | export interface CostFact { | |
| 29 | readonly label: string; | |
| 30 | readonly value: string; | |
| 31 | } | |
| 32 | ||
| 33 | /** | |
| 34 | * Took, peak (with its share of the limit) and the limit; one row saying none was kept, for an | |
| 35 | * older drill. | |
| 36 | */ | |
| 37 | export function costFacts(d: Pick<DrillResult, "cost">): CostFact[] { | |
| 38 | const c = d.cost; | |
| 39 | if (c === undefined) return [{ label: "Drill cost", value: "elapsed time and peak memory not recorded for this drill" }]; | |
| 40 | return [ | |
| 41 | { label: "Took", value: formatSpan(c.elapsedMs) }, | |
| 42 | { label: "Peak memory", value: `${formatBytes(c.peakBytes)}, ${share(c.peakBytes, c.limitBytes)}% of the limit` }, | |
| 43 | { label: "Limit", value: `${formatBytes(c.limitBytes)}: ${LIMIT_RULE}` }, | |
| 44 | ]; | |
| 45 | } |
src/snapshot/drill_job.ts +49 −10
| @@ -11,14 +11,17 @@ | ||
| 11 | 11 | * |
| 12 | 12 | * The child's memory is bounded: `--smol`, and its resident set is read every |
| 13 | 13 | * few seconds and the child is killed, the drill red, above `DRILL_MEMORY_BYTES`. |
| 14 | * Every drill run here keeps its `cost` (elapsed, peak resident, limit; #13253), | |
| 15 | * and a child that ends without a result is a red drill whose reason names the | |
| 16 | * cause (memory, time, or how it exited), never only a log line. | |
| 14 | 17 | */ |
| 15 | 18 | |
| 16 | 19 | import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; |
| 17 | 20 | import { totalmem } from "node:os"; |
| 18 | 21 | import { join } from "node:path"; |
| 19 | 22 | import { during } from "../perf/jobs.ts"; |
| 20 | import { describeProgress, DRILL_MAX_MS, type DrillResult } from "./drill.ts"; | |
| 21 | import type { DrillJob, DrillLine } from "./drill_child.ts"; | |
| 23 | import { describeProgress, DRILL_MAX_MS, type DrillCost, type DrillResult } from "./drill.ts"; | |
| 24 | import type { ChildCost, DrillJob, DrillLine } from "./drill_child.ts"; | |
| 22 | 25 | import type { Progress } from "./restore.ts"; |
| 23 | 26 | |
| 24 | 27 | const GIB = 1024 ** 3; |
| @@ -28,8 +31,8 @@ const GIB = 1024 ** 3; | ||
| 28 | 31 | * snapshot held 9 to 13 GiB in the serving process, so a fixed small number would be a drill that |
| 29 | 32 | * kills itself: the limit is 60% of the machine's memory, between 3 and 16 GiB, or |
| 30 | 33 | * `FORGE_DRILL_MEMORY_MIB` when that is a whole number of at least 512. It is stated on the Backups |
| 31 | * page. Not yet measured on a real-size store: the child reads the bound and logs what it held | |
| 32 | * (`restore drill working`), so the first real drill tunes this number. | |
| 34 | * page beside the last drill's measured peak (`drill_cost.ts`), so each drill says how near it | |
| 35 | * came. | |
| 33 | 36 | */ |
| 34 | 37 | export function drillMemoryLimit(env: Record<string, string | undefined> = process.env, machine: number = totalmem()): number { |
| 35 | 38 | const set = env["FORGE_DRILL_MEMORY_MIB"] ?? ""; |
| @@ -117,10 +120,30 @@ export function cutDrill(cut: { id: string; at: string }): DrillResult { | ||
| 117 | 120 | }; |
| 118 | 121 | } |
| 119 | 122 | |
| 123 | /** | |
| 124 | * The red result of a drill whose process gave none: the cause is the reason, and its cost is kept. | |
| 125 | */ | |
| 126 | export function deadDrill(at: string, cause: string, cost: DrillCost): DrillResult { | |
| 127 | return { | |
| 128 | at, | |
| 129 | snapshotId: null, | |
| 130 | ms: cost.elapsedMs, | |
| 131 | counts: { repositories: 0, issues: 0, users: 0, pulls: 0, apps: 0 }, | |
| 132 | health: "unreached", | |
| 133 | verdict: "red", | |
| 134 | failedStep: "restore", | |
| 135 | phases: [], | |
| 136 | budgetMs: 0, | |
| 137 | reason: cause, | |
| 138 | cost, | |
| 139 | }; | |
| 140 | } | |
| 141 | ||
| 120 | 142 | /** |
| 121 | 143 | * Run the drill in its own process. `watch` hears each state change (at most |
| 122 | 144 | * about one a second, and one per memory reading); the promise answers with the |
| 123 | * drill's result, or rejects with why the process gave none. The scratch is | |
| 145 | * drill's result, carrying its `cost`; when the process gave none (killed over | |
| 146 | * its memory or time, or it died), a red result naming why. The scratch is | |
| 124 | 147 | * removed whatever the end. `options` are test seams: the limit, how often memory is read, and the |
| 125 | 148 | * script the child runs. |
| 126 | 149 | */ |
| @@ -141,6 +164,8 @@ export async function runDrillInChild( | ||
| 141 | 164 | mkdirSync(parent(job.dataDir), { recursive: true }); |
| 142 | 165 | writeFileSync(marker(job.dataDir), JSON.stringify({ id, at })); |
| 143 | 166 | log("restore drill started", { id, limitMiB: Math.round(limit / 1024 ** 2) }); |
| 167 | const began = performance.now(); | |
| 168 | let peak = 0; | |
| 144 | 169 | // Its own session, so the end of the drill ends the restored forge it booted too. |
| 145 | 170 | const child = Bun.spawn([process.execPath, "--smol", options.script ?? CHILD], { |
| 146 | 171 | stdin: "pipe", |
| @@ -162,9 +187,10 @@ export async function runDrillInChild( | ||
| 162 | 187 | let lastLog = 0; |
| 163 | 188 | const watcher = setInterval(() => void during("drill-watch", () => { |
| 164 | 189 | const rss = residentBytes(child.pid); |
| 190 | peak = Math.max(peak, rss); | |
| 165 | 191 | set({ rss }); |
| 166 | 192 | if (rss > limit) { |
| 167 | killed = `the drill process held ${Math.round(rss / 1024 ** 2)} MiB, over its ${Math.round(limit / 1024 ** 2)} MiB limit`; | |
| 193 | killed = `memory: the drill process held ${Math.round(rss / 1024 ** 2)} MiB, over its ${Math.round(limit / 1024 ** 2)} MiB limit`; | |
| 168 | 194 | stop(); |
| 169 | 195 | } |
| 170 | 196 | if (Date.now() - lastLog >= 15_000) { |
| @@ -173,10 +199,11 @@ export async function runDrillInChild( | ||
| 173 | 199 | } |
| 174 | 200 | }), options.watchMs ?? WATCH_MS); |
| 175 | 201 | const deadline = setTimeout(() => { |
| 176 | killed = "the drill process outlived every budget"; | |
| 202 | killed = "time: the drill process outlived every budget"; | |
| 177 | 203 | stop(); |
| 178 | 204 | }, DRILL_MAX_MS + 5 * 60_000); |
| 179 | 205 | let result: DrillResult | undefined; |
| 206 | let childCost: ChildCost | undefined; | |
| 180 | 207 | let stderr = ""; |
| 181 | 208 | const reading = (async () => { |
| 182 | 209 | let buf = ""; |
| @@ -190,7 +217,10 @@ export async function runDrillInChild( | ||
| 190 | 217 | try { |
| 191 | 218 | const v = JSON.parse(line) as DrillLine; |
| 192 | 219 | if ("progress" in v) set({ phase: v.progress.phase, done: v.progress.done, total: v.progress.total, bytes: v.progress.bytes }); |
| 193 | else if ("result" in v) result = v.result; | |
| 220 | else if ("result" in v) { | |
| 221 | result = v.result; | |
| 222 | childCost = v.cost; | |
| 223 | } | |
| 194 | 224 | } catch { |
| 195 | 225 | // not ours |
| 196 | 226 | } |
| @@ -211,13 +241,22 @@ export async function runDrillInChild( | ||
| 211 | 241 | // Whatever the end, nothing of the drill stays on disk. |
| 212 | 242 | sweepDrills(job.dataDir); |
| 213 | 243 | } |
| 244 | // getrusage of the reaped child counts every peak, even one between readings or at a kill. | |
| 245 | const reaped = child.resourceUsage()?.maxRSS ?? 0; | |
| 246 | const cost: DrillCost = { | |
| 247 | elapsedMs: childCost?.elapsedMs ?? Math.round(performance.now() - began), | |
| 248 | peakBytes: Math.max(peak, reaped, childCost?.peakBytes ?? 0), | |
| 249 | limitBytes: limit, | |
| 250 | }; | |
| 251 | log("restore drill cost", { id, elapsedMs: cost.elapsedMs, peakMiB: Math.round(cost.peakBytes / 1024 ** 2), limitMiB: Math.round(limit / 1024 ** 2) }); | |
| 214 | 252 | if (result !== undefined) { |
| 215 | 253 | log("restore drill ended", { id, verdict: result.verdict }); |
| 216 | return result; | |
| 254 | return { ...result, cost }; | |
| 217 | 255 | } |
| 218 | 256 | const why = killed ?? `the drill process exited with ${child.exitCode ?? "no code"}${scrubbed(stderr) ? `: ${scrubbed(stderr)}` : ""}`; |
| 219 | 257 | const where = state.phase === "starting" ? "" : ` in ${describeProgress({ phase: state.phase as Progress["phase"], done: state.done, total: state.total, bytes: state.bytes })}`; |
| 220 | throw new Error(`${why}${where}`); | |
| 258 | log("restore drill ended", { id, verdict: "red", cause: why }); | |
| 259 | return deadDrill(at, `${why}${where}`, cost); | |
| 221 | 260 | } |
| 222 | 261 | |
| 223 | 262 | /** A marker is left only while a drill runs; true when one is. */ |
src/snapshot/drills.ts +30 −3
| @@ -5,7 +5,7 @@ | ||
| 5 | 5 | */ |
| 6 | 6 | |
| 7 | 7 | import type { Database } from "bun:sqlite"; |
| 8 | import type { DrillResult, DrillStep, PhaseTime } from "./drill.ts"; | |
| 8 | import type { DrillCost, DrillResult, DrillStep, PhaseTime } from "./drill.ts"; | |
| 9 | 9 | |
| 10 | 10 | interface Row { |
| 11 | 11 | at: string; |
| @@ -24,7 +24,10 @@ interface Row { | ||
| 24 | 24 | |
| 25 | 25 | const PHASES = new Set(["manifest", "files", "unpack", "bundles", "key", "open", "boot"]); |
| 26 | 26 | |
| 27 | /** Where a kept drill restored from; empty when it was kept before that was recorded, or the JSON is not ours. */ | |
| 27 | /** | |
| 28 | * Where a kept drill restored from; empty when it was kept before that was recorded, or the JSON is | |
| 29 | * not ours. | |
| 30 | */ | |
| 28 | 31 | export function parseFrom(raw: string | null): { from?: string; offSite?: boolean; format?: number } { |
| 29 | 32 | try { |
| 30 | 33 | const v = JSON.parse(raw ?? "null") as { from?: unknown; offSite?: unknown; format?: unknown } | null; |
| @@ -36,7 +39,27 @@ export function parseFrom(raw: string | null): { from?: string; offSite?: boolea | ||
| 36 | 39 | } |
| 37 | 40 | } |
| 38 | 41 | |
| 39 | /** The stored JSON as phases; anything that is not a list of known phases with whole ms reads as none. */ | |
| 42 | /** | |
| 43 | * A kept drill's cost (#13253); absent when it kept none, or any of it is not a whole, finite, | |
| 44 | * non-negative number. | |
| 45 | */ | |
| 46 | export function parseCost(raw: string | null): { cost?: DrillCost } { | |
| 47 | try { | |
| 48 | const c = (JSON.parse(raw ?? "null") as { cost?: unknown } | null)?.cost as Record<string, unknown> | undefined; | |
| 49 | if (typeof c !== "object" || c === null) return {}; | |
| 50 | const whole = (v: unknown): v is number => typeof v === "number" && Number.isSafeInteger(v) && v >= 0; | |
| 51 | const { elapsedMs, peakBytes, limitBytes } = c; | |
| 52 | if (!whole(elapsedMs) || !whole(peakBytes) || !whole(limitBytes)) return {}; | |
| 53 | return { cost: { elapsedMs, peakBytes, limitBytes } }; | |
| 54 | } catch { | |
| 55 | return {}; | |
| 56 | } | |
| 57 | } | |
| 58 | ||
| 59 | /** | |
| 60 | * The stored JSON as phases; anything that is not a list of known phases with whole ms reads as | |
| 61 | * none. | |
| 62 | */ | |
| 40 | 63 | export function parsePhases(raw: string | null): { phases: PhaseTime[]; budgetMs: number } { |
| 41 | 64 | const none = { phases: [], budgetMs: 0 }; |
| 42 | 65 | if (raw === null) return none; |
| @@ -79,6 +102,9 @@ export function recordDrill(db: Database, result: DrillResult): void { | ||
| 79 | 102 | budgetMs: result.budgetMs, |
| 80 | 103 | ...(result.from === undefined ? {} : { from: result.from, offSite: result.offSite === true }), |
| 81 | 104 | ...(result.format === undefined ? {} : { format: result.format }), |
| 105 | ...(result.cost === undefined | |
| 106 | ? {} | |
| 107 | : { cost: { elapsedMs: Math.round(result.cost.elapsedMs), peakBytes: Math.round(result.cost.peakBytes), limitBytes: Math.round(result.cost.limitBytes) } }), | |
| 82 | 108 | }), |
| 83 | 109 | ); |
| 84 | 110 | } |
| @@ -119,5 +145,6 @@ function asResult(row: Row | null): DrillResult | undefined { | ||
| 119 | 145 | phases: kept.phases, |
| 120 | 146 | budgetMs: kept.budgetMs, |
| 121 | 147 | ...parseFrom(row.phases), |
| 148 | ...parseCost(row.phases), | |
| 122 | 149 | }; |
| 123 | 150 | } |
src/web/pages/backups.ts +10 −2
| @@ -25,6 +25,7 @@ import { mirrorSealedSet, mirrorNeed, MIRROR_SEALED } from "../../snapshot/mirro | ||
| 25 | 25 | import { resolveSnapshots, sealedSet } from "../../snapshot/destination.ts"; |
| 26 | 26 | import { keyOf } from "../../snapshot/snapshot.ts"; |
| 27 | 27 | import { formatBytes, formatMinutes, formatSpan, timeByPart, type DrillResult } from "../../snapshot/drill.ts"; |
| 28 | import { costFacts } from "../../snapshot/drill_cost.ts"; | |
| 28 | 29 | import { lastDrill } from "../../snapshot/drills.ts"; |
| 29 | 30 | import { backupVerdict } from "../../snapshot/bucket.ts"; |
| 30 | 31 | import { bodiesOf, type Store } from "../../snapshot/store.ts"; |
| @@ -124,6 +125,11 @@ function drillTiming(d: DrillResult): string { | ||
| 124 | 125 | function drillLine(ctx: Ctx): Html { |
| 125 | 126 | const d = lastDrill(ctx.app.db); |
| 126 | 127 | const timing = d === undefined ? "" : drillTiming(d); |
| 128 | // The drill process's cost beside its limit, a fact a row (#13253). | |
| 129 | const cost = | |
outdated
Design principles 1 and 7: a one-sentence paragraph carrying three facts plus the limit rule. Render took / peak / limit as dl rows, and say the rule once. |
||
| 130 | d === undefined | |
| 131 | ? "" | |
| 132 | : html`<dl class="status-rows" id="drill-cost">${costFacts(d).map((f) => html`<div class="status-row"><dt>${f.label}</dt><dd>${f.value}</dd></div>`)}</dl>`; | |
| 127 | 133 | const line = |
| 128 | 134 | d === undefined |
| 129 | 135 | ? html`<p class="break">No restore drill has run yet.</p>` |
| @@ -150,7 +156,7 @@ function drillLine(ctx: Ctx): Html { | ||
| 150 | 156 | : html`<p class="break">${badge("Running", { tone: "attn", icon: "clock" })} ${says}${"bytes" in running |
| 151 | 157 | ? html` <span class="muted small">${formatBytes(running.bytes)} read; its process holds ${formatBytes(running.rss)} of at most ${formatBytes(running.limit)}.</span>` |
| 152 | 158 | : ""}</p>`; |
| 153 | return html`${line}${now} | |
| 159 | return html`${line}${cost}${now} | |
| 154 | 160 | <form method="post" action="/settings/backups/drill" class="mt"> |
| 155 | 161 | ${btn("Run a drill", { type: "submit", icon: "clock" })} |
| 156 | 162 | </form>`; |
| @@ -495,7 +501,9 @@ function destinationCard(ctx: Ctx): Html { | ||
| 495 | 501 | </section>`; |
| 496 | 502 | } |
| 497 | 503 | |
| 498 | /** The mirror: its age and reachability from the last pass, and the write-only form that seals it. */ | |
| 504 | /** | |
| 505 | * The mirror: its age and reachability from the last pass, and the write-only form that seals it. | |
| 506 | */ | |
| 499 | 507 | function mirrorCard(ctx: Ctx): Html { |
| 500 | 508 | const svc = snapshotsOf(ctx.app); |
| 501 | 509 | const state = svc?.mirrored(); |
test/backups-drill-cost.test.ts added +78
| @@ -0,0 +1,78 @@ | ||
| 1 | /** | |
| 2 | * The Backups page says the last drill's elapsed time and peak memory beside its limit, as `dl` | |
| 3 | * rows (#13253). | |
| 4 | */ | |
| 5 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 6 | import { mkdtempSync, rmSync } from "node:fs"; | |
| 7 | import { tmpdir } from "node:os"; | |
| 8 | import { join } from "node:path"; | |
| 9 | import { createApp, type App } from "../src/app.ts"; | |
| 10 | import { createUser } from "../src/core/users.ts"; | |
| 11 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 12 | import type { DrillResult } from "../src/snapshot/drill.ts"; | |
| 13 | import { recordDrill } from "../src/snapshot/drills.ts"; | |
| 14 | import { startSnapshots, type SnapshotsHandle } from "../src/snapshot/service.ts"; | |
| 15 | import { dirStore } from "../src/snapshot/store.ts"; | |
| 16 | import { quietLogs } from "./helpers/globals.ts"; | |
| 17 | ||
| 18 | quietLogs(); | |
| 19 | ||
| 20 | let root = ""; | |
| 21 | let app: App; | |
| 22 | let svc: SnapshotsHandle; | |
| 23 | let servers: Servers; | |
| 24 | let cookie = ""; | |
| 25 | ||
| 26 | const MiB = 1024 ** 2; | |
| 27 | const drill: DrillResult = { | |
| 28 | at: "2026-11-15T08:10:00.000Z", snapshotId: "s", ms: 60_000, counts: { repositories: 1, issues: 0, users: 1, pulls: 0, apps: 0 }, | |
| 29 | health: "ok", verdict: "green", failedStep: null, phases: [], budgetMs: 600_000, reason: null, | |
| 30 | }; | |
| 31 | ||
| 32 | beforeAll(async () => { | |
| 33 | root = mkdtempSync(join(tmpdir(), "forge-backups-cost-")); | |
| 34 | app = createApp({ dataDir: root, port: 0 }); | |
| 35 | const u = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true }); | |
| 36 | if (!u.ok) throw new Error(u.error); | |
| 37 | svc = startSnapshots(app, dirStore(join(root, "store")), { version: "t", nightly: false }); | |
| 38 | servers = await serve(app); | |
| 39 | const r = await fetch(`${servers.url}/signin`, { | |
| 40 | method: "POST", | |
| 41 | headers: { origin: servers.url }, | |
| 42 | body: new URLSearchParams({ username: "alice", password: "a-good-passphrase" }), | |
| 43 | redirect: "manual", | |
| 44 | }); | |
| 45 | cookie = (r.headers.get("set-cookie") ?? "").split(";")[0]!; | |
| 46 | }); | |
| 47 | ||
| 48 | afterAll(async () => { | |
| 49 | servers.stop(); | |
| 50 | await svc.stop(); | |
| 51 | app.close(); | |
| 52 | rmSync(root, { recursive: true, force: true }); | |
| 53 | }); | |
| 54 | ||
| 55 | const costShown = async (): Promise<{ rows: string[]; page: string }> => { | |
| 56 | const page = await (await fetch(`${servers.url}/settings/backups`, { headers: { cookie } })).text(); | |
| 57 | const dl = /<dl class="status-rows" id="drill-cost">([\s\S]*?)<\/dl>/.exec(page)?.[1] ?? ""; | |
| 58 | const rows = [...dl.matchAll(/<dt>([^<]*)<\/dt><dd>([^<]*)<\/dd>/g)].map((m) => `${m[1]}: ${m[2]}`); | |
| 59 | return { rows, page }; | |
| 60 | }; | |
| 61 | ||
| 62 | test("a measured drill: took, peak with its share, and the limit with its rule, a fact a row", async () => { | |
| 63 | recordDrill(app.db, { ...drill, cost: { elapsedMs: 62_000, peakBytes: 412 * MiB, limitBytes: 3072 * MiB } }); | |
| 64 | const { rows, page } = await costShown(); | |
| 65 | // 412 / 3072 = 13.4%, rounded up to 14%; the apostrophe is escaped on the way out. | |
| 66 | expect(rows).toEqual([ | |
| 67 | "Took: 1 min 2 s", | |
| 68 | "Peak memory: 412 MiB, 14% of the limit", | |
| 69 | "Limit: 3 GiB: 60% of the machine's memory, kept between 3 and 16 GiB, or FORGE_DRILL_MEMORY_MIB", | |
| 70 | ]); | |
| 71 | // The rule is said once on the page. | |
| 72 | expect(page.split("between 3 and 16 GiB").length - 1).toBe(1); | |
| 73 | }); | |
| 74 | ||
| 75 | test("a drill kept before the cost was measured says so in one row", async () => { | |
| 76 | recordDrill(app.db, drill); | |
| 77 | expect((await costShown()).rows).toEqual(["Drill cost: elapsed time and peak memory not recorded for this drill"]); | |
| 78 | }); |
test/drill-job.test.ts +124 −7
| @@ -6,8 +6,10 @@ import { join } from "node:path"; | ||
| 6 | 6 | import { createApp } from "../src/app.ts"; |
| 7 | 7 | import { judge, type LaneRead } from "../src/snapshot/bucket.ts"; |
| 8 | 8 | import type { DrillResult } from "../src/snapshot/drill.ts"; |
| 9 | import type { DrillLine } from "../src/snapshot/drill_child.ts"; | |
| 9 | 10 | import { cutDrill, drillMemoryLimit, markerLeft, residentBytes, runDrillInChild, scrubbed, sweepDrills } from "../src/snapshot/drill_job.ts"; |
| 10 | import { lastDrill } from "../src/snapshot/drills.ts"; | |
| 11 | import { costFacts, costLine, LIMIT_RULE } from "../src/snapshot/drill_cost.ts"; | |
| 12 | import { lastDrill, parseCost, recordDrill } from "../src/snapshot/drills.ts"; | |
| 11 | 13 | import { startSnapshots } from "../src/snapshot/service.ts"; |
| 12 | 14 | import { dirStore } from "../src/snapshot/store.ts"; |
| 13 | 15 | |
| @@ -43,25 +45,140 @@ test("a drill with an empty store ends red in its own process, leaving no direct | ||
| 43 | 45 | expect(result.reason).toContain("no snapshot"); |
| 44 | 46 | expect(existsSync(join(root, "drill", "running.json"))).toBe(false); |
| 45 | 47 | expect(existsSync(join(root, "drill")) ? readdirSync(join(root, "drill")).join(" ") : "").toBe(""); |
| 48 | // A real drill in the real child, measured: a Bun process holds tens of MiB at the least. | |
| 49 | expect(result.cost!.elapsedMs).toBeGreaterThan(0); | |
| 50 | expect(result.cost!.peakBytes).toBeGreaterThan(16 * 1024 ** 2); | |
| 51 | expect(result.cost!.peakBytes).toBeLessThan(result.cost!.limitBytes); | |
| 46 | 52 | }); |
| 47 | 53 | |
| 48 | test("a drill over its memory limit is killed and says so", async () => { | |
| 54 | test("the drill's own process ends with its result and its own elapsed time and peak memory", async () => { | |
| 55 | // drill_child.ts run as the job runs it: one JSON job on stdin, a real runDrill on an empty | |
| 56 | // store. | |
| 57 | const child = Bun.spawn([process.execPath, "--smol", join(import.meta.dir, "..", "src", "snapshot", "drill_child.ts")], { | |
| 58 | stdin: "pipe", | |
| 59 | stdout: "pipe", | |
| 60 | stderr: "ignore", | |
| 61 | env: { PATH: process.env["PATH"] ?? "" }, | |
| 62 | }); | |
| 63 | child.stdin.write(JSON.stringify({ dataDir: join(root, "child"), store: { kind: "dir", root: join(root, "store") } })); | |
| 64 | await child.stdin.end(); | |
| 65 | const out = await new Response(child.stdout).text(); | |
| 66 | expect(await child.exited).toBe(0); | |
| 67 | const last = JSON.parse(out.trim().split("\n").at(-1)!) as DrillLine; | |
| 68 | if (!("result" in last)) throw new Error(`no result line: ${out}`); | |
| 69 | expect(last.result.verdict).toBe("red"); | |
| 70 | expect(last.result.reason).toContain("no snapshot"); | |
| 71 | // process.uptime in ms, and getrusage's maxRSS in bytes: whole, and of a real Bun process. | |
| 72 | expect(Number.isSafeInteger(last.cost?.elapsedMs)).toBe(true); | |
| 73 | expect(last.cost!.elapsedMs).toBeGreaterThan(0); | |
| 74 | expect(last.cost!.elapsedMs).toBeLessThan(60_000); | |
| 75 | expect(last.cost!.peakBytes).toBeGreaterThan(16 * 1024 ** 2); | |
| 76 | expect(last.cost!.peakBytes % 1024).toBe(0); | |
| 77 | }); | |
| 78 | ||
| 79 | test("a drill over its memory limit is a red drill naming memory as the cause, with what it held", async () => { | |
| 49 | 80 | // A child that holds still, a limit of one byte and a quick reading: killed at the first reading. |
| 50 | 81 | const script = join(root, "idle.ts"); |
| 51 | 82 | writeFileSync(script, "await Bun.stdin.text(); await Bun.sleep(30_000);"); // real-timer: a real child process is the thing under test |
| 52 | const err = await runDrillInChild( | |
| 83 | const result = await runDrillInChild( | |
| 53 | 84 | { dataDir: root, store: { kind: "dir", root: join(root, "store") } }, |
| 54 | 85 | () => {}, |
| 55 | 86 | () => {}, |
| 56 | 87 | { limit: 1, watchMs: 50, script }, |
| 57 | ).then( | |
| 58 | () => "no error", | |
| 59 | (e) => String(e), | |
| 60 | 88 | ); |
| 61 | expect(err).toContain("over its 0 MiB limit"); | |
| 89 | expect(result.verdict).toBe("red"); | |
| 90 | expect(result.failedStep).toBe("restore"); | |
| 91 | expect(result.reason).toStartWith("memory: the drill process held "); | |
| 92 | expect(result.reason).toContain("over its 0 MiB limit"); | |
| 93 | // The kill is measured too: getrusage of the reaped child, and the time to the kill. | |
| 94 | expect(result.cost?.limitBytes).toBe(1); | |
| 95 | expect(result.cost!.peakBytes).toBeGreaterThan(1024 ** 2); | |
| 96 | expect(result.cost!.elapsedMs).toBeGreaterThan(0); | |
| 97 | expect(result.cost!.elapsedMs).toBeLessThan(30_000); | |
| 62 | 98 | expect(markerLeft(root)).toBe(false); |
| 63 | 99 | }); |
| 64 | 100 | |
| 101 | test("a child that dies without a result is a red drill saying how it exited, never a silent end", async () => { | |
| 102 | const script = join(root, "dies.ts"); | |
| 103 | writeFileSync(script, "await Bun.stdin.text(); process.exit(7);"); | |
| 104 | const result = await runDrillInChild({ dataDir: root, store: { kind: "dir", root: join(root, "store") } }, () => {}, () => {}, { script }); | |
| 105 | expect(result.verdict).toBe("red"); | |
| 106 | expect(result.reason).toContain("the drill process exited with 7"); | |
| 107 | expect(result.cost?.peakBytes).toBeGreaterThan(0); | |
| 108 | }); | |
| 109 | ||
| 110 | test("a drill's own process reads back its elapsed time and peak memory beside its limit", async () => { | |
| 111 | // The child speaks the real protocol (drill_child.ts): a progress line, then its result and | |
| 112 | // cost, the peak read by the real `peakSoFar`. A full restore and boot in the child takes | |
| 113 | // seconds; what is under test is the cost crossing the process boundary and being kept. | |
| 114 | const script = join(root, "speaks.ts"); | |
| 115 | const child = join(import.meta.dir, "..", "src", "snapshot", "drill_child.ts"); | |
| 116 | const drilled: DrillResult = { | |
| 117 | at: "2026-11-15T08:00:00.000Z", snapshotId: "s-1", ms: 5, counts: { repositories: 1, issues: 0, users: 1, pulls: 0, apps: 0 }, | |
| 118 | health: "ok", verdict: "green", failedStep: null, phases: [], budgetMs: 0, reason: null, | |
| 119 | }; | |
| 120 | writeFileSync( | |
| 121 | script, | |
| 122 | `import { peakSoFar } from ${JSON.stringify(child)}; | |
| 123 | await Bun.stdin.text(); | |
| 124 | const held = new Uint8Array(32 * 1024 ** 2).fill(1); | |
| 125 | console.log(JSON.stringify({ progress: { phase: "download", done: 1, total: 1, bytes: held.length } })); | |
| 126 | console.log(JSON.stringify({ result: ${JSON.stringify(drilled)}, cost: { elapsedMs: 1234, peakBytes: peakSoFar(0) } })); | |
| 127 | process.exit(0);`, | |
| 128 | ); | |
| 129 | const limit = 3 * 1024 ** 3; | |
| 130 | const logged: string[] = []; | |
| 131 | const phases: string[] = []; | |
| 132 | const result = await runDrillInChild( | |
| 133 | { dataDir: join(root, "drill-data"), store: { kind: "dir", root: join(root, "store") } }, | |
| 134 | (s) => phases.push(s.phase), | |
| 135 | (m) => logged.push(m), | |
| 136 | { limit, script }, | |
| 137 | ); | |
| 138 | expect(result.verdict).toBe("green"); | |
| 139 | expect(result.counts.repositories).toBe(1); | |
| 140 | expect(phases).toContain("download"); | |
| 141 | const cost = result.cost!; | |
| 142 | expect(cost.limitBytes).toBe(limit); | |
| 143 | // The child's own elapsed time is the one kept; its peak includes the 32 MiB it held. | |
| 144 | expect(cost.elapsedMs).toBe(1234); | |
| 145 | expect(cost.peakBytes).toBeGreaterThan(32 * 1024 ** 2); | |
| 146 | expect(cost.peakBytes).toBeLessThan(limit); | |
| 147 | expect(logged).toContain("restore drill cost"); | |
| 148 | expect(markerLeft(join(root, "drill-data"))).toBe(false); | |
| 149 | // Kept and read back whole, the same numbers. | |
| 150 | const app = createApp({ dataDir: join(root, "kept"), port: 0 }); | |
| 151 | try { | |
| 152 | recordDrill(app.db, result); | |
| 153 | expect(lastDrill(app.db)?.cost).toEqual(cost); | |
| 154 | } finally { | |
| 155 | app.close(); | |
| 156 | } | |
| 157 | }); | |
| 158 | ||
| 159 | test("the cost line says elapsed, peak, the limit and its share; the facts say the rule once", () => { | |
| 160 | const MiB = 1024 ** 2; | |
| 161 | const d = { cost: { elapsedMs: 62_000, peakBytes: 412 * MiB, limitBytes: 3072 * MiB } }; | |
| 162 | // 412 MiB of 3 GiB = 13.41%, rounded up to 14%. | |
| 163 | expect(costLine(d)).toBe("took 1 min 2 s, peak 412 MiB of its 3 GiB limit (14%)"); | |
| 164 | expect(costLine({})).toBe("elapsed time and peak memory not recorded for this drill"); | |
| 165 | expect(costFacts(d)).toEqual([ | |
| 166 | { label: "Took", value: "1 min 2 s" }, | |
| 167 | { label: "Peak memory", value: "412 MiB, 14% of the limit" }, | |
| 168 | { label: "Limit", value: `3 GiB: ${LIMIT_RULE}` }, | |
| 169 | ]); | |
| 170 | expect(LIMIT_RULE).toContain("between 3 and 16 GiB"); | |
| 171 | expect(costFacts({})).toEqual([{ label: "Drill cost", value: "elapsed time and peak memory not recorded for this drill" }]); | |
| 172 | }); | |
| 173 | ||
| 174 | test("a kept cost that is not three whole numbers reads as none", () => { | |
| 175 | expect(parseCost(JSON.stringify({ cost: { elapsedMs: 1, peakBytes: 2, limitBytes: 3 } }))).toEqual({ cost: { elapsedMs: 1, peakBytes: 2, limitBytes: 3 } }); | |
| 176 | for (const bad of [{ elapsedMs: -1, peakBytes: 2, limitBytes: 3 }, { elapsedMs: 1, peakBytes: "2", limitBytes: 3 }, { elapsedMs: 1.5, peakBytes: 2, limitBytes: 3 }, null, "x"]) | |
| 177 | expect(parseCost(JSON.stringify({ cost: bad }))).toEqual({}); | |
| 178 | expect(parseCost("not json")).toEqual({}); | |
| 179 | expect(parseCost(null)).toEqual({}); | |
| 180 | }); | |
| 181 | ||
| 65 | 182 | test("the limit follows the machine, within bounds, and an override of at least 512 MiB", () => { |
| 66 | 183 | const GiB = 1024 ** 3; |
| 67 | 184 | expect(drillMemoryLimit({}, 2 * GiB)).toBe(3 * GiB); |
test/mcp-restore-drill.test.ts +6
| @@ -64,6 +64,7 @@ describe("MCP restore_drill", () => { | ||
| 64 | 64 | phases: [], |
| 65 | 65 | budgetMs: 0, |
| 66 | 66 | reason: green ? null : "a sealed value does not open", |
| 67 | cost: { elapsedMs: 62_000, peakBytes: 412 * 1024 ** 2, limitBytes: 3072 * 1024 ** 2 }, | |
| 67 | 68 | }; |
| 68 | 69 | }, |
| 69 | 70 | }); |
| @@ -118,6 +119,11 @@ describe("MCP restore_drill", () => { | ||
| 118 | 119 | expect(verdict.says).toBe("The drill restored green."); |
| 119 | 120 | expect(lastDrill(app.db)?.snapshotId).toBe("snap-1"); |
| 120 | 121 | }); |
| 122 | test("status prints the drill's elapsed time and peak memory in one line beside its limit (#13253)", async () => { | |
| 123 | const { verdict } = await drillAndRead(token("alice")); | |
| 124 | expect(verdict.cost).toBe("took 1 min 2 s, peak 412 MiB of its 3 GiB limit (14%)"); | |
| 125 | expect([verdict.elapsedMs, verdict.peakMiB, verdict.limitMiB]).toEqual([62_000, 412, 3072]); | |
| 126 | }); | |
| 121 | 127 | test("a second ask while one runs says so and does not start another", async () => { |
| 122 | 128 | const t = token("alice"); |
| 123 | 129 | let open!: () => void; |
test/tui-drill.test.ts +15
| @@ -127,6 +127,21 @@ test("a restore drill says its outcome in words and leaves the domain row alone" | ||
| 127 | 127 | h.app.stop(); |
| 128 | 128 | }); |
| 129 | 129 | |
| 130 | test("the verdict says the drill process's cost beside its limit (#13253)", async () => { | |
| 131 | await handle?.stop(); | |
| 132 | handle = startSnapshots(forge, dirStore(join(dir, "store")), { | |
| 133 | version: "0", | |
| 134 | nightly: false, | |
| 135 | drill: async () => ({ ...result("green"), cost: { elapsedMs: 62_000, peakBytes: 412 * 1024 ** 2, limitBytes: 3 * 1024 ** 3 } }), | |
| 136 | }); | |
| 137 | const h = await open(alice); | |
| 138 | await h.press("d"); | |
| 139 | await h.press("left"); | |
| 140 | const text = await h.pressUntil("enter", "peak 412 MiB"); | |
| 141 | expect(text).toContain("The drill restored green. Its process took 1 min 2 s, peak 412 MiB of its 3 GiB limit (14%)."); | |
| 142 | h.app.stop(); | |
| 143 | }); | |
| 144 | ||
| 130 | 145 | test("declining runs nothing", async () => { |
| 131 | 146 | const h = await open(alice); |
| 132 | 147 | const before = runs; |
test/tui-resilience-drill.test.ts +13
| @@ -131,6 +131,19 @@ describe("a restore drill from the terminal's Resilience dialog", () => { | ||
| 131 | 131 | h.stop(); |
| 132 | 132 | }); |
| 133 | 133 | |
| 134 | test("the verdict says the drill process's cost beside its limit, as the Backups page does (#13253)", async () => { | |
| 135 | verdict = { ...GREEN, cost: { elapsedMs: 62_000, peakBytes: 412 * 1024 ** 2, limitBytes: 3 * 1024 ** 3 } }; | |
| 136 | const h = boot("alice"); | |
| 137 | await h.open(); | |
| 138 | await h.press("r", "y"); | |
| 139 | await h.waitFor("peak 412 MiB"); | |
| 140 | // The dialog wraps the line; its words, border and padding removed, read as one. | |
| 141 | const said = h.text().replace(/[│╭╮╰╯─]/g, " ").replace(/\s+/g, " "); | |
| 142 | expect(said).toContain("The drill restored green. Its process took 1 min 2 s, peak 412 MiB of its 3 GiB limit (14%)."); | |
| 143 | verdict = GREEN; | |
| 144 | h.stop(); | |
| 145 | }); | |
| 146 | ||
| 134 | 147 | test("r then n, or esc, runs nothing", async () => { |
| 135 | 148 | const h = boot("alice"); |
| 136 | 149 | await h.open(); |
new above
offline
Principle 1: this rule text is repeated on every drill line and in the MCP answer, so the page explains itself twice. Keep it in one place.