Public: anyone can read this; the crew can change it.
The restore drill: cut by the edge, buffered in the serving process, lost on a restart, and silent while it runs #12600
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/12592-the-restore-drill-cut-by-the-edge-buffer → main 10 commits fork 1 thread · 1 unresolved
23 files
changed +700
−70
docs/MCP.md +1 −1
| @@ -187,7 +187,7 @@ bytes in `tools/list`. After changing a tool run `bun scripts/mcp-table.ts`; | ||
| 187 | 187 | | `repo_labels` | — | Repo labels: list (default), create, update, remove (admin token) | 351 | |
| 188 | 188 | | `repo_settings` | — | Read, change, rename (name) or move (newOwner) a repo: PATCH /api/v1/repos/:owner/:name (admin); mirrorPacks mirrors the packs' upstreams | 484 | |
| 189 | 189 | | `reports` | `accept`, `dismiss` | Outsiders' reports; `id`+`action`: accept or dismiss | 355 | |
| 190 | | `restore_drill` | — | Drill-restore the newest snapshot | 216 | | |
| 190 | | `restore_drill` | `start`, `status` | Start a restore drill of the newest snapshot, or read how it is going | 304 | | |
| 191 | 191 | | `returns` | — | What crew tokens bought in `days` (1–90, default 7): dollars, runs, landed, cost/change | 284 | |
| 192 | 192 | | `review` | — | Approve a pull, or request changes (`reason` required); your own pull is refused | 350 | |
| 193 | 193 | | `rollback` | — | Plan, then apply, a rollback to a kept release (admin) | 209 | |
docs/features/backup-truth.md +5
| @@ -134,3 +134,8 @@ Not done here: the drill itself, the dump decision, and the lane rows. Each | ||
| 134 | 134 | needs the box or the owner; this section is what they follow. |
| 135 | 135 | |
| 136 | 136 | An app's own data has its own encrypted backup and restore proof: `app-backup.md` (#11701). |
| 137 | ||
| 138 | ## The drill is a job (#12592) | |
| 139 | A restore drill is asked for and answers at once with an id (MCP `restore_drill`, `action: start`; the Backups page's button; in the terminal, `d` on the Failure domains screen and `r` on the Resilience screen, which start the same job and follow it, saying "A drill is running: <phase>, X of Y." until the verdict). One runs at a time; a second ask says one is running and where. It runs in its own process (`src/snapshot/drill_child.ts`, started by `drill_job.ts` with `--smol`), so the restore is never held by the serving process; that process's resident memory is read every 3 s and it is killed, the drill red, above its limit: 60% of the machine's memory, between 3 and 16 GiB, or `FORGE_DRILL_MEMORY_MIB` (at least 512). The limit is stated on the Backups page. It is not yet measured on a store of real size: the first real drill's `restore drill working` lines say what the child held, and the number is tuned from them. The restore itself already streams chunks to disk with at most 24 reads in flight; what the serving process held on 2026-11-14 is read memory the collector had not yet given back, so the child collects after each progress report. Where it is (phase, done of total, bytes, memory) is on the Backups page, in MCP `action: status`, and in the log (`restore drill working`, every 15 s). `action: status` answers the verdict once it ends. | |
| 140 | ||
| 141 | A marker (`drill/running.json`) is left while a drill runs. A boot that finds it records a red drill "cut by a restart", removes every `drill/restore-*` directory (none runs at boot), and drills again. The status line says the proof's age ("restore drill 2026-11-08 passed (6 d ago)") and goes amber when the newest snapshot is more than 3 days newer than the drill. A snapshot whose format differs from the last drilled one is drilled by the forge on its own. |
scripts/unused-exports-allow.json +1
| @@ -158,6 +158,7 @@ | ||
| 158 | 158 | "src/runs/log.ts runLogListenerCount", |
| 159 | 159 | "src/runs/log.ts runLogTapCount", |
| 160 | 160 | "src/runs/stoppable.ts runningRun", |
| 161 | "src/snapshot/drill_job.ts markerLeft", | |
| 161 | 162 | "src/spawn/child-apps.ts childRuntime", |
| 162 | 163 | "src/spawn/child_db.ts childDbReaders", |
| 163 | 164 | "src/spawn/clients.ts monthsSettled", |
src/actions/backups_now.ts +45
| @@ -11,6 +11,7 @@ import type { App } from "../app.ts"; | ||
| 11 | 11 | import type { User } from "../core/users.ts"; |
| 12 | 12 | import { err, ok, type Result } from "../kernel/result.ts"; |
| 13 | 13 | import type { DrillStep } from "../snapshot/drill.ts"; |
| 14 | import { lastDrill } from "../snapshot/drills.ts"; | |
| 14 | 15 | import { snapshotsOf } from "../snapshot/service.ts"; |
| 15 | 16 | import { fileSize } from "../web/ui.ts"; |
| 16 | 17 | |
| @@ -118,6 +119,50 @@ export async function drillNow( | ||
| 118 | 119 | } |
| 119 | 120 | } |
| 120 | 121 | |
| 122 | /** Ask for a drill as a job and answer at once; one running is said, not doubled (#12592). */ | |
| 123 | export function startDrillNow(app: App, actor: User | undefined): Result<{ started: boolean; says: string }, DrillRefusal> { | |
| 124 | if (!isAdmin(actor)) return err({ code: "not_found", says: "Only a forge admin runs a restore drill." }); | |
| 125 | const svc = snapshotsOf(app); | |
| 126 | if (svc?.store === undefined) return err({ code: "no_store", says: "This forge has no snapshots to drill." }); | |
| 127 | const { started } = svc.startDrill(); | |
| 128 | return ok({ | |
| 129 | started, | |
| 130 | says: started | |
| 131 | ? "A restore drill started; this page shows where it is, and its verdict when it ends." | |
| 132 | : "A restore drill is already running; one runs at a time.", | |
| 133 | }); | |
| 134 | } | |
| 135 | ||
| 136 | /** | |
| 137 | * Where a running drill stands in one sentence, the page's and the terminal's; undefined when | |
| 138 | * none runs. | |
| 139 | */ | |
| 140 | export function drillStatus(app: App): string | undefined { | |
| 141 | const running = snapshotsOf(app)?.drillState(); | |
| 142 | if (running === undefined) return undefined; | |
| 143 | if (!("bytes" in running)) return "A drill is waiting for the snapshot in flight."; | |
| 144 | return `A drill is running: ${running.phase}${running.total > 0 ? `, ${running.done} of ${running.total}` : ""}.`; | |
| 145 | } | |
| 146 | ||
| 147 | /** | |
| 148 | * Wait for the running drill, telling `tick` where it stands each `everyMs`, and answer its | |
| 149 | * verdict in the words `drillNow` uses; the terminal's door to a drill that is a job (#12592). | |
| 150 | */ | |
| 151 | export async function followDrill(app: App, tick: (says: string) => void, everyMs = 250): Promise<string> { | |
| 152 | for (let says = drillStatus(app); says !== undefined; says = drillStatus(app)) { | |
| 153 | tick(says); | |
| 154 | await Bun.sleep(everyMs); | |
| 155 | } | |
| 156 | let d: ReturnType<typeof lastDrill>; | |
| 157 | try { | |
| 158 | d = lastDrill(app.db); | |
| 159 | } catch { | |
| 160 | d = undefined; // the forge closed while the drill ran | |
| 161 | } | |
| 162 | if (d === undefined) return "The drill ended without a verdict."; | |
| 163 | return d.verdict === "green" ? "The drill restored green." : `The drill failed at ${d.failedStep}.`; | |
| 164 | } | |
| 165 | ||
| 121 | 166 | /** |
| 122 | 167 | * The newest snapshots first, one line each: `name · size`; empty when there |
| 123 | 168 | * is no store. A store that cannot be listed is `unreadable`, never an empty |
src/http/mcp/tools/restore_drill.ts +31 −16
| @@ -1,32 +1,47 @@ | ||
| 1 | 1 | import { scopeAllows } from "../../../core/tokens.ts"; |
| 2 | import { snapshotsOf } from "../../../snapshot/service.ts"; | |
| 2 | import { lastDrill } from "../../../snapshot/drills.ts"; | |
| 3 | import { snapshotsOf, type SnapshotsHandle } from "../../../snapshot/service.ts"; | |
| 3 | 4 | import { good, bad, type Tool, mayBoards } from "../shared.ts"; |
| 4 | 5 | |
| 6 | const phaseWords = (st: ReturnType<SnapshotsHandle["startDrill"]>["state"]): Record<string, string | number> => | |
| 7 | "bytes" in st | |
| 8 | ? { id: st.id, phase: st.phase, done: st.done, total: st.total, bytes: st.bytes, rssMiB: Math.round(st.rss / 1024 ** 2), limitMiB: Math.round(st.limit / 1024 ** 2), startedAt: st.startedAt } | |
| 9 | : { id: st.id, phase: st.phase }; | |
| 10 | ||
| 5 | 11 | export const tool: Tool = { |
| 6 | 12 | name: "restore_drill", |
| 7 | doc: "no input: restore the newest snapshot as a drill and answer `{ verdict, failedStep?, says, from?, offSite? }` (from: the store it restored, the off-site lane when there is one), worded as the Backups page's button; no store, a failure or a red drill is an answer, not an error; a forge admin's, with an admin token", | |
| 8 | description: "Drill-restore the newest snapshot.", | |
| 9 | inputSchema: { type: "object", properties: {} }, | |
| 13 | doc: "`action` start (default) answers at once `{ started, id, phase, bytes, ... }`; one drill runs at a time, a second ask says `started: false` and where the running one is; `status` answers the running drill's phase and bytes, or the newest verdict `{ verdict, failedStep?, says, at, from?, offSite? }` (red, not an error, when it failed); it runs in its own process; a forge admin's, with an admin token", | |
| 14 | description: "Start a restore drill of the newest snapshot, or read how it is going.", | |
| 15 | inputSchema: { type: "object", properties: { action: { type: "string", enum: ["start", "status"] } } }, | |
| 10 | 16 | writes: true, |
| 11 | 17 | adminOnly: true, |
| 12 | async run(ctx, user) { | |
| 18 | async run(ctx, user, args) { | |
| 13 | 19 | if (!mayBoards(user)) return bad("forbidden", "a restore drill is a forge admin's to run"); |
| 14 | 20 | if (!scopeAllows(ctx.auth.scope ?? "ro", "admin")) |
| 15 | 21 | return bad("forbidden", "a drill needs an admin token"); |
| 22 | const action = args["action"] ?? "start"; | |
| 23 | if (action !== "start" && action !== "status") return bad("invalid", "action is start or status"); | |
| 16 | 24 | const svc = snapshotsOf(ctx.app); |
| 17 | 25 | if (svc === undefined || svc.store === undefined) |
| 18 | 26 | return good({ verdict: "none", says: "This forge has no snapshots to drill." }); |
| 19 | try { | |
| 20 | const d = await svc.drill(); | |
| 21 | // Where it restored from (the off-site lane when the forge has one, #11805), never a key. | |
| 22 | const from = d.from === undefined ? {} : { from: d.from, offSite: d.offSite === true }; | |
| 23 | const answer = | |
| 24 | d.verdict === "green" | |
| 25 | ? good({ verdict: "green", says: "The drill restored green.", ...from }) | |
| 26 | : good({ verdict: "red", failedStep: d.failedStep, says: `The drill failed at ${d.failedStep}.`, ...from }); | |
| 27 | return { ...answer, card: { at: d.at } }; | |
| 28 | } catch (error) { | |
| 29 | return good({ verdict: "none", says: `The drill could not run: ${String(error)}` }); | |
| 27 | if (action === "start") { | |
| 28 | const { started, state } = svc.startDrill(); | |
| 29 | return good({ | |
| 30 | started, | |
| 31 | ...phaseWords(state), | |
| 32 | says: started ? "A drill started; read it with action status." : "A drill is already running; one runs at a time.", | |
| 33 | }); | |
| 30 | 34 | } |
| 35 | const running = svc.drillState(); | |
| 36 | if (running !== undefined) return good({ running: true, ...phaseWords(running), says: `A drill is running: ${running.phase}.` }); | |
| 37 | const d = lastDrill(ctx.app.db); | |
| 38 | if (d === undefined) return good({ verdict: "none", says: "No drill has run." }); | |
| 39 | // Where it restored from (the off-site lane when the forge has one, #11805), never a key. | |
| 40 | const from = d.from === undefined ? {} : { from: d.from, offSite: d.offSite === true }; | |
| 41 | const answer = | |
| 42 | d.verdict === "green" | |
| 43 | ? good({ verdict: "green", at: d.at, says: "The drill restored green.", ...from }) | |
| 44 | : good({ verdict: "red", at: d.at, failedStep: d.failedStep, says: `The drill failed at ${d.failedStep}${d.reason ? `: ${d.reason}` : ""}.`, ...from }); | |
| 45 | return { ...answer, card: { at: d.at } }; | |
| 31 | 46 | }, |
| 32 | 47 | }; |
src/snapshot/bucket.ts +10 −1
| @@ -146,6 +146,9 @@ const watching = new Map<string, string>(); | ||
| 146 | 146 | /** Whether a bucket watch has been started for this data directory. */ |
| 147 | 147 | export const isWatched = (dataDir: string): boolean => watching.has(dataDir); |
| 148 | 148 | |
| 149 | /** A passed drill this far behind the newest snapshot no longer vouches for it. */ | |
| 150 | export const DRILL_BEHIND_MS = 3 * 86_400_000; | |
| 151 | ||
| 149 | 152 | export function ageWords(ms: number): string { |
| 150 | 153 | const min = Math.max(0, Math.floor(ms / 60_000)); |
| 151 | 154 | if (min < 1) return "just now"; |
| @@ -298,11 +301,14 @@ export function judge( | ||
| 298 | 301 | let drillSays: BackupVerdict["drill"] = null; |
| 299 | 302 | if (drill !== undefined) { |
| 300 | 303 | const ok = drillCounts(drill); |
| 301 | const says = ok | |
| 304 | let says = ok | |
| 302 | 305 | ? `restore drill ${drill.at.slice(0, 10)} passed` |
| 303 | 306 | : drill.verdict === "green" |
| 304 | 307 | ? `restore drill ${drill.at.slice(0, 10)} brought back no data` |
| 305 | 308 | : `restore drill ${drill.at.slice(0, 10)} failed${drill.failedStep ? ` at ${drill.failedStep}` : ""}`; |
| 309 | // The proof's age, said: "passed" for six days is not the same as passed today (#12592). | |
| 310 | const proofAge = ageWords(Math.max(0, now - Date.parse(drill.at))); | |
| 311 | if (ok) says = `${says} (${proofAge} ago)`; | |
| 306 | 312 | drillSays = { at: drill.at, ok, says }; |
| 307 | 313 | if (!ok) { |
| 308 | 314 | causes.push(drill.verdict === "green" ? "drill restored nothing" : "drill failed"); |
| @@ -310,6 +316,9 @@ export function judge( | ||
| 310 | 316 | } |
| 311 | 317 | } else if (checkedAt !== null) unknowns.push("No restore drill has run, so nothing shows a snapshot restores."); |
| 312 | 318 | const newest = lanes.map((l) => l.newestAt).filter((t): t is string => t !== null).sort().at(-1) ?? null; |
| 319 | // Amber: the newest snapshot is newer than the drill that vouches for it by more than a few days. | |
| 320 | if (drillSays?.ok && newest !== null && Date.parse(newest) - Date.parse(drillSays.at) > DRILL_BEHIND_MS) | |
| 321 | unknowns.push(`The last restore drill is ${ageWords(now - Date.parse(drillSays.at))} old; the newest snapshot has not been proved to restore.`); | |
| 313 | 322 | const primary = lanes[0]; |
| 314 | 323 | const head = |
| 315 | 324 | primary?.newestAt != null && primary.error === undefined |
src/snapshot/drill.ts +8
| @@ -139,6 +139,8 @@ export interface DrillDeps { | ||
| 139 | 139 | readonly restore?: typeof restoreSnapshot | undefined; |
| 140 | 140 | /** Test seam: boot a data directory on a side port and say whether it is healthy. */ |
| 141 | 141 | readonly boot?: typeof bootRestored | undefined; |
| 142 | /** Told as the restore moves (phase, counts, bytes), so a job can show where it is. */ | |
| 143 | readonly onProgress?: ((p: Progress) => void) | undefined; | |
| 142 | 144 | } |
| 143 | 145 | |
| 144 | 146 | export type { DrillCounts }; |
| @@ -162,6 +164,8 @@ export interface DrillResult { | ||
| 162 | 164 | readonly from?: string | undefined; |
| 163 | 165 | /** Whether that store is off the forge's machine (`isOffBox`). */ |
| 164 | 166 | readonly offSite?: boolean | undefined; |
| 167 | /** The manifest format of the snapshot drilled (1 before chunks); absent on older drills. */ | |
| 168 | readonly format?: number | undefined; | |
| 165 | 169 | } |
| 166 | 170 | |
| 167 | 171 | /** The restored forge's front door: `/` answers below 400 (a redirect to sign in is an answer), or this throws. */ |
| @@ -201,6 +205,7 @@ export async function runDrill(deps: DrillDeps): Promise<DrillResult> { | ||
| 201 | 205 | const restore = deps.restore ?? restoreSnapshot; |
| 202 | 206 | const counts: { -readonly [K in keyof DrillCounts]: number } = { repositories: 0, issues: 0, users: 0, pulls: 0, apps: 0 }; |
| 203 | 207 | let snapshotId: string | null = null; |
| 208 | let format: number | undefined; | |
| 204 | 209 | let step: DrillStep = "fetch"; |
| 205 | 210 | let scratch: string | undefined; |
| 206 | 211 | let reason: string | null = null; |
| @@ -219,6 +224,7 @@ export async function runDrill(deps: DrillDeps): Promise<DrillResult> { | ||
| 219 | 224 | const newest = (await within(listSnapshots(store), left(), budget))[0]; |
| 220 | 225 | if (newest === undefined) throw new Error("the store holds no snapshot"); |
| 221 | 226 | snapshotId = newest.name; |
| 227 | format = newest.format ?? 1; | |
| 222 | 228 | sized = snapshotBytes(newest); |
| 223 | 229 | if (deps.timeoutMs === undefined) budgetMs = drillBudgetMs(sized); |
| 224 | 230 | |
| @@ -232,6 +238,7 @@ export async function runDrill(deps: DrillDeps): Promise<DrillResult> { | ||
| 232 | 238 | passphrase: deps.passphrase, |
| 233 | 239 | onProgress: (p) => { |
| 234 | 240 | progress = p; |
| 241 | deps.onProgress?.(p); | |
| 235 | 242 | if (began[began.length - 1]?.phase !== p.phase) began.push({ phase: p.phase, at: tick() }); |
| 236 | 243 | }, |
| 237 | 244 | }), |
| @@ -340,6 +347,7 @@ export async function runDrill(deps: DrillDeps): Promise<DrillResult> { | ||
| 340 | 347 | reason, |
| 341 | 348 | from: deps.store.where, |
| 342 | 349 | offSite: isOffBox(deps.store), |
| 350 | ...(format === undefined ? {} : { format }), | |
| 343 | 351 | }; |
| 344 | 352 | } |
| 345 | 353 | } |
src/snapshot/drill_child.ts added +46
| @@ -0,0 +1,46 @@ | ||
| 1 | /** | |
| 2 | * The restore drill's own process (`drill_job.ts` starts it). The serving | |
| 3 | * process held a whole restore in memory (9 to 13 GiB on 2026-11-14, #12592); | |
| 4 | * here the restore streams into the drill's scratch directory and this process | |
| 5 | * ends with the drill, so whatever it held goes back to the machine. | |
| 6 | * | |
| 7 | * One JSON job on stdin ({ dataDir, store, passphrase }), so no secret is on a | |
| 8 | * command line or in the environment. One JSON object per line on stdout: | |
| 9 | * `{ progress }` as the restore moves, then `{ result }`. | |
| 10 | */ | |
| 11 | ||
| 12 | import { runDrill, type DrillResult } from "./drill.ts"; | |
| 13 | import { storeFrom } from "./remake.ts"; | |
| 14 | import type { Progress } from "./restore.ts"; | |
| 15 | import type { StoreSpec } from "./store.ts"; | |
| 16 | ||
| 17 | export interface DrillJob { | |
| 18 | readonly dataDir: string; | |
| 19 | readonly store: StoreSpec; | |
| 20 | readonly passphrase?: string | undefined; | |
| 21 | } | |
| 22 | ||
| 23 | export type DrillLine = { readonly progress: Progress } | { readonly result: DrillResult }; | |
| 24 | ||
| 25 | const say = (line: DrillLine): void => void process.stdout.write(`${JSON.stringify(line)}\n`); | |
| 26 | ||
| 27 | if (import.meta.main) { | |
| 28 | const job = JSON.parse(await Bun.stdin.text()) as DrillJob; | |
| 29 | let last = 0; | |
| 30 | const result = await runDrill({ | |
| 31 | store: storeFrom(job.store), | |
| 32 | dataDir: job.dataDir, | |
| 33 | passphrase: job.passphrase, | |
| 34 | // A restore reports often; a line a second says as much. | |
| 35 | onProgress: (progress) => { | |
| 36 | const now = Date.now(); | |
| 37 | if (now - last < 1_000) return; | |
| 38 | last = now; | |
| 39 | // Buffers of finished reads are given back at once, not at the collector's leisure. | |
| 40 | Bun.gc(false); | |
| 41 | say({ progress }); | |
| 42 | }, | |
| 43 | }); | |
| 44 | say({ result }); | |
| 45 | process.exit(0); | |
| 46 | } |
src/snapshot/drill_job.ts added +224
| @@ -0,0 +1,224 @@ | ||
| 1 | /** | |
| 2 | * The restore drill as a job: asked for, it answers at once with an id; it runs | |
| 3 | * in its own process (`drill_child.ts`) so a restore is never held in the | |
| 4 | * serving process; one runs at a time; where it is (phase, bytes, the child's | |
| 5 | * memory) is read by the Backups page, the MCP `restore_drill` tool and the log. | |
| 6 | * (#12592) | |
| 7 | * | |
| 8 | * A marker file under the drill's scratch parent says a drill is running; a | |
| 9 | * boot that finds it knows the drill was cut by the restart (`sweepDrills`), | |
| 10 | * and removes every scratch directory, since no drill is running at boot. | |
| 11 | * | |
| 12 | * The child's memory is bounded: `--smol`, and its resident set is read every | |
| 13 | * few seconds and the child is killed, the drill red, above `DRILL_MEMORY_BYTES`. | |
| 14 | */ | |
| 15 | ||
| 16 | import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; | |
| 17 | import { totalmem } from "node:os"; | |
| 18 | import { join } from "node:path"; | |
| 19 | import { during } from "../perf/jobs.ts"; | |
| 20 | import { describeProgress, DRILL_MAX_MS, type DrillResult } from "./drill.ts"; | |
| 21 | import type { DrillJob, DrillLine } from "./drill_child.ts"; | |
| 22 | import type { Progress } from "./restore.ts"; | |
| 23 | ||
| 24 | const GIB = 1024 ** 3; | |
| 25 | ||
| 26 | /** | |
| 27 | * The most resident memory the drill's process may hold. The 2026-11-14 drill of the first chunked | |
| 28 | * snapshot held 9 to 13 GiB in the serving process, so a fixed small number would be a drill that | |
| 29 | * kills itself: the limit is 60% of the machine's memory, between 3 and 16 GiB, or | |
| 30 | * `FORGE_DRILL_MEMORY_MIB` when that is a whole number of at least 512. It is stated on the Backups | |
| 31 | * page. Not yet measured on a real-size store: the child reads the bound and logs what it held | |
| 32 | * (`restore drill working`), so the first real drill tunes this number. | |
| 33 | */ | |
| 34 | export function drillMemoryLimit(env: Record<string, string | undefined> = process.env, machine: number = totalmem()): number { | |
| 35 | const set = env["FORGE_DRILL_MEMORY_MIB"] ?? ""; | |
| 36 | if (/^[0-9]{3,6}$/.test(set) && Number(set) >= 512) return Number(set) * 1024 ** 2; | |
| 37 | return Math.min(16 * GIB, Math.max(3 * GIB, Math.floor(machine * 0.6))); | |
| 38 | } | |
| 39 | export const DRILL_MEMORY_BYTES = drillMemoryLimit(); | |
| 40 | const WATCH_MS = 3_000; | |
| 41 | const CHILD = join(import.meta.dir, "drill_child.ts"); | |
| 42 | ||
| 43 | /** What the child's last words may say in a recorded error: no paths, one short line. */ | |
| 44 | export function scrubbed(text: string): string { | |
| 45 | return text.replace(/\S*\/\S*/g, "<path>").replace(/\s+/g, " ").trim().slice(-200); | |
| 46 | } | |
| 47 | ||
| 48 | /** Where a running drill stands, as plain data. */ | |
| 49 | export interface DrillState { | |
| 50 | readonly id: string; | |
| 51 | readonly startedAt: string; | |
| 52 | readonly phase: Progress["phase"] | "starting" | "checking"; | |
| 53 | readonly done: number; | |
| 54 | readonly total: number; | |
| 55 | readonly bytes: number; | |
| 56 | /** The drill process's resident memory, bytes; 0 until read. */ | |
| 57 | readonly rss: number; | |
| 58 | readonly limit: number; | |
| 59 | } | |
| 60 | ||
| 61 | export type DrillLog = (message: string, fields: Record<string, string | number | boolean | undefined>) => void; | |
| 62 | ||
| 63 | const parent = (dataDir: string): string => join(dataDir, "drill"); | |
| 64 | const marker = (dataDir: string): string => join(parent(dataDir), "running.json"); | |
| 65 | ||
| 66 | /** Resident bytes of a process on Linux; 0 where /proc has none. */ | |
| 67 | export function residentBytes(pid: number): number { | |
| 68 | try { | |
| 69 | const m = /VmRSS:\s+(\d+)\s+kB/.exec(readFileSync(`/proc/${pid}/status`, "utf8")); | |
| 70 | return m ? Number(m[1]) * 1024 : 0; | |
| 71 | } catch { | |
| 72 | return 0; | |
| 73 | } | |
| 74 | } | |
| 75 | ||
| 76 | /** A drill's id: when it began, readable and sortable. */ | |
| 77 | export const drillId = (now = new Date()): string => `drill-${now.toISOString().replace(/[:.]/g, "-")}`; | |
| 78 | ||
| 79 | /** | |
| 80 | * At boot: no drill is running, so every scratch directory is stale. Removes | |
| 81 | * them, and says whether a drill was cut by the restart (the marker was left). | |
| 82 | */ | |
| 83 | export function sweepDrills(dataDir: string): { cut: { id: string; at: string } | undefined; removed: number } { | |
| 84 | let cut: { id: string; at: string } | undefined; | |
| 85 | try { | |
| 86 | const m = JSON.parse(readFileSync(marker(dataDir), "utf8")) as { id?: unknown; at?: unknown }; | |
| 87 | cut = { id: typeof m.id === "string" ? m.id : "unknown", at: typeof m.at === "string" ? m.at : "" }; | |
| 88 | } catch { | |
| 89 | // none left | |
| 90 | } | |
| 91 | let removed = 0; | |
| 92 | try { | |
| 93 | for (const name of readdirSync(parent(dataDir))) { | |
| 94 | if (name !== "running.json" && !name.startsWith("restore-")) continue; | |
| 95 | rmSync(join(parent(dataDir), name), { recursive: true, force: true }); | |
| 96 | if (name !== "running.json") removed++; | |
| 97 | } | |
| 98 | } catch { | |
| 99 | // no scratch parent yet | |
| 100 | } | |
| 101 | return { cut, removed }; | |
| 102 | } | |
| 103 | ||
| 104 | /** The record of a drill a restart cut: red, and saying so. */ | |
| 105 | export function cutDrill(cut: { id: string; at: string }): DrillResult { | |
| 106 | return { | |
| 107 | at: cut.at || new Date().toISOString(), | |
| 108 | snapshotId: null, | |
| 109 | ms: 0, | |
| 110 | counts: { repositories: 0, issues: 0, users: 0, pulls: 0, apps: 0 }, | |
| 111 | health: "unreached", | |
| 112 | verdict: "red", | |
| 113 | failedStep: "restore", | |
| 114 | phases: [], | |
| 115 | budgetMs: 0, | |
| 116 | reason: `drill ${cut.id} was cut by a restart; it is started again`, | |
| 117 | }; | |
| 118 | } | |
| 119 | ||
| 120 | /** | |
| 121 | * Run the drill in its own process. `watch` hears each state change (at most | |
| 122 | * about one a second, and one per memory reading); the promise answers with the | |
| 123 | * drill's result, or rejects with why the process gave none. The scratch is | |
| 124 | * removed whatever the end. `options` are test seams: the limit, how often memory is read, and the | |
| 125 | * script the child runs. | |
| 126 | */ | |
| 127 | export async function runDrillInChild( | |
| 128 | job: DrillJob, | |
| 129 | watch: (state: DrillState) => void, | |
| 130 | log: DrillLog, | |
| 131 | options: { readonly limit?: number; readonly watchMs?: number; readonly script?: string } = {}, | |
| 132 | ): Promise<DrillResult> { | |
| 133 | const limit = options.limit ?? DRILL_MEMORY_BYTES; | |
| 134 | const id = drillId(); | |
| 135 | const at = new Date().toISOString(); | |
| 136 | let state: DrillState = { id, startedAt: at, phase: "starting", done: 0, total: 0, bytes: 0, rss: 0, limit }; | |
| 137 | const set = (patch: Partial<DrillState>): void => { | |
| 138 | state = { ...state, ...patch }; | |
| 139 | watch(state); | |
| 140 | }; | |
| 141 | mkdirSync(parent(job.dataDir), { recursive: true }); | |
| 142 | writeFileSync(marker(job.dataDir), JSON.stringify({ id, at })); | |
| 143 | log("restore drill started", { id, limitMiB: Math.round(limit / 1024 ** 2) }); | |
| 144 | // Its own session, so the end of the drill ends the restored forge it booted too. | |
| 145 | const child = Bun.spawn([process.execPath, "--smol", options.script ?? CHILD], { | |
| 146 | stdin: "pipe", | |
| 147 | stdout: "pipe", | |
| 148 | stderr: "pipe", | |
| 149 | detached: true, | |
| 150 | env: { PATH: process.env["PATH"] ?? "", ...(process.env["FORGE_TEST_SOCKET"] ? { FORGE_TEST_SOCKET: process.env["FORGE_TEST_SOCKET"] } : {}) }, | |
| 151 | }); | |
| 152 | const stop = (): void => { | |
| 153 | try { | |
| 154 | process.kill(-child.pid, "SIGKILL"); | |
| 155 | } catch { | |
| 156 | child.kill("SIGKILL"); | |
| 157 | } | |
| 158 | }; | |
| 159 | child.stdin.write(JSON.stringify(job)); | |
| 160 | await child.stdin.end(); | |
| 161 | let killed: string | undefined; | |
| 162 | let lastLog = 0; | |
| 163 | const watcher = setInterval(() => void during("drill-watch", () => { | |
| 164 | const rss = residentBytes(child.pid); | |
| 165 | set({ rss }); | |
| 166 | if (rss > limit) { | |
| 167 | killed = `the drill process held ${Math.round(rss / 1024 ** 2)} MiB, over its ${Math.round(limit / 1024 ** 2)} MiB limit`; | |
| 168 | stop(); | |
| 169 | } | |
| 170 | if (Date.now() - lastLog >= 15_000) { | |
| 171 | lastLog = Date.now(); | |
| 172 | log("restore drill working", { id, phase: state.phase, done: state.done, total: state.total, bytes: state.bytes, rssMiB: Math.round(rss / 1024 ** 2) }); | |
| 173 | } | |
| 174 | }), options.watchMs ?? WATCH_MS); | |
| 175 | const deadline = setTimeout(() => { | |
| 176 | killed = "the drill process outlived every budget"; | |
| 177 | stop(); | |
| 178 | }, DRILL_MAX_MS + 5 * 60_000); | |
| 179 | let result: DrillResult | undefined; | |
| 180 | let stderr = ""; | |
| 181 | const reading = (async () => { | |
| 182 | let buf = ""; | |
| 183 | const decoder = new TextDecoder(); | |
| 184 | for await (const chunk of child.stdout) { | |
| 185 | buf += decoder.decode(chunk, { stream: true }); | |
| 186 | let nl: number; | |
| 187 | while ((nl = buf.indexOf("\n")) >= 0) { | |
| 188 | const line = buf.slice(0, nl); | |
| 189 | buf = buf.slice(nl + 1); | |
| 190 | try { | |
| 191 | const v = JSON.parse(line) as DrillLine; | |
| 192 | if ("progress" in v) set({ phase: v.progress.phase, done: v.progress.done, total: v.progress.total, bytes: v.progress.bytes }); | |
| 193 | else if ("result" in v) result = v.result; | |
| 194 | } catch { | |
| 195 | // not ours | |
| 196 | } | |
| 197 | } | |
| 198 | } | |
| 199 | })().catch(() => undefined); | |
| 200 | const errors = (async () => { | |
| 201 | const decoder = new TextDecoder(); | |
| 202 | for await (const chunk of child.stderr) stderr = (stderr + decoder.decode(chunk, { stream: true })).slice(-2_000); | |
| 203 | })().catch(() => undefined); | |
| 204 | try { | |
| 205 | await child.exited; | |
| 206 | await Promise.all([reading, errors]); | |
| 207 | } finally { | |
| 208 | clearInterval(watcher); | |
| 209 | clearTimeout(deadline); | |
| 210 | stop(); | |
| 211 | // Whatever the end, nothing of the drill stays on disk. | |
| 212 | sweepDrills(job.dataDir); | |
| 213 | } | |
| 214 | if (result !== undefined) { | |
| 215 | log("restore drill ended", { id, verdict: result.verdict }); | |
| 216 | return result; | |
| 217 | } | |
| 218 | const why = killed ?? `the drill process exited with ${child.exitCode ?? "no code"}${scrubbed(stderr) ? `: ${scrubbed(stderr)}` : ""}`; | |
| 219 | const where = state.phase === "starting" ? "" : ` in ${describeProgress({ phase: state.phase as Progress["phase"], done: state.done, total: state.total, bytes: state.bytes })}`; | |
| 220 | throw new Error(`${why}${where}`); | |
| 221 | } | |
| 222 | ||
| 223 | /** A marker is left only while a drill runs; true when one is. */ | |
| 224 | export const markerLeft = (dataDir: string): boolean => existsSync(marker(dataDir)); |
src/snapshot/drills.ts +6 −4
| @@ -25,11 +25,12 @@ interface Row { | ||
| 25 | 25 | const PHASES = new Set(["manifest", "files", "unpack", "bundles", "key", "open", "boot"]); |
| 26 | 26 | |
| 27 | 27 | /** Where a kept drill restored from; empty when it was kept before that was recorded, or the JSON is not ours. */ |
| 28 | export function parseFrom(raw: string | null): { from?: string; offSite?: boolean } { | |
| 28 | export function parseFrom(raw: string | null): { from?: string; offSite?: boolean; format?: number } { | |
| 29 | 29 | try { |
| 30 | const v = JSON.parse(raw ?? "null") as { from?: unknown; offSite?: unknown } | null; | |
| 31 | if (typeof v?.from !== "string" || v.from.length > 300) return {}; | |
| 32 | return { from: v.from, offSite: v.offSite === true }; | |
| 30 | const v = JSON.parse(raw ?? "null") as { from?: unknown; offSite?: unknown; format?: unknown } | null; | |
| 31 | const format = typeof v?.format === "number" && Number.isInteger(v.format) ? { format: v.format } : {}; | |
| 32 | if (typeof v?.from !== "string" || v.from.length > 300) return format; | |
| 33 | return { from: v.from, offSite: v.offSite === true, ...format }; | |
| 33 | 34 | } catch { |
| 34 | 35 | return {}; |
| 35 | 36 | } |
| @@ -77,6 +78,7 @@ export function recordDrill(db: Database, result: DrillResult): void { | ||
| 77 | 78 | phases: result.phases, |
| 78 | 79 | budgetMs: result.budgetMs, |
| 79 | 80 | ...(result.from === undefined ? {} : { from: result.from, offSite: result.offSite === true }), |
| 81 | ...(result.format === undefined ? {} : { format: result.format }), | |
| 80 | 82 | }), |
| 81 | 83 | ); |
| 82 | 84 | } |
src/snapshot/service.ts +61 −7
| @@ -30,10 +30,11 @@ import { sealedTarget } from "./destination.ts"; | ||
| 30 | 30 | import { watchBucket } from "./bucket.ts"; |
| 31 | 31 | import { readDeclaredLanes } from "./lanes.ts"; |
| 32 | 32 | import { lastDrill, recordDrill } from "./drills.ts"; |
| 33 | import { cutDrill, runDrillInChild, sweepDrills, type DrillState } from "./drill_job.ts"; | |
| 33 | 34 | import { lastMirror, mirrorOffThread, mirrorTarget, MIRROR_EVERY_MS, recordMirror, type MirrorState } from "./mirror.ts"; |
| 34 | 35 | import { isOffBox } from "./off_box.ts"; |
| 35 | 36 | import { noteOffBox, recordSnapshotOutcome } from "./outcome.ts"; |
| 36 | import { s3FromEnv, type Store } from "./store.ts"; | |
| 37 | import { remakeOf, s3FromEnv, type Store } from "./store.ts"; | |
| 37 | 38 | import { verifySnapshot } from "./verify.ts"; |
| 38 | 39 | import { during } from "../perf/jobs.ts"; |
| 39 | 40 | |
| @@ -76,6 +77,13 @@ export interface SnapshotsHandle { | ||
| 76 | 77 | readonly last: () => { manifest?: Manifest; error?: string; at?: string }; |
| 77 | 78 | /** Drill the newest snapshot now and keep the result; waits for any snapshot in flight. */ |
| 78 | 79 | readonly drill: () => Promise<DrillResult>; |
| 80 | /** | |
| 81 | * Ask for a drill as a job: answers at once. `started` is false when one is already | |
| 82 | * running (one at a time); `state` is where the running one stands. | |
| 83 | */ | |
| 84 | readonly startDrill: () => { readonly started: boolean; readonly state: DrillState | { readonly id: string; readonly phase: "waiting" } }; | |
| 85 | /** Where the running drill stands; undefined when none runs. */ | |
| 86 | readonly drillState: () => DrillState | { readonly id: string; readonly phase: "waiting" } | undefined; | |
| 79 | 87 | /** Drill if this week's slot has no drill yet, or the last one has stood red long enough; says whether it did. */ |
| 80 | 88 | readonly drillIfDue: () => Promise<boolean>; |
| 81 | 89 | /** The off-box mirror's store; undefined when the forge has none. */ |
| @@ -243,6 +251,12 @@ export function startSnapshots( | ||
| 243 | 251 | // Retention has swept the store: the snapshot just announced is still whole there, or this |
| 244 | 252 | // run failed and says so. |
| 245 | 253 | await verifySnapshot(store, manifest.name); |
| 254 | // A snapshot in a format the last drill did not prove is drilled on its own (#12592). | |
| 255 | const proved = lastDrill(app.db)?.format; | |
| 256 | if (options.nightly !== false && !stopped && proved !== undefined && proved !== (manifest.format ?? 1)) { | |
| 257 | log.info("restore drill owed: the snapshot format changed", { was: proved, now: manifest.format ?? 1 }); | |
| 258 | startDrill(); | |
| 259 | } | |
| 246 | 260 | return manifest; |
| 247 | 261 | }); |
| 248 | 262 | // The mirror hears of every snapshot at once; a failure there is its own |
| @@ -369,6 +383,18 @@ export function startSnapshots( | ||
| 369 | 383 | // Single-flight: a restore of every repository is heavy, so a second call |
| 370 | 384 | // while one runs joins it instead of filling the disk with another. |
| 371 | 385 | let drilling: Promise<DrillResult> | undefined; |
| 386 | let drillNow: DrillState | undefined; | |
| 387 | let drillSeq = 0; | |
| 388 | let drillWaiting: { id: string; phase: "waiting" } | undefined; | |
| 389 | const drillState = (): DrillState | { readonly id: string; readonly phase: "waiting" } | undefined => | |
| 390 | drilling === undefined ? undefined : (drillNow ?? drillWaiting); | |
| 391 | const startDrill: SnapshotsHandle["startDrill"] = () => { | |
| 392 | const running = drillState(); | |
| 393 | if (running !== undefined) return { started: false, state: running }; | |
| 394 | drillWaiting = { id: `drill-${Date.now()}-${++drillSeq}`, phase: "waiting" }; | |
| 395 | drill().catch((error) => log.error("restore drill failed to run", { error: String(error) })); | |
| 396 | return { started: true, state: drillNow ?? drillWaiting }; | |
| 397 | }; | |
| 372 | 398 | const drill = (): Promise<DrillResult> => { |
| 373 | 399 | drilling ??= (async (): Promise<DrillResult> => { |
| 374 | 400 | await (inFlight ?? Promise.resolve()).catch(() => {}); |
| @@ -381,11 +407,22 @@ export function startSnapshots( | ||
| 381 | 407 | throw new Error("this forge has nowhere to drill from"); |
| 382 | 408 | // A drill that could not run is not a drill that passed: it is kept red, |
| 383 | 409 | // so the board and the alarm say so instead of the log alone (#8742). |
| 384 | const result = await (options.drill ?? runDrill)({ | |
| 385 | store, | |
| 386 | dataDir: app.config.dataDir, | |
| 387 | passphrase: options.passphrase, | |
| 388 | }).catch( | |
| 410 | const deps: DrillDeps = { store, dataDir: app.config.dataDir, passphrase: options.passphrase }; | |
| 411 | const spec = remakeOf(store); | |
| 412 | // A drill is its own process whenever the store can be made again there. | |
| 413 | const run = | |
| 414 | options.drill ?? | |
| 415 | (spec === undefined | |
| 416 | ? runDrill | |
| 417 | : (d: DrillDeps) => | |
| 418 | runDrillInChild( | |
| 419 | { dataDir: d.dataDir, store: spec, passphrase: d.passphrase }, | |
| 420 | (st) => { | |
| 421 | drillNow = st; | |
| 422 | }, | |
| 423 | (message, fields) => log.info(message, fields), | |
| 424 | )); | |
| 425 | const result = await run(deps).catch( | |
| 389 | 426 | (error): DrillResult => ({ |
| 390 | 427 | at: new Date().toISOString(), |
| 391 | 428 | snapshotId: null, |
| @@ -407,6 +444,7 @@ export function startSnapshots( | ||
| 407 | 444 | return result; |
| 408 | 445 | })().finally(() => { |
| 409 | 446 | drilling = undefined; |
| 447 | drillNow = undefined; | |
| 410 | 448 | // However it was asked for, a red drill is tried again on its own. |
| 411 | 449 | armRedRetry(); |
| 412 | 450 | }); |
| @@ -472,13 +510,27 @@ export function startSnapshots( | ||
| 472 | 510 | await drill().catch((error) => log.error("restore drill failed to run", { error: String(error) })); |
| 473 | 511 | return true; |
| 474 | 512 | }; |
| 513 | // No drill runs at boot: a marker left is a drill the restart cut. Say so, and drill again. | |
| 514 | let cutAtBoot = false; | |
| 515 | try { | |
| 516 | const swept = sweepDrills(app.config.dataDir); | |
| 517 | if (swept.cut !== undefined) { | |
| 518 | cutAtBoot = true; | |
| 519 | recordDrill(app.db, cutDrill(swept.cut)); | |
| 520 | log.warn("restore drill was cut by a restart; recorded, and started again", { id: swept.cut.id }); | |
| 521 | } | |
| 522 | if (swept.removed > 0) log.info("restore drill scratch swept", { directories: swept.removed }); | |
| 523 | } catch (error) { | |
| 524 | log.warn("drill: scratch not swept", { error: String(error) }); | |
| 525 | } | |
| 475 | 526 | scheduleDrill(); |
| 476 | 527 | // A forge that was down at the slot, or whose last drill was red, drills |
| 477 | 528 | // shortly after boot; a red one is looked at again until one passes. |
| 478 | 529 | if (options.nightly !== false) { |
| 479 | 530 | catchUp = setTimeout( |
| 480 | 531 | () => |
| 481 | void during("drill", () => runIfDue(options.bootRedFloorMs ?? BOOT_RED_FLOOR_MS), undefined, app) | |
| 532 | void during("drill", () => (cutAtBoot ? drill().then(() => true) : runIfDue(options.bootRedFloorMs ?? BOOT_RED_FLOOR_MS)), undefined, app) | |
| 533 | .catch((error) => log.error("restore drill failed to run", { error: String(error) })) | |
| 482 | 534 | .finally(armRedRetry), |
| 483 | 535 | catchUpMs, |
| 484 | 536 | ); |
| @@ -544,6 +596,8 @@ export function startSnapshots( | ||
| 544 | 596 | wraps: !!options.passphrase, |
| 545 | 597 | now: take, |
| 546 | 598 | drill, |
| 599 | startDrill, | |
| 600 | drillState, | |
| 547 | 601 | drillIfDue: () => runIfDue(), |
| 548 | 602 | list: async () => { |
| 549 | 603 | const store = current(); |
src/tui/screens/domains.ts +13 −12
| @@ -5,7 +5,7 @@ | ||
| 5 | 5 | * empty screen that says nothing exists. |
| 6 | 6 | */ |
| 7 | 7 | |
| 8 | import { snapshotsOf } from "../../snapshot/service.ts"; | |
| 8 | import { followDrill, startDrillNow } from "../../actions/backups_now.ts"; | |
| 9 | 9 | import { listResilience, type ResilienceRow } from "../../actions/resilience.ts"; |
| 10 | 10 | import type { Ctx, Screen } from "../app/app.ts"; |
| 11 | 11 | import { binding } from "../app/keys.ts"; |
| @@ -60,18 +60,19 @@ export function domainsScreen(f: Session, ctx: Ctx): Screen { | ||
| 60 | 60 | } |
| 61 | 61 | drilling = true; |
| 62 | 62 | ctx.invalidate(); |
| 63 | // A drill is a job: asked for, then followed, as the Backups page and MCP do (#12592). | |
| 64 | const started = startDrillNow(f.app, f.user); | |
| 63 | 65 | let says: string; |
| 64 | 66 | let red = true; |
| 65 | try { | |
| 66 | const svc = snapshotsOf(f.app); | |
| 67 | if (svc === undefined || svc.store === undefined) says = "This forge has no snapshots to drill."; | |
| 68 | else { | |
| 69 | const d = await svc.drill(); | |
| 70 | red = d.verdict !== "green"; | |
| 71 | says = red ? `The drill failed at ${d.failedStep}.` : "The drill restored green."; | |
| 72 | } | |
| 73 | } catch (error) { | |
| 74 | says = `The drill could not run: ${String(error)}`; | |
| 67 | if (!started.ok) says = started.error.says; | |
| 68 | else { | |
| 69 | verdict = started.value.says; | |
| 70 | ctx.toast(verdict); | |
| 71 | says = await followDrill(f.app, (now) => { | |
| 72 | verdict = now; | |
| 73 | ctx.invalidate(); | |
| 74 | }); | |
| 75 | red = !says.endsWith("green."); | |
| 75 | 76 | } |
| 76 | 77 | drilling = false; |
| 77 | 78 | verdict = says; |
| @@ -111,7 +112,7 @@ export function domainsScreen(f: Session, ctx: Ctx): Screen { | ||
| 111 | 112 | : domains.length === 0 |
| 112 | 113 | ? "Nothing here." |
| 113 | 114 | : drilling |
| 114 | ? "Restore drill running…" | |
| 115 | ? verdict || "Restore drill running…" | |
| 115 | 116 | : verdict !== "" |
| 116 | 117 | ? verdict |
| 117 | 118 | : `${domains.length} failure domains, and each one's last drill`; |
src/tui/screens/resilience.ts +11 −2
| @@ -24,7 +24,7 @@ | ||
| 24 | 24 | * `Flaky:` and `Red:` issues link to. |
| 25 | 25 | */ |
| 26 | 26 | |
| 27 | import { drillNow, mirrorNow, recentSnapshots, snapshotNow } from "../../actions/backups_now.ts"; | |
| 27 | import { followDrill, mirrorNow, recentSnapshots, snapshotNow, startDrillNow } from "../../actions/backups_now.ts"; | |
| 28 | 28 | import { readResilience } from "../../checks/resilience.ts"; |
| 29 | 29 | import { |
| 30 | 30 | forgetDestination, |
| @@ -185,8 +185,17 @@ export function resilienceScreen(f: Session, ctx: Ctx): Screen { | ||
| 185 | 185 | const taken = await snapshotNow(f.app, f.user); |
| 186 | 186 | said = taken.ok ? taken.value.says : taken.error.says; |
| 187 | 187 | } else if (what === "drill") { |
| 188 | const ran = await drillNow(f.app, f.user); | |
| 188 | // A drill is a job: asked for, then followed, so no key press waits on a request | |
| 189 | // (#12592). | |
| 190 | const ran = startDrillNow(f.app, f.user); | |
| 189 | 191 | said = ran.ok ? ran.value.says : ran.error.says; |
| 192 | if (ran.ok) { | |
| 193 | ctx.invalidate(); | |
| 194 | said = await followDrill(f.app, (says) => { | |
| 195 | said = says; | |
| 196 | ctx.invalidate(); | |
| 197 | }); | |
| 198 | } | |
| 190 | 199 | } else { |
| 191 | 200 | const level = await mirrorNow(f.app, f.user); |
| 192 | 201 | said = level.ok ? level.value.says : level.error.says; |
src/web/pages/backups.ts +13 −7
| @@ -24,12 +24,12 @@ import { listResilience, type ResilienceRow } from "../../actions/resilience.ts" | ||
| 24 | 24 | import { mirrorSealedSet, mirrorNeed, MIRROR_SEALED } from "../../snapshot/mirror.ts"; |
| 25 | 25 | import { resolveSnapshots, sealedSet } from "../../snapshot/destination.ts"; |
| 26 | 26 | import { keyOf } from "../../snapshot/snapshot.ts"; |
| 27 | import { formatMinutes, formatSpan, timeByPart, type DrillResult } from "../../snapshot/drill.ts"; | |
| 27 | import { formatBytes, formatMinutes, formatSpan, timeByPart, type DrillResult } from "../../snapshot/drill.ts"; | |
| 28 | 28 | import { lastDrill } from "../../snapshot/drills.ts"; |
| 29 | 29 | import { backupVerdict } from "../../snapshot/bucket.ts"; |
| 30 | 30 | import { bodiesOf, type Store } from "../../snapshot/store.ts"; |
| 31 | 31 | import { snapshotsOf, type SnapshotsHandle } from "../../snapshot/service.ts"; |
| 32 | import { drillNow, mirrorNow, snapshotNow } from "../../actions/backups_now.ts"; | |
| 32 | import { drillStatus, mirrorNow, snapshotNow, startDrillNow } from "../../actions/backups_now.ts"; | |
| 33 | 33 | import { lastSnapshotOutcome } from "../../snapshot/outcome.ts"; |
| 34 | 34 | import type { Ctx } from "../../http/context.ts"; |
| 35 | 35 | import { |
| @@ -142,7 +142,15 @@ function drillLine(ctx: Ctx): Html { | ||
| 142 | 142 | ? html`: ${d.reason}` |
| 143 | 143 | : ""}.${timing ? html` <span class="muted small">${timing}.</span>` : ""} |
| 144 | 144 | </p>`; |
| 145 | return html`${line} | |
| 145 | const running = snapshotsOf(ctx.app)?.drillState(); | |
| 146 | const says = drillStatus(ctx.app); | |
| 147 | const now = | |
| 148 | running === undefined || says === undefined | |
| 149 | ? "" | |
| 150 | : html`<p class="break">${badge("Running", { tone: "attn", icon: "clock" })} ${says}${"bytes" in running | |
| 151 | ? html` <span class="muted small">${formatBytes(running.bytes)} read; its process holds ${formatBytes(running.rss)} of at most ${formatBytes(running.limit)}.</span>` | |
| 152 | : ""}</p>`; | |
| 153 | return html`${line}${now} | |
| 146 | 154 | <form method="post" action="/settings/backups/drill" class="mt"> |
| 147 | 155 | ${btn("Run a drill", { type: "submit", icon: "clock" })} |
| 148 | 156 | </form>`; |
| @@ -650,13 +658,11 @@ export async function backupsDrill(ctx: Ctx): Promise<Response> { | ||
| 650 | 658 | }); |
| 651 | 659 | const form = await readForm(ctx, "admin"); |
| 652 | 660 | if (form instanceof Response) return form; |
| 653 | const ran = await drillNow(ctx.app, ctx.auth.user); | |
| 661 | const ran = startDrillNow(ctx.app, ctx.auth.user); | |
| 654 | 662 | return flashTo( |
| 655 | 663 | ctx, |
| 656 | 664 | "/settings/backups", |
| 657 | ran.ok && ran.value.verdict === "green" | |
| 658 | ? { message: ran.value.says } | |
| 659 | : { tone: "danger", message: ran.ok ? ran.value.says : ran.error.says }, | |
| 665 | ran.ok ? { message: ran.value.says } : { tone: "danger", message: ran.error.says }, | |
| 660 | 666 | ); |
| 661 | 667 | } |
| 662 | 668 |
test/drill-job.test.ts added +118
| @@ -0,0 +1,118 @@ | ||
| 1 | /** The restore drill as a job (#12592): its own process, a marker, a sweep, a cut recorded. */ | |
| 2 | import { afterEach, beforeEach, expect, test } from "bun:test"; | |
| 3 | import { existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { createApp } from "../src/app.ts"; | |
| 7 | import { judge, type LaneRead } from "../src/snapshot/bucket.ts"; | |
| 8 | import type { DrillResult } from "../src/snapshot/drill.ts"; | |
| 9 | import { cutDrill, drillMemoryLimit, markerLeft, residentBytes, runDrillInChild, scrubbed, sweepDrills } from "../src/snapshot/drill_job.ts"; | |
| 10 | import { lastDrill } from "../src/snapshot/drills.ts"; | |
| 11 | import { startSnapshots } from "../src/snapshot/service.ts"; | |
| 12 | import { dirStore } from "../src/snapshot/store.ts"; | |
| 13 | ||
| 14 | let root = ""; | |
| 15 | beforeEach(() => { | |
| 16 | root = mkdtempSync(join(tmpdir(), "forge-drill-job-")); | |
| 17 | }); | |
| 18 | afterEach(() => rmSync(root, { recursive: true, force: true })); | |
| 19 | ||
| 20 | test("a boot sweeps every scratch directory and reports a drill the restart cut", () => { | |
| 21 | mkdirSync(join(root, "drill", "restore-abc", "data"), { recursive: true }); | |
| 22 | mkdirSync(join(root, "drill", "restore-def"), { recursive: true }); | |
| 23 | writeFileSync(join(root, "drill", "running.json"), JSON.stringify({ id: "drill-1", at: "2026-11-14T09:10:00.000Z" })); | |
| 24 | const swept = sweepDrills(root); | |
| 25 | expect(swept.removed).toBe(2); | |
| 26 | expect(swept.cut).toEqual({ id: "drill-1", at: "2026-11-14T09:10:00.000Z" }); | |
| 27 | expect(existsSync(join(root, "drill", "restore-abc"))).toBe(false); | |
| 28 | expect(markerLeft(root)).toBe(false); | |
| 29 | const rec = cutDrill(swept.cut!); | |
| 30 | expect(rec.verdict).toBe("red"); | |
| 31 | expect(rec.reason).toContain("cut by a restart"); | |
| 32 | expect(sweepDrills(root)).toEqual({ cut: undefined, removed: 0 }); | |
| 33 | }); | |
| 34 | ||
| 35 | test("a drill with an empty store ends red in its own process, leaving no directory or marker", async () => { | |
| 36 | const states: string[] = []; | |
| 37 | const result = await runDrillInChild( | |
| 38 | { dataDir: root, store: { kind: "dir", root: join(root, "store") } }, | |
| 39 | (s) => states.push(s.phase), | |
| 40 | () => {}, | |
| 41 | ); | |
| 42 | expect(result.verdict).toBe("red"); | |
| 43 | expect(result.reason).toContain("no snapshot"); | |
| 44 | expect(existsSync(join(root, "drill", "running.json"))).toBe(false); | |
| 45 | expect(existsSync(join(root, "drill")) ? readdirSync(join(root, "drill")).join(" ") : "").toBe(""); | |
| 46 | }); | |
| 47 | ||
| 48 | test("a drill over its memory limit is killed and says so", async () => { | |
| 49 | // A child that holds still, a limit of one byte and a quick reading: killed at the first reading. | |
| 50 | const script = join(root, "idle.ts"); | |
| 51 | writeFileSync(script, "await Bun.stdin.text(); await Bun.sleep(30_000);"); // real-timer: a real child process is the thing under test | |
| 52 | const err = await runDrillInChild( | |
| 53 | { dataDir: root, store: { kind: "dir", root: join(root, "store") } }, | |
| 54 | () => {}, | |
| 55 | () => {}, | |
| 56 | { limit: 1, watchMs: 50, script }, | |
| 57 | ).then( | |
| 58 | () => "no error", | |
| 59 | (e) => String(e), | |
| 60 | ); | |
| 61 | expect(err).toContain("over its 0 MiB limit"); | |
| 62 | expect(markerLeft(root)).toBe(false); | |
| 63 | }); | |
| 64 | ||
| 65 | test("the limit follows the machine, within bounds, and an override of at least 512 MiB", () => { | |
| 66 | const GiB = 1024 ** 3; | |
| 67 | expect(drillMemoryLimit({}, 2 * GiB)).toBe(3 * GiB); | |
| 68 | expect(drillMemoryLimit({}, 20 * GiB)).toBe(Math.floor(12 * GiB)); | |
| 69 | expect(drillMemoryLimit({}, 64 * GiB)).toBe(16 * GiB); | |
| 70 | expect(drillMemoryLimit({ FORGE_DRILL_MEMORY_MIB: "20000" }, 2 * GiB)).toBe(20000 * 1024 ** 2); | |
| 71 | expect(drillMemoryLimit({ FORGE_DRILL_MEMORY_MIB: "10" }, 20 * GiB)).toBe(12 * GiB); | |
| 72 | expect(drillMemoryLimit({ FORGE_DRILL_MEMORY_MIB: "lots" }, 20 * GiB)).toBe(12 * GiB); | |
| 73 | }); | |
| 74 | ||
| 75 | test("a child's last words in an error carry no path and are short", () => { | |
| 76 | const says = scrubbed(`boom at /data/forge/drill/restore-x/secret.key\n${"x ".repeat(400)}`); | |
| 77 | expect(says).not.toContain("/data"); | |
| 78 | expect(says.length).toBeLessThanOrEqual(200); | |
| 79 | expect(scrubbed("open /data/a/b failed")).toBe("open <path> failed"); | |
| 80 | }); | |
| 81 | ||
| 82 | test("resident memory is read for a live process and 0 for none", () => { | |
| 83 | expect(residentBytes(process.pid)).toBeGreaterThan(0); | |
| 84 | expect(residentBytes(2 ** 22 + 1)).toBe(0); | |
| 85 | }); | |
| 86 | ||
| 87 | test("the status says the proof's age and goes amber when the drill is days behind the newest snapshot", () => { | |
| 88 | const day = 86_400_000; | |
| 89 | const now = Date.parse("2026-11-14T12:00:00Z"); | |
| 90 | const drill: DrillResult = { | |
| 91 | at: "2026-11-08T10:00:00.000Z", snapshotId: "s", ms: 1, counts: { repositories: 1, issues: 1, users: 1, pulls: 0, apps: 0 }, | |
| 92 | health: "ok", verdict: "green", failedStep: null, phases: [], budgetMs: 0, reason: null, | |
| 93 | }; | |
| 94 | const lane: LaneRead = { lane: "primary", where: "dir", offBox: false, newestAt: new Date(now - 3_600_000).toISOString(), newestName: "n", newestBytes: 10, count: 1, readAt: new Date(now).toISOString() }; | |
| 95 | const v = judge({ lanes: [lane], checkedAt: new Date(now).toISOString() }, drill, now); | |
| 96 | expect(v.drill?.says).toBe("restore drill 2026-11-08 passed (6 d ago)"); | |
| 97 | expect(v.unknowns.join(" ")).toContain("not been proved"); | |
| 98 | expect(v.state).toBe("unknown"); | |
| 99 | const fresh = judge({ lanes: [lane], checkedAt: new Date(now).toISOString() }, { ...drill, at: new Date(now - day).toISOString() }, now); | |
| 100 | expect(fresh.unknowns.join(" ")).not.toContain("not been proved"); | |
| 101 | }); | |
| 102 | ||
| 103 | test("a restart mid-drill leaves a recorded outcome and no directory", async () => { | |
| 104 | mkdirSync(join(root, "drill", "restore-x", "data"), { recursive: true }); | |
| 105 | writeFileSync(join(root, "drill", "running.json"), JSON.stringify({ id: "drill-9", at: "2026-11-14T09:10:00.000Z" })); | |
| 106 | const app = createApp({ dataDir: root, port: 0 }); | |
| 107 | const svc = startSnapshots(app, dirStore(join(root, "store")), { version: "t", nightly: false }); | |
| 108 | try { | |
| 109 | const d = lastDrill(app.db); | |
| 110 | expect(d?.verdict).toBe("red"); | |
| 111 | expect(d?.reason).toContain("drill-9 was cut by a restart"); | |
| 112 | expect(existsSync(join(root, "drill", "restore-x"))).toBe(false); | |
| 113 | expect(markerLeft(root)).toBe(false); | |
| 114 | } finally { | |
| 115 | await svc.stop(); | |
| 116 | app.close(); | |
| 117 | } | |
| 118 | }); |
test/job-slices.test.ts +1
| @@ -245,6 +245,7 @@ function jobs(): Record<string, () => unknown> { | ||
| 245 | 245 | * The jobs not timed here, each with why. The release keeper's are off unless a cluster turns them on. |
| 246 | 246 | */ |
| 247 | 247 | const UNTIMED: Record<string, string> = { |
| 248 | "drill-watch": "a /proc read of the drill child's memory and a state write every WATCH_MS, only while a restore drill runs in its own process; drill-job tests drive runDrillInChild directly", | |
| 248 | 249 | desk: "expires notes past their time: one indexed delete on the desk table every 30 s (actions/desk.ts); the fixture has no notes", |
| 249 | 250 | ledger: "walks the cost ledger's hash chain at boot and once a day (#12273): file reads proportional to the ledger, off the request path; the fixture has no ledger", |
| 250 | 251 | "full-disk-probe": "one 64 KiB write every 2 s, only while the disk is full; the full-disk test drives probeFull directly", |
test/mcp-ops-tool-cards.test.ts +6 −1
| @@ -71,7 +71,12 @@ const call = async (name: string, args: Record<string, unknown>): Promise<any> = | ||
| 71 | 71 | }; |
| 72 | 72 | |
| 73 | 73 | test("restore_drill: the card has the verdict, the failed step and the When row", async () => { |
| 74 | const r = await call("restore_drill", {}); | |
| 74 | await call("restore_drill", {}); | |
| 75 | let r = await call("restore_drill", { action: "status" }); | |
| 76 | for (let i = 0; i < 100 && r.structuredContent?.card === undefined; i++) { | |
| 77 | await Bun.sleep(20); // real-timer: the drill runs in its own task | |
| 78 | r = await call("restore_drill", { action: "status" }); | |
| 79 | } | |
| 75 | 80 | expect(r.structuredContent.card.title).toBe("Restore drill"); |
| 76 | 81 | expect(r.structuredContent.card.rows).toEqual([ |
| 77 | 82 | { label: "Verdict", value: "red" }, |
test/mcp-restore-drill.test.ts +50 −11
| @@ -23,12 +23,13 @@ describe("MCP restore_drill", () => { | ||
| 23 | 23 | let servers: Servers; |
| 24 | 24 | let svc: SnapshotsHandle | undefined; |
| 25 | 25 | |
| 26 | let gate: Promise<void> = Promise.resolve(); | |
| 26 | 27 | let mode: "green" | "red" | "throw" = "green"; |
| 27 | const call = async (token: string) => { | |
| 28 | const call = async (token: string, action?: string) => { | |
| 28 | 29 | const res = await fetch(`${servers.url}/mcp`, { |
| 29 | 30 | method: "POST", |
| 30 | 31 | headers: { "content-type": "application/json", accept: "application/json, text/event-stream", authorization: `Bearer ${token}` }, |
| 31 | body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name: "restore_drill", arguments: {} } }), | |
| 32 | body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name: "restore_drill", arguments: action === undefined ? {} : { action } } }), | |
| 32 | 33 | }); |
| 33 | 34 | const body = (await res.json()) as any; |
| 34 | 35 | return { error: body.result?.isError === true, rpc: body.error?.message as string | undefined, text: (body.result?.content?.[0]?.text ?? "") as string }; |
| @@ -37,6 +38,7 @@ describe("MCP restore_drill", () => { | ||
| 37 | 38 | |
| 38 | 39 | const boot = async (withStore: boolean) => { |
| 39 | 40 | mode = "green"; |
| 41 | gate = Promise.resolve(); | |
| 40 | 42 | root = mkdtempSync(join(tmpdir(), "forge-mcp-drill-")); |
| 41 | 43 | app = createApp({ dataDir: root, port: 0 }); |
| 42 | 44 | for (const [username, adminIfFirst] of [["alice", true], ["bob", false]] as const) { |
| @@ -48,6 +50,7 @@ describe("MCP restore_drill", () => { | ||
| 48 | 50 | version: "t", |
| 49 | 51 | nightly: false, |
| 50 | 52 | drill: async (): Promise<DrillResult> => { |
| 53 | await gate; | |
| 51 | 54 | if (mode === "throw") throw new Error("store went away"); |
| 52 | 55 | const green = mode === "green"; |
| 53 | 56 | return { |
| @@ -89,26 +92,62 @@ describe("MCP restore_drill", () => { | ||
| 89 | 92 | }); |
| 90 | 93 | }); |
| 91 | 94 | |
| 95 | /** Wait for the running drill, which runs in its own task: a drill that never ends fails here. */ | |
| 96 | const settled = async (t: string): Promise<void> => { | |
| 97 | for (let i = 0; ; i++) { | |
| 98 | if (JSON.parse((await call(t, "status")).text).running !== true) return; | |
| 99 | if (i >= 100) throw new Error("the drill was still running after 2 s"); | |
| 100 | await Bun.sleep(20); // real-timer: polls a job running in its own task | |
| 101 | } | |
| 102 | }; | |
| 103 | ||
| 104 | /** Start a drill and read its verdict by status, as a harness does through the edge. */ | |
| 105 | const drillAndRead = async (t: string) => { | |
| 106 | const started = JSON.parse((await call(t)).text); | |
| 107 | await settled(t); | |
| 108 | return { started, verdict: JSON.parse((await call(t, "status")).text) }; | |
| 109 | }; | |
| 110 | ||
| 92 | 111 | describe("with a local store", () => { |
| 93 | 112 | beforeEach(() => boot(true)); |
| 94 | test("an admin gets a verdict, and the page reads the kept drill", async () => { | |
| 95 | const r = await call(token("alice")); | |
| 96 | expect(JSON.parse(r.text)).toEqual({ verdict: "green", says: "The drill restored green." }); | |
| 113 | test("start answers at once with an id; status then reads the verdict", async () => { | |
| 114 | const { started, verdict } = await drillAndRead(token("alice")); | |
| 115 | expect(started.started).toBe(true); | |
| 116 | expect(started.id).toMatch(/^drill-/); | |
| 117 | expect(verdict.verdict).toBe("green"); | |
| 118 | expect(verdict.says).toBe("The drill restored green."); | |
| 97 | 119 | expect(lastDrill(app.db)?.snapshotId).toBe("snap-1"); |
| 98 | 120 | }); |
| 121 | test("a second ask while one runs says so and does not start another", async () => { | |
| 122 | const t = token("alice"); | |
| 123 | let open!: () => void; | |
| 124 | gate = new Promise<void>((r) => (open = r)); | |
| 125 | const first = JSON.parse((await call(t)).text); | |
| 126 | const second = JSON.parse((await call(t)).text); | |
| 127 | expect(first.started).toBe(true); | |
| 128 | expect(second.started).toBe(false); | |
| 129 | expect(second.says).toContain("already running"); | |
| 130 | expect(second.id).toBe(first.id); | |
| 131 | open(); | |
| 132 | await settled(t); | |
| 133 | }); | |
| 99 | 134 | test("a red drill names its failed step", async () => { |
| 100 | 135 | mode = "red"; |
| 101 | const r = await call(token("alice")); | |
| 102 | expect(r.error).toBe(false); | |
| 103 | expect(JSON.parse(r.text)).toEqual({ verdict: "red", failedStep: "unseal", says: "The drill failed at unseal." }); | |
| 136 | const { verdict } = await drillAndRead(token("alice")); | |
| 137 | expect(verdict.verdict).toBe("red"); | |
| 138 | expect(verdict.failedStep).toBe("unseal"); | |
| 104 | 139 | }); |
| 105 | 140 | test("a drill that throws is kept as a red drill, not as nothing (#8742)", async () => { |
| 106 | 141 | mode = "throw"; |
| 107 | const r = await call(token("alice")); | |
| 108 | expect(r.error).toBe(false); | |
| 109 | expect(JSON.parse(r.text)).toEqual({ verdict: "red", failedStep: "restore", says: "The drill failed at restore." }); | |
| 142 | const { verdict } = await drillAndRead(token("alice")); | |
| 143 | expect(verdict.verdict).toBe("red"); | |
| 144 | expect(verdict.failedStep).toBe("restore"); | |
| 110 | 145 | expect(lastDrill(app.db)?.reason).toContain("store went away"); |
| 111 | 146 | }); |
| 147 | test("an unknown action is refused", async () => { | |
| 148 | const r = await call(token("alice"), "bogus"); | |
| 149 | expect(r.text).toContain("action is start or status"); | |
| 150 | }); | |
| 112 | 151 | test("an admin's read-only token is refused and nothing runs", async () => { |
| 113 | 152 | const r = await call(token("alice", "ro")); |
| 114 | 153 | expect(r.error).toBe(true); |
test/snapshot-drill-page.test.ts +11 −5
| @@ -83,12 +83,16 @@ describe("the drill on the Backups page", () => { | ||
| 83 | 83 | rmSync(root, { recursive: true, force: true }); |
| 84 | 84 | }); |
| 85 | 85 | |
| 86 | const post = (cookie: string, origin = servers.url) => | |
| 87 | fetch(`${servers.url}/settings/backups/drill`, { | |
| 86 | // The button starts a job and answers at once (#12592); a test wanting the verdict waits. | |
| 87 | const post = async (cookie: string, origin = servers.url, wait = true) => { | |
| 88 | const res = await fetch(`${servers.url}/settings/backups/drill`, { | |
| 88 | 89 | method: "POST", |
| 89 | 90 | headers: { cookie, origin }, |
| 90 | 91 | redirect: "manual", |
| 91 | 92 | }); |
| 93 | if (wait) while (svc.drillState() !== undefined) await Bun.sleep(10); // real-timer: polls a job running in its own task | |
| 94 | return res; | |
| 95 | }; | |
| 92 | 96 | const page = async (cookie: string) => |
| 93 | 97 | (await fetch(`${servers.url}/settings/backups`, { headers: { cookie } })).text(); |
| 94 | 98 | |
| @@ -144,10 +148,12 @@ describe("the drill on the Backups page", () => { | ||
| 144 | 148 | const rows = () => |
| 145 | 149 | (app.db.query("SELECT count(*) AS n FROM drill").get() as { n: number }).n; |
| 146 | 150 | const rowsBefore = rows(); |
| 147 | const both = Promise.all([post(alice), post(alice)]); | |
| 148 | await Bun.sleep(200); | |
| 149 | release(); | |
| 151 | const both = Promise.all([post(alice, servers.url, false), post(alice, servers.url, false)]); | |
| 150 | 152 | const [a, b] = await both; |
| 153 | // Held: the page says a drill is running, with where it stands, and no second one began. | |
| 154 | expect(await page(alice)).toMatch(/A drill is (running|waiting)/); | |
| 155 | release(); | |
| 156 | while (svc.drillState() !== undefined) await Bun.sleep(10); // real-timer: polls a job running in its own task | |
| 151 | 157 | gate = undefined; |
| 152 | 158 | expect([a.status, b.status]).toEqual([303, 303]); |
| 153 | 159 | expect(ran - before).toBe(1); |
test/spawn-pipes.test.ts +1
| @@ -19,6 +19,7 @@ const PIPING: Record<string, { sites: string[]; reason: string }> = { | ||
| 19 | 19 | "src/apps/backup_door.ts": { sites: ["stdout+stderr"], reason: "both read to their end alongside the exit: stdout's last line is the proof, stderr the refusal" }, |
| 20 | 20 | "src/apps/contain.ts": { sites: ["stderr"], reason: "stderr awaited with the exit; stdout ignored" }, |
| 21 | 21 | "src/apps/process.ts": { sites: ["stdout+stderr", "stdout+stderr"], reason: "pump() reads stdout and stderr of the app until exit" }, |
| 22 | "src/snapshot/drill_job.ts": { sites: ["stdout+stderr"], reason: "the drill child's stdout is read line by line for progress and its result, stderr to its end for the failure text, alongside the exit" }, | |
| 22 | 23 | "src/spawn/archive.ts": { sites: ["stdout+stderr"], reason: "tar() reads the listing from stdout and the failure text from stderr, then waits for exit" }, |
| 23 | 24 | "src/spawn/cell.ts": { sites: ["stdout+stderr"], reason: "drain() reads the child forge's stdout and stderr to its log until exit" }, |
| 24 | 25 | "src/chaos/faults.ts": { sites: ["stdout"], reason: "stdout is read until the kill closes it, then cancelled; stderr is ignored" }, |
test/spawn-sites.test.ts +2
| @@ -83,6 +83,8 @@ const SITES: Record<string, string> = { | ||
| 83 | 83 | "bun (process.execPath) on src/main.ts inside a cgroup cell, and git with fixed argv in a scratch directory it just made, the token in the environment only", |
| 84 | 84 | "src/snapshot/boot.ts": |
| 85 | 85 | "bun (process.execPath) on src/main.ts in the snapshot tree", |
| 86 | "src/snapshot/drill_job.ts": | |
| 87 | "runs bun on the forge's own drill_child.ts with the job as JSON on stdin; argv holds only the bun path and a fixed script, nothing from a request", | |
| 86 | 88 | "src/spawn/archive.ts": |
| 87 | 89 | "tar with fixed flags on the forge's own data directory and an archive path it derived from a validated name", |
| 88 | 90 | "src/spawn/child_db.ts": |
test/timers-named.test.ts +1
| @@ -153,6 +153,7 @@ const ALLOWED: readonly Allowed[] = [ | ||
| 153 | 153 | ]), |
| 154 | 154 | ...allow("setTimeout", KILL, [ |
| 155 | 155 | ["src/checks/install.ts", "stop"], |
| 156 | ["src/snapshot/drill_job.ts", "() => { killed ="], | |
| 156 | 157 | ["src/apps/process.ts", "() => { if (proc.exitCode === null"], |
| 157 | 158 | ["src/release/launch.ts", "() => child.kill(\"SIGKILL\")"], |
| 158 | 159 | ["src/release/checkout.ts", "() => proc.kill(\"SIGKILL\")"], |
test/tui-drill.test.ts +35 −3
| @@ -68,7 +68,17 @@ async function open(user: User) { | ||
| 68 | 68 | await settleApp(app); |
| 69 | 69 | return app.screen.toText(); |
| 70 | 70 | }; |
| 71 | return { app, press, text: () => app.screen.toText() }; | |
| 71 | /** Press `key`, then wait (up to 5 s) for the screen to say `words`: a drill is followed. */ | |
| 72 | const pressUntil = async (key: string, words: string): Promise<string> => { | |
| 73 | let text = await press(key); | |
| 74 | for (let i = 0; i < 50 && !text.includes(words); i++) { | |
| 75 | await Bun.sleep(100); // real-timer: a real child process drills in the background | |
| 76 | await settleApp(app); | |
| 77 | text = app.screen.toText(); | |
| 78 | } | |
| 79 | return text; | |
| 80 | }; | |
| 81 | return { app, press, pressUntil, text: () => app.screen.toText() }; | |
| 72 | 82 | } |
| 73 | 83 | |
| 74 | 84 | test("a non-admin is not offered d and a press asks nothing", async () => { |
| @@ -102,7 +112,7 @@ test("a restore drill says its outcome in words and leaves the domain row alone" | ||
| 102 | 112 | expect(h.text()).toMatch(/The snapshot store +never run/); |
| 103 | 113 | await h.press("d"); |
| 104 | 114 | await h.press("left"); |
| 105 | let text = await h.press("enter"); | |
| 115 | let text = await h.pressUntil("enter", "The drill restored green."); | |
| 106 | 116 | expect(runs).toBe(1); |
| 107 | 117 | expect(text).toContain("The drill restored green."); |
| 108 | 118 | expect(text).toMatch(/The snapshot store +never run/); // the domain row is the fault drill's, not this one's |
| @@ -110,7 +120,7 @@ test("a restore drill says its outcome in words and leaves the domain row alone" | ||
| 110 | 120 | next = "red"; |
| 111 | 121 | await h.press("d"); |
| 112 | 122 | await h.press("left"); |
| 113 | text = await h.press("enter"); | |
| 123 | text = await h.pressUntil("enter", "The drill failed at fetch."); | |
| 114 | 124 | expect(runs).toBe(2); |
| 115 | 125 | expect(text).toContain("The drill failed at fetch."); |
| 116 | 126 | expect(text).toMatch(/The snapshot store +never run/); |
| @@ -142,3 +152,25 @@ test("an admin demoted after load runs nothing", async () => { | ||
| 142 | 152 | expect(text).not.toContain("restored green"); |
| 143 | 153 | h.app.stop(); |
| 144 | 154 | }); |
| 155 | ||
| 156 | test("a drill that is running is said on the screen at once, and the key waits for no request", async () => { | |
| 157 | let release: () => void = () => {}; | |
| 158 | const held = new Promise<void>((r) => (release = r)); | |
| 159 | handle = startSnapshots(forge, dirStore(join(dir, "store")), { | |
| 160 | version: "0", | |
| 161 | nightly: false, | |
| 162 | drill: async () => { | |
| 163 | await held; | |
| 164 | return result("green"); | |
| 165 | }, | |
| 166 | }); | |
| 167 | const h = await open(alice); | |
| 168 | await h.press("d"); | |
| 169 | await h.press("left"); | |
| 170 | const text = await h.pressUntil("enter", "A drill is waiting for the snapshot in flight."); | |
| 171 | expect(text).toContain("A drill is waiting for the snapshot in flight."); | |
| 172 | expect(h.text()).not.toMatch(/d +restore drill/); // one at a time: not offered while it runs | |
| 173 | release(); | |
| 174 | expect(await h.pressUntil("x", "The drill restored green.")).toContain("The drill restored green."); | |
| 175 | h.app.stop(); | |
| 176 | }); |
new above
offline
3 GiB is below what restore.ts buffers (9 to 13 GiB per #12592), so a real drill is killed red. Fix the restore's buffering or justify and verify this number on a real snapshot.