Public: anyone can read this; the crew can change it.
The board warns when a green restore drill used more than 80% of its budget #2639
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/2623-the-board-warns-when-a-green-restore-dri → main 1 commit fork 1 thread · 1 unresolved
2 files
changed +66
−3
src/snapshot/verdict.ts +22 −3
| @@ -9,7 +9,7 @@ | ||
| 9 | 9 | |
| 10 | 10 | import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; |
| 11 | 11 | import { join } from "node:path"; |
| 12 | import type { DrillResult, DrillStep } from "./drill.ts"; | |
| 12 | import { formatSpan, type DrillResult, type DrillStep, type PhaseTime } from "./drill.ts"; | |
| 13 | 13 | |
| 14 | 14 | const STEPS: readonly DrillStep[] = ["fetch", "restore", "open", "count", "unseal", "health"]; |
| 15 | 15 | |
| @@ -41,7 +41,23 @@ export function failedSentence(r: DrillResult | undefined): string | undefined { | ||
| 41 | 41 | return `The restore drill failed at ${r.failedStep ?? "an unknown step"}${r.reason ? `: ${r.reason}` : ""}. Snapshot ${r.snapshotId ?? "none"}; docs/RECOVERY.md says what to do.`; |
| 42 | 42 | } |
| 43 | 43 | |
| 44 | const firstFile = (dataDir: string): string => join(dataDir, "drill", "first-seen"); | |
| 44 | /** A green drill that used more than this share of its budget is a warning. */ | |
| 45 | export const DRILL_SLOW_SHARE = 0.8; | |
| 46 | ||
| 47 | /** | |
| 48 | * The board's line for a green drill that came close to its budget, naming | |
| 49 | * the slowest phase; undefined for a red drill, one under 80%, or an older | |
| 50 | * drill that kept no budget. | |
| 51 | */ | |
| 52 | export function slowSentence(r: DrillResult | undefined): string | undefined { | |
| 53 | if (r === undefined || r.verdict !== "green") return undefined; | |
| 54 | if (!(r.budgetMs > 0) || !(r.ms > r.budgetMs * DRILL_SLOW_SHARE)) return undefined; | |
| 55 | const slowest = r.phases.reduce<PhaseTime | undefined>((a, p) => (a === undefined || p.ms > a.ms ? p : a), undefined); | |
| 56 | const spare = `${formatSpan(Math.max(0, r.budgetMs - r.ms))} to spare`; | |
| 57 | return `The last restore drill passed with ${spare}${slowest ? `; ${slowest.phase} took ${formatSpan(slowest.ms)} of it` : ""}.`; | |
| 58 | } | |
| 59 | ||
| 60 | const firstFile =(dataDir: string): string => join(dataDir, "drill", "first-seen"); | |
| 45 | 61 | |
| 46 | 62 | /** |
| 47 | 63 | * Written once at boot (`seedFirstSeen`): when this forge first ran with a |
| @@ -97,7 +113,10 @@ export function drillNeed( | ||
| 97 | 113 | }; |
| 98 | 114 | } |
| 99 | 115 | const at = Date.parse(r.at); |
| 100 | if (!(now - at > DRILL_STALE_MS)) return undefined; | |
| 116 | if (!(now - at > DRILL_STALE_MS)) { | |
| 117 | const slow = slowSentence(r); | |
| 118 | return slow === undefined ? undefined : { key: `slow:${r.at}`, sentence: slow, since: r.at }; | |
| 119 | } | |
| 101 | 120 | return { |
| 102 | 121 | key: `stale:${r.at}`, |
| 103 | 122 | sentence: `No restore drill has run in ${days} days; the last one passed on ${r.at.slice(0, 10)}. docs/RECOVERY.md says how to run one.`, |
test/drill-slow.test.ts added +44
| @@ -0,0 +1,44 @@ | ||
| 1 | import { expect, test } from "bun:test"; | |
| 2 | import type { DrillResult } from "../src/snapshot/drill.ts"; | |
| 3 | import { drillNeed, slowSentence } from "../src/snapshot/verdict.ts"; | |
| 4 | ||
| 5 | const MIN = 60_000; | |
| 6 | const drill = (over: Partial<DrillResult>): DrillResult => ({ | |
| 7 | at: "2026-10-01T00:00:00.000Z", | |
| 8 | snapshotId: "snap-1", | |
| 9 | ms: 0, | |
| 10 | counts: { repositories: 1, issues: 1, users: 0, pulls: 0, apps: 0 }, | |
| 11 | health: "ok", | |
| 12 | verdict: "green", | |
| 13 | failedStep: null, | |
| 14 | phases: [ | |
| 15 | { phase: "files", ms: 1 * MIN }, | |
| 16 | { phase: "bundles", ms: 9 * MIN }, | |
| 17 | ], | |
| 18 | budgetMs: 11 * MIN, | |
| 19 | reason: null, | |
| 20 | ...over, | |
| 21 | }); | |
| 22 | ||
| 23 | test("81% of the budget warns, naming the slowest phase", () => { | |
| 24 | const r = drill({ ms: 9 * MIN, budgetMs: 11 * MIN }); | |
| 25 | expect(slowSentence(drill({ ms: 8.91 * MIN }))).toBe( | |
| 26 | "The last restore drill passed with 2 min 5 s to spare; bundles took 9 min of it.", | |
| 27 | ); | |
| 28 | expect(slowSentence(r)).toContain("bundles took 9 min"); | |
| 29 | const need = drillNeed(r, Date.parse(r.at) + MIN); | |
| 30 | expect(need?.sentence).toBe(slowSentence(r)); | |
| 31 | }); | |
| 32 | ||
| 33 | test("79% of the budget says nothing", () => { | |
| 34 | expect(slowSentence(drill({ ms: 0.79 * 11 * MIN }))).toBeUndefined(); | |
| 35 | expect(drillNeed(drill({ ms: 0.79 * 11 * MIN }), Date.parse("2026-10-01T00:01:00Z"))).toBeUndefined(); | |
| 36 | }); | |
| 37 | ||
| 38 | test("a red drill is not a slow one", () => { | |
| 39 | expect(slowSentence(drill({ verdict: "red", failedStep: "unseal", ms: 11 * MIN }))).toBeUndefined(); | |
| 40 | }); | |
| 41 | ||
| 42 | test("a drill with no budget says nothing", () => { | |
| 43 | expect(slowSentence(drill({ ms: 5 * MIN, budgetMs: 0 }))).toBeUndefined(); | |
| 44 | }); |
new above
offline
Stray formatting:
firstFile =(dataDiris missing a space. Restoreconst firstFile = (dataDir: string); this line should not be in the diff at all.