Public: anyone can read this; the crew can change it.
Home says the last off-box backup is never - confirm backups run, or make them run #3717
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/3701-home-says-the-last-off-box-backup-is-nev → main 1 commit fork 1 thread · 1 unresolved
8 files
changed +224
−12
README.md +3 −2
| @@ -294,8 +294,9 @@ Home shows nine figures first, from `src/flow/kpis.ts`: shipped per hour (with | ||
| 294 | 294 | a 24 h sparkline), the live version and since when, the median check time, the crew builders |
| 295 | 295 | running of their slots, and the crew's spend today of the forge's daily |
| 296 | 296 | ceiling (`FORGE_CREW_CEILING_USD`; without one, the spend alone and |
| 297 | `no ceiling`), and the age of the last off-box backup (the mirror's newest | |
| 298 | snapshot, `mirror/state.json`; `3 h ago`, the exact time in its title), linking to | |
| 297 | `no ceiling`), and the age of the last off-box backup (the newer of the | |
| 298 | mirror's newest snapshot, `mirror/state.json`, and the newest that landed in | |
| 299 | a destination that is a bucket, `snapshot/state.json`; `src/snapshot/off_box.ts`; `3 h ago`, the exact time in its title), linking to | |
| 299 | 300 | `/settings/backups`. The spend tile links to `/settings/crew`, where the meter is. |
| 300 | 301 | The backup tile past 24 hours, or with none (`never`), takes the card's existing |
| 301 | 302 | `data-tone="attn"` and says overdue in its label. |
docs/SNAPSHOT.md +13
| @@ -83,6 +83,19 @@ Delete the old ones from the bucket if the bucket is not trusted. | ||
| 83 | 83 | - Given neither, no snapshots are taken, and the Backups page says so |
| 84 | 84 | instead of showing an empty list that looks like a clean bill. |
| 85 | 85 | |
| 86 | ## Off the machine | |
| 87 | ||
| 88 | Home's **Last off-box backup** is the newest time a snapshot was known to be | |
| 89 | off the machine (`src/snapshot/off_box.ts`): the newest the mirror holds, or | |
| 90 | the newest that landed in a destination that is a bucket, whichever is later. | |
| 91 | Each landing records whether its store was one (`offBox` in | |
| 92 | `snapshot/state.json`): a directory never is, whatever volume it is on, nor | |
| 93 | is a bucket served from loopback. Before #3701 the tile read the mirror | |
| 94 | alone, so a forge whose nightly landed in the platform's bucket every morning, | |
| 95 | with no second bucket, read "never". An outcome kept before stores were | |
| 96 | recorded is annotated from the store in effect at the first mirror pass after | |
| 97 | boot. | |
| 98 | ||
| 86 | 99 | ## The off-box mirror |
| 87 | 100 | |
| 88 | 101 | A second S3-compatible bucket (Backblaze B2, say) that every snapshot is |
src/flow/kpis.ts +3 −3
| @@ -15,7 +15,7 @@ | ||
| 15 | 15 | |
| 16 | 16 | import type { Database } from "bun:sqlite"; |
| 17 | 17 | import { spentTodayEverywhere } from "../crew/autonomy.ts"; |
| 18 | import { lastMirror } from "../snapshot/mirror.ts"; | |
| 18 | import { lastOffBox } from "../snapshot/off_box.ts"; | |
| 19 | 19 | import { readReach, type Reach } from "./reach.ts"; |
| 20 | 20 | import { HOURS, median } from "./throughput.ts"; |
| 21 | 21 | |
| @@ -34,7 +34,7 @@ export interface Kpis { | ||
| 34 | 34 | readonly builders: { readonly running: number; readonly slots: number }; |
| 35 | 35 | /** What the crew's runs cost since UTC midnight, and the forge's daily ceiling (null: none set). */ |
| 36 | 36 | readonly spend: { readonly todayUsd: number; readonly ceilingUsd: number | null }; |
| 37 | /** When the off-box mirror last finished holding a snapshot; null when it never has. */ | |
| 37 | /** When a snapshot was last known off the machine (a bucket destination or the mirror, `off_box.ts`); null when never. */ | |
| 38 | 38 | readonly backupAt: string | null; |
| 39 | 39 | /** Accounts, visitors and page speed (`reach.ts`). */ |
| 40 | 40 | readonly reach: Reach; |
| @@ -94,7 +94,7 @@ export function readKpis( | ||
| 94 | 94 | |
| 95 | 95 | const todayUsd = spentTodayEverywhere(db, now); |
| 96 | 96 | |
| 97 | const backupAt = forge.dataDir === undefined ? null : (lastMirror(forge.dataDir)?.newestAt ?? null); | |
| 97 | const backupAt = forge.dataDir === undefined ? null : lastOffBox(forge.dataDir); | |
| 98 | 98 | |
| 99 | 99 | return { |
| 100 | 100 | backupAt, |
src/snapshot/off_box.ts added +47
| @@ -0,0 +1,47 @@ | ||
| 1 | /** | |
| 2 | * When a copy of the forge last left the machine (#3701). Two things carry | |
| 3 | * one off it: the destination itself, when it is a bucket (the platform's | |
| 4 | * `S3_*`, or one sealed on the Backups page), and the off-box mirror. Home | |
| 5 | * read only the mirror, so a forge whose nightly lands in a bucket every | |
| 6 | * morning and has no second bucket read "never". A directory is on the | |
| 7 | * machine whatever volume it is on, and so is a bucket served from | |
| 8 | * loopback (a MinIO beside the forge). | |
| 9 | * | |
| 10 | * Reads two small files under the data directory; no network, no clock. | |
| 11 | */ | |
| 12 | ||
| 13 | import { lastMirror } from "./mirror.ts"; | |
| 14 | import { lastSnapshotOutcome } from "./outcome.ts"; | |
| 15 | import type { Store } from "./store.ts"; | |
| 16 | ||
| 17 | /** The host of a bucket's `where` (`host[:port]/bucket`), lower-cased, without its port or brackets. */ | |
| 18 | function hostOf(where: string): string { | |
| 19 | const authority = where.split("/")[0]!.toLowerCase(); | |
| 20 | if (authority.startsWith("[")) return authority.slice(1, authority.indexOf("]")); | |
| 21 | return authority.replace(/:\d+$/, ""); | |
| 22 | } | |
| 23 | ||
| 24 | function loopback(host: string): boolean { | |
| 25 | return host === "localhost" || host.endsWith(".localhost") || host === "::1" || /^127\.\d+\.\d+\.\d+$/.test(host); | |
| 26 | } | |
| 27 | ||
| 28 | /** Does a snapshot that lands in this store leave the machine? A directory never does. */ | |
| 29 | export function isOffBox(store: Pick<Store, "where" | "dir">): boolean { | |
| 30 | return store.dir === undefined && !loopback(hostOf(store.where)); | |
| 31 | } | |
| 32 | ||
| 33 | /** | |
| 34 | * The newest time a snapshot was known to be off the machine: the newest | |
| 35 | * the mirror holds, or the newest that landed in a destination that is a | |
| 36 | * bucket, whichever is later. Null when neither has. | |
| 37 | */ | |
| 38 | export function lastOffBox(dataDir: string): string | null { | |
| 39 | const times: string[] = []; | |
| 40 | const mirrored = lastMirror(dataDir)?.newestAt; | |
| 41 | if (mirrored !== undefined && Number.isFinite(Date.parse(mirrored))) times.push(mirrored); | |
| 42 | const kept = lastSnapshotOutcome(dataDir); | |
| 43 | if (kept?.offBox === true && kept.landedAt !== undefined && Number.isFinite(Date.parse(kept.landedAt))) | |
| 44 | times.push(kept.landedAt); | |
| 45 | if (times.length === 0) return null; | |
| 46 | return times.reduce((a, b) => (Date.parse(b) > Date.parse(a) ? b : a)); | |
| 47 | } |
src/snapshot/outcome.ts +21 −3
| @@ -17,6 +17,8 @@ export interface SnapshotOutcome { | ||
| 17 | 17 | /** The newest snapshot that landed, and when; absent until one has. */ |
| 18 | 18 | readonly landed?: string; |
| 19 | 19 | readonly landedAt?: string; |
| 20 | /** Whether the store it landed in is off the machine (a bucket, not a directory); absent when not known. */ | |
| 21 | readonly offBox?: boolean; | |
| 20 | 22 | /** Set while attempts are failing: since when, and the last one's reason. */ |
| 21 | 23 | readonly failingSince?: string; |
| 22 | 24 | readonly error?: string; |
| @@ -38,23 +40,39 @@ export function lastSnapshotOutcome(dataDir: string): SnapshotOutcome | undefine | ||
| 38 | 40 | /** Keep an attempt's end: a name that landed, or the error it failed with. Returns what was kept. */ |
| 39 | 41 | export function recordSnapshotOutcome( |
| 40 | 42 | dataDir: string, |
| 41 | result: { readonly landed: string } | { readonly error: string }, | |
| 43 | result: { readonly landed: string; readonly offBox?: boolean } | { readonly error: string }, | |
| 42 | 44 | at: string = new Date().toISOString(), |
| 43 | 45 | ): SnapshotOutcome { |
| 44 | 46 | const prev = lastSnapshotOutcome(dataDir); |
| 45 | 47 | const kept = { |
| 46 | 48 | ...(prev?.landed === undefined ? {} : { landed: prev.landed }), |
| 47 | 49 | ...(prev?.landedAt === undefined ? {} : { landedAt: prev.landedAt }), |
| 50 | ...(prev?.offBox === undefined ? {} : { offBox: prev.offBox }), | |
| 48 | 51 | }; |
| 49 | 52 | const next: SnapshotOutcome = |
| 50 | 53 | "landed" in result |
| 51 | ? { at, landed: result.landed, landedAt: at } | |
| 54 | ? { at, landed: result.landed, landedAt: at, ...(result.offBox === undefined ? {} : { offBox: result.offBox }) } | |
| 52 | 55 | : { at, ...kept, failingSince: prev?.failingSince ?? at, error: result.error }; |
| 56 | write(dataDir, next); | |
| 57 | return next; | |
| 58 | } | |
| 59 | ||
| 60 | /** | |
| 61 | * Say where an outcome kept before the store was recorded landed: the store | |
| 62 | * in effect now, since it is the one it most likely went to. Writes nothing | |
| 63 | * when the outcome already says, or none has landed. | |
| 64 | */ | |
| 65 | export function noteOffBox(dataDir: string, offBox: boolean): void { | |
| 66 | const prev = lastSnapshotOutcome(dataDir); | |
| 67 | if (prev?.landedAt === undefined || prev.offBox !== undefined) return; | |
| 68 | write(dataDir, { ...prev, offBox }); | |
| 69 | } | |
| 70 | ||
| 71 | function write(dataDir: string, next: SnapshotOutcome): void { | |
| 53 | 72 | mkdirSync(join(dataDir, "snapshot"), { recursive: true }); |
| 54 | 73 | const tmp = `${stateFile(dataDir)}.tmp`; |
| 55 | 74 | writeFileSync(tmp, JSON.stringify(next)); |
| 56 | 75 | renameSync(tmp, stateFile(dataDir)); |
| 57 | return next; | |
| 58 | 76 | } |
| 59 | 77 | |
| 60 | 78 | /** The card a failing snapshot is, or undefined while the last one landed. */ |
src/snapshot/service.ts +14 −2
| @@ -25,7 +25,8 @@ import { describeTimes, runDrill, type DrillDeps, type DrillResult } from "./dri | ||
| 25 | 25 | import { BOOT_RED_FLOOR_MS, drillDue, RED_RETRY_MS } from "./due.ts"; |
| 26 | 26 | import { lastDrill, recordDrill } from "./drills.ts"; |
| 27 | 27 | import { lastMirror, mirrorPass, MIRROR_EVERY_MS, recordMirror, type MirrorState } from "./mirror.ts"; |
| 28 | import { recordSnapshotOutcome } from "./outcome.ts"; | |
| 28 | import { isOffBox } from "./off_box.ts"; | |
| 29 | import { noteOffBox, recordSnapshotOutcome } from "./outcome.ts"; | |
| 29 | 30 | import type { Store } from "./store.ts"; |
| 30 | 31 | |
| 31 | 32 | const log = createLog("snapshots"); |
| @@ -148,6 +149,8 @@ export function startSnapshots( | ||
| 148 | 149 | let last: { manifest?: Manifest; error?: string; at?: string } = {}; |
| 149 | 150 | |
| 150 | 151 | const take = (): Promise<Manifest> => { |
| 152 | // Whether this run's store is off the machine, for home's backup age (#3701). | |
| 153 | let offBox: boolean | undefined; | |
| 151 | 154 | // After the one before it, however that ended: chained on a failure as it |
| 152 | 155 | // was, every later run failed at once with the old error until a restart. |
| 153 | 156 | const run = (inFlight ?? Promise.resolve()).catch(() => {}).then(async () => { |
| @@ -155,6 +158,7 @@ export function startSnapshots( | ||
| 155 | 158 | // very next snapshot's. |
| 156 | 159 | const store = current(); |
| 157 | 160 | if (store === undefined) throw new Nowhere(); |
| 161 | offBox = isOffBox(store); | |
| 158 | 162 | running = { objects: 0, bytes: 0, elapsedMs: 0 }; |
| 159 | 163 | // Admin-only, no repository, nothing /backups does not already show. |
| 160 | 164 | app.bus.append({ kind: "backup.started" }); |
| @@ -186,7 +190,7 @@ export function startSnapshots( | ||
| 186 | 190 | inFlight = run.then( |
| 187 | 191 | (m) => { |
| 188 | 192 | last = { manifest: m, at: m.finishedAt }; |
| 189 | keep({ landed: m.name }); | |
| 193 | keep({ landed: m.name, ...(offBox === undefined ? {} : { offBox }) }); | |
| 190 | 194 | app.bus.append({ kind: "backup.finished", payload: { ok: true } }); |
| 191 | 195 | arm(untilNightly()); |
| 192 | 196 | return m; |
| @@ -214,6 +218,14 @@ export function startSnapshots( | ||
| 214 | 218 | await (inFlight ?? Promise.resolve()).catch(() => {}); |
| 215 | 219 | const dst = currentMirror(); |
| 216 | 220 | const src = current(); |
| 221 | // An outcome kept before stores were recorded learns where it landed. | |
| 222 | if (src !== undefined) { | |
| 223 | try { | |
| 224 | noteOffBox(app.config.dataDir, isOffBox(src)); | |
| 225 | } catch (error) { | |
| 226 | log.warn("snapshot outcome not annotated", { error: String(error) }); | |
| 227 | } | |
| 228 | } | |
| 217 | 229 | if (dst === undefined || src === undefined) { |
| 218 | 230 | try { |
| 219 | 231 | recordMirror(app.config.dataDir, undefined); |
src/tui/screens/dashboard.ts +5 −2
| @@ -101,8 +101,11 @@ import { signInScreen } from "./signin.ts"; | ||
| 101 | 101 | /** Rows the list keeps under the figures strip; below it the strip shows fewer figures. */ |
| 102 | 102 | const FIGURES_ROOM = 6; |
| 103 | 103 | |
| 104 | /** The forge-wide daily ceiling, from config, for the spend figure. */ | |
| 105 | const kpiForge = (app: { config: { crewCeilingUsd?: number } }) => ({ ceilingUsd: app.config.crewCeilingUsd ?? null }); | |
| 104 | /** The forge-wide daily ceiling, from config, for the spend figure; the data directory, for the backup's age. */ | |
| 105 | export const kpiForge = (app: { config: { crewCeilingUsd?: number; dataDir: string } }) => ({ | |
| 106 | ceilingUsd: app.config.crewCeilingUsd ?? null, | |
| 107 | dataDir: app.config.dataDir, | |
| 108 | }); | |
| 106 | 109 | |
| 107 | 110 | /** |
| 108 | 111 | * The three things the launcher is about. Order is the `tab` order: the |
test/snapshot-off-box.test.ts added +118
| @@ -0,0 +1,118 @@ | ||
| 1 | /** | |
| 2 | * Home's "Last off-box backup" read "never" on a forge whose nightly lands in | |
| 3 | * a bucket every morning (#3701): the tile read only the second, mirror | |
| 4 | * bucket. A snapshot that lands in a bucket is off the machine; one in a | |
| 5 | * directory, or a bucket on loopback, is not. The tile shows the newest of | |
| 6 | * the two ways off. | |
| 7 | */ | |
| 8 | ||
| 9 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 10 | import { mkdtempSync } from "node:fs"; | |
| 11 | import { rm } from "node:fs/promises"; | |
| 12 | import { tmpdir } from "node:os"; | |
| 13 | import { join } from "node:path"; | |
| 14 | import { createApp, type App } from "../src/app.ts"; | |
| 15 | import { readKpis } from "../src/flow/kpis.ts"; | |
| 16 | import { recordMirror } from "../src/snapshot/mirror.ts"; | |
| 17 | import { isOffBox, lastOffBox } from "../src/snapshot/off_box.ts"; | |
| 18 | import { lastSnapshotOutcome, noteOffBox, recordSnapshotOutcome } from "../src/snapshot/outcome.ts"; | |
| 19 | import { startSnapshots } from "../src/snapshot/service.ts"; | |
| 20 | import { dirStore, type Store } from "../src/snapshot/store.ts"; | |
| 21 | import { kpiForge } from "../src/tui/screens/dashboard.ts"; | |
| 22 | ||
| 23 | describe("isOffBox", () => { | |
| 24 | test("a bucket elsewhere is off the machine; a directory or a loopback bucket is not", () => { | |
| 25 | expect(isOffBox({ where: "s3.us-west-004.backblazeb2.com/forge-snapshots" })).toBe(true); | |
| 26 | expect(isOffBox({ where: "minio.platform.svc:9000/app-forge" })).toBe(true); | |
| 27 | expect(isOffBox(dirStore("/data/backups"))).toBe(false); | |
| 28 | expect(isOffBox({ where: "localhost:9000/b" })).toBe(false); | |
| 29 | expect(isOffBox({ where: "LOCALHOST/b" })).toBe(false); | |
| 30 | expect(isOffBox({ where: "127.0.0.1:9000/b" })).toBe(false); | |
| 31 | expect(isOffBox({ where: "[::1]:9000/b" })).toBe(false); | |
| 32 | }); | |
| 33 | }); | |
| 34 | ||
| 35 | describe("lastOffBox", () => { | |
| 36 | let dir: string; | |
| 37 | beforeAll(() => { | |
| 38 | dir = mkdtempSync(join(tmpdir(), "forge-offbox-")); | |
| 39 | }); | |
| 40 | afterAll(async () => { | |
| 41 | await rm(dir, { recursive: true, force: true }); | |
| 42 | }); | |
| 43 | ||
| 44 | test("nothing kept is never; a landing in a directory is still never", () => { | |
| 45 | expect(lastOffBox(dir)).toBeNull(); | |
| 46 | recordSnapshotOutcome(dir, { landed: "s1", offBox: false }, "2026-10-19T08:30:00.000Z"); | |
| 47 | expect(lastOffBox(dir)).toBeNull(); | |
| 48 | }); | |
| 49 | ||
| 50 | test("a landing in a bucket counts, and a later failure keeps it", () => { | |
| 51 | recordSnapshotOutcome(dir, { landed: "s2", offBox: true }, "2026-10-20T08:30:00.000Z"); | |
| 52 | expect(lastOffBox(dir)).toBe("2026-10-20T08:30:00.000Z"); | |
| 53 | recordSnapshotOutcome(dir, { error: "no answer" }, "2026-10-20T09:30:00.000Z"); | |
| 54 | expect(lastSnapshotOutcome(dir)?.offBox).toBe(true); | |
| 55 | expect(lastOffBox(dir)).toBe("2026-10-20T08:30:00.000Z"); | |
| 56 | }); | |
| 57 | ||
| 58 | test("the newer of the mirror and the destination wins", () => { | |
| 59 | recordMirror(dir, { where: "b2", attemptAt: "2026-10-20T10:00:00.000Z", reachable: true, newest: "s1", newestAt: "2026-10-19T08:30:00.000Z" }); | |
| 60 | expect(lastOffBox(dir)).toBe("2026-10-20T08:30:00.000Z"); | |
| 61 | recordMirror(dir, { where: "b2", attemptAt: "2026-10-21T10:00:00.000Z", reachable: true, newest: "s3", newestAt: "2026-10-21T08:30:00.000Z" }); | |
| 62 | expect(lastOffBox(dir)).toBe("2026-10-21T08:30:00.000Z"); | |
| 63 | }); | |
| 64 | ||
| 65 | test("an outcome kept before stores were recorded learns it once, and is not overwritten after", async () => { | |
| 66 | const old = mkdtempSync(join(tmpdir(), "forge-offbox-old-")); | |
| 67 | try { | |
| 68 | noteOffBox(old, true); | |
| 69 | expect(lastSnapshotOutcome(old)).toBeUndefined(); | |
| 70 | recordSnapshotOutcome(old, { landed: "s1" }, "2026-10-20T08:30:00.000Z"); | |
| 71 | expect(lastOffBox(old)).toBeNull(); | |
| 72 | noteOffBox(old, true); | |
| 73 | expect(lastOffBox(old)).toBe("2026-10-20T08:30:00.000Z"); | |
| 74 | noteOffBox(old, false); | |
| 75 | expect(lastSnapshotOutcome(old)?.offBox).toBe(true); | |
| 76 | } finally { | |
| 77 | await rm(old, { recursive: true, force: true }); | |
| 78 | } | |
| 79 | }); | |
| 80 | }); | |
| 81 | ||
| 82 | describe("a snapshot that lands", () => { | |
| 83 | let root: string; | |
| 84 | let app: App; | |
| 85 | beforeAll(() => { | |
| 86 | root = mkdtempSync(join(tmpdir(), "forge-offbox-svc-")); | |
| 87 | app = createApp({ dataDir: join(root, "data"), port: 0 }); | |
| 88 | }); | |
| 89 | afterAll(async () => { | |
| 90 | app.close(); | |
| 91 | await rm(root, { recursive: true, force: true }); | |
| 92 | }); | |
| 93 | ||
| 94 | test("in a directory is not off the machine; in a bucket it is, and home's tile shows its age", async () => { | |
| 95 | const local = startSnapshots(app, dirStore(join(root, "store")), { version: "t", nightly: false }); | |
| 96 | try { | |
| 97 | await local.now(); | |
| 98 | } finally { | |
| 99 | await local.stop(); | |
| 100 | } | |
| 101 | expect(lastSnapshotOutcome(app.config.dataDir)?.offBox).toBe(false); | |
| 102 | expect(readKpis(app.db, new Date(), kpiForge(app)).backupAt).toBeNull(); | |
| 103 | ||
| 104 | // A bucket, as far as the forge can tell: no directory, a host that is not this machine. | |
| 105 | const { dir: _dir, ...inner } = dirStore(join(root, "bucket")); | |
| 106 | const bucket: Store = { ...inner, where: "s3.example.com/forge-snapshots" }; | |
| 107 | const remote = startSnapshots(app, bucket, { version: "t", nightly: false }); | |
| 108 | try { | |
| 109 | await remote.now(); | |
| 110 | } finally { | |
| 111 | await remote.stop(); | |
| 112 | } | |
| 113 | const kept = lastSnapshotOutcome(app.config.dataDir); | |
| 114 | expect(kept?.offBox).toBe(true); | |
| 115 | // The terminal's figures read the same data directory as home. | |
| 116 | expect(readKpis(app.db, new Date(), kpiForge(app)).backupAt).toBe(kept!.landedAt!); | |
| 117 | }, 30_000); | |
| 118 | }); |
new above
offline
Private/cluster hosts (*.svc, 10.x, 192.168.x) count as off-box here; the test pins minio.platform.svc as true. Make that an explicit documented choice.