Public: anyone can read this; the crew can change it.
A failed nightly backup says why in the system card: cause word, bytes sent and seconds spent #13074
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/13046-a-failed-nightly-backup-says-why-in-the → main 7 commits fork 3 threads · 3 unresolved
docs/features/offsite-lane.md +19
| @@ -55,3 +55,22 @@ as it stands. Each drill is kept with the store it restored from. | ||
| 55 | 55 | `{ health, says, where, sealed, lastSuccess, newest, error, drill }`. |
| 56 | 56 | |
| 57 | 57 | All three are a forge admin's alone. |
| 58 | ||
| 59 | ## When the nightly fails (#13046) | |
| 60 | ||
| 61 | A failed nightly backup says why, in one line: `timeout after 42 s, 1.5 MiB | |
| 62 | sent`. The word is the cause (`timeout`, `auth`, `disk`, `refused`, or | |
| 63 | `unknown` when the error names none), then the seconds the attempt ran and | |
| 64 | the bytes that landed before it failed. It is read once from the error | |
| 65 | (`src/snapshot/cause.ts`) and kept with the snapshot outcome, so a landed | |
| 66 | snapshot clears it. | |
| 67 | ||
| 68 | - **On `/system`**, under "Off-site backup", as "Last nightly backup". | |
| 69 | - **In the terminal**, on the Off-site screen, as `Failed` under "Last nightly | |
| 70 | backup" (the palette's Off-site entry). | |
| 71 | - **Over MCP**, the `system` tool with `{ "kind": "offsite" }` returns | |
| 72 | `backupFailure: { cause, bytes, seconds, says }`, or `null` while the last | |
| 73 | snapshot landed. | |
| 74 | ||
| 75 | All three are a forge admin's, a person's, as the rest of the lane is. Only the | |
| 76 | cause word and two numbers are shown, never the error text. |
src/http/mcp/tools/system.ts +1 −1
Diff not shown — a line 2,332 characters wide (limit 2,000). +1 −1
View filesrc/snapshot/cause.ts added +28
| @@ -0,0 +1,28 @@ | ||
| 1 | /** | |
| 2 | * Why a snapshot failed, as one word the owner can act on. The error text | |
| 3 | * is a store's or the OS's, never ours to parse twice: this reads it once, | |
| 4 | * where the failure is kept, and the board, /system and the MCP `system` | |
| 5 | * tool (`kind: offsite`) print the word instead of "failed". | |
| 6 | */ | |
| 7 | ||
| 8 | export type FailureCause = "timeout" | "auth" | "disk" | "refused" | "unknown"; | |
| 9 | ||
| 10 | /** | |
| 11 | * The cause the error text names; "unknown" when it names none. | |
| 12 | * Order matters: a refused login is auth. | |
| 13 | */ | |
| 14 | export function failureCause(error: unknown): FailureCause { | |
| 15 | const e = error as { code?: unknown; message?: unknown } | null; | |
| 16 | const text = `${typeof e?.code === "string" ? e.code : ""} ${error instanceof Error ? error.message : String(error)}`; | |
| 17 | if (/ETIMEDOUT|timed? ?out|timeout|deadline|no answer in/i.test(text)) return "timeout"; | |
| 18 | if (/\b40[13]\b|forbidden|unauthori[sz]ed|access ?denied|invalid.*(credential|key|signature)|signature.*match|not authenticated|passphrase/i.test(text)) return "auth"; | |
| 19 | if (/\b(ENOSPC|EDQUOT|EROFS|EIO)\b|no space|disk full|disk quota/i.test(text)) return "disk"; | |
| 20 | if (/ECONNREFUSED|ECONNRESET|EHOSTUNREACH|ENETUNREACH|ENOTFOUND|refused|unreachable/i.test(text)) return "refused"; | |
| 21 | return "unknown"; | |
| 22 | } | |
| 23 | ||
| 24 | /** One line: `timeout after 42 s, 1.5 MiB sent`. */ | |
| 25 | export function failureLine(f: { readonly cause: FailureCause; readonly bytes: number; readonly seconds: number }): string { | |
| 26 | const size = f.bytes >= 1048576 ? `${(f.bytes / 1048576).toFixed(1)} MiB` : f.bytes >= 1024 ? `${(f.bytes / 1024).toFixed(1)} KiB` : `${f.bytes} B`; | |
| 27 | return `${f.cause} after ${f.seconds} s, ${size} sent`; | |
| 28 | } |
src/snapshot/offsite_lane.ts +18 −3
| @@ -14,6 +14,8 @@ | ||
| 14 | 14 | import type { Database } from "bun:sqlite"; |
| 15 | 15 | import type { DrillResult } from "./drill.ts"; |
| 16 | 16 | import { lastDrill } from "./drills.ts"; |
| 17 | import { failureLine } from "./cause.ts"; | |
| 18 | import { lastSnapshotOutcome, type SnapshotOutcome } from "./outcome.ts"; | |
| 17 | 19 | import { lastMirror, MIRROR_STALE_MS, type MirrorState } from "./mirror.ts"; |
| 18 | 20 | |
| 19 | 21 | export type LaneHealth = "none" | "ok" | "late" | "failing" | "unsealed"; |
| @@ -29,6 +31,11 @@ export interface OffsiteLane { | ||
| 29 | 31 | readonly newest: string | null; |
| 30 | 32 | /** Why passes are failing now, one line. */ |
| 31 | 33 | readonly error: string | null; |
| 34 | /** | |
| 35 | * The nightly's failure in one line (`timeout after 42 s, 1.5 MiB sent`) with its parts; | |
| 36 | * null while the last snapshot landed. | |
| 37 | */ | |
| 38 | readonly backupFailure: (NonNullable<SnapshotOutcome["failure"]> & { readonly says: string }) | null; | |
| 32 | 39 | readonly drill: { |
| 33 | 40 | readonly at: string; |
| 34 | 41 | readonly verdict: "green" | "red"; |
| @@ -38,12 +45,19 @@ export interface OffsiteLane { | ||
| 38 | 45 | } | null; |
| 39 | 46 | } |
| 40 | 47 | |
| 41 | export function offsiteLane(state: MirrorState | undefined, drill: DrillResult | undefined, now: number): OffsiteLane { | |
| 48 | export function offsiteLane( | |
| 49 | state: MirrorState | undefined, | |
| 50 | drill: DrillResult | undefined, | |
| 51 | now: number, | |
| 52 | snapshot?: SnapshotOutcome, | |
| 53 | ): OffsiteLane { | |
| 54 | const f = snapshot?.failingSince !== undefined ? snapshot.failure : undefined; | |
| 55 | const backupFailure = f === undefined ? null : { ...f, says: failureLine(f) }; | |
| 42 | 56 | const drillSeen = drill === undefined |
| 43 | 57 | ? null |
| 44 | 58 | : { at: drill.at, verdict: drill.verdict, from: drill.from ?? null, offSite: drill.offSite === true, reason: drill.reason }; |
| 45 | 59 | if (state === undefined) |
| 46 | return { health: "none", where: null, sealed: false, lastSuccess: null, newest: null, error: null, drill: drillSeen }; | |
| 60 | return { health: "none", where: null, sealed: false, lastSuccess: null, newest: null, error: null, backupFailure, drill: drillSeen }; | |
| 47 | 61 | const sealed = state.sealed === true; |
| 48 | 62 | const synced = Date.parse(state.syncedAt ?? ""); |
| 49 | 63 | // A lane without the passphrase refuses every write (#11805); that, not its failing passes, is what to fix. |
| @@ -59,13 +73,14 @@ export function offsiteLane(state: MirrorState | undefined, drill: DrillResult | | ||
| 59 | 73 | lastSuccess: state.syncedAt ?? null, |
| 60 | 74 | newest: state.newest ?? null, |
| 61 | 75 | error: state.failingSince === undefined ? null : (state.error ?? "unknown reason"), |
| 76 | backupFailure, | |
| 62 | 77 | drill: drillSeen, |
| 63 | 78 | }; |
| 64 | 79 | } |
| 65 | 80 | |
| 66 | 81 | /** The lane now, from what the last pass and the last drill kept: one small file and one row, no network. */ |
| 67 | 82 | export function readOffsiteLane(dataDir: string, db: Database, now = Date.now()): OffsiteLane { |
| 68 | return offsiteLane(lastMirror(dataDir), lastDrill(db), now); | |
| 83 | return offsiteLane(lastMirror(dataDir), lastDrill(db), now, lastSnapshotOutcome(dataDir)); | |
| 69 | 84 | } |
| 70 | 85 | |
| 71 | 86 | /** One sentence for the lane, as `/system` heads it. */ |
src/snapshot/outcome.ts +8 −2
| @@ -10,6 +10,7 @@ | ||
| 10 | 10 | |
| 11 | 11 | import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; |
| 12 | 12 | import { join } from "node:path"; |
| 13 | import type { FailureCause } from "./cause.ts"; | |
| 13 | 14 | |
| 14 | 15 | export interface SnapshotOutcome { |
| 15 | 16 | /** When the last attempt ended. */ |
| @@ -27,6 +28,11 @@ export interface SnapshotOutcome { | ||
| 27 | 28 | /** Set while attempts are failing: since when, and the last one's reason. */ |
| 28 | 29 | readonly failingSince?: string; |
| 29 | 30 | readonly error?: string; |
| 31 | /** | |
| 32 | * Of the last failed attempt: its cause word, bytes sent and seconds spent | |
| 33 | * (absent in outcomes kept before #13046). | |
| 34 | */ | |
| 35 | readonly failure?: { readonly cause: FailureCause; readonly bytes: number; readonly seconds: number }; | |
| 30 | 36 | /** Of the snapshot that landed: refs it could not hold, their commits missing, by repository. */ |
| 31 | 37 | readonly brokenRefs?: readonly { readonly repo: string; readonly refs: readonly string[] }[]; |
| 32 | 38 | /** |
| @@ -65,7 +71,7 @@ export function recordSnapshotOutcome( | ||
| 65 | 71 | readonly brokenRefs?: SnapshotOutcome["brokenRefs"]; |
| 66 | 72 | readonly absent?: SnapshotOutcome["absent"]; |
| 67 | 73 | } |
| 68 | | { readonly error: string }, | |
| 74 | | { readonly error: string; readonly failure?: SnapshotOutcome["failure"] }, | |
| 69 | 75 | at: string = new Date().toISOString(), |
| 70 | 76 | ): SnapshotOutcome { |
| 71 | 77 | const prev = lastSnapshotOutcome(dataDir); |
| @@ -86,7 +92,7 @@ export function recordSnapshotOutcome( | ||
| 86 | 92 | ...(result.brokenRefs === undefined || result.brokenRefs.length === 0 ? {} : { brokenRefs: result.brokenRefs }), |
| 87 | 93 | ...(result.absent === undefined || result.absent.length === 0 ? {} : { absent: result.absent }), |
| 88 | 94 | } |
| 89 | : { at, ...kept, failingSince: prev?.failingSince ?? at, error: result.error }; | |
| 95 | : { at, ...kept, failingSince: prev?.failingSince ?? at, error: result.error, ...(result.failure === undefined ? {} : { failure: result.failure }) }; | |
| 90 | 96 | write(dataDir, next); |
| 91 | 97 | return next; |
| 92 | 98 | } |
src/snapshot/service.ts +12 −1
| @@ -33,6 +33,7 @@ import { lastDrill, recordDrill } from "./drills.ts"; | ||
| 33 | 33 | import { cutDrill, runDrillInChild, sweepDrills, type DrillState } from "./drill_job.ts"; |
| 34 | 34 | import { lastMirror, mirrorOffThread, mirrorTarget, MIRROR_EVERY_MS, recordMirror, type MirrorState } from "./mirror.ts"; |
| 35 | 35 | import { isOffBox } from "./off_box.ts"; |
| 36 | import { failureCause } from "./cause.ts"; | |
| 36 | 37 | import { noteOffBox, recordSnapshotOutcome } from "./outcome.ts"; |
| 37 | 38 | import { remakeOf, s3FromEnv, type Store } from "./store.ts"; |
| 38 | 39 | import { verifySnapshot } from "./verify.ts"; |
| @@ -211,6 +212,8 @@ export function startSnapshots( | ||
| 211 | 212 | // Whether this run's store is off the machine, for home's backup age (#3701). |
| 212 | 213 | let offBox: boolean | undefined; |
| 213 | 214 | let began = new Date().toISOString(); |
| 215 | // Bytes landed so far, kept past the run's end so a failure can say how far it got (#13046). | |
| 216 | let sent = 0; | |
| 214 | 217 | // After the one before it, however that ended: chained on a failure as it |
| 215 | 218 | // was, every later run failed at once with the old error until a restart. |
| 216 | 219 | wait.release("snapshot"); |
| @@ -235,6 +238,7 @@ export function startSnapshots( | ||
| 235 | 238 | passphrase: options.passphrase, |
| 236 | 239 | onLand: (p) => { |
| 237 | 240 | running = p; |
| 241 | sent = p.bytes; | |
| 238 | 242 | if (Date.now() - toldAt < (options.progressEventMs ?? PROGRESS_EVENT_MS)) return; |
| 239 | 243 | toldAt = Date.now(); |
| 240 | 244 | app.bus.append({ kind: "backup.progress", payload: { objects: p.objects, bytes: p.bytes } }); |
| @@ -284,7 +288,14 @@ export function startSnapshots( | ||
| 284 | 288 | } |
| 285 | 289 | appendSnapshot({ startedAt: began, endedAt: new Date().toISOString(), failed: true }); |
| 286 | 290 | log.error("snapshot failed", { error: String(error) }); |
| 287 | keep({ error: error instanceof Error ? error.message : String(error) }); | |
| 291 | keep({ | |
| 292 | error: error instanceof Error ? error.message : String(error), | |
| 293 | failure: { | |
| 294 | cause: failureCause(error), | |
| 295 | bytes: sent, | |
| 296 | seconds: Math.max(0, Math.round((Date.now() - Date.parse(began)) / 1000)), | |
| 297 | }, | |
| 298 | }); | |
| 288 | 299 | app.bus.append({ kind: "backup.finished", payload: { ok: false } }); |
| 289 | 300 | arm(Math.min(options.retryMs ?? RETRY_MS, untilNightly())); |
| 290 | 301 | throw error; |
src/tui/screens/offsite.ts +1
| @@ -34,6 +34,7 @@ export function offsiteLines(session: Session, l: OffsiteLane = readOffsiteLane( | ||
| 34 | 34 | ...(l.newest === null ? [] : [fact("Newest", l.newest)]), |
| 35 | 35 | "", |
| 36 | 36 | ); |
| 37 | if (l.backupFailure !== null) lines.push("Last nightly backup", fact("Failed", l.backupFailure.says), ""); | |
| 37 | 38 | const d = l.drill; |
| 38 | 39 | lines.push("Restore drill"); |
| 39 | 40 | if (d === null) lines.push(fact("Last", "never run")); |
src/web/pages/system.ts +5
| @@ -385,6 +385,11 @@ export function offsiteSection(l: OffsiteLane): Html { | ||
| 385 | 385 | ], |
| 386 | 386 | data: { lane: "offsite", sealed: String(l.sealed), ...(l.lastSuccess === null ? {} : { success: l.lastSuccess }) }, |
| 387 | 387 | })} |
| 388 | ${l.backupFailure === null ? null : row({ | |
| 389 | title: "Last nightly backup", | |
| 390 | facts: [{ label: "Failed", value: l.backupFailure.says }], | |
| 391 | data: { backup: "failed", cause: l.backupFailure.cause }, | |
| 392 | })} | |
| 388 | 393 | ${row({ |
| 389 | 394 | title: "Restore drill", |
| 390 | 395 | facts: d === null |
test/backup-failure-cause.test.ts added +44
| @@ -0,0 +1,44 @@ | ||
| 1 | import { describe, expect, test } from "bun:test"; | |
| 2 | import { mkdtempSync } from "node:fs"; | |
| 3 | import { tmpdir } from "node:os"; | |
| 4 | import { join } from "node:path"; | |
| 5 | import { failureCause, failureLine, type FailureCause } from "../src/snapshot/cause.ts"; | |
| 6 | import { offsiteLane } from "../src/snapshot/offsite_lane.ts"; | |
| 7 | import { lastSnapshotOutcome, recordSnapshotOutcome } from "../src/snapshot/outcome.ts"; | |
| 8 | import { offsiteSection } from "../src/web/pages/system.ts"; | |
| 9 | ||
| 10 | const cases: [string, Error, FailureCause][] = [ | |
| 11 | ["timeout", new Error("request timed out after 30000 ms"), "timeout"], | |
| 12 | ["auth", new Error("PUT snapshot: 403 Forbidden"), "auth"], | |
| 13 | ["disk", Object.assign(new Error("write failed"), { code: "ENOSPC" }), "disk"], | |
| 14 | ["refused", new Error("connect ECONNREFUSED 10.0.0.1:9000"), "refused"], | |
| 15 | ["unknown", new Error("something odd"), "unknown"], | |
| 16 | ["a word holding eio is not disk", new Error("serious violation of the seieio protocol"), "unknown"], | |
| 17 | ["a rate-limit quota is not disk", new Error("API quota exceeded, slow down"), "unknown"], | |
| 18 | ["EIO as a code is disk", new Error("read failed: EIO"), "disk"], | |
| 19 | ]; | |
| 20 | ||
| 21 | describe("a failed backup says why (#13046)", () => { | |
| 22 | for (const [word, err, cause] of cases) | |
| 23 | test(`${word}: kept, shown on the card and the lane`, () => { | |
| 24 | expect(failureCause(err)).toBe(cause); | |
| 25 | const dir = mkdtempSync(join(tmpdir(), "bf-")); | |
| 26 | recordSnapshotOutcome(dir, { error: err.message, failure: { cause, bytes: 1572864, seconds: 42 } }); | |
| 27 | const kept = lastSnapshotOutcome(dir)!; | |
| 28 | expect(kept.failure).toEqual({ cause, bytes: 1572864, seconds: 42 }); | |
| 29 | const lane = offsiteLane(undefined, undefined, Date.now(), kept); | |
| 30 | expect(lane.backupFailure?.says).toBe(`${cause} after 42 s, 1.5 MiB sent`); | |
| 31 | expect(String(offsiteSection(lane))).toContain(`${cause} after 42 s, 1.5 MiB sent`); | |
| 32 | }); | |
| 33 | ||
| 34 | test("a landed snapshot clears it", () => { | |
| 35 | const dir = mkdtempSync(join(tmpdir(), "bf-")); | |
| 36 | recordSnapshotOutcome(dir, { error: "x", failure: { cause: "disk", bytes: 0, seconds: 1 } }); | |
| 37 | recordSnapshotOutcome(dir, { landed: "s1" }); | |
| 38 | expect(offsiteLane(undefined, undefined, Date.now(), lastSnapshotOutcome(dir)).backupFailure).toBeNull(); | |
| 39 | }); | |
| 40 | ||
| 41 | test("the line reads small sizes", () => { | |
| 42 | expect(failureLine({ cause: "auth", bytes: 0, seconds: 0 })).toBe("auth after 0 s, 0 B sent"); | |
| 43 | }); | |
| 44 | }); |
test/offsite-lane.test.ts +10
| @@ -23,6 +23,7 @@ import { listSnapshots, SNAPSHOT_PREFIX } from "../src/snapshot/snapshot.ts"; | ||
| 23 | 23 | import { dirStore, type Store } from "../src/snapshot/store.ts"; |
| 24 | 24 | import { tool as systemTool } from "../src/http/mcp/tools/system.ts"; |
| 25 | 25 | import { Session } from "../src/tui/forge.ts"; |
| 26 | import { lastSnapshotOutcome, recordSnapshotOutcome } from "../src/snapshot/outcome.ts"; | |
| 26 | 27 | import { offsiteLines } from "../src/tui/screens/offsite.ts"; |
| 27 | 28 | import { FOOTING } from "../src/system/footing.ts"; |
| 28 | 29 | import { offsiteSection } from "../src/web/pages/system.ts"; |
| @@ -262,11 +263,20 @@ describe("the off-site lane end to end", () => { | ||
| 262 | 263 | expect(lines.join("\n")).toContain(`Last success ${lane.lastSuccess!}`); |
| 263 | 264 | expect(lines.join("\n")).toContain(`From ${lane.drill!.from!} (off-site)`); |
| 264 | 265 | expect(offsiteLines(new Session(app, ident(member)))).toEqual([]); |
| 266 | const failed = { ...lane, backupFailure: { cause: "timeout" as const, bytes: 1572864, seconds: 42, says: "timeout after 42 s, 1.5 MiB sent" } }; | |
outdated
This checks the terminal with a hand-built lane. The MCP read just below it never asserts backupFailure; add an assertion that systemTool kind offsite returns backupFailure with cause, bytes, seconds and says (null when none). Without it box 3 is untested. |
||
| 267 | expect(offsiteLines(new Session(app, ident(admin)), failed).join("\n")).toContain("Failed timeout after 42 s, 1.5 MiB sent"); | |
| 265 | 268 | |
| 269 | const before = lastSnapshotOutcome(app.config.dataDir); | |
| 270 | recordSnapshotOutcome(app.config.dataDir, { error: "no answer in 42 s", failure: { cause: "timeout", bytes: 1572864, seconds: 42 } }); | |
| 271 | const failure = { cause: "timeout", bytes: 1572864, seconds: 42, says: "timeout after 42 s, 1.5 MiB sent" }; | |
| 266 | 272 | const read = await systemTool.run({ app } as never, admin, { kind: "offsite" }); |
| 267 | 273 | expect(JSON.stringify(read)).toContain(`"lastSuccess":"${lane.lastSuccess!}"`); |
| 268 | 274 | expect(JSON.stringify(read)).toContain('"says":"The off-site lane is level and sealed."'); |
| 269 | 275 | expect(JSON.stringify(await systemTool.run({ app } as never, member, { kind: "offsite" }))).toContain("forbidden"); |
| 276 | ||
| 277 | // The nightly's failure is on the MCP read too, with its parts; a landed one clears it. | |
| 278 | expect(read).toMatchObject({ ok: true, value: { backupFailure: failure } }); | |
| 279 | recordSnapshotOutcome(app.config.dataDir, { landed: before?.landed ?? "restored" }); | |
| 270 | 280 | }); |
| 271 | 281 | |
| 272 | 282 | test("a mirror pass that never ends holds a drill no longer than the drill's wait", async () => { |
test/snapshot-outcome.test.ts +12 −2
| @@ -85,11 +85,14 @@ describe("a nightly that fails", () => { | ||
| 85 | 85 | test("is a card on vitals and a red line on Backups at once, and the next try lands and clears both", async () => { |
| 86 | 86 | const inner = dirStore(join(root, "store")); |
| 87 | 87 | let broken = true; |
| 88 | let landedBytes = 0; | |
| 88 | 89 | const store: Store = { |
| 89 | 90 | ...inner, |
| 90 | 91 | put: async (key, body) => { |
| 91 | if (broken && key.startsWith("chunks/")) throw new Error("no answer in 180 s"); | |
| 92 | return inner.put(key, body); | |
| 92 | // The chunks land, the manifest gets no answer: the failure has bytes behind it. | |
| 93 | if (broken && key.endsWith("manifest.json")) throw new Error("no answer in 180 s"); | |
| 94 | await inner.put(key, body); | |
| 95 | if (broken) landedBytes += typeof body === "string" ? Buffer.byteLength(body) : body.byteLength; | |
| 93 | 96 | }, |
| 94 | 97 | }; |
| 95 | 98 | const svc = startSnapshots(app, store, { version: "t", retryMs: 300 }); |
| @@ -99,6 +102,13 @@ describe("a nightly that fails", () => { | ||
| 99 | 102 | expect(card?.sentence).toContain("The last snapshot failed"); |
| 100 | 103 | expect(card?.sentence).toContain("no answer in 180 s"); |
| 101 | 104 | expect(card?.href).toBe("/settings/backups"); |
| 105 | // The cause word, bytes and seconds are kept with it (#13046). | |
| 106 | const f = lastSnapshotOutcome(app.config.dataDir)?.failure; | |
outdated
typeof bytes === number would pass with a hard-coded 0. Make the failing store land some bytes before failing (or assert seconds >= 0 and bytes equals what onLand reported) so the sent-tracking in service.ts is actually covered. |
||
| 107 | expect(f?.cause).toBe("timeout"); | |
| 108 | // What landed before the failure is what it says was sent, not a stand-in 0. | |
| 109 | expect(landedBytes).toBeGreaterThan(0); | |
| 110 | expect(f?.bytes).toBe(landedBytes); | |
| 111 | expect(f?.seconds).toBeGreaterThanOrEqual(0); | |
| 102 | 112 | // The terminal's dashboard prints the forge's needs: the card is there too. |
| 103 | 113 | expect(forgeNeeds(app, admin)?.some((l) => l.includes("The last snapshot failed"))).toBe(true); |
| 104 | 114 | const page = await (await fetch(`${servers.url}/settings/backups`, { headers: { cookie } })).text(); |
With the /i flag, EIO matches inside ordinary words in an error message and would misreport as disk; anchor it as a word (\bEIO\b, case-sensitive for errno codes) and drop bare
quota, which also names API rate limits that are not disk.