Public: anyone can read this; the crew can change it.
A nightly backup that doubles in size or duration within a week files one defect with both numbers #13055
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/13032-a-nightly-backup-that-doubles-in-size-or → main 6 commits fork 2 threads · 2 unresolved
13 files
changed +325
−10
docs/features/README.md +1
| @@ -369,6 +369,7 @@ Read with the guide page [Agents: MCP, git, or the terminal](../guide-agents.md) | ||
| 369 | 369 | |
| 370 | 370 | Read with the guide page [](../guide-backups-moving.md). |
| 371 | 371 | |
| 372 | - [](backup-growth.md) | |
| 372 | 373 | - [](backup-refs.md) |
| 373 | 374 | - [](forge-origin.md) |
| 374 | 375 | - [](origin-pull-state.md) |
docs/features/backup-growth.md added +18
| @@ -0,0 +1,18 @@ | ||
| 1 | # A nightly backup that doubles | |
| 2 | ||
| 3 | The `backup-growth` maintenance duty (#13032) files one defect when the last nightly | |
| 4 | backup took over twice what the nights before it took. | |
| 5 | ||
| 6 | - **What it watches.** The last landed night against the median of the previous seven, | |
| 7 | in bytes (what a restore writes) and in seconds (start to finish). Either number | |
| 8 | over a factor of 2 counts; the defect names both numbers and the backup's destination | |
| 9 | (the store's page-safe `where`, never a secret). | |
| 10 | - **Where the record lives.** `snapshot/state.json` under `FORGE_APP_DATA`: one entry a | |
| 11 | UTC day, newest last, a later backup the same day replacing that day's entry. | |
| 12 | - **Minimum.** Nothing is read until three nights are recorded, the last included. | |
| 13 | - **Freshness.** Only a night from the last two days is read: a backup that stopped | |
| 14 | landing is another alarm's. | |
| 15 | - **One defect.** While one is open, a later run files none (`fileIssueOnce`). | |
| 16 | - **Closing it.** Closing the defect silences that night only: a defect closed since | |
| 17 | that night landed is not filed again for it. A later night that grows files a new one. | |
| 18 | - The duty runs hourly and spends no money. |
src/maintain/budget.ts +22
| @@ -49,6 +49,28 @@ export function budgetOf(db: Database, duty: Duty): Budget { | ||
| 49 | 49 | }; |
| 50 | 50 | } |
| 51 | 51 | |
| 52 | /** The budget in force for each of `ds`, by duty name: one read for all of them, for a page that | |
| 53 | * lists every duty (#13032). */ | |
| 54 | export function budgetsOf(db: Database, ds: readonly Duty[]): Map<string, Budget> { | |
| 55 | const rows = db.query("SELECT key, value FROM forge_setting WHERE key LIKE 'maintain.%'").all() as { key: string; value: string }[]; | |
| 56 | const kept = new Map(rows.map((r) => [r.key, r.value])); | |
| 57 | const read = (d: Duty, kind: BudgetKind): number => { | |
| 58 | const v = kept.get(settingKey(d.name, kind)); | |
| 59 | return (v === undefined ? undefined : parseBudget(kind, v)) ?? declared(d, kind); | |
| 60 | }; | |
| 61 | return new Map(ds.map((d) => [d.name, { runsPerDay: read(d, "runs"), dollarsPerDay: read(d, "dollars") }])); | |
| 62 | } | |
| 63 | ||
| 64 | /** What every duty has used today, by duty name, in one read; a duty with no run today is | |
| 65 | * absent. */ | |
| 66 | export function usedTodayAll(db: Database, now: Date): Map<string, { runs: number; dollars: number }> { | |
| 67 | const { from, to } = dayBounds(now); | |
| 68 | const rows = db | |
| 69 | .query("SELECT duty, COUNT(*) AS runs, COALESCE(SUM(dollars), 0) AS dollars FROM maintenance_run WHERE started >= ?1 AND started < ?2 AND outcome NOT LIKE 'budget:%' GROUP BY duty") | |
| 70 | .all(from, to) as { duty: string; runs: number; dollars: number }[]; | |
| 71 | return new Map(rows.map((r) => [r.duty, { runs: r.runs, dollars: r.dollars }])); | |
| 72 | } | |
| 73 | ||
| 52 | 74 | /** Keep a new value for a budget (`null`, or the duty's own number, goes back to the duty's own), and who moved it. */ |
| 53 | 75 | export function setBudget(db: Database, duty: Duty, kind: BudgetKind, given: number | null, actorId: string): void { |
| 54 | 76 | // Only a value that differs from the declared one is stored, so a later change of the default still takes. |
src/maintain/duties/backup_growth.ts added +59
| @@ -0,0 +1,59 @@ | ||
| 1 | /** | |
| 2 | * Backup growth (#13032): when the last nightly backup took over twice the | |
| 3 | * median of the seven before it, in bytes or in seconds, file one defect naming | |
| 4 | * both numbers and the destination. One open defect stands for the growth; a | |
| 5 | * night that already had one filed (and closed) is not filed again. | |
| 6 | */ | |
| 7 | ||
| 8 | import { fileIssueOnce } from "../../checks/flake.ts"; | |
| 9 | import { getRepo } from "../../core/repos.ts"; | |
| 10 | import { ownRepoPath } from "../../kernel/own_repo.ts"; | |
| 11 | import { growthOf, type Growth, type Measure } from "../../snapshot/growth.ts"; | |
| 12 | import { lastSnapshotOutcome } from "../../snapshot/outcome.ts"; | |
| 13 | import { sizeSaid } from "../../system/db_size.ts"; | |
| 14 | import type { Duty } from "../duty.ts"; | |
| 15 | ||
| 16 | export const TITLE = "The nightly backup doubled in size or duration within a week"; | |
| 17 | const DAY_MS = 86_400_000; | |
| 18 | ||
| 19 | const bytesSaid = (m: Measure): string => `${sizeSaid(m.last)} against a median of ${sizeSaid(m.median)}`; | |
| 20 | const secondsSaid = (m: Measure): string => `${m.last} s against a median of ${Math.round(m.median)} s`; | |
| 21 | ||
| 22 | /** The defect's body: both numbers whichever passed, and where the backup went. */ | |
| 23 | export function bodyOf(g: Growth, destination: string): string { | |
| 24 | return ( | |
| 25 | `_Filed by the backup growth duty from the backup record, not written by a person._\n\n` + | |
| 26 | `The night of ${g.night.at.slice(0, 10)} is over twice the median of the nights before it.\n\n` + | |
| 27 | `- Size: ${bytesSaid(g.bytes)}${g.over.includes("bytes") ? " (over twice)" : ""}\n` + | |
| 28 | `- Duration: ${secondsSaid(g.seconds)}${g.over.includes("seconds") ? " (over twice)" : ""}\n` + | |
| 29 | `- Destination: ${destination}` | |
| 30 | ); | |
| 31 | } | |
| 32 | ||
| 33 | export const duty: Duty = { | |
| 34 | name: "backup-growth", | |
| 35 | intervalMs: 3_600_000, | |
| 36 | budget: { runsPerDay: 30, dollarsPerDay: 1, reason: "hourly is 24 a day; the rest is room for a catch-up, and it spends no money" }, | |
| 37 | run({ db, bus, now, app }) { | |
| 38 | if (app === undefined) return { findings: [], actions: [] }; | |
| 39 | const outcome = lastSnapshotOutcome(app.config.dataDir); | |
| 40 | const g = growthOf(outcome?.nights ?? []); | |
| 41 | // A backup that stopped landing is another alarm's; only a night from the last two days | |
| 42 | // is read. | |
| 43 | if (g === undefined || g.over.length === 0 || now.getTime() - Date.parse(g.night.at) > 2 * DAY_MS) | |
| 44 | return { findings: [], actions: [] }; | |
| 45 | const destination = outcome?.destination ?? "unknown"; | |
| 46 | const summary = `last backup ${bytesSaid(g.bytes)}, ${secondsSaid(g.seconds)}; destination ${destination}`; | |
| 47 | const finding = { summary, probe: { kind: "measurement" as const, what: "bytes of the last nightly backup", number: g.bytes.last, at: now.toISOString() } }; | |
| 48 | const [owner = "", name = ""] = ownRepoPath().split("/"); | |
| 49 | const repo = getRepo(db, owner, name); | |
| 50 | if (repo === undefined) return { findings: [finding], actions: [] }; | |
| 51 | // Closed since this night landed: that was a person's answer to this night, not a new defect. | |
| 52 | const seen = db | |
| 53 | .query<{ n: number }, [string, string, string]>("SELECT COUNT(*) AS n FROM issue WHERE repo_id = ?1 AND title = ?2 AND created_at >= ?3") | |
| 54 | .get(repo.id, TITLE, g.night.at)!.n; | |
| 55 | if (seen > 0) return { findings: [finding], actions: [] }; | |
| 56 | const filed = fileIssueOnce(db, bus, repo, { title: TITLE, body: bodyOf(g, destination) }); | |
| 57 | return { findings: [finding], actions: filed.opened ? [`filed backup growth issue #${filed.number}`] : [] }; | |
| 58 | }, | |
| 59 | }; |
src/maintain/registry.ts +2 −1
| @@ -6,6 +6,7 @@ import { duty as ruleWatcher } from "./duties/rule_watcher.ts"; | ||
| 6 | 6 | import { duty as efficiencyWatcher } from "./duties/efficiency_watcher.ts"; |
| 7 | 7 | import { duty as flowKeeper } from "./duties/flow_keeper.ts"; |
| 8 | 8 | import { duty as visitorRetention } from "./duties/visitor_retention.ts"; |
| 9 | import { duty as backupGrowth } from "./duties/backup_growth.ts"; | |
| 9 | 10 | import { duty as v1Proven } from "./duties/v1_proven.ts"; |
| 10 | 11 | |
| 11 | export const duties: readonly Duty[] = [pullsWaiting, ruleWatcher, efficiencyWatcher, flowKeeper, visitorRetention, v1Proven]; | |
| 12 | export const duties: readonly Duty[] = [pullsWaiting, ruleWatcher, efficiencyWatcher, flowKeeper, visitorRetention, backupGrowth, v1Proven]; |
src/snapshot/growth.ts added +46
| @@ -0,0 +1,46 @@ | ||
| 1 | /** | |
| 2 | * Whether the nightly backup is outgrowing its night (#13032): the last landed | |
| 3 | * night against the median of the seven before it, in bytes and in seconds. | |
| 4 | * Pure: nights in, a reading out. The record is `outcome.ts`. | |
| 5 | */ | |
| 6 | ||
| 7 | import type { Night } from "./outcome.ts"; | |
| 8 | ||
| 9 | /** Nights compared with. */ | |
| 10 | export const COMPARE_NIGHTS = 7; | |
| 11 | /** No reading until this many nights are recorded, last one included. */ | |
| 12 | export const MIN_NIGHTS = 3; | |
| 13 | /** Over this many times the median is growth worth a defect. */ | |
| 14 | export const FACTOR = 2; | |
| 15 | ||
| 16 | export interface Measure { | |
| 17 | readonly last: number; | |
| 18 | readonly median: number; | |
| 19 | } | |
| 20 | ||
| 21 | export interface Growth { | |
| 22 | readonly night: Night; | |
| 23 | readonly bytes: Measure; | |
| 24 | readonly seconds: Measure; | |
| 25 | /** Which of the two passed `FACTOR` times its median. */ | |
| 26 | readonly over: readonly ("bytes" | "seconds")[]; | |
| 27 | } | |
| 28 | ||
| 29 | const median = (xs: readonly number[]): number => { | |
| 30 | const s = [...xs].sort((a, b) => a - b); | |
| 31 | const m = s.length >> 1; | |
| 32 | return s.length % 2 === 1 ? s[m]! : (s[m - 1]! + s[m]!) / 2; | |
| 33 | }; | |
| 34 | ||
| 35 | /** The reading of `nights` (oldest first), or undefined while too few are recorded. */ | |
| 36 | export function growthOf(nights: readonly Night[]): Growth | undefined { | |
| 37 | if (nights.length < MIN_NIGHTS) return undefined; | |
| 38 | const night = nights[nights.length - 1]!; | |
| 39 | const before = nights.slice(-1 - COMPARE_NIGHTS, -1); | |
| 40 | const bytes = { last: night.bytes, median: median(before.map((n) => n.bytes)) }; | |
| 41 | const seconds = { last: night.seconds, median: median(before.map((n) => n.seconds)) }; | |
| 42 | const over: ("bytes" | "seconds")[] = []; | |
| 43 | if (bytes.median > 0 && bytes.last > FACTOR * bytes.median) over.push("bytes"); | |
| 44 | if (seconds.median > 0 && seconds.last > FACTOR * seconds.median) over.push("seconds"); | |
| 45 | return { night, bytes, seconds, over }; | |
| 46 | } |
src/snapshot/outcome.ts +32
| @@ -12,12 +12,29 @@ import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; | ||
| 12 | 12 | import { join } from "node:path"; |
| 13 | 13 | import type { FailureCause } from "./cause.ts"; |
| 14 | 14 | |
| 15 | /** One landed snapshot's size and duration: bytes a restore writes, seconds from start to | |
| 16 | * finish. */ | |
| 17 | export interface Night { | |
| 18 | readonly at: string; | |
| 19 | readonly bytes: number; | |
| 20 | readonly seconds: number; | |
| 21 | } | |
| 22 | ||
| 23 | /** Nights kept in the record: a week to compare with, and a few spare. */ | |
| 24 | export const NIGHTS_KEPT = 14; | |
| 25 | ||
| 15 | 26 | export interface SnapshotOutcome { |
| 16 | 27 | /** When the last attempt ended. */ |
| 17 | 28 | readonly at: string; |
| 18 | 29 | /** The newest snapshot that landed, and when; absent until one has. */ |
| 19 | 30 | readonly landed?: string; |
| 20 | 31 | readonly landedAt?: string; |
| 32 | /** Where the newest snapshot went (the store's `where`, never a secret); absent before it | |
| 33 | * was kept. */ | |
| 34 | readonly destination?: string; | |
| 35 | /** The nights that landed, newest last, at most `NIGHTS_KEPT`: what each took, for | |
| 36 | * `growth.ts`. */ | |
| 37 | readonly nights?: readonly Night[]; | |
| 21 | 38 | /** Whether the store it landed in is off the machine (a bucket, not a directory); absent when not known. */ |
| 22 | 39 | readonly offBox?: boolean; |
| 23 | 40 | /** |
| @@ -61,12 +78,21 @@ export function offBoxAtOf(s: SnapshotOutcome | undefined): string | undefined { | ||
| 61 | 78 | return s.offBox === true ? s.landedAt : undefined; |
| 62 | 79 | } |
| 63 | 80 | |
| 81 | /** The nights with `night` added; a second landing on the same UTC day replaces the first. */ | |
| 82 | function nightsWith(nights: readonly Night[] | undefined, night: Night): Night[] { | |
| 83 | const day = night.at.slice(0, 10); | |
| 84 | return [...(nights ?? []).filter((n) => n.at.slice(0, 10) !== day), night].slice(-NIGHTS_KEPT); | |
| 85 | } | |
| 86 | ||
| 64 | 87 | /** Keep an attempt's end: a name that landed, or the error it failed with. Returns what was kept. */ |
| 65 | 88 | export function recordSnapshotOutcome( |
| 66 | 89 | dataDir: string, |
| 67 | 90 | result: |
| 68 | 91 | | { |
| 69 | 92 | readonly landed: string; |
| 93 | readonly bytes?: number; | |
| 94 | readonly seconds?: number; | |
| 95 | readonly destination?: string; | |
| 70 | 96 | readonly offBox?: boolean; |
| 71 | 97 | readonly brokenRefs?: SnapshotOutcome["brokenRefs"]; |
| 72 | 98 | readonly absent?: SnapshotOutcome["absent"]; |
| @@ -80,6 +106,8 @@ export function recordSnapshotOutcome( | ||
| 80 | 106 | ...(prev?.landedAt === undefined ? {} : { landedAt: prev.landedAt }), |
| 81 | 107 | ...(prev?.offBox === undefined ? {} : { offBox: prev.offBox }), |
| 82 | 108 | ...(offBoxAtOf(prev) === undefined ? {} : { offBoxAt: offBoxAtOf(prev)! }), |
| 109 | ...(prev?.nights === undefined ? {} : { nights: prev.nights }), | |
| 110 | ...(prev?.destination === undefined ? {} : { destination: prev.destination }), | |
| 83 | 111 | }; |
| 84 | 112 | const next: SnapshotOutcome = |
| 85 | 113 | "landed" in result |
| @@ -87,6 +115,10 @@ export function recordSnapshotOutcome( | ||
| 87 | 115 | at, |
| 88 | 116 | landed: result.landed, |
| 89 | 117 | landedAt: at, |
| 118 | ...(result.destination === undefined ? {} : { destination: result.destination }), | |
| 119 | ...(result.bytes === undefined || result.seconds === undefined | |
| 120 | ? prev?.nights === undefined ? {} : { nights: prev.nights } | |
| 121 | : { nights: nightsWith(prev?.nights, { at, bytes: result.bytes, seconds: result.seconds }) }), | |
| 90 | 122 | ...(result.offBox === undefined ? {} : { offBox: result.offBox }), |
| 91 | 123 | ...(result.offBox === true ? { offBoxAt: at } : offBoxAtOf(prev) === undefined ? {} : { offBoxAt: offBoxAtOf(prev)! }), |
| 92 | 124 | ...(result.brokenRefs === undefined || result.brokenRefs.length === 0 ? {} : { brokenRefs: result.brokenRefs }), |
src/snapshot/service.ts +5
| @@ -211,6 +211,7 @@ export function startSnapshots( | ||
| 211 | 211 | const take = (): Promise<Manifest> => { |
| 212 | 212 | // Whether this run's store is off the machine, for home's backup age (#3701). |
| 213 | 213 | let offBox: boolean | undefined; |
| 214 | let destination: string | undefined; | |
| 214 | 215 | let began = new Date().toISOString(); |
| 215 | 216 | // Bytes landed so far, kept past the run's end so a failure can say how far it got (#13046). |
| 216 | 217 | let sent = 0; |
| @@ -223,6 +224,7 @@ export function startSnapshots( | ||
| 223 | 224 | const store = current(); |
| 224 | 225 | if (store === undefined) throw new Nowhere(); |
| 225 | 226 | offBox = isOffBox(store); |
| 227 | destination = store.where; | |
| 226 | 228 | running = { objects: 0, bytes: 0, elapsedMs: 0 }; |
| 227 | 229 | startedAt = Date.now(); |
| 228 | 230 | began = new Date(startedAt).toISOString(); |
| @@ -272,6 +274,9 @@ export function startSnapshots( | ||
| 272 | 274 | last = { manifest: m, at: m.finishedAt }; |
| 273 | 275 | keep({ |
| 274 | 276 | landed: m.name, |
| 277 | bytes: m.bytes, | |
| 278 | seconds: Math.max(0, Math.round((Date.parse(m.finishedAt) - Date.parse(m.startedAt)) / 1000)), | |
| 279 | ...(destination === undefined ? {} : { destination }), | |
| 275 | 280 | ...(offBox === undefined ? {} : { offBox }), |
| 276 | 281 | ...(m.brokenRefs === undefined ? {} : { brokenRefs: m.brokenRefs }), |
| 277 | 282 | absent: notInSnapshot(m).map((a) => ({ repo: a.repo, why: a.why })), |
src/system/db_size.ts +1 −1
| @@ -332,7 +332,7 @@ export function dbGrowth(db: Database, bytesNow: number, now: number = Date.now( | ||
| 332 | 332 | } |
| 333 | 333 | |
| 334 | 334 | /** Bytes said short: GiB with one decimal from 1 GiB, else whole MiB. */ |
| 335 | const sizeSaid = (n: number): string => (n >= 1024 ** 3 ? `${(n / 1024 ** 3).toFixed(1)} GiB` : `${Math.round(n / 1024 ** 2)} MiB`); | |
| 335 | export const sizeSaid = (n: number): string => (n >= 1024 ** 3 ? `${(n / 1024 ** 3).toFixed(1)} GiB` : `${Math.round(n / 1024 ** 2)} MiB`); | |
| 336 | 336 | |
| 337 | 337 | /** Rows said short: 812, 41k, 3.2M. */ |
| 338 | 338 | const rowsSaid = (n: number): string => |
src/web/pages/maintenance.ts +2 −2
| @@ -17,7 +17,7 @@ import { icon } from "../icons.ts"; | ||
| 17 | 17 | import { flashTo, page } from "../layout.ts"; |
| 18 | 18 | import { badge, btn, disclosure, field } from "../ui.ts"; |
| 19 | 19 | import { STALE_DAYS, STALE_WORDS, closeStale, nudge, stalePulls } from "../../maintain/stale_pulls.ts"; |
| 20 | import { budgetOfRow } from "./maintenance_budgets.ts"; | |
| 20 | import { budgetRows } from "./maintenance_budgets.ts"; | |
| 21 | 21 | |
| 22 | 22 | function admin(ctx: Ctx): Response | undefined { |
| 23 | 23 | if (!ctx.auth.user) return signInFirst(ctx); |
| @@ -76,7 +76,7 @@ export async function maintenancePage(ctx: Ctx): Promise<Response> { | ||
| 76 | 76 | </form>`), |
| 77 | 77 | )} |
| 78 | 78 | </dl>`)} |
| 79 | ${fold("maintenance-budgets", "Budgets", String(states.length), false, html`<dl class="status-rows">${states.map((d) => budgetOfRow(ctx.app, d.name))}</dl>`)} | |
| 79 | ${fold("maintenance-budgets", "Budgets", String(states.length), false, html`<dl class="status-rows">${budgetRows(ctx.app, states.map((d) => d.name))}</dl>`)} | |
| 80 | 80 | ${stale > 0 |
| 81 | 81 | ? fold("stale-pulls", `Pulls untouched for ${String(STALE_DAYS)} days`, String(stale), true, staleRows(ctx.auth.user!.kind !== "agent", stalePending)) |
| 82 | 82 | : null} |
src/web/pages/maintenance_budgets.ts +14 −6
| @@ -4,7 +4,8 @@ | ||
| 4 | 4 | * row to move either. A blank field, or the duty's own number, takes the duty's own back. |
| 5 | 5 | */ |
| 6 | 6 | |
| 7 | import { budgetOf, parseBudget, setBudget, usedToday, DOLLARS_MAX, DOLLARS_MIN, RUNS_MAX } from "../../maintain/budget.ts"; | |
| 7 | import { budgetsOf, parseBudget, setBudget, usedTodayAll, DOLLARS_MAX, DOLLARS_MIN, RUNS_MAX, type Budget } from "../../maintain/budget.ts"; | |
| 8 | import type { Duty } from "../../maintain/duty.ts"; | |
| 8 | 9 | import { duties } from "../../maintain/registry.ts"; |
| 9 | 10 | import type { App } from "../../app.ts"; |
| 10 | 11 | import type { Ctx } from "../../http/context.ts"; |
| @@ -13,12 +14,19 @@ import { html, type Html } from "../html.ts"; | ||
| 13 | 14 | import { flashTo } from "../layout.ts"; |
| 14 | 15 | import { btn, field as formControl } from "../ui.ts"; |
| 15 | 16 | |
| 17 | /** Every duty's budget row, in `names` order: two reads for all of them, not three a duty. */ | |
| 18 | export function budgetRows(app: App, names: readonly string[], now = new Date()): Html[] { | |
| 19 | const budgets = budgetsOf(app.db, duties); | |
| 20 | const used = usedTodayAll(app.db, now); | |
| 21 | return names.map((name) => { | |
| 22 | const d = duties.find((x) => x.name === name); | |
| 23 | const b = budgets.get(name); | |
| 24 | return d === undefined || b === undefined ? html`` : budgetOfRow(d, b, used.get(name) ?? { runs: 0, dollars: 0 }); | |
| 25 | }); | |
| 26 | } | |
| 27 | ||
| 16 | 28 | /** What a duty's row says of its budget: use today, the reason, and the two inputs. */ |
| 17 | export function budgetOfRow(app: App, name: string, now = new Date()): Html { | |
| 18 | const d = duties.find((x) => x.name === name); | |
| 19 | if (d === undefined) return html``; | |
| 20 | const b = budgetOf(app.db, d); | |
| 21 | const u = usedToday(app.db, d.name, now); | |
| 29 | function budgetOfRow(d: Duty, b: Budget, u: { runs: number; dollars: number }): Html { | |
| 22 | 30 | const row = (label: string, value: Html): Html => html`<div class="status-row"><dt>${label}</dt><dd>${value}</dd></div>`; |
| 23 | 31 | return html`${row(`${d.name} budget`, html`<span id="budget-${d.name}">${String(u.runs)} of ${String(b.runsPerDay)} runs, $${u.dollars.toFixed(2)} of $${b.dollarsPerDay.toFixed(2)} today</span>`)} |
| 24 | 32 | ${row("Why", html`<span class="muted">${d.budget.reason}</span>`)} |
test/backup-growth.test.ts added +99
| @@ -0,0 +1,99 @@ | ||
| 1 | /** Backup growth (#13032): steady files nothing, a doubled night files one defect with both | |
| 2 | * numbers, a repeat files none. */ | |
| 3 | import { expect, test } from "bun:test"; | |
| 4 | import { mkdtempSync } from "node:fs"; | |
| 5 | import { rm } from "node:fs/promises"; | |
| 6 | import { tmpdir } from "node:os"; | |
| 7 | import { join } from "node:path"; | |
| 8 | import { createApp } from "../src/app.ts"; | |
| 9 | import { startSnapshots } from "../src/snapshot/service.ts"; | |
| 10 | import { dirStore, type Store } from "../src/snapshot/store.ts"; | |
| 11 | import { rulesWorld } from "./helpers/rules-app.ts"; | |
| 12 | import { duty, TITLE } from "../src/maintain/duties/backup_growth.ts"; | |
| 13 | import { growthOf } from "../src/snapshot/growth.ts"; | |
outdated
START is the real clock. The duty's |
||
| 14 | import { lastSnapshotOutcome, recordSnapshotOutcome } from "../src/snapshot/outcome.ts"; | |
| 15 | ||
| 16 | process.env.FORGE_SENTINEL_REPO = "alice/rules"; | |
| 17 | const w = rulesWorld(); | |
| 18 | const MB = 1024 * 1024; | |
| 19 | const DAY = 86_400_000; | |
| 20 | // Nights end today, so an issue filed on the real clock (its `created_at` is not faked) is | |
| 21 | // later than the night it answers: the "closed for that night" test relies on that. | |
| 22 | const START = Date.now() - 14.5 * DAY; | |
| 23 | const WHERE = "https://bucket.example/forge"; | |
| 24 | ||
| 25 | /** Night `i` (0 is 1 Nov) lands with this size and duration. */ | |
| 26 | const night = (i: number, mb: number, seconds: number) => | |
| 27 | recordSnapshotOutcome(w.app.config.dataDir, { landed: `n${i}`, bytes: mb * MB, seconds, destination: WHERE }, new Date(START + i * DAY).toISOString()); | |
| 28 | const run = async (i: number) => await duty.run({ db: w.app.db, bus: w.app.bus, app: w.app, now: new Date(START + i * DAY + 3_600_000) }); | |
| 29 | const defects = () => w.titles().filter((t) => t === TITLE); | |
| 30 | ||
| 31 | test("the record keeps bytes and seconds a night, one a UTC day, newest last", async () => { | |
| 32 | night(0, 300, 600); | |
| 33 | night(0, 310, 620); | |
| 34 | night(1, 320, 640); | |
| 35 | const nights = lastSnapshotOutcome(w.app.config.dataDir)!.nights!; | |
| 36 | expect(nights.map((n) => [n.bytes / MB, n.seconds])).toEqual([[310, 620], [320, 640]]); | |
| 37 | }); | |
| 38 | ||
| 39 | test("steady nights, and fewer than three, file nothing; a doubled night files one defect", async () => { | |
| 40 | expect(growthOf([])).toBeUndefined(); | |
| 41 | expect((await run(1)).actions).toEqual([]); // two nights: too few | |
| 42 | night(2, 330, 650); | |
| 43 | expect((await run(2)).actions).toEqual([]); // three, steady | |
| 44 | expect(defects()).toHaveLength(0); | |
| 45 | night(3, 650, 1300); // over twice the median of 310, 320, 330 in both numbers | |
| 46 | expect((await run(3)).actions).toHaveLength(1); | |
| 47 | expect(defects()).toHaveLength(1); | |
| 48 | }); | |
| 49 | ||
| 50 | test("the defect names both numbers and the destination, and is not filed again while open", async () => { | |
| 51 | const body = w.app.db.query<{ body: string }, [string]>("SELECT body FROM issue WHERE title = ?1").get(TITLE)!.body; | |
| 52 | expect(body).toContain("650 MiB against a median of 320 MiB"); | |
| 53 | expect(body).toContain("1300 s against a median of 640 s"); | |
| 54 | expect(body).toContain(WHERE); | |
| 55 | night(4, 700, 1400); | |
| 56 | expect((await run(4)).actions).toEqual([]); | |
| 57 | expect(defects()).toHaveLength(1); | |
| 58 | }); | |
| 59 | ||
| 60 | test("a defect a person closed for a night is not filed again for that night", async () => { | |
| 61 | night(3, 650, 1300); // the doubled night is the last again | |
| 62 | w.app.db.query("UPDATE issue SET state = 'closed' WHERE title = ?1").run(TITLE); | |
| 63 | expect((await run(3)).actions).toEqual([]); | |
| 64 | expect(defects()).toHaveLength(1); | |
| 65 | }); | |
| 66 | ||
| 67 | test("a steady week after files nothing", async () => { | |
| 68 | const before = defects().length; | |
| 69 | for (let i = 5; i < 14; i++) night(i, 700, 1400); | |
| 70 | expect((await run(13)).actions).toEqual([]); | |
| 71 | expect(defects()).toHaveLength(before); | |
| 72 | }); | |
| 73 | ||
| 74 | test("duration alone is growth too, and a steady week reads as none", () => { | |
| 75 | const at = (i: number) => new Date(START + i * DAY).toISOString(); | |
| 76 | const nights = [10, 11, 12, 13].map((i) => ({ at: at(i), bytes: 700 * MB, seconds: 1400 })); | |
| 77 | expect(growthOf([...nights, { at: at(14), bytes: 710 * MB, seconds: 4000 }])!.over).toEqual(["seconds"]); | |
| 78 | expect(growthOf([...nights, { at: at(14), bytes: 710 * MB, seconds: 1500 }])!.over).toEqual([]); | |
| 79 | }); | |
| 80 | ||
| 81 | test("a snapshot the service takes records its bytes, seconds and destination as tonight's night", async () => { | |
| 82 | const root = mkdtempSync(join(tmpdir(), "forge-growth-svc-")); | |
| 83 | const app = createApp({ dataDir: join(root, "data"), port: 0 }); | |
| 84 | const store: Store = { ...dirStore(join(root, "store")), where: "growth-test-store" }; | |
| 85 | const svc = startSnapshots(app, store, { version: "t", nightly: false }); | |
| 86 | try { | |
| 87 | const m = await svc.now(); | |
| 88 | const nights = lastSnapshotOutcome(app.config.dataDir)!.nights!; | |
| 89 | expect(nights).toHaveLength(1); | |
| 90 | expect(nights[0]!.bytes).toBe(m.bytes); | |
| 91 | expect(m.bytes).toBeGreaterThan(0); | |
| 92 | expect(nights[0]!.seconds).toBe(Math.round((Date.parse(m.finishedAt) - Date.parse(m.startedAt)) / 1000)); | |
| 93 | expect(lastSnapshotOutcome(app.config.dataDir)!.destination).toBe("growth-test-store"); | |
| 94 | } finally { | |
| 95 | await svc.stop(); | |
| 96 | app.close(); | |
| 97 | await rm(root, { recursive: true, force: true }); | |
| 98 | } | |
| 99 | }, 30_000); |
test/maintain.test.ts +24
| @@ -352,3 +352,27 @@ test("the budget screen's keys move the chosen duty's budget, and a refused sess | ||
| 352 | 352 | press(agent, "+"); press(agent, "]"); |
| 353 | 353 | expect(budgetOf(app.db, d)).toEqual(own); |
| 354 | 354 | }); |
| 355 | ||
| 356 | test("every duty's budget and today's use, read in two statements, are what the per-duty reads say (#13032)", async () => { | |
| 357 | clear(); | |
| 358 | const { budgetOf, budgetsOf, setBudget, usedToday, usedTodayAll } = await import("../src/maintain/budget.ts"); | |
| 359 | const now = new Date("2026-11-11T12:00:00Z"); | |
| 360 | const ins = app.db.query("INSERT INTO maintenance_run (duty, started, finished, findings, actions, probes, outcome, dollars) VALUES (?1, ?2, ?2, '[]', '[]', '[]', ?3, ?4)"); | |
| 361 | ins.run(duties[0]!.name, "2026-11-11T01:00:00.000Z", "ok", 0.25); | |
| 362 | ins.run(duties[0]!.name, "2026-11-11T02:00:00.000Z", "ok", 0.5); | |
| 363 | ins.run(duties[0]!.name, "2026-11-11T03:00:00.000Z", "budget: runs per day, 2 of 2 used", 0); | |
| 364 | ins.run(duties[1]!.name, "2026-11-10T23:00:00.000Z", "ok", 1); | |
| 365 | setBudget(app.db, duties[1]!, "runs", 9, "u"); | |
| 366 | app.db.query("INSERT INTO forge_setting (key, value, updated_by, updated_at) VALUES (?1, 'nonsense', 'u', '')").run(`maintain.${duties[2]!.name}.dollars`); | |
| 367 | const budgets = budgetsOf(app.db, duties); | |
| 368 | const used = usedTodayAll(app.db, now); | |
| 369 | for (const d of duties) { | |
| 370 | expect(budgets.get(d.name)).toEqual(budgetOf(app.db, d)); | |
| 371 | expect(used.get(d.name) ?? { runs: 0, dollars: 0 }).toEqual(usedToday(app.db, d.name, now)); | |
| 372 | } | |
| 373 | expect(budgets.get(duties[1]!.name)!.runsPerDay).toBe(9); | |
| 374 | expect(used.get(duties[0]!.name)).toEqual({ runs: 2, dollars: 0.75 }); | |
| 375 | setBudget(app.db, duties[1]!, "runs", null, "u"); | |
| 376 | app.db.query("DELETE FROM forge_setting WHERE key = ?1").run(`maintain.${duties[2]!.name}.dollars`); | |
| 377 | clear(); | |
| 378 | }); |
new above
offline
Untested: bytes, seconds and destination reach the record only here. Add a test through startSnapshots that fails when these lines are removed.