Public: anyone can read this; the crew can change it.

A nightly backup that doubles in size or duration within a week files one defect with both numbers #13055

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/13032-a-nightly-backup-that-doubles-in-size-or → main 6 commits fork 2 threads · 2 unresolved

  • Merged. mason/13032-a-nightly-backup-that-doubles-in-size-or was merged into main as bf242d9.

Review

Sign in to review this change.
13 files changed +325 −10
docs/features/README.md +1
@@ -369,6 +369,7 @@ Read with the guide page [Agents: MCP, git, or the terminal](../guide-agents.md)
369369
370370 Read with the guide page [Backups and moving](../guide-backups-moving.md).
371371
372 - [A nightly backup that doubles](backup-growth.md)
372373 - [A branch that cannot be backed up](backup-refs.md)
373374 - [A forge is an origin](forge-origin.md)
374375 - [A pull proposed to an origin, and its state coming back](origin-pull-state.md)
docs/features/backup-growth.md added +18
@@ -0,0 +1,18 @@
1 # A nightly backup that doubles
2
3 The `backup-growth` maintenance duty (#13032) files one defect when the last nightly
4 backup took over twice what the nights before it took.
5
6 - **What it watches.** The last landed night against the median of the previous seven,
7 in bytes (what a restore writes) and in seconds (start to finish). Either number
8 over a factor of 2 counts; the defect names both numbers and the backup's destination
9 (the store's page-safe `where`, never a secret).
10 - **Where the record lives.** `snapshot/state.json` under `FORGE_APP_DATA`: one entry a
11 UTC day, newest last, a later backup the same day replacing that day's entry.
12 - **Minimum.** Nothing is read until three nights are recorded, the last included.
13 - **Freshness.** Only a night from the last two days is read: a backup that stopped
14 landing is another alarm's.
15 - **One defect.** While one is open, a later run files none (`fileIssueOnce`).
16 - **Closing it.** Closing the defect silences that night only: a defect closed since
17 that night landed is not filed again for it. A later night that grows files a new one.
18 - The duty runs hourly and spends no money.
src/maintain/budget.ts +22
@@ -49,6 +49,28 @@ export function budgetOf(db: Database, duty: Duty): Budget {
4949 };
5050 }
5151
52 /** The budget in force for each of `ds`, by duty name: one read for all of them, for a page that
53 * lists every duty (#13032). */
54 export function budgetsOf(db: Database, ds: readonly Duty[]): Map<string, Budget> {
55 const rows = db.query("SELECT key, value FROM forge_setting WHERE key LIKE 'maintain.%'").all() as { key: string; value: string }[];
56 const kept = new Map(rows.map((r) => [r.key, r.value]));
57 const read = (d: Duty, kind: BudgetKind): number => {
58 const v = kept.get(settingKey(d.name, kind));
59 return (v === undefined ? undefined : parseBudget(kind, v)) ?? declared(d, kind);
60 };
61 return new Map(ds.map((d) => [d.name, { runsPerDay: read(d, "runs"), dollarsPerDay: read(d, "dollars") }]));
62 }
63
64 /** What every duty has used today, by duty name, in one read; a duty with no run today is
65 * absent. */
66 export function usedTodayAll(db: Database, now: Date): Map<string, { runs: number; dollars: number }> {
67 const { from, to } = dayBounds(now);
68 const rows = db
69 .query("SELECT duty, COUNT(*) AS runs, COALESCE(SUM(dollars), 0) AS dollars FROM maintenance_run WHERE started >= ?1 AND started < ?2 AND outcome NOT LIKE 'budget:%' GROUP BY duty")
70 .all(from, to) as { duty: string; runs: number; dollars: number }[];
71 return new Map(rows.map((r) => [r.duty, { runs: r.runs, dollars: r.dollars }]));
72 }
73
5274 /** Keep a new value for a budget (`null`, or the duty's own number, goes back to the duty's own), and who moved it. */
5375 export function setBudget(db: Database, duty: Duty, kind: BudgetKind, given: number | null, actorId: string): void {
5476 // Only a value that differs from the declared one is stored, so a later change of the default still takes.
src/maintain/duties/backup_growth.ts added +59
@@ -0,0 +1,59 @@
1 /**
2 * Backup growth (#13032): when the last nightly backup took over twice the
3 * median of the seven before it, in bytes or in seconds, file one defect naming
4 * both numbers and the destination. One open defect stands for the growth; a
5 * night that already had one filed (and closed) is not filed again.
6 */
7
8 import { fileIssueOnce } from "../../checks/flake.ts";
9 import { getRepo } from "../../core/repos.ts";
10 import { ownRepoPath } from "../../kernel/own_repo.ts";
11 import { growthOf, type Growth, type Measure } from "../../snapshot/growth.ts";
12 import { lastSnapshotOutcome } from "../../snapshot/outcome.ts";
13 import { sizeSaid } from "../../system/db_size.ts";
14 import type { Duty } from "../duty.ts";
15
16 export const TITLE = "The nightly backup doubled in size or duration within a week";
17 const DAY_MS = 86_400_000;
18
19 const bytesSaid = (m: Measure): string => `${sizeSaid(m.last)} against a median of ${sizeSaid(m.median)}`;
20 const secondsSaid = (m: Measure): string => `${m.last} s against a median of ${Math.round(m.median)} s`;
21
22 /** The defect's body: both numbers whichever passed, and where the backup went. */
23 export function bodyOf(g: Growth, destination: string): string {
24 return (
25 `_Filed by the backup growth duty from the backup record, not written by a person._\n\n` +
26 `The night of ${g.night.at.slice(0, 10)} is over twice the median of the nights before it.\n\n` +
27 `- Size: ${bytesSaid(g.bytes)}${g.over.includes("bytes") ? " (over twice)" : ""}\n` +
28 `- Duration: ${secondsSaid(g.seconds)}${g.over.includes("seconds") ? " (over twice)" : ""}\n` +
29 `- Destination: ${destination}`
30 );
31 }
32
33 export const duty: Duty = {
34 name: "backup-growth",
35 intervalMs: 3_600_000,
36 budget: { runsPerDay: 30, dollarsPerDay: 1, reason: "hourly is 24 a day; the rest is room for a catch-up, and it spends no money" },
37 run({ db, bus, now, app }) {
38 if (app === undefined) return { findings: [], actions: [] };
39 const outcome = lastSnapshotOutcome(app.config.dataDir);
40 const g = growthOf(outcome?.nights ?? []);
41 // A backup that stopped landing is another alarm's; only a night from the last two days
42 // is read.
43 if (g === undefined || g.over.length === 0 || now.getTime() - Date.parse(g.night.at) > 2 * DAY_MS)
44 return { findings: [], actions: [] };
45 const destination = outcome?.destination ?? "unknown";
46 const summary = `last backup ${bytesSaid(g.bytes)}, ${secondsSaid(g.seconds)}; destination ${destination}`;
47 const finding = { summary, probe: { kind: "measurement" as const, what: "bytes of the last nightly backup", number: g.bytes.last, at: now.toISOString() } };
48 const [owner = "", name = ""] = ownRepoPath().split("/");
49 const repo = getRepo(db, owner, name);
50 if (repo === undefined) return { findings: [finding], actions: [] };
51 // Closed since this night landed: that was a person's answer to this night, not a new defect.
52 const seen = db
53 .query<{ n: number }, [string, string, string]>("SELECT COUNT(*) AS n FROM issue WHERE repo_id = ?1 AND title = ?2 AND created_at >= ?3")
54 .get(repo.id, TITLE, g.night.at)!.n;
55 if (seen > 0) return { findings: [finding], actions: [] };
56 const filed = fileIssueOnce(db, bus, repo, { title: TITLE, body: bodyOf(g, destination) });
57 return { findings: [finding], actions: filed.opened ? [`filed backup growth issue #${filed.number}`] : [] };
58 },
59 };
src/maintain/registry.ts +2 −1
@@ -6,6 +6,7 @@ import { duty as ruleWatcher } from "./duties/rule_watcher.ts";
66 import { duty as efficiencyWatcher } from "./duties/efficiency_watcher.ts";
77 import { duty as flowKeeper } from "./duties/flow_keeper.ts";
88 import { duty as visitorRetention } from "./duties/visitor_retention.ts";
9 import { duty as backupGrowth } from "./duties/backup_growth.ts";
910 import { duty as v1Proven } from "./duties/v1_proven.ts";
1011
11 export const duties: readonly Duty[] = [pullsWaiting, ruleWatcher, efficiencyWatcher, flowKeeper, visitorRetention, v1Proven];
12 export const duties: readonly Duty[] = [pullsWaiting, ruleWatcher, efficiencyWatcher, flowKeeper, visitorRetention, backupGrowth, v1Proven];
src/snapshot/growth.ts added +46
@@ -0,0 +1,46 @@
1 /**
2 * Whether the nightly backup is outgrowing its night (#13032): the last landed
3 * night against the median of the seven before it, in bytes and in seconds.
4 * Pure: nights in, a reading out. The record is `outcome.ts`.
5 */
6
7 import type { Night } from "./outcome.ts";
8
9 /** Nights compared with. */
10 export const COMPARE_NIGHTS = 7;
11 /** No reading until this many nights are recorded, last one included. */
12 export const MIN_NIGHTS = 3;
13 /** Over this many times the median is growth worth a defect. */
14 export const FACTOR = 2;
15
16 export interface Measure {
17 readonly last: number;
18 readonly median: number;
19 }
20
21 export interface Growth {
22 readonly night: Night;
23 readonly bytes: Measure;
24 readonly seconds: Measure;
25 /** Which of the two passed `FACTOR` times its median. */
26 readonly over: readonly ("bytes" | "seconds")[];
27 }
28
29 const median = (xs: readonly number[]): number => {
30 const s = [...xs].sort((a, b) => a - b);
31 const m = s.length >> 1;
32 return s.length % 2 === 1 ? s[m]! : (s[m - 1]! + s[m]!) / 2;
33 };
34
35 /** The reading of `nights` (oldest first), or undefined while too few are recorded. */
36 export function growthOf(nights: readonly Night[]): Growth | undefined {
37 if (nights.length < MIN_NIGHTS) return undefined;
38 const night = nights[nights.length - 1]!;
39 const before = nights.slice(-1 - COMPARE_NIGHTS, -1);
40 const bytes = { last: night.bytes, median: median(before.map((n) => n.bytes)) };
41 const seconds = { last: night.seconds, median: median(before.map((n) => n.seconds)) };
42 const over: ("bytes" | "seconds")[] = [];
43 if (bytes.median > 0 && bytes.last > FACTOR * bytes.median) over.push("bytes");
44 if (seconds.median > 0 && seconds.last > FACTOR * seconds.median) over.push("seconds");
45 return { night, bytes, seconds, over };
46 }
src/snapshot/outcome.ts +32
@@ -12,12 +12,29 @@ import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
1212 import { join } from "node:path";
1313 import type { FailureCause } from "./cause.ts";
1414
15 /** One landed snapshot's size and duration: bytes a restore writes, seconds from start to
16 * finish. */
17 export interface Night {
18 readonly at: string;
19 readonly bytes: number;
20 readonly seconds: number;
21 }
22
23 /** Nights kept in the record: a week to compare with, and a few spare. */
24 export const NIGHTS_KEPT = 14;
25
1526 export interface SnapshotOutcome {
1627 /** When the last attempt ended. */
1728 readonly at: string;
1829 /** The newest snapshot that landed, and when; absent until one has. */
1930 readonly landed?: string;
2031 readonly landedAt?: string;
32 /** Where the newest snapshot went (the store's `where`, never a secret); absent before it
33 * was kept. */
34 readonly destination?: string;
35 /** The nights that landed, newest last, at most `NIGHTS_KEPT`: what each took, for
36 * `growth.ts`. */
37 readonly nights?: readonly Night[];
2138 /** Whether the store it landed in is off the machine (a bucket, not a directory); absent when not known. */
2239 readonly offBox?: boolean;
2340 /**
@@ -61,12 +78,21 @@ export function offBoxAtOf(s: SnapshotOutcome | undefined): string | undefined {
6178 return s.offBox === true ? s.landedAt : undefined;
6279 }
6380
81 /** The nights with `night` added; a second landing on the same UTC day replaces the first. */
82 function nightsWith(nights: readonly Night[] | undefined, night: Night): Night[] {
83 const day = night.at.slice(0, 10);
84 return [...(nights ?? []).filter((n) => n.at.slice(0, 10) !== day), night].slice(-NIGHTS_KEPT);
85 }
86
6487 /** Keep an attempt's end: a name that landed, or the error it failed with. Returns what was kept. */
6588 export function recordSnapshotOutcome(
6689 dataDir: string,
6790 result:
6891 | {
6992 readonly landed: string;
93 readonly bytes?: number;
94 readonly seconds?: number;
95 readonly destination?: string;
7096 readonly offBox?: boolean;
7197 readonly brokenRefs?: SnapshotOutcome["brokenRefs"];
7298 readonly absent?: SnapshotOutcome["absent"];
@@ -80,6 +106,8 @@ export function recordSnapshotOutcome(
80106 ...(prev?.landedAt === undefined ? {} : { landedAt: prev.landedAt }),
81107 ...(prev?.offBox === undefined ? {} : { offBox: prev.offBox }),
82108 ...(offBoxAtOf(prev) === undefined ? {} : { offBoxAt: offBoxAtOf(prev)! }),
109 ...(prev?.nights === undefined ? {} : { nights: prev.nights }),
110 ...(prev?.destination === undefined ? {} : { destination: prev.destination }),
83111 };
84112 const next: SnapshotOutcome =
85113 "landed" in result
@@ -87,6 +115,10 @@ export function recordSnapshotOutcome(
87115 at,
88116 landed: result.landed,
89117 landedAt: at,
118 ...(result.destination === undefined ? {} : { destination: result.destination }),
119 ...(result.bytes === undefined || result.seconds === undefined
120 ? prev?.nights === undefined ? {} : { nights: prev.nights }
121 : { nights: nightsWith(prev?.nights, { at, bytes: result.bytes, seconds: result.seconds }) }),
90122 ...(result.offBox === undefined ? {} : { offBox: result.offBox }),
91123 ...(result.offBox === true ? { offBoxAt: at } : offBoxAtOf(prev) === undefined ? {} : { offBoxAt: offBoxAtOf(prev)! }),
92124 ...(result.brokenRefs === undefined || result.brokenRefs.length === 0 ? {} : { brokenRefs: result.brokenRefs }),
src/snapshot/service.ts +5
@@ -211,6 +211,7 @@ export function startSnapshots(
211211 const take = (): Promise<Manifest> => {
212212 // Whether this run's store is off the machine, for home's backup age (#3701).
213213 let offBox: boolean | undefined;
214 let destination: string | undefined;
214215 let began = new Date().toISOString();
215216 // Bytes landed so far, kept past the run's end so a failure can say how far it got (#13046).
216217 let sent = 0;
@@ -223,6 +224,7 @@ export function startSnapshots(
223224 const store = current();
224225 if (store === undefined) throw new Nowhere();
225226 offBox = isOffBox(store);
227 destination = store.where;
226228 running = { objects: 0, bytes: 0, elapsedMs: 0 };
227229 startedAt = Date.now();
228230 began = new Date(startedAt).toISOString();
@@ -272,6 +274,9 @@ export function startSnapshots(
272274 last = { manifest: m, at: m.finishedAt };
outdated

Untested: bytes, seconds and destination reach the record only here. Add a test through startSnapshots that fails when these lines are removed.

273275 keep({
274276 landed: m.name,
277 bytes: m.bytes,
278 seconds: Math.max(0, Math.round((Date.parse(m.finishedAt) - Date.parse(m.startedAt)) / 1000)),
279 ...(destination === undefined ? {} : { destination }),
275280 ...(offBox === undefined ? {} : { offBox }),
276281 ...(m.brokenRefs === undefined ? {} : { brokenRefs: m.brokenRefs }),
277282 absent: notInSnapshot(m).map((a) => ({ repo: a.repo, why: a.why })),
src/system/db_size.ts +1 −1
@@ -332,7 +332,7 @@ export function dbGrowth(db: Database, bytesNow: number, now: number = Date.now(
332332 }
333333
334334 /** Bytes said short: GiB with one decimal from 1 GiB, else whole MiB. */
335 const sizeSaid = (n: number): string => (n >= 1024 ** 3 ? `${(n / 1024 ** 3).toFixed(1)} GiB` : `${Math.round(n / 1024 ** 2)} MiB`);
335 export const sizeSaid = (n: number): string => (n >= 1024 ** 3 ? `${(n / 1024 ** 3).toFixed(1)} GiB` : `${Math.round(n / 1024 ** 2)} MiB`);
336336
337337 /** Rows said short: 812, 41k, 3.2M. */
338338 const rowsSaid = (n: number): string =>
src/web/pages/maintenance.ts +2 −2
@@ -17,7 +17,7 @@ import { icon } from "../icons.ts";
1717 import { flashTo, page } from "../layout.ts";
1818 import { badge, btn, disclosure, field } from "../ui.ts";
1919 import { STALE_DAYS, STALE_WORDS, closeStale, nudge, stalePulls } from "../../maintain/stale_pulls.ts";
20 import { budgetOfRow } from "./maintenance_budgets.ts";
20 import { budgetRows } from "./maintenance_budgets.ts";
2121
2222 function admin(ctx: Ctx): Response | undefined {
2323 if (!ctx.auth.user) return signInFirst(ctx);
@@ -76,7 +76,7 @@ export async function maintenancePage(ctx: Ctx): Promise<Response> {
7676 </form>`),
7777 )}
7878 </dl>`)}
79 ${fold("maintenance-budgets", "Budgets", String(states.length), false, html`<dl class="status-rows">${states.map((d) => budgetOfRow(ctx.app, d.name))}</dl>`)}
79 ${fold("maintenance-budgets", "Budgets", String(states.length), false, html`<dl class="status-rows">${budgetRows(ctx.app, states.map((d) => d.name))}</dl>`)}
8080 ${stale > 0
8181 ? fold("stale-pulls", `Pulls untouched for ${String(STALE_DAYS)} days`, String(stale), true, staleRows(ctx.auth.user!.kind !== "agent", stalePending))
8282 : null}
src/web/pages/maintenance_budgets.ts +14 −6
@@ -4,7 +4,8 @@
44 * row to move either. A blank field, or the duty's own number, takes the duty's own back.
55 */
66
7 import { budgetOf, parseBudget, setBudget, usedToday, DOLLARS_MAX, DOLLARS_MIN, RUNS_MAX } from "../../maintain/budget.ts";
7 import { budgetsOf, parseBudget, setBudget, usedTodayAll, DOLLARS_MAX, DOLLARS_MIN, RUNS_MAX, type Budget } from "../../maintain/budget.ts";
8 import type { Duty } from "../../maintain/duty.ts";
89 import { duties } from "../../maintain/registry.ts";
910 import type { App } from "../../app.ts";
1011 import type { Ctx } from "../../http/context.ts";
@@ -13,12 +14,19 @@ import { html, type Html } from "../html.ts";
1314 import { flashTo } from "../layout.ts";
1415 import { btn, field as formControl } from "../ui.ts";
1516
17 /** Every duty's budget row, in `names` order: two reads for all of them, not three a duty. */
18 export function budgetRows(app: App, names: readonly string[], now = new Date()): Html[] {
19 const budgets = budgetsOf(app.db, duties);
20 const used = usedTodayAll(app.db, now);
21 return names.map((name) => {
22 const d = duties.find((x) => x.name === name);
23 const b = budgets.get(name);
24 return d === undefined || b === undefined ? html`` : budgetOfRow(d, b, used.get(name) ?? { runs: 0, dollars: 0 });
25 });
26 }
27
1628 /** What a duty's row says of its budget: use today, the reason, and the two inputs. */
17 export function budgetOfRow(app: App, name: string, now = new Date()): Html {
18 const d = duties.find((x) => x.name === name);
19 if (d === undefined) return html``;
20 const b = budgetOf(app.db, d);
21 const u = usedToday(app.db, d.name, now);
29 function budgetOfRow(d: Duty, b: Budget, u: { runs: number; dollars: number }): Html {
2230 const row = (label: string, value: Html): Html => html`<div class="status-row"><dt>${label}</dt><dd>${value}</dd></div>`;
2331 return html`${row(`${d.name} budget`, html`<span id="budget-${d.name}">${String(u.runs)} of ${String(b.runsPerDay)} runs, $${u.dollars.toFixed(2)} of $${b.dollarsPerDay.toFixed(2)} today</span>`)}
2432 ${row("Why", html`<span class="muted">${d.budget.reason}</span>`)}
test/backup-growth.test.ts added +99
@@ -0,0 +1,99 @@
1 /** Backup growth (#13032): steady files nothing, a doubled night files one defect with both
2 * numbers, a repeat files none. */
3 import { expect, test } from "bun:test";
4 import { mkdtempSync } from "node:fs";
5 import { rm } from "node:fs/promises";
6 import { tmpdir } from "node:os";
7 import { join } from "node:path";
8 import { createApp } from "../src/app.ts";
9 import { startSnapshots } from "../src/snapshot/service.ts";
10 import { dirStore, type Store } from "../src/snapshot/store.ts";
11 import { rulesWorld } from "./helpers/rules-app.ts";
12 import { duty, TITLE } from "../src/maintain/duties/backup_growth.ts";
13 import { growthOf } from "../src/snapshot/growth.ts";
outdated

START is the real clock. The duty's now is fake, but the issue's created_at is real, so the "closed for that night" test depends on real time being later than the fake nights. Note it, or inject the clock.

14 import { lastSnapshotOutcome, recordSnapshotOutcome } from "../src/snapshot/outcome.ts";
15
16 process.env.FORGE_SENTINEL_REPO = "alice/rules";
17 const w = rulesWorld();
18 const MB = 1024 * 1024;
19 const DAY = 86_400_000;
20 // Nights end today, so an issue filed on the real clock (its `created_at` is not faked) is
21 // later than the night it answers: the "closed for that night" test relies on that.
22 const START = Date.now() - 14.5 * DAY;
23 const WHERE = "https://bucket.example/forge";
24
25 /** Night `i` (0 is 1 Nov) lands with this size and duration. */
26 const night = (i: number, mb: number, seconds: number) =>
27 recordSnapshotOutcome(w.app.config.dataDir, { landed: `n${i}`, bytes: mb * MB, seconds, destination: WHERE }, new Date(START + i * DAY).toISOString());
28 const run = async (i: number) => await duty.run({ db: w.app.db, bus: w.app.bus, app: w.app, now: new Date(START + i * DAY + 3_600_000) });
29 const defects = () => w.titles().filter((t) => t === TITLE);
30
31 test("the record keeps bytes and seconds a night, one a UTC day, newest last", async () => {
32 night(0, 300, 600);
33 night(0, 310, 620);
34 night(1, 320, 640);
35 const nights = lastSnapshotOutcome(w.app.config.dataDir)!.nights!;
36 expect(nights.map((n) => [n.bytes / MB, n.seconds])).toEqual([[310, 620], [320, 640]]);
37 });
38
39 test("steady nights, and fewer than three, file nothing; a doubled night files one defect", async () => {
40 expect(growthOf([])).toBeUndefined();
41 expect((await run(1)).actions).toEqual([]); // two nights: too few
42 night(2, 330, 650);
43 expect((await run(2)).actions).toEqual([]); // three, steady
44 expect(defects()).toHaveLength(0);
45 night(3, 650, 1300); // over twice the median of 310, 320, 330 in both numbers
46 expect((await run(3)).actions).toHaveLength(1);
47 expect(defects()).toHaveLength(1);
48 });
49
50 test("the defect names both numbers and the destination, and is not filed again while open", async () => {
51 const body = w.app.db.query<{ body: string }, [string]>("SELECT body FROM issue WHERE title = ?1").get(TITLE)!.body;
52 expect(body).toContain("650 MiB against a median of 320 MiB");
53 expect(body).toContain("1300 s against a median of 640 s");
54 expect(body).toContain(WHERE);
55 night(4, 700, 1400);
56 expect((await run(4)).actions).toEqual([]);
57 expect(defects()).toHaveLength(1);
58 });
59
60 test("a defect a person closed for a night is not filed again for that night", async () => {
61 night(3, 650, 1300); // the doubled night is the last again
62 w.app.db.query("UPDATE issue SET state = 'closed' WHERE title = ?1").run(TITLE);
63 expect((await run(3)).actions).toEqual([]);
64 expect(defects()).toHaveLength(1);
65 });
66
67 test("a steady week after files nothing", async () => {
68 const before = defects().length;
69 for (let i = 5; i < 14; i++) night(i, 700, 1400);
70 expect((await run(13)).actions).toEqual([]);
71 expect(defects()).toHaveLength(before);
72 });
73
74 test("duration alone is growth too, and a steady week reads as none", () => {
75 const at = (i: number) => new Date(START + i * DAY).toISOString();
76 const nights = [10, 11, 12, 13].map((i) => ({ at: at(i), bytes: 700 * MB, seconds: 1400 }));
77 expect(growthOf([...nights, { at: at(14), bytes: 710 * MB, seconds: 4000 }])!.over).toEqual(["seconds"]);
78 expect(growthOf([...nights, { at: at(14), bytes: 710 * MB, seconds: 1500 }])!.over).toEqual([]);
79 });
80
81 test("a snapshot the service takes records its bytes, seconds and destination as tonight's night", async () => {
82 const root = mkdtempSync(join(tmpdir(), "forge-growth-svc-"));
83 const app = createApp({ dataDir: join(root, "data"), port: 0 });
84 const store: Store = { ...dirStore(join(root, "store")), where: "growth-test-store" };
85 const svc = startSnapshots(app, store, { version: "t", nightly: false });
86 try {
87 const m = await svc.now();
88 const nights = lastSnapshotOutcome(app.config.dataDir)!.nights!;
89 expect(nights).toHaveLength(1);
90 expect(nights[0]!.bytes).toBe(m.bytes);
91 expect(m.bytes).toBeGreaterThan(0);
92 expect(nights[0]!.seconds).toBe(Math.round((Date.parse(m.finishedAt) - Date.parse(m.startedAt)) / 1000));
93 expect(lastSnapshotOutcome(app.config.dataDir)!.destination).toBe("growth-test-store");
94 } finally {
95 await svc.stop();
96 app.close();
97 await rm(root, { recursive: true, force: true });
98 }
99 }, 30_000);
test/maintain.test.ts +24
@@ -352,3 +352,27 @@ test("the budget screen's keys move the chosen duty's budget, and a refused sess
352352 press(agent, "+"); press(agent, "]");
353353 expect(budgetOf(app.db, d)).toEqual(own);
354354 });
355
356 test("every duty's budget and today's use, read in two statements, are what the per-duty reads say (#13032)", async () => {
357 clear();
358 const { budgetOf, budgetsOf, setBudget, usedToday, usedTodayAll } = await import("../src/maintain/budget.ts");
359 const now = new Date("2026-11-11T12:00:00Z");
360 const ins = app.db.query("INSERT INTO maintenance_run (duty, started, finished, findings, actions, probes, outcome, dollars) VALUES (?1, ?2, ?2, '[]', '[]', '[]', ?3, ?4)");
361 ins.run(duties[0]!.name, "2026-11-11T01:00:00.000Z", "ok", 0.25);
362 ins.run(duties[0]!.name, "2026-11-11T02:00:00.000Z", "ok", 0.5);
363 ins.run(duties[0]!.name, "2026-11-11T03:00:00.000Z", "budget: runs per day, 2 of 2 used", 0);
364 ins.run(duties[1]!.name, "2026-11-10T23:00:00.000Z", "ok", 1);
365 setBudget(app.db, duties[1]!, "runs", 9, "u");
366 app.db.query("INSERT INTO forge_setting (key, value, updated_by, updated_at) VALUES (?1, 'nonsense', 'u', '')").run(`maintain.${duties[2]!.name}.dollars`);
367 const budgets = budgetsOf(app.db, duties);
368 const used = usedTodayAll(app.db, now);
369 for (const d of duties) {
370 expect(budgets.get(d.name)).toEqual(budgetOf(app.db, d));
371 expect(used.get(d.name) ?? { runs: 0, dollars: 0 }).toEqual(usedToday(app.db, d.name, now));
372 }
373 expect(budgets.get(duties[1]!.name)!.runsPerDay).toBe(9);
374 expect(used.get(duties[0]!.name)).toEqual({ runs: 2, dollars: 0.75 });
375 setBudget(app.db, duties[1]!, "runs", null, "u");
376 app.db.query("DELETE FROM forge_setting WHERE key = ?1").run(`maintain.${duties[2]!.name}.dollars`);
377 clear();
378 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs