Public: anyone can read this; the crew can change it.
One slow backup night is a finding, not an issue: the growth duty files on duration only after two slow nights in a row #15233
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/15232-one-slow-backup-night-is-a-finding-not-a → main 1 commit fork
3 files
changed +68
−13
src/maintain/duties/backup_growth.ts +22 −12
| @@ -1,8 +1,8 @@ | ||
| 1 | 1 | /** |
| 2 | 2 | * Backup growth (#13032): when the last nightly backup took over twice the |
| 3 | * median of the seven before it, in bytes or in seconds, file one defect naming | |
| 4 | * both numbers and the destination. One open defect stands for the growth; a | |
| 5 | * night that already had one filed (and closed) is not filed again. | |
| 3 | * median of the seven before it in bytes, or the last two nights each were in seconds | |
| 4 | * (#15232), file one defect naming the numbers and the destination. One open defect stands for the | |
| 5 | * growth; a night that already had one filed (and closed) is not filed again. | |
| 6 | 6 | */ |
| 7 | 7 | |
| 8 | 8 | import { fileIssueOnce } from "../../checks/flake.ts"; |
| @@ -19,15 +19,25 @@ const DAY_MS = 86_400_000; | ||
| 19 | 19 | const bytesSaid = (m: Measure): string => `${sizeSaid(m.last)} against a median of ${sizeSaid(m.median)}`; |
| 20 | 20 | const secondsSaid = (m: Measure): string => `${m.last} s against a median of ${Math.round(m.median)} s`; |
| 21 | 21 | |
| 22 | /** The defect's body: both numbers whichever passed, and where the backup went. */ | |
| 22 | /** | |
| 23 | * The defect's body: which measure tripped, its numbers (both nights for duration), and where to | |
| 24 | * look. | |
| 25 | */ | |
| 23 | 26 | export function bodyOf(g: Growth, destination: string): string { |
| 24 | return ( | |
| 25 | `_Filed by the backup growth duty from the backup record, not written by a person._\n\n` + | |
| 26 | `The night of ${g.night.at.slice(0, 10)} is over twice the median of the nights before it.\n\n` + | |
| 27 | `- Size: ${bytesSaid(g.bytes)}${g.over.includes("bytes") ? " (over twice)" : ""}\n` + | |
| 28 | `- Duration: ${secondsSaid(g.seconds)}${g.over.includes("seconds") ? " (over twice)" : ""}\n` + | |
| 29 | `- Destination: ${destination}` | |
| 30 | ); | |
| 27 | const size = g.files.includes("bytes"); | |
| 28 | const slow = g.files.includes("seconds"); | |
| 29 | const prev = g.previous; | |
| 30 | const facts = [ | |
| 31 | `- Size, night of ${g.night.at.slice(0, 10)}: ${bytesSaid(g.bytes)}${size ? " (over twice)" : ""}`, | |
| 32 | `- Duration, night of ${g.night.at.slice(0, 10)}: ${secondsSaid(g.seconds)}${g.over.includes("seconds") ? " (over twice)" : ""}`, | |
| 33 | ]; | |
| 34 | if (slow && prev !== undefined) facts.push(`- Duration, night of ${prev.night.at.slice(0, 10)}: ${secondsSaid(prev.seconds)} (over twice)`); | |
| 35 | facts.push(`- Destination: ${destination}`); | |
| 36 | const tripped = [size ? "size (one night over twice its median)" : "", slow ? "duration (two nights in a row over twice their median)" : ""].filter(Boolean); | |
| 37 | const look = slow && !size | |
| 38 | ? `Look at the upload path to the destination (network, bucket, throttling), not the code: the size barely moved.` | |
| 39 | : `Look at what grew in the data directory, then at the upload path to the destination.`; | |
| 40 | return [`_Filed by the backup growth duty from the backup record, not written by a person._`, `Tripped: ${tripped.join("; ")}.`, facts.join("\n"), look].join("\n\n"); | |
| 31 | 41 | } |
| 32 | 42 | |
| 33 | 43 | export const duty: Duty = { |
| @@ -47,7 +57,7 @@ export const duty: Duty = { | ||
| 47 | 57 | const finding = { summary, probe: { kind: "measurement" as const, what: "bytes of the last nightly backup", number: g.bytes.last, at: now.toISOString() } }; |
| 48 | 58 | const [owner = "", name = ""] = ownRepoPath().split("/"); |
| 49 | 59 | const repo = getRepo(db, owner, name); |
| 50 | if (repo === undefined) return { findings: [finding], actions: [] }; | |
| 60 | if (repo === undefined || g.files.length === 0) return { findings: [finding], actions: [] }; | |
| 51 | 61 | // Closed since this night landed: that was a person's answer to this night, not a new defect. |
| 52 | 62 | const seen = db |
| 53 | 63 | .query<{ n: number }, [string, string, string]>("SELECT COUNT(*) AS n FROM issue WHERE repo_id = ?1 AND title = ?2 AND created_at >= ?3") |
src/snapshot/growth.ts +12 −1
| @@ -24,6 +24,15 @@ export interface Growth { | ||
| 24 | 24 | readonly seconds: Measure; |
| 25 | 25 | /** Which of the two passed `FACTOR` times its median. */ |
| 26 | 26 | readonly over: readonly ("bytes" | "seconds")[]; |
| 27 | /** | |
| 28 | * The night before the last, read against the seven before it; undefined while too few nights. | |
| 29 | */ | |
| 30 | readonly previous?: Growth | undefined; | |
| 31 | /** | |
| 32 | * What is worth a defect: bytes on a single night over `FACTOR`, seconds only when the night | |
| 33 | * before was over `FACTOR` too (#15232). One slow upload night is a finding, not a defect. | |
| 34 | */ | |
| 35 | readonly files: readonly ("bytes" | "seconds")[]; | |
| 27 | 36 | } |
| 28 | 37 | |
| 29 | 38 | const median = (xs: readonly number[]): number => { |
| @@ -42,5 +51,7 @@ export function growthOf(nights: readonly Night[]): Growth | undefined { | ||
| 42 | 51 | const over: ("bytes" | "seconds")[] = []; |
| 43 | 52 | if (bytes.median > 0 && bytes.last > FACTOR * bytes.median) over.push("bytes"); |
| 44 | 53 | if (seconds.median > 0 && seconds.last > FACTOR * seconds.median) over.push("seconds"); |
| 45 | return { night, bytes, seconds, over }; | |
| 54 | const previous = growthOf(nights.slice(0, -1)); | |
| 55 | const files = over.filter((m) => m === "bytes" || previous?.over.includes("seconds") === true); | |
| 56 | return { night, bytes, seconds, over, previous, files }; | |
| 46 | 57 | } |
test/backup-growth.test.ts +34
| @@ -75,9 +75,43 @@ test("duration alone is growth too, and a steady week reads as none", () => { | ||
| 75 | 75 | const at = (i: number) => new Date(START + i * DAY).toISOString(); |
| 76 | 76 | const nights = [10, 11, 12, 13].map((i) => ({ at: at(i), bytes: 700 * MB, seconds: 1400 })); |
| 77 | 77 | expect(growthOf([...nights, { at: at(14), bytes: 710 * MB, seconds: 4000 }])!.over).toEqual(["seconds"]); |
| 78 | expect(growthOf([...nights, { at: at(14), bytes: 710 * MB, seconds: 4000 }])!.files).toEqual([]); // one slow night: a finding only | |
| 78 | 79 | expect(growthOf([...nights, { at: at(14), bytes: 710 * MB, seconds: 1500 }])!.over).toEqual([]); |
| 79 | 80 | }); |
| 80 | 81 | |
| 82 | const mk = (secs: number[], mb: number[] = secs.map(() => 700)) => | |
| 83 | secs.map((seconds, i) => ({ at: new Date(START + i * DAY).toISOString(), bytes: mb[i]! * MB, seconds })); | |
| 84 | ||
| 85 | test("duration files only when two nights in a row are over twice the median; size files on one", () => { | |
| 86 | const steady = [1400, 1400, 1400, 1400, 1400]; | |
| 87 | expect(growthOf(mk([...steady, 4000]))!.files).toEqual([]); // one slow night | |
| 88 | expect(growthOf(mk([...steady, 4000, 1400]))!.files).toEqual([]); // slow, then normal | |
| 89 | const two = growthOf(mk([...steady, 4000, 4100]))!; | |
| 90 | expect(two.files).toEqual(["seconds"]); | |
| 91 | expect(two.previous!.seconds.last).toBe(4000); | |
| 92 | expect(growthOf(mk([...steady.map(() => 1400), 1400], [700, 700, 700, 700, 700, 2000]))!.files).toEqual(["bytes"]); | |
| 93 | }); | |
| 94 | ||
| 95 | test("one slow night is a finding and opens no issue; the second in a row files one, with both nights", async () => { | |
| 96 | w.app.db.query("UPDATE issue SET created_at = ?2 WHERE title = ?1").run(TITLE, new Date(START).toISOString()); // the earlier night's defect is old news | |
| 97 | const before = defects().length; | |
| 98 | for (let i = 5; i < 12; i++) night(i, 700, 1400); | |
| 99 | night(12, 710, 4000); | |
| 100 | const one = await run(12); | |
| 101 | expect(one.findings).toHaveLength(1); | |
| 102 | expect(one.actions).toEqual([]); | |
| 103 | expect(defects()).toHaveLength(before); | |
| 104 | night(13, 710, 4200); | |
| 105 | const two = await run(13); | |
| 106 | expect(two.actions).toHaveLength(1); | |
| 107 | expect(defects()).toHaveLength(before + 1); | |
| 108 | const bodies = w.app.db.query<{ body: string }, [string]>("SELECT body FROM issue WHERE title = ?1 ORDER BY id DESC").get(TITLE)!.body; | |
| 109 | expect(bodies).toContain("duration"); | |
| 110 | expect(bodies).toContain("4000 s"); | |
| 111 | expect(bodies).toContain("4200 s"); | |
| 112 | expect(bodies).toContain("upload path"); | |
| 113 | }); | |
| 114 | ||
| 81 | 115 | test("a snapshot the service takes records its bytes, seconds and destination as tonight's night", async () => { |
| 82 | 116 | const root = mkdtempSync(join(tmpdir(), "forge-growth-svc-")); |
| 83 | 117 | const app = createApp({ dataDir: join(root, "data"), port: 0 }); |
new above
offline