Public: anyone can read this; the crew can change it.

One slow backup night is a finding, not an issue: the growth duty files on duration only after two slow nights in a row #15233

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/15232-one-slow-backup-night-is-a-finding-not-a → main 1 commit fork

  • Merged. mason/15232-one-slow-backup-night-is-a-finding-not-a was merged into main as bb1c5a6.

Review

Sign in to review this change.
3 files changed +68 −13
src/maintain/duties/backup_growth.ts +22 −12
@@ -1,8 +1,8 @@
11 /**
22 * Backup growth (#13032): when the last nightly backup took over twice the
3 * median of the seven before it, in bytes or in seconds, file one defect naming
4 * both numbers and the destination. One open defect stands for the growth; a
5 * night that already had one filed (and closed) is not filed again.
3 * median of the seven before it in bytes, or the last two nights each were in seconds
4 * (#15232), file one defect naming the numbers and the destination. One open defect stands for the
5 * growth; a night that already had one filed (and closed) is not filed again.
66 */
77
88 import { fileIssueOnce } from "../../checks/flake.ts";
@@ -19,15 +19,25 @@ const DAY_MS = 86_400_000;
1919 const bytesSaid = (m: Measure): string => `${sizeSaid(m.last)} against a median of ${sizeSaid(m.median)}`;
2020 const secondsSaid = (m: Measure): string => `${m.last} s against a median of ${Math.round(m.median)} s`;
2121
22 /** The defect's body: both numbers whichever passed, and where the backup went. */
22 /**
23 * The defect's body: which measure tripped, its numbers (both nights for duration), and where to
24 * look.
25 */
2326 export function bodyOf(g: Growth, destination: string): string {
24 return (
25 `_Filed by the backup growth duty from the backup record, not written by a person._\n\n` +
26 `The night of ${g.night.at.slice(0, 10)} is over twice the median of the nights before it.\n\n` +
27 `- Size: ${bytesSaid(g.bytes)}${g.over.includes("bytes") ? " (over twice)" : ""}\n` +
28 `- Duration: ${secondsSaid(g.seconds)}${g.over.includes("seconds") ? " (over twice)" : ""}\n` +
29 `- Destination: ${destination}`
30 );
27 const size = g.files.includes("bytes");
28 const slow = g.files.includes("seconds");
29 const prev = g.previous;
30 const facts = [
31 `- Size, night of ${g.night.at.slice(0, 10)}: ${bytesSaid(g.bytes)}${size ? " (over twice)" : ""}`,
32 `- Duration, night of ${g.night.at.slice(0, 10)}: ${secondsSaid(g.seconds)}${g.over.includes("seconds") ? " (over twice)" : ""}`,
33 ];
34 if (slow && prev !== undefined) facts.push(`- Duration, night of ${prev.night.at.slice(0, 10)}: ${secondsSaid(prev.seconds)} (over twice)`);
35 facts.push(`- Destination: ${destination}`);
36 const tripped = [size ? "size (one night over twice its median)" : "", slow ? "duration (two nights in a row over twice their median)" : ""].filter(Boolean);
37 const look = slow && !size
38 ? `Look at the upload path to the destination (network, bucket, throttling), not the code: the size barely moved.`
39 : `Look at what grew in the data directory, then at the upload path to the destination.`;
40 return [`_Filed by the backup growth duty from the backup record, not written by a person._`, `Tripped: ${tripped.join("; ")}.`, facts.join("\n"), look].join("\n\n");
3141 }
3242
3343 export const duty: Duty = {
@@ -47,7 +57,7 @@ export const duty: Duty = {
4757 const finding = { summary, probe: { kind: "measurement" as const, what: "bytes of the last nightly backup", number: g.bytes.last, at: now.toISOString() } };
4858 const [owner = "", name = ""] = ownRepoPath().split("/");
4959 const repo = getRepo(db, owner, name);
50 if (repo === undefined) return { findings: [finding], actions: [] };
60 if (repo === undefined || g.files.length === 0) return { findings: [finding], actions: [] };
5161 // Closed since this night landed: that was a person's answer to this night, not a new defect.
5262 const seen = db
5363 .query<{ n: number }, [string, string, string]>("SELECT COUNT(*) AS n FROM issue WHERE repo_id = ?1 AND title = ?2 AND created_at >= ?3")
src/snapshot/growth.ts +12 −1
@@ -24,6 +24,15 @@ export interface Growth {
2424 readonly seconds: Measure;
2525 /** Which of the two passed `FACTOR` times its median. */
2626 readonly over: readonly ("bytes" | "seconds")[];
27 /**
28 * The night before the last, read against the seven before it; undefined while too few nights.
29 */
30 readonly previous?: Growth | undefined;
31 /**
32 * What is worth a defect: bytes on a single night over `FACTOR`, seconds only when the night
33 * before was over `FACTOR` too (#15232). One slow upload night is a finding, not a defect.
34 */
35 readonly files: readonly ("bytes" | "seconds")[];
2736 }
2837
2938 const median = (xs: readonly number[]): number => {
@@ -42,5 +51,7 @@ export function growthOf(nights: readonly Night[]): Growth | undefined {
4251 const over: ("bytes" | "seconds")[] = [];
4352 if (bytes.median > 0 && bytes.last > FACTOR * bytes.median) over.push("bytes");
4453 if (seconds.median > 0 && seconds.last > FACTOR * seconds.median) over.push("seconds");
45 return { night, bytes, seconds, over };
54 const previous = growthOf(nights.slice(0, -1));
55 const files = over.filter((m) => m === "bytes" || previous?.over.includes("seconds") === true);
56 return { night, bytes, seconds, over, previous, files };
4657 }
test/backup-growth.test.ts +34
@@ -75,9 +75,43 @@ test("duration alone is growth too, and a steady week reads as none", () => {
7575 const at = (i: number) => new Date(START + i * DAY).toISOString();
7676 const nights = [10, 11, 12, 13].map((i) => ({ at: at(i), bytes: 700 * MB, seconds: 1400 }));
7777 expect(growthOf([...nights, { at: at(14), bytes: 710 * MB, seconds: 4000 }])!.over).toEqual(["seconds"]);
78 expect(growthOf([...nights, { at: at(14), bytes: 710 * MB, seconds: 4000 }])!.files).toEqual([]); // one slow night: a finding only
7879 expect(growthOf([...nights, { at: at(14), bytes: 710 * MB, seconds: 1500 }])!.over).toEqual([]);
7980 });
8081
82 const mk = (secs: number[], mb: number[] = secs.map(() => 700)) =>
83 secs.map((seconds, i) => ({ at: new Date(START + i * DAY).toISOString(), bytes: mb[i]! * MB, seconds }));
84
85 test("duration files only when two nights in a row are over twice the median; size files on one", () => {
86 const steady = [1400, 1400, 1400, 1400, 1400];
87 expect(growthOf(mk([...steady, 4000]))!.files).toEqual([]); // one slow night
88 expect(growthOf(mk([...steady, 4000, 1400]))!.files).toEqual([]); // slow, then normal
89 const two = growthOf(mk([...steady, 4000, 4100]))!;
90 expect(two.files).toEqual(["seconds"]);
91 expect(two.previous!.seconds.last).toBe(4000);
92 expect(growthOf(mk([...steady.map(() => 1400), 1400], [700, 700, 700, 700, 700, 2000]))!.files).toEqual(["bytes"]);
93 });
94
95 test("one slow night is a finding and opens no issue; the second in a row files one, with both nights", async () => {
96 w.app.db.query("UPDATE issue SET created_at = ?2 WHERE title = ?1").run(TITLE, new Date(START).toISOString()); // the earlier night's defect is old news
97 const before = defects().length;
98 for (let i = 5; i < 12; i++) night(i, 700, 1400);
99 night(12, 710, 4000);
100 const one = await run(12);
101 expect(one.findings).toHaveLength(1);
102 expect(one.actions).toEqual([]);
103 expect(defects()).toHaveLength(before);
104 night(13, 710, 4200);
105 const two = await run(13);
106 expect(two.actions).toHaveLength(1);
107 expect(defects()).toHaveLength(before + 1);
108 const bodies = w.app.db.query<{ body: string }, [string]>("SELECT body FROM issue WHERE title = ?1 ORDER BY id DESC").get(TITLE)!.body;
109 expect(bodies).toContain("duration");
110 expect(bodies).toContain("4000 s");
111 expect(bodies).toContain("4200 s");
112 expect(bodies).toContain("upload path");
113 });
114
81115 test("a snapshot the service takes records its bytes, seconds and destination as tonight's night", async () => {
82116 const root = mkdtempSync(join(tmpdir(), "forge-growth-svc-"));
83117 const app = createApp({ dataDir: join(root, "data"), port: 0 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs