Public: anyone can read this; the crew can change it.

The disk free duty files crew work only when the forge's own data fills the volume, and otherwise names the space outside the forge #14039

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/14038-the-disk-free-duty-files-crew-work-only → main 1 commit fork 1 thread · 1 unresolved

  • Merged. scout/14038-the-disk-free-duty-files-crew-work-only was merged into main as 2f85628.

Review

Sign in to review this change.
2 files changed +125 −19
src/maintain/duties/disk_free.ts +79 −14
@@ -1,8 +1,12 @@
11 /**
2 * Disk free (#13320): when the data volume has under 15% free, file one issue
3 * naming the number and the three largest directories under the data
4 * directory. One open issue stands for the shortage, however many ticks see it.
5 * Sizes are read with fs only; no process is started.
2 * Disk free (#13320): when the data volume has under 15% free, measure what
3 * the forge's own data holds of the used space (#14038). When it is half or
4 * more, file one issue naming the number and every top-level entry over 1 GiB
5 * under the data directory, so a builder can see what to prune. When it is
6 * less, the space is held outside the forge and no change to the code frees
7 * it: the duty returns only the finding, saying so. One open issue stands for
8 * the shortage, however many ticks see it. Sizes are read with fs only; no
9 * process is started.
610 */
711
812 import { lstatSync, readdirSync } from "node:fs";
@@ -49,19 +53,74 @@ export function largestDirs(dataDir: string, count = 3): DirSize[] {
4953 .slice(0, count);
5054 }
5155
52 const percent = (r: DiskReading): number => (r.totalBytes > 0 ? (r.freeBytes / r.totalBytes) * 100 : 100);
56 /** Bytes in plain files directly under the data directory (the database among them). */
57 function looseBytes(dataDir: string): number {
58 try {
59 return readdirSync(dataDir, { withFileTypes: true })
60 .filter((e) => e.isFile())
61 .reduce((sum, e) => sum + bytesUnder(join(dataDir, e.name)), 0);
62 } catch {
63 return 0;
64 }
65 }
5366
54 /** The issue's body: the free number and where the space went. */
55 export function bodyOf(r: DiskReading, dirs: readonly DirSize[]): string {
56 const list = dirs.length === 0 ? "- (none readable)" : dirs.map((d) => `- \`${d.name}\`: ${sizeSaid(d.bytes)}`).join("\n");
67 /** The name the files directly under the data directory are listed by. */
68 export const LOOSE = "(files directly under the data directory)";
69
70 /** Every top-level directory under `dataDir`, and its loose files as one entry, largest first. */
71 export function dataSizes(dataDir: string): DirSize[] {
72 const all = largestDirs(dataDir, Infinity);
73 const loose = looseBytes(dataDir);
74 if (loose > 0) all.push({ name: LOOSE, bytes: loose });
75 return all.sort((a, b) => b.bytes - a.bytes || a.name.localeCompare(b.name));
76 }
77
78 /** Entries at or above this size are listed in the issue. */
79 export const LISTED = 1024 ** 3;
80
81 /** What the forge's data holds of the used space, and what is held outside it. */
82 export interface Share {
83 readonly forgeBytes: number;
84 readonly usedBytes: number;
85 readonly outsideBytes: number;
86 /** The forge's data over the used space, 0..1; 1 when nothing is used. */
87 readonly fraction: number;
88 }
89
90 export function shareOf(r: DiskReading, dirs: readonly DirSize[]): Share {
91 const forgeBytes = dirs.reduce((sum, d) => sum + d.bytes, 0);
92 const usedBytes = Math.max(0, r.totalBytes - r.freeBytes);
93 const outsideBytes = Math.max(0, usedBytes - forgeBytes);
94 return { forgeBytes, usedBytes, outsideBytes, fraction: usedBytes > 0 ? Math.min(1, forgeBytes / usedBytes) : 1 };
95 }
96
97 /** `the forge's data: X of Y used (Z%); outside the forge's data directory: W` */
98 export function shareSaid(s: Share): string {
5799 return (
58 `_Filed by the disk free duty from the volume's own count, not written by a person._\n\n` +
59 `${percent(r).toFixed(1)}% of the data volume is free: ${sizeSaid(r.freeBytes)} of ${sizeSaid(r.totalBytes)}.\n\n` +
60 `Largest directories under the data directory:\n${list}`
100 `the forge's data: ${sizeSaid(s.forgeBytes)} of ${sizeSaid(s.usedBytes)} used (${(s.fraction * 100).toFixed(1)}%); ` +
101 `outside the forge's data directory: ${sizeSaid(s.outsideBytes)}`
61102 );
62103 }
63104
64 export function makeDuty(read: (dir: string) => DiskReading = readDisk, sizes: (dir: string) => DirSize[] = largestDirs): Duty {
105 /** Below this share of the used space, the space is held outside the forge. */
106 export const OURS = 0.5;
107 export const OUTSIDE = "held outside the forge: a person frees it on the machine";
108
109 const percent = (r: DiskReading): number => (r.totalBytes > 0 ? (r.freeBytes / r.totalBytes) * 100 : 100);
110
111 /** The issue's body: the free number, the forge's share, and every entry over 1 GiB. */
112 export function bodyOf(r: DiskReading, dirs: readonly DirSize[]): string {
113 const big = [...dirs].sort((a, b) => b.bytes - a.bytes || a.name.localeCompare(b.name)).filter((d) => d.bytes >= LISTED);
114 const list = big.length === 0 ? "- (none over 1 GiB)" : big.map((d) => `- \`${d.name}\`: ${sizeSaid(d.bytes)}`).join("\n");
115 return (
116 `_Filed by the disk free duty from the volume's own count, not written by a person._\n\n` +
117 `${percent(r).toFixed(1)}% of the data volume is free: ${sizeSaid(r.freeBytes)} of ${sizeSaid(r.totalBytes)}.\n\n` +
118 `${shareSaid(shareOf(r, dirs))}.\n\n` +
119 `Top-level entries under the data directory over 1 GiB, largest first:\n${list}`
120 );
121 }
122
123 export function makeDuty(read: (dir: string) => DiskReading = readDisk, sizes: (dir: string) => DirSize[] = dataSizes): Duty {
65124 return {
66125 name: "disk-free",
67126 intervalMs: 3_600_000,
@@ -76,14 +135,20 @@ export function makeDuty(read: (dir: string) => DiskReading = readDisk, sizes: (
76135 }
77136 const pct = percent(r);
78137 if (r.totalBytes <= 0 || pct >= LINE * 100) return { findings: [], actions: [] };
138 const dirs = sizes(app.config.dataDir);

Empty or unreadable dirs gives fraction 0, which is reported as 'held outside the forge'. Treat a measurement of nothing as unknown and say so.

139 const share = shareOf(r, dirs);
140 const outside = share.fraction < OURS;
79141 const finding = {
80 summary: `${pct.toFixed(1)}% of the data volume free (${sizeSaid(r.freeBytes)} of ${sizeSaid(r.totalBytes)})`,
142 summary:
143 `${pct.toFixed(1)}% of the data volume free (${sizeSaid(r.freeBytes)} of ${sizeSaid(r.totalBytes)}); ${shareSaid(share)}` +
144 (outside ? `; ${OUTSIDE}` : ""),
81145 probe: { kind: "measurement" as const, what: "percent of the data volume free", number: pct, at: now.toISOString() },
82146 };
147 if (outside) return { findings: [finding], actions: [] };
83148 const [owner = "", name = ""] = ownRepoPath().split("/");
84149 const repo = getRepo(db, owner, name);
85150 if (repo === undefined) return { findings: [finding], actions: [] };
86 const filed = fileIssueOnce(db, bus, repo, { title: TITLE, body: bodyOf(r, sizes(app.config.dataDir)) });
151 const filed = fileIssueOnce(db, bus, repo, { title: TITLE, body: bodyOf(r, dirs) });
87152 return { findings: [finding], actions: filed.opened ? [`filed disk free issue #${filed.number}`] : [] };
88153 },
89154 };
test/maintain-disk-free.test.ts +46 −5
@@ -1,11 +1,12 @@
1 /** Disk free (#13320): above the line nothing is filed; under it one issue with the number and the
2 * three largest directories, and a second tick files none. */
1 /** Disk free (#13320, #14038): above the line nothing is filed; under it, when the forge's data is
2 * half or more of the used space, one issue with the number and every top-level entry over 1 GiB,
3 * and a second tick files none; when the space is held outside the forge, only the finding. */
34 import { afterAll, expect, test } from "bun:test";
45 import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
56 import { tmpdir } from "node:os";
67 import { join } from "node:path";
78 import { rulesWorld } from "./helpers/rules-app.ts";
8 import { largestDirs, makeDuty, TITLE } from "../src/maintain/duties/disk_free.ts";
9 import { dataSizes, largestDirs, LOOSE, makeDuty, OUTSIDE, TITLE } from "../src/maintain/duties/disk_free.ts";
910 import { duties } from "../src/maintain/registry.ts";
1011
1112 process.env.FORGE_SENTINEL_REPO = "alice/rules";
@@ -16,6 +17,8 @@ const duty = makeDuty(() => ({ freeBytes: free, totalBytes: 100 * GB }), () => [
1617 { name: "repos", bytes: 30 * GB },
1718 { name: "backups", bytes: 20 * GB },
1819 { name: "runs", bytes: 10 * GB },
20 { name: "drop", bytes: 2 * GB },
21 { name: "tiny", bytes: 100 * 1024 ** 2 },
1922 ]);
2023 const run = async () => await duty.run({ db: w.app.db, bus: w.app.bus, app: w.app, now: new Date() });
2124 const filed = () => w.titles().filter((t) => t === TITLE);
@@ -27,7 +30,24 @@ test("above the line it files nothing", async () => {
2730 expect(filed()).toHaveLength(0);
2831 });
2932
30 test("under the line it files one issue with the number and three directories, once", async () => {
33 test("space held outside the forge files nothing and says so (#14037's reading)", async () => {
34 const outside = makeDuty(() => ({ freeBytes: 304 * GB, totalBytes: 2197 * GB }), () => [
35 { name: "repos", bytes: 30 * GB },
36 { name: "runs", bytes: 20 * GB },
37 { name: "backups", bytes: 10 * GB },
38 ]);
39 const out = await outside.run({ db: w.app.db, bus: w.app.bus, app: w.app, now: new Date() });
40 expect(out.actions).toEqual([]);
41 expect(out.findings).toHaveLength(1);
42 const summary = out.findings[0]!.summary;
43 expect(summary).toContain("the forge's data: 60.0 GiB of 1893.0 GiB used (3.2%)");
44 expect(summary).toContain("outside the forge's data directory: 1833.0 GiB");
45 expect(summary).toContain("outside the forge's data directory: ");
46 expect(summary.endsWith(OUTSIDE)).toBe(true);
47 expect(filed()).toHaveLength(0);
48 });
49
50 test("under the line it files one issue with the number, the share and every entry over 1 GiB, once", async () => {
3151 free = 10 * GB;
3252 const first = await run();
3353 expect(first.actions).toHaveLength(1);
@@ -36,7 +56,12 @@ test("under the line it files one issue with the number and three directories, o
3656 expect(filed()).toHaveLength(1);
3757 const body = w.app.db.query<{ body: string }, [string]>("SELECT body FROM issue WHERE title = ?1").get(TITLE)!.body;
3858 expect(body).toContain("10.0%");
39 for (const d of ["repos", "backups", "runs"]) expect(body).toContain(`\`${d}\``);
59 expect(body).toContain("the forge's data: 62.1 GiB of 90.0 GiB used (69.0%)");
60 expect(body).toContain("outside the forge's data directory: 27.9 GiB");
61 const at = ["repos", "backups", "runs", "drop"].map((d) => body.indexOf(`\`${d}\``));
62 expect(at.every((i) => i > 0)).toBe(true);
63 expect([...at].sort((a, b) => a - b)).toEqual(at); // largest first
64 expect(body).not.toContain("`tiny`");
4065 });
4166
4267 const made: string[] = [];
@@ -67,6 +92,22 @@ test("largestDirs reads sizes with fs, largest first, at most three", () => {
6792 expect(largestDirs(root).map((d) => d.name)).toEqual(["d", "b", "c"]);
6893 });
6994
95 test("dataSizes sums every top-level directory and the loose files, largest first", () => {
96 const root = mkdtemp();
97 for (const [n, b] of [["a", 1], ["b", 300], ["c", 20], ["d", 4000]] as const) {
98 mkdirSync(join(root, n), { recursive: true });
99 writeFileSync(join(root, n, "f"), "z".repeat(b));
100 }
101 writeFileSync(join(root, "forge.db"), "z".repeat(500));
102 expect(dataSizes(root)).toEqual([
103 { name: "d", bytes: 4000 },
104 { name: LOOSE, bytes: 500 },
105 { name: "b", bytes: 300 },
106 { name: "c", bytes: 20 },
107 { name: "a", bytes: 1 },
108 ]);
109 });
110
70111 test("the duty is in the registry", () => {
71112 expect(duties.map((d) => d.name)).toContain("disk-free");
72113 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs