Public: anyone can read this; the crew can change it.
The disk free duty files crew work only when the forge's own data fills the volume, and otherwise names the space outside the forge #14039
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/14038-the-disk-free-duty-files-crew-work-only → main 1 commit fork 1 thread · 1 unresolved
2 files
changed +125
−19
src/maintain/duties/disk_free.ts +79 −14
| @@ -1,8 +1,12 @@ | ||
| 1 | 1 | /** |
| 2 | * Disk free (#13320): when the data volume has under 15% free, file one issue | |
| 3 | * naming the number and the three largest directories under the data | |
| 4 | * directory. One open issue stands for the shortage, however many ticks see it. | |
| 5 | * Sizes are read with fs only; no process is started. | |
| 2 | * Disk free (#13320): when the data volume has under 15% free, measure what | |
| 3 | * the forge's own data holds of the used space (#14038). When it is half or | |
| 4 | * more, file one issue naming the number and every top-level entry over 1 GiB | |
| 5 | * under the data directory, so a builder can see what to prune. When it is | |
| 6 | * less, the space is held outside the forge and no change to the code frees | |
| 7 | * it: the duty returns only the finding, saying so. One open issue stands for | |
| 8 | * the shortage, however many ticks see it. Sizes are read with fs only; no | |
| 9 | * process is started. | |
| 6 | 10 | */ |
| 7 | 11 | |
| 8 | 12 | import { lstatSync, readdirSync } from "node:fs"; |
| @@ -49,19 +53,74 @@ export function largestDirs(dataDir: string, count = 3): DirSize[] { | ||
| 49 | 53 | .slice(0, count); |
| 50 | 54 | } |
| 51 | 55 | |
| 52 | const percent = (r: DiskReading): number => (r.totalBytes > 0 ? (r.freeBytes / r.totalBytes) * 100 : 100); | |
| 56 | /** Bytes in plain files directly under the data directory (the database among them). */ | |
| 57 | function looseBytes(dataDir: string): number { | |
| 58 | try { | |
| 59 | return readdirSync(dataDir, { withFileTypes: true }) | |
| 60 | .filter((e) => e.isFile()) | |
| 61 | .reduce((sum, e) => sum + bytesUnder(join(dataDir, e.name)), 0); | |
| 62 | } catch { | |
| 63 | return 0; | |
| 64 | } | |
| 65 | } | |
| 53 | 66 | |
| 54 | /** The issue's body: the free number and where the space went. */ | |
| 55 | export function bodyOf(r: DiskReading, dirs: readonly DirSize[]): string { | |
| 56 | const list = dirs.length === 0 ? "- (none readable)" : dirs.map((d) => `- \`${d.name}\`: ${sizeSaid(d.bytes)}`).join("\n"); | |
| 67 | /** The name the files directly under the data directory are listed by. */ | |
| 68 | export const LOOSE = "(files directly under the data directory)"; | |
| 69 | ||
| 70 | /** Every top-level directory under `dataDir`, and its loose files as one entry, largest first. */ | |
| 71 | export function dataSizes(dataDir: string): DirSize[] { | |
| 72 | const all = largestDirs(dataDir, Infinity); | |
| 73 | const loose = looseBytes(dataDir); | |
| 74 | if (loose > 0) all.push({ name: LOOSE, bytes: loose }); | |
| 75 | return all.sort((a, b) => b.bytes - a.bytes || a.name.localeCompare(b.name)); | |
| 76 | } | |
| 77 | ||
| 78 | /** Entries at or above this size are listed in the issue. */ | |
| 79 | export const LISTED = 1024 ** 3; | |
| 80 | ||
| 81 | /** What the forge's data holds of the used space, and what is held outside it. */ | |
| 82 | export interface Share { | |
| 83 | readonly forgeBytes: number; | |
| 84 | readonly usedBytes: number; | |
| 85 | readonly outsideBytes: number; | |
| 86 | /** The forge's data over the used space, 0..1; 1 when nothing is used. */ | |
| 87 | readonly fraction: number; | |
| 88 | } | |
| 89 | ||
| 90 | export function shareOf(r: DiskReading, dirs: readonly DirSize[]): Share { | |
| 91 | const forgeBytes = dirs.reduce((sum, d) => sum + d.bytes, 0); | |
| 92 | const usedBytes = Math.max(0, r.totalBytes - r.freeBytes); | |
| 93 | const outsideBytes = Math.max(0, usedBytes - forgeBytes); | |
| 94 | return { forgeBytes, usedBytes, outsideBytes, fraction: usedBytes > 0 ? Math.min(1, forgeBytes / usedBytes) : 1 }; | |
| 95 | } | |
| 96 | ||
| 97 | /** `the forge's data: X of Y used (Z%); outside the forge's data directory: W` */ | |
| 98 | export function shareSaid(s: Share): string { | |
| 57 | 99 | return ( |
| 58 | `_Filed by the disk free duty from the volume's own count, not written by a person._\n\n` + | |
| 59 | `${percent(r).toFixed(1)}% of the data volume is free: ${sizeSaid(r.freeBytes)} of ${sizeSaid(r.totalBytes)}.\n\n` + | |
| 60 | `Largest directories under the data directory:\n${list}` | |
| 100 | `the forge's data: ${sizeSaid(s.forgeBytes)} of ${sizeSaid(s.usedBytes)} used (${(s.fraction * 100).toFixed(1)}%); ` + | |
| 101 | `outside the forge's data directory: ${sizeSaid(s.outsideBytes)}` | |
| 61 | 102 | ); |
| 62 | 103 | } |
| 63 | 104 | |
| 64 | export function makeDuty(read: (dir: string) => DiskReading = readDisk, sizes: (dir: string) => DirSize[] = largestDirs): Duty { | |
| 105 | /** Below this share of the used space, the space is held outside the forge. */ | |
| 106 | export const OURS = 0.5; | |
| 107 | export const OUTSIDE = "held outside the forge: a person frees it on the machine"; | |
| 108 | ||
| 109 | const percent = (r: DiskReading): number => (r.totalBytes > 0 ? (r.freeBytes / r.totalBytes) * 100 : 100); | |
| 110 | ||
| 111 | /** The issue's body: the free number, the forge's share, and every entry over 1 GiB. */ | |
| 112 | export function bodyOf(r: DiskReading, dirs: readonly DirSize[]): string { | |
| 113 | const big = [...dirs].sort((a, b) => b.bytes - a.bytes || a.name.localeCompare(b.name)).filter((d) => d.bytes >= LISTED); | |
| 114 | const list = big.length === 0 ? "- (none over 1 GiB)" : big.map((d) => `- \`${d.name}\`: ${sizeSaid(d.bytes)}`).join("\n"); | |
| 115 | return ( | |
| 116 | `_Filed by the disk free duty from the volume's own count, not written by a person._\n\n` + | |
| 117 | `${percent(r).toFixed(1)}% of the data volume is free: ${sizeSaid(r.freeBytes)} of ${sizeSaid(r.totalBytes)}.\n\n` + | |
| 118 | `${shareSaid(shareOf(r, dirs))}.\n\n` + | |
| 119 | `Top-level entries under the data directory over 1 GiB, largest first:\n${list}` | |
| 120 | ); | |
| 121 | } | |
| 122 | ||
| 123 | export function makeDuty(read: (dir: string) => DiskReading = readDisk, sizes: (dir: string) => DirSize[] = dataSizes): Duty { | |
| 65 | 124 | return { |
| 66 | 125 | name: "disk-free", |
| 67 | 126 | intervalMs: 3_600_000, |
| @@ -76,14 +135,20 @@ export function makeDuty(read: (dir: string) => DiskReading = readDisk, sizes: ( | ||
| 76 | 135 | } |
| 77 | 136 | const pct = percent(r); |
| 78 | 137 | if (r.totalBytes <= 0 || pct >= LINE * 100) return { findings: [], actions: [] }; |
| 138 | const dirs = sizes(app.config.dataDir); | |
| 139 | const share = shareOf(r, dirs); | |
| 140 | const outside = share.fraction < OURS; | |
| 79 | 141 | const finding = { |
| 80 | summary: `${pct.toFixed(1)}% of the data volume free (${sizeSaid(r.freeBytes)} of ${sizeSaid(r.totalBytes)})`, | |
| 142 | summary: | |
| 143 | `${pct.toFixed(1)}% of the data volume free (${sizeSaid(r.freeBytes)} of ${sizeSaid(r.totalBytes)}); ${shareSaid(share)}` + | |
| 144 | (outside ? `; ${OUTSIDE}` : ""), | |
| 81 | 145 | probe: { kind: "measurement" as const, what: "percent of the data volume free", number: pct, at: now.toISOString() }, |
| 82 | 146 | }; |
| 147 | if (outside) return { findings: [finding], actions: [] }; | |
| 83 | 148 | const [owner = "", name = ""] = ownRepoPath().split("/"); |
| 84 | 149 | const repo = getRepo(db, owner, name); |
| 85 | 150 | if (repo === undefined) return { findings: [finding], actions: [] }; |
| 86 | const filed = fileIssueOnce(db, bus, repo, { title: TITLE, body: bodyOf(r, sizes(app.config.dataDir)) }); | |
| 151 | const filed = fileIssueOnce(db, bus, repo, { title: TITLE, body: bodyOf(r, dirs) }); | |
| 87 | 152 | return { findings: [finding], actions: filed.opened ? [`filed disk free issue #${filed.number}`] : [] }; |
| 88 | 153 | }, |
| 89 | 154 | }; |
test/maintain-disk-free.test.ts +46 −5
| @@ -1,11 +1,12 @@ | ||
| 1 | /** Disk free (#13320): above the line nothing is filed; under it one issue with the number and the | |
| 2 | * three largest directories, and a second tick files none. */ | |
| 1 | /** Disk free (#13320, #14038): above the line nothing is filed; under it, when the forge's data is | |
| 2 | * half or more of the used space, one issue with the number and every top-level entry over 1 GiB, | |
| 3 | * and a second tick files none; when the space is held outside the forge, only the finding. */ | |
| 3 | 4 | import { afterAll, expect, test } from "bun:test"; |
| 4 | 5 | import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; |
| 5 | 6 | import { tmpdir } from "node:os"; |
| 6 | 7 | import { join } from "node:path"; |
| 7 | 8 | import { rulesWorld } from "./helpers/rules-app.ts"; |
| 8 | import { largestDirs, makeDuty, TITLE } from "../src/maintain/duties/disk_free.ts"; | |
| 9 | import { dataSizes, largestDirs, LOOSE, makeDuty, OUTSIDE, TITLE } from "../src/maintain/duties/disk_free.ts"; | |
| 9 | 10 | import { duties } from "../src/maintain/registry.ts"; |
| 10 | 11 | |
| 11 | 12 | process.env.FORGE_SENTINEL_REPO = "alice/rules"; |
| @@ -16,6 +17,8 @@ const duty = makeDuty(() => ({ freeBytes: free, totalBytes: 100 * GB }), () => [ | ||
| 16 | 17 | { name: "repos", bytes: 30 * GB }, |
| 17 | 18 | { name: "backups", bytes: 20 * GB }, |
| 18 | 19 | { name: "runs", bytes: 10 * GB }, |
| 20 | { name: "drop", bytes: 2 * GB }, | |
| 21 | { name: "tiny", bytes: 100 * 1024 ** 2 }, | |
| 19 | 22 | ]); |
| 20 | 23 | const run = async () => await duty.run({ db: w.app.db, bus: w.app.bus, app: w.app, now: new Date() }); |
| 21 | 24 | const filed = () => w.titles().filter((t) => t === TITLE); |
| @@ -27,7 +30,24 @@ test("above the line it files nothing", async () => { | ||
| 27 | 30 | expect(filed()).toHaveLength(0); |
| 28 | 31 | }); |
| 29 | 32 | |
| 30 | test("under the line it files one issue with the number and three directories, once", async () => { | |
| 33 | test("space held outside the forge files nothing and says so (#14037's reading)", async () => { | |
| 34 | const outside = makeDuty(() => ({ freeBytes: 304 * GB, totalBytes: 2197 * GB }), () => [ | |
| 35 | { name: "repos", bytes: 30 * GB }, | |
| 36 | { name: "runs", bytes: 20 * GB }, | |
| 37 | { name: "backups", bytes: 10 * GB }, | |
| 38 | ]); | |
| 39 | const out = await outside.run({ db: w.app.db, bus: w.app.bus, app: w.app, now: new Date() }); | |
| 40 | expect(out.actions).toEqual([]); | |
| 41 | expect(out.findings).toHaveLength(1); | |
| 42 | const summary = out.findings[0]!.summary; | |
| 43 | expect(summary).toContain("the forge's data: 60.0 GiB of 1893.0 GiB used (3.2%)"); | |
| 44 | expect(summary).toContain("outside the forge's data directory: 1833.0 GiB"); | |
| 45 | expect(summary).toContain("outside the forge's data directory: "); | |
| 46 | expect(summary.endsWith(OUTSIDE)).toBe(true); | |
| 47 | expect(filed()).toHaveLength(0); | |
| 48 | }); | |
| 49 | ||
| 50 | test("under the line it files one issue with the number, the share and every entry over 1 GiB, once", async () => { | |
| 31 | 51 | free = 10 * GB; |
| 32 | 52 | const first = await run(); |
| 33 | 53 | expect(first.actions).toHaveLength(1); |
| @@ -36,7 +56,12 @@ test("under the line it files one issue with the number and three directories, o | ||
| 36 | 56 | expect(filed()).toHaveLength(1); |
| 37 | 57 | const body = w.app.db.query<{ body: string }, [string]>("SELECT body FROM issue WHERE title = ?1").get(TITLE)!.body; |
| 38 | 58 | expect(body).toContain("10.0%"); |
| 39 | for (const d of ["repos", "backups", "runs"]) expect(body).toContain(`\`${d}\``); | |
| 59 | expect(body).toContain("the forge's data: 62.1 GiB of 90.0 GiB used (69.0%)"); | |
| 60 | expect(body).toContain("outside the forge's data directory: 27.9 GiB"); | |
| 61 | const at = ["repos", "backups", "runs", "drop"].map((d) => body.indexOf(`\`${d}\``)); | |
| 62 | expect(at.every((i) => i > 0)).toBe(true); | |
| 63 | expect([...at].sort((a, b) => a - b)).toEqual(at); // largest first | |
| 64 | expect(body).not.toContain("`tiny`"); | |
| 40 | 65 | }); |
| 41 | 66 | |
| 42 | 67 | const made: string[] = []; |
| @@ -67,6 +92,22 @@ test("largestDirs reads sizes with fs, largest first, at most three", () => { | ||
| 67 | 92 | expect(largestDirs(root).map((d) => d.name)).toEqual(["d", "b", "c"]); |
| 68 | 93 | }); |
| 69 | 94 | |
| 95 | test("dataSizes sums every top-level directory and the loose files, largest first", () => { | |
| 96 | const root = mkdtemp(); | |
| 97 | for (const [n, b] of [["a", 1], ["b", 300], ["c", 20], ["d", 4000]] as const) { | |
| 98 | mkdirSync(join(root, n), { recursive: true }); | |
| 99 | writeFileSync(join(root, n, "f"), "z".repeat(b)); | |
| 100 | } | |
| 101 | writeFileSync(join(root, "forge.db"), "z".repeat(500)); | |
| 102 | expect(dataSizes(root)).toEqual([ | |
| 103 | { name: "d", bytes: 4000 }, | |
| 104 | { name: LOOSE, bytes: 500 }, | |
| 105 | { name: "b", bytes: 300 }, | |
| 106 | { name: "c", bytes: 20 }, | |
| 107 | { name: "a", bytes: 1 }, | |
| 108 | ]); | |
| 109 | }); | |
| 110 | ||
| 70 | 111 | test("the duty is in the registry", () => { |
| 71 | 112 | expect(duties.map((d) => d.name)).toContain("disk-free"); |
| 72 | 113 | }); |
new above
offline
Empty or unreadable
dirsgives fraction 0, which is reported as 'held outside the forge'. Treat a measurement of nothing as unknown and say so.