Public: anyone can read this; the crew can change it.
A maintenance duty watches free space on the data volume and files one issue under 15% free #13337
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/13320-a-maintenance-duty-watches-free-space-on → main 5 commits fork 3 threads · 3 unresolved
5 files
changed +184
−1
docs/features/README.md +1
| @@ -383,6 +383,7 @@ Read with the guide page [Agents: MCP, git, or the terminal](../guide-agents.md) | ||
| 383 | 383 | Read with the guide page [](../guide-backups-moving.md). |
| 384 | 384 | |
| 385 | 385 | - [](backup-growth.md) |
| 386 | - [](disk-free.md) | |
| 386 | 387 | - [](backup-refs.md) |
| 387 | 388 | - [](forge-origin.md) |
| 388 | 389 | - [](origin-pull-state.md) |
docs/features/disk-free.md added +17
| @@ -0,0 +1,17 @@ | ||
| 1 | # A data volume that is filling up | |
| 2 | ||
| 3 | The `disk-free` maintenance duty (#13320) files one issue when the volume under | |
| 4 | `FORGE_APP_DATA` has under 15% free, so the forge says it is filling up days before it is full. | |
| 5 | ||
| 6 | - **What it watches.** Free space as statfs reports it (what an unprivileged writer can | |
| 7 | still use) against the volume's size, read once per hourly tick. A volume that cannot be | |
| 8 | read (the data directory is missing) yields no finding and does not fail the run. | |
| 9 | - **The line.** Under 15% free files; exactly 15% or more is quiet. | |
| 10 | - **What the issue says.** "The data volume has under 15% free": the percent, the free and | |
| 11 | total size, and the three largest directories directly under the data directory, sized | |
| 12 | with fs only (symlinks not followed, no process started). | |
| 13 | - **One issue.** While one is open, later ticks file none (`fileIssueOnce`). | |
| 14 | - **Closing it.** Closed while the disk is still under the line, the next tick files a new | |
| 15 | one: the duty remembers nothing but the open issue. | |
| 16 | - **Where it shows.** One row, `disk-free`, on the maintenance board (`/maintenance`, admin). | |
| 17 | - The duty runs hourly and spends no money. |
src/maintain/duties/disk_free.ts added +92
| @@ -0,0 +1,92 @@ | ||
| 1 | /** | |
| 2 | * Disk free (#13320): when the data volume has under 15% free, file one issue | |
| 3 | * naming the number and the three largest directories under the data | |
| 4 | * directory. One open issue stands for the shortage, however many ticks see it. | |
| 5 | * Sizes are read with fs only; no process is started. | |
| 6 | */ | |
| 7 | ||
| 8 | import { lstatSync, readdirSync } from "node:fs"; | |
| 9 | import { join } from "node:path"; | |
| 10 | import { fileIssueOnce } from "../../checks/flake.ts"; | |
| 11 | import { getRepo } from "../../core/repos.ts"; | |
| 12 | import { readDisk, type DiskReading } from "../../kernel/disk_reading.ts"; | |
| 13 | import { ownRepoPath } from "../../kernel/own_repo.ts"; | |
| 14 | import { sizeSaid } from "../../system/db_size.ts"; | |
| 15 | import type { Duty } from "../duty.ts"; | |
| 16 | ||
| 17 | export const TITLE = "The data volume has under 15% free"; | |
| 18 | export const LINE = 0.15; | |
| 19 | ||
| 20 | export interface DirSize { | |
| 21 | readonly name: string; | |
| 22 | readonly bytes: number; | |
| 23 | } | |
| 24 | ||
| 25 | /** Bytes under `path`, symlinks not followed, unreadable entries counted as nothing. */ | |
| 26 | function bytesUnder(path: string): number { | |
| 27 | try { | |
| 28 | const st = lstatSync(path); | |
| 29 | if (!st.isDirectory()) return st.isSymbolicLink() ? 0 : st.size; | |
| 30 | let sum = 0; | |
| 31 | for (const e of readdirSync(path)) sum += bytesUnder(join(path, e)); | |
| 32 | return sum; | |
| 33 | } catch { | |
| 34 | return 0; | |
| 35 | } | |
| 36 | } | |
| 37 | ||
| 38 | /** The `count` largest directories directly under `dataDir`, largest first. */ | |
| 39 | export function largestDirs(dataDir: string, count = 3): DirSize[] { | |
| 40 | let names: string[]; | |
| 41 | try { | |
| 42 | names = readdirSync(dataDir, { withFileTypes: true }).filter((e) => e.isDirectory()).map((e) => e.name); | |
| 43 | } catch { | |
| 44 | return []; | |
| 45 | } | |
| 46 | return names | |
| 47 | .map((name) => ({ name, bytes: bytesUnder(join(dataDir, name)) })) | |
| 48 | .sort((a, b) => b.bytes - a.bytes || a.name.localeCompare(b.name)) | |
| 49 | .slice(0, count); | |
| 50 | } | |
| 51 | ||
| 52 | const percent = (r: DiskReading): number => (r.totalBytes > 0 ? (r.freeBytes / r.totalBytes) * 100 : 100); | |
| 53 | ||
| 54 | /** The issue's body: the free number and where the space went. */ | |
| 55 | export function bodyOf(r: DiskReading, dirs: readonly DirSize[]): string { | |
| 56 | const list = dirs.length === 0 ? "- (none readable)" : dirs.map((d) => `- \`${d.name}\`: ${sizeSaid(d.bytes)}`).join("\n"); | |
| 57 | return ( | |
| 58 | `_Filed by the disk free duty from the volume's own count, not written by a person._\n\n` + | |
| 59 | `${percent(r).toFixed(1)}% of the data volume is free: ${sizeSaid(r.freeBytes)} of ${sizeSaid(r.totalBytes)}.\n\n` + | |
| 60 | `Largest directories under the data directory:\n${list}` | |
| 61 | ); | |
| 62 | } | |
| 63 | ||
| 64 | export function makeDuty(read: (dir: string) => DiskReading = readDisk, sizes: (dir: string) => DirSize[] = largestDirs): Duty { | |
| 65 | return { | |
| 66 | name: "disk-free", | |
| 67 | intervalMs: 3_600_000, | |
| 68 | budget: { runsPerDay: 30, dollarsPerDay: 1, reason: "hourly is 24 a day; the rest is room for a catch-up, and it spends no money" }, | |
| 69 | run({ db, bus, now, app }) { | |
| 70 | if (app === undefined) return { findings: [], actions: [] }; | |
Build |
||
| 71 | let r: DiskReading; | |
| 72 | try { | |
| 73 | r = read(app.config.dataDir); | |
| 74 | } catch { | |
| 75 | return { findings: [], actions: [] }; // a data directory that cannot be read is not a number | |
| 76 | } | |
| 77 | const pct = percent(r); | |
| 78 | if (r.totalBytes <= 0 || pct >= LINE * 100) return { findings: [], actions: [] }; | |
| 79 | const finding = { | |
| 80 | summary: `${pct.toFixed(1)}% of the data volume free (${sizeSaid(r.freeBytes)} of ${sizeSaid(r.totalBytes)})`, | |
| 81 | probe: { kind: "measurement" as const, what: "percent of the data volume free", number: pct, at: now.toISOString() }, | |
| 82 | }; | |
| 83 | const [owner = "", name = ""] = ownRepoPath().split("/"); | |
| 84 | const repo = getRepo(db, owner, name); | |
| 85 | if (repo === undefined) return { findings: [finding], actions: [] }; | |
| 86 | const filed = fileIssueOnce(db, bus, repo, { title: TITLE, body: bodyOf(r, sizes(app.config.dataDir)) }); | |
| 87 | return { findings: [finding], actions: filed.opened ? [`filed disk free issue #${filed.number}`] : [] }; | |
| 88 | }, | |
| 89 | }; | |
| 90 | } | |
| 91 | ||
| 92 | export const duty: Duty = makeDuty(); |
src/maintain/registry.ts +2 −1
| @@ -8,5 +8,6 @@ import { duty as flowKeeper } from "./duties/flow_keeper.ts"; | ||
| 8 | 8 | import { duty as visitorRetention } from "./duties/visitor_retention.ts"; |
| 9 | 9 | import { duty as backupGrowth } from "./duties/backup_growth.ts"; |
| 10 | 10 | import { duty as v1Proven } from "./duties/v1_proven.ts"; |
| 11 | import { duty as diskFree } from "./duties/disk_free.ts"; | |
| 11 | 12 | |
| 12 | export const duties: readonly Duty[] = [pullsWaiting, ruleWatcher, efficiencyWatcher, flowKeeper, visitorRetention, backupGrowth, v1Proven]; | |
| 13 | export const duties: readonly Duty[] = [pullsWaiting, ruleWatcher, efficiencyWatcher, flowKeeper, visitorRetention, backupGrowth, v1Proven, diskFree]; |
test/maintain-disk-free.test.ts added +72
| @@ -0,0 +1,72 @@ | ||
| 1 | /** Disk free (#13320): above the line nothing is filed; under it one issue with the number and the | |
| 2 | * three largest directories, and a second tick files none. */ | |
| 3 | import { afterAll, expect, test } from "bun:test"; | |
| 4 | import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; | |
| 5 | import { tmpdir } from "node:os"; | |
| 6 | import { join } from "node:path"; | |
| 7 | import { rulesWorld } from "./helpers/rules-app.ts"; | |
| 8 | import { largestDirs, makeDuty, TITLE } from "../src/maintain/duties/disk_free.ts"; | |
| 9 | import { duties } from "../src/maintain/registry.ts"; | |
| 10 | ||
| 11 | process.env.FORGE_SENTINEL_REPO = "alice/rules"; | |
| 12 | const w = rulesWorld(); | |
| 13 | const GB = 1024 ** 3; | |
| 14 | let free = 50 * GB; | |
| 15 | const duty = makeDuty(() => ({ freeBytes: free, totalBytes: 100 * GB }), () => [ | |
| 16 | { name: "repos", bytes: 30 * GB }, | |
| 17 | { name: "backups", bytes: 20 * GB }, | |
| 18 | { name: "runs", bytes: 10 * GB }, | |
| 19 | ]); | |
| 20 | const run = async () => await duty.run({ db: w.app.db, bus: w.app.bus, app: w.app, now: new Date() }); | |
| 21 | const filed = () => w.titles().filter((t) => t === TITLE); | |
| 22 | ||
| 23 | test("above the line it files nothing", async () => { | |
| 24 | expect((await run()).actions).toEqual([]); | |
| 25 | free = 15 * GB; // exactly 15% is not under | |
| 26 | expect((await run()).actions).toEqual([]); | |
| 27 | expect(filed()).toHaveLength(0); | |
| 28 | }); | |
| 29 | ||
| 30 | test("under the line it files one issue with the number and three directories, once", async () => { | |
| 31 | free = 10 * GB; | |
| 32 | const first = await run(); | |
| 33 | expect(first.actions).toHaveLength(1); | |
| 34 | expect(first.findings[0]!.probe!.number).toBeCloseTo(10); | |
| 35 | expect((await run()).actions).toEqual([]); | |
| 36 | expect(filed()).toHaveLength(1); | |
| 37 | const body = w.app.db.query<{ body: string }, [string]>("SELECT body FROM issue WHERE title = ?1").get(TITLE)!.body; | |
| 38 | expect(body).toContain("10.0%"); | |
| 39 | for (const d of ["repos", "backups", "runs"]) expect(body).toContain(`\`${d}\``); | |
| 40 | }); | |
outdated
Use mkdtempSync and remove the directory afterwards; this leaves files in TMPDIR. |
||
| 41 | ||
| 42 | const made: string[] = []; | |
| 43 | const mkdtemp = (): string => { | |
| 44 | const d = mkdtempSync(join(tmpdir(), "dfree-")); | |
| 45 | made.push(d); | |
| 46 | return d; | |
| 47 | }; | |
| 48 | afterAll(() => { | |
| 49 | for (const d of made) rmSync(d, { recursive: true, force: true }); | |
| 50 | }); | |
| 51 | ||
| 52 | test("a volume that cannot be read files nothing and does not throw", async () => { | |
| 53 | const broken = makeDuty(() => { | |
| 54 | throw new Error("ENOENT: statfs"); | |
| 55 | }); | |
| 56 | const out = await broken.run({ db: w.app.db, bus: w.app.bus, app: w.app, now: new Date() }); | |
| 57 | expect(out).toEqual({ findings: [], actions: [] }); | |
| 58 | }); | |
| 59 | ||
| 60 | test("largestDirs reads sizes with fs, largest first, at most three", () => { | |
| 61 | const root = mkdtemp(); | |
| 62 | for (const [n, b] of [["a", 1], ["b", 300], ["c", 20], ["d", 4000]] as const) { | |
| 63 | mkdirSync(join(root, n, "x"), { recursive: true }); | |
| 64 | writeFileSync(join(root, n, "x", "f"), "z".repeat(b)); | |
| 65 | } | |
| 66 | writeFileSync(join(root, "loose"), "z".repeat(99999)); | |
| 67 | expect(largestDirs(root).map((d) => d.name)).toEqual(["d", "b", "c"]); | |
| 68 | }); | |
| 69 | ||
| 70 | test("the duty is in the registry", () => { | |
| 71 | expect(duties.map((d) => d.name)).toContain("disk-free"); | |
| 72 | }); |
new above
offline
statfs can throw if the data dir is missing; catch and return no findings, with a test.