Public: anyone can read this; the crew can change it.
Sentinel: disk was degraded 1072 minutes today: Disk is nearly full: 9% free. #6240
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/6238-sentinel-disk-was-degraded-1072-minutes → main 1 commit fork 1 thread · 1 unresolved
3 files
changed +68
−0
src/crew/dispatch.ts +3
| @@ -29,6 +29,7 @@ import { | ||
| 29 | 29 | } from "../core/issues.ts"; |
| 30 | 30 | import { getPull, type Pull } from "../core/pulls.ts"; |
| 31 | 31 | import { claimedClones, sweepOrphanClones } from "./orphans.ts"; |
| 32 | import { pruneRunFiles } from "../runs/prune.ts"; | |
| 32 | 33 | import { crewRoot } from "./workdir.ts"; |
| 33 | 34 | import { resolveOpen } from "../core/decisions.ts"; |
| 34 | 35 | import { accountActive } from "../core/accounts.ts"; |
| @@ -229,6 +230,8 @@ export async function startCrew( | ||
| 229 | 230 | if (count > 0) log.info("removed clones nothing claims", { count }); |
| 230 | 231 | }) |
| 231 | 232 | .catch((error) => log.warn("clone sweep failed", { error: String(error) })); |
| 233 | const pruned = pruneRunFiles(app.config.dataDir); | |
| 234 | if (pruned > 0) log.info("removed old run logs", { count: pruned }); | |
| 232 | 235 | }; |
| 233 | 236 | sweepClones(); |
| 234 | 237 | const cloneTimer = setInterval(() => !stopped && sweepClones(), 60 * 60_000); |
src/runs/prune.ts added +41
| @@ -0,0 +1,41 @@ | ||
| 1 | /** | |
| 2 | * Run logs nothing reads any more (#6238). Every run writes `<id>.log` and | |
| 3 | * a check `<id>.vitals.json` under `runs/`, and nothing removed them, so the | |
| 4 | * state volume only grew. A file whose last write is older than `maxAgeMs` | |
| 5 | * is removed; the pages that link a log already say "no log" for a missing one. | |
| 6 | */ | |
| 7 | ||
| 8 | import { readdirSync, rmSync, statSync } from "node:fs"; | |
| 9 | import { join } from "node:path"; | |
| 10 | import { runsRoot } from "./log.ts"; | |
| 11 | ||
| 12 | /** Logs and vitals are kept this long after their last write. */ | |
| 13 | export const RUN_FILES_KEEP_MS = 14 * 86_400_000; | |
| 14 | ||
| 15 | /** Remove the regular files of `runs/` last written before `now - maxAgeMs`; how many went. */ | |
| 16 | export function pruneRunFiles( | |
| 17 | dataDir: string, | |
| 18 | now: number = Date.now(), | |
| 19 | maxAgeMs: number = RUN_FILES_KEEP_MS, | |
| 20 | ): number { | |
| 21 | const root = runsRoot(dataDir); | |
| 22 | let removed = 0; | |
| 23 | let names: string[]; | |
| 24 | try { | |
| 25 | names = readdirSync(root); | |
| 26 | } catch { | |
| 27 | return 0; | |
| 28 | } | |
| 29 | for (const name of names) { | |
| 30 | const path = join(root, name); | |
| 31 | try { | |
| 32 | const st = statSync(path); | |
| 33 | if (!st.isFile() || now - st.mtimeMs <= maxAgeMs) continue; | |
| 34 | rmSync(path, { force: true }); | |
| 35 | removed++; | |
| 36 | } catch { | |
| 37 | // gone already, or busy: the next sweep looks again | |
| 38 | } | |
| 39 | } | |
| 40 | return removed; | |
| 41 | } |
test/run-prune.test.ts added +24
| @@ -0,0 +1,24 @@ | ||
| 1 | import { expect, test } from "bun:test"; | |
| 2 | import { existsSync, mkdirSync, mkdtempSync, utimesSync, writeFileSync } from "node:fs"; | |
| 3 | import { join } from "node:path"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { runsRoot } from "../src/runs/log.ts"; | |
| 6 | import { pruneRunFiles } from "../src/runs/prune.ts"; | |
| 7 | ||
| 8 | test("old run files go, recent ones stay", () => { | |
| 9 | const dir = mkdtempSync(join(tmpdir(), "prune-")); | |
| 10 | mkdirSync(runsRoot(dir), { recursive: true }); | |
| 11 | const old = join(runsRoot(dir), "a.log"); | |
| 12 | const fresh = join(runsRoot(dir), "b.log"); | |
| 13 | writeFileSync(old, "x"); | |
| 14 | writeFileSync(fresh, "x"); | |
| 15 | const past = new Date(Date.now() - 30 * 86_400_000); | |
| 16 | utimesSync(old, past, past); | |
| 17 | expect(pruneRunFiles(dir)).toBe(1); | |
| 18 | expect(existsSync(old)).toBe(false); | |
| 19 | expect(existsSync(fresh)).toBe(true); | |
| 20 | }); | |
| 21 | ||
| 22 | test("no runs directory is nothing to prune", () => { | |
| 23 | expect(pruneRunFiles(mkdtempSync(join(tmpdir(), "prune-")))).toBe(0); | |
| 24 | }); |
new above
offline
A fixed 14 days is a guess; if the disk stays under the floor, make the age shrink when free space is low rather than adding a second sweep.