Public: anyone can read this; the crew can change it.
Sentinel: disk-red was degraded 83 minutes today: The check machine's disk was full when 5 checks failed: 9% free, under the 10% floor — the machine's red, not the pulls… #6946
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/6945-sentinel-disk-red-was-degraded-83-minute → main 2 commits fork 2 threads · 2 unresolved
3 files
changed +117
−1
src/checks/service.ts +13 −1
| @@ -58,7 +58,7 @@ import { keepCache, restoreCache } from "./cache.ts"; | ||
| 58 | 58 | import { MAX_FILED, clearedFiles, fileOrderDependent } from "./order_dependent.ts"; |
| 59 | 59 | import { parseBunTestFailure } from "./bun-test.ts"; |
| 60 | 60 | import { passedAlone, recordFlakes, rerunCommand, rerunnable } from "./flake.ts"; |
| 61 | import { withDisk } from "./disk_red.ts"; | |
| 61 | import { diskRecovered, withDisk } from "./disk_red.ts"; | |
| 62 | 62 | import { ASKED, redMain, unheld } from "./red_main.ts"; |
| 63 | 63 | import { proposeRevert } from "./revert_red.ts"; |
| 64 | 64 | import { rebasesFirst } from "./finishing_first.ts"; |
| @@ -309,6 +309,17 @@ export function sweepCheckouts(app: App, now: number = Date.now()): void { | ||
| 309 | 309 | } |
| 310 | 310 | } |
| 311 | 311 | |
| 312 | /** | |
| 313 | * Under the free floor, give back what no check is using before taking more | |
| 314 | * (#6945): checkouts leaked by a crash sat on the disk until the next boot | |
| 315 | * while every check failed red on it. Answers whether it swept. | |
| 316 | */ | |
| 317 | export function reclaimIfFull(app: App, now: number = Date.now()): boolean { | |
| 318 | if (diskRecovered(app.config.dataDir)) return false; | |
| 319 | sweepCheckouts(app, now); | |
| 320 | return true; | |
| 321 | } | |
| 322 | ||
| 312 | 323 | const handles = new WeakMap<App, ChecksHandle>(); |
| 313 | 324 | |
| 314 | 325 | /** The checks service of this forge, if one was started. */ |
| @@ -853,6 +864,7 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle { | ||
| 853 | 864 | write(`already proven green by ${earlier}`); |
| 854 | 865 | return { ok: true, proven: earlier }; |
| 855 | 866 | } |
| 867 | if (reclaimIfFull(app)) write("disk under its free floor: swept unused checkouts"); | |
| 856 | 868 | const dir = join(checksRoot(app.config.dataDir), run.id); |
| 857 | 869 | mkdirSync(dir, { recursive: true }); |
| 858 | 870 | write(mergeLine === "" ? `checking out ${run.commitOid.slice(0, 7)}` : mergeLine); |
test/check-reclaim-full-disk.test.ts added +79
| @@ -0,0 +1,79 @@ | ||
| 1 | /** | |
| 2 | * A check that starts on a disk under its free floor first sweeps the | |
| 3 | * checkouts nobody is using (#6945) — and only those: a checkout whose run | |
| 4 | * is running with a fresh lease is another check's, and stays. | |
| 5 | */ | |
| 6 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 7 | import { existsSync, mkdirSync } from "node:fs"; | |
| 8 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 9 | import { tmpdir } from "node:os"; | |
| 10 | import { join } from "node:path"; | |
| 11 | import { createApp, type App } from "../src/app.ts"; | |
| 12 | import { commitToBranch, createRepo } from "../src/actions/repos.ts"; | |
| 13 | import { diskReading } from "../src/checks/disk_red.ts"; | |
| 14 | import { startChecks, type ChecksHandle } from "../src/checks/service.ts"; | |
| 15 | import { claimRun, createRun, listRuns } from "../src/core/runs.ts"; | |
| 16 | import { createUser } from "../src/core/users.ts"; | |
| 17 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 18 | import { statDisk } from "../src/kernel/vitals.ts"; | |
| 19 | import { readRunLog } from "../src/runs/log.ts"; | |
| 20 | ||
| 21 | let dataDir: string; | |
| 22 | let app: App; | |
| 23 | let servers: Servers; | |
| 24 | let checks: ChecksHandle; | |
| 25 | ||
| 26 | beforeAll(async () => { | |
| 27 | dataDir = await mkdtemp(join(tmpdir(), "forge-check-reclaim-")); | |
| 28 | app = createApp({ dataDir, port: 0 }); | |
| 29 | servers = await serve(app); | |
| 30 | checks = startChecks(app, { forgeUrl: servers.url, concurrency: 1 }); | |
| 31 | }); | |
| 32 | afterAll(async () => { | |
| 33 | diskReading.read = statDisk; | |
| 34 | await checks.stop(); | |
| 35 | servers.stop(); | |
| 36 | app.close(); | |
| 37 | await rm(dataDir, { recursive: true, force: true }); | |
| 38 | }); | |
| 39 | ||
| 40 | test("a check starting under the floor sweeps the orphan checkout, keeps a live one, and says so", async () => { | |
| 41 | const alice = await createUser(app.db, { | |
| 42 | origin: "seed", | |
| 43 | trusted: true, | |
| 44 | username: "alice", | |
| 45 | email: "[email protected]", | |
| 46 | password: "a-good-passphrase", | |
| 47 | adminIfFirst: true, | |
| 48 | }); | |
| 49 | if (!alice.ok) throw new Error(alice.error); | |
| 50 | const made = await createRepo(app, alice.value, { name: "full", visibility: "public" }); | |
| 51 | if (!made.ok) throw new Error(String(made.error)); | |
| 52 | const repo = made.value; | |
| 53 | ||
| 54 | const live = createRun(app.db, { repoId: repo.id, kind: "check" }); | |
| 55 | expect(claimRun(app.db, live.id)?.state).toBe("running"); | |
| 56 | const liveDir = join(dataDir, "checks", live.id); | |
| 57 | const orphan = join(dataDir, "checks", "orphan"); | |
| 58 | mkdirSync(liveDir, { recursive: true }); | |
| 59 | mkdirSync(orphan, { recursive: true }); | |
| 60 | ||
| 61 | diskReading.read = () => ({ freeBytes: 50, totalBytes: 1000 }); | |
| 62 | const c = await commitToBranch(app, alice.value, repo, { | |
| 63 | message: "Declare a check", | |
| 64 | files: [{ path: "forge.json", content: '{"check": {"run": "exit 0"}}\n' }], | |
| 65 | }); | |
| 66 | if (!c.ok) throw new Error(JSON.stringify(c.error)); | |
| 67 | ||
| 68 | const deadline = Date.now() + 30_000; | |
| 69 | let mine = listRuns(app.db, repo.id, { kind: "check", limit: 50 }).filter((r) => r.id !== live.id); | |
| 70 | while (Date.now() < deadline && !mine.some((r) => r.state !== "queued" && r.state !== "running")) { | |
| 71 | await Bun.sleep(50); | |
| 72 | mine = listRuns(app.db, repo.id, { kind: "check", limit: 50 }).filter((r) => r.id !== live.id); | |
| 73 | } | |
| 74 | expect(mine.length).toBeGreaterThan(0); | |
| 75 | const log = readRunLog(dataDir, mine[0]!.id).lines; | |
| 76 | expect(log).toContain("disk under its free floor: swept unused checkouts"); | |
| 77 | expect(existsSync(orphan)).toBe(false); | |
| 78 | expect(existsSync(liveDir)).toBe(true); | |
| 79 | }, 60_000); |
test/checks-disk-red.test.ts +25
| @@ -153,3 +153,28 @@ describe("a red on a full check machine", () => { | ||
| 153 | 153 | expect(cards()[0]!.sentence).toBe(before); |
| 154 | 154 | }); |
| 155 | 155 | }); |
| 156 | ||
| 157 | describe("a full check machine gives back unused checkouts", () => { | |
| 158 | test("under the floor the sweep removes a checkout no run owns; above it, nothing", async () => { | |
| 159 | const { mkdirSync, existsSync, rmSync } = await import("node:fs"); | |
| 160 | const { join } = await import("node:path"); | |
| 161 | const { reclaimIfFull } = await import("../src/checks/service.ts"); | |
| 162 | const standIn = await startStandInCrew(); | |
| 163 | const dataDir = join(process.env.TMPDIR ?? "/tmp", `reclaim-${Date.now()}`); | |
| 164 | try { | |
| 165 | const app = { ...standIn.app, config: { ...standIn.app.config, dataDir } } as App; | |
| 166 | const orphan = join(dataDir, "checks", "orphan"); | |
| 167 | mkdirSync(orphan, { recursive: true }); | |
| 168 | diskReading.read = () => ({ freeBytes: 500, totalBytes: 1000 }); | |
| 169 | expect(reclaimIfFull(app)).toBe(false); | |
| 170 | expect(existsSync(orphan)).toBe(true); | |
| 171 | diskReading.read = () => ({ freeBytes: 50, totalBytes: 1000 }); | |
| 172 | expect(reclaimIfFull(app)).toBe(true); | |
| 173 | expect(existsSync(orphan)).toBe(false); | |
| 174 | } finally { | |
Add a second checkout owned by a running run with a fresh lease and assert it survives the sweep. Also rmSync dataDir in finally. |
||
| 175 | diskReading.read = statDisk; | |
| 176 | rmSync(dataDir, { recursive: true, force: true }); | |
| 177 | await stopStandInCrew(standIn); | |
| 178 | } | |
| 179 | }, 30_000); | |
| 180 | }); |
new above
offline
This wiring is the behaviour and nothing tests it. Cover it through the service, asserting the log line and that the orphan is gone before the checkout starts.