Public: anyone can read this; the crew can change it.
The data volume has under 15% free #13515
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/13514-the-data-volume-has-under-15-free → main 1 commit fork
4 files
changed +122
−17
docs/features/disk-free.md +7
| @@ -15,3 +15,10 @@ The `disk-free` maintenance duty (#13320) files one issue when the volume under | ||
| 15 | 15 | one: the duty remembers nothing but the open issue. |
| 16 | 16 | - **Where it shows.** One row, `disk-free`, on the maintenance board (`/maintenance`, admin). |
| 17 | 17 | - The duty runs hourly and spends no money. |
| 18 | ||
| 19 | ## What gives space back (#13514) | |
| 20 | ||
| 21 | The hourly clone sweep (`src/crew/dispatch.ts`) also removes `node_modules` from a clone kept for | |
| 22 | an open issue or pull that nothing has touched for a day and no live run holds | |
| 23 | (`sweepIdleDependencies`, `src/crew/orphans.ts`). It is about two thirds of a clone; sources and | |
| 24 | `.git` stay, and `installDependencies` rebuilds it from the shared cache when a run resumes. |
src/crew/dispatch.ts +36 −16
| @@ -2,10 +2,10 @@ | ||
| 2 | 2 | * How work reaches the crew. The dispatcher listens to the outbox and |
| 3 | 3 | * sweeps behind it: an issue assigned to an agent becomes a run; a person |
| 4 | 4 | * who may direct the crew (`direct.ts`) speaking on an agent's pull — a |
| 5 | * comment, a verdict, a line comment — becomes a follow-up; a failed deploy of an agent's pull becomes a | |
| 6 | * bounded follow-up. Runs are rows first (`queued`), and a worker tick | |
| 7 | * takes them in order up to the concurrency limit, one per issue at a | |
| 8 | * time. A missed wake-up costs a tick, never a run. | |
| 5 | * comment, a verdict, a line comment — becomes a follow-up; a failed deploy of an agent's pull | |
| 6 | * becomes a bounded follow-up. Runs are rows first (`queued`), and a worker tick takes them in | |
| 7 | * order up to the concurrency limit, one per issue at a time. A missed wake-up costs a tick, never | |
| 8 | * a run. | |
| 9 | 9 | */ |
| 10 | 10 | |
| 11 | 11 | import { ceilingHold, pausedUntil } from "../core/planner_pause.ts"; |
| @@ -40,7 +40,7 @@ import { | ||
| 40 | 40 | type Issue, |
| 41 | 41 | } from "../core/issues.ts"; |
| 42 | 42 | import { closesFromBody, getPull, pullDir, type Pull } from "../core/pulls.ts"; |
| 43 | import { claimedClones, sweepOrphanClones } from "./orphans.ts"; | |
| 43 | import { claimedClones, sweepIdleDependencies, sweepOrphanClones } from "./orphans.ts"; | |
| 44 | 44 | import { pruneRunFiles } from "../runs/prune.ts"; |
| 45 | 45 | import { crewRoot, isClaimed } from "./workdir.ts"; |
| 46 | 46 | import { resolveOpen } from "../core/decisions.ts"; |
| @@ -316,6 +316,10 @@ export async function startCrew( | ||
| 316 | 316 | .then((count) => { |
| 317 | 317 | if (count > 0) log.info("removed clones nothing claims", { count }); |
| 318 | 318 | }) |
| 319 | .then(() => sweepIdleDependencies(crewRoot(app.config.dataDir), isClaimed)) | |
| 320 | .then((count) => { | |
| 321 | if (count > 0) log.info("removed installed packages of idle clones", { count }); | |
| 322 | }) | |
| 319 | 323 | .catch((error) => log.warn("clone sweep failed", { error: String(error) })); |
| 320 | 324 | const pruned = pruneRunFiles(app.config.dataDir); |
| 321 | 325 | if (pruned > 0) log.info("removed old run logs", { count: pruned }); |
| @@ -417,7 +421,8 @@ export function outsideSeat( | ||
| 417 | 421 | agent: Agent, |
| 418 | 422 | reason: RunReason, |
| 419 | 423 | ): boolean { |
| 420 | // Every reason but a review is building work: a first go, feedback, a red check, a conflict, a converge. | |
| 424 | // Every reason but a review is building work: a first go, feedback, a red check, a conflict, a | |
| 425 | // converge. | |
| 421 | 426 | const seat = |
| 422 | 427 | reason === "review" |
| 423 | 428 | ? "reviewer" |
| @@ -425,7 +430,8 @@ export function outsideSeat( | ||
| 425 | 430 | ? "planner" |
| 426 | 431 | : "builder"; |
| 427 | 432 | const held = crewRole(app.db, repo.id, agent.user.id); |
| 428 | // A pull assigned to its reviewer is asked for a look, a builder's for a fix: a planner and a maintainer are outside. | |
| 433 | // A pull assigned to its reviewer is asked for a look, a builder's for a fix: a planner and a | |
| 434 | // maintainer are outside. | |
| 429 | 435 | if (reason !== "review" && issue.kind === "pull") { |
| 430 | 436 | const own = held ?? agent.roles[0]; |
| 431 | 437 | return own === "planner" || own === "maintainer"; |
| @@ -469,7 +475,10 @@ export function queueAgentRun( | ||
| 469 | 475 | again?: { notBefore: string; attempt: number; summary: string; interruptions?: number }, |
| 470 | 476 | variantGroup?: string, |
| 471 | 477 | retry = false, |
| 472 | /** The run is this agent's own on the issue: another agent's run does not stand in its way (team reviews). */ | |
| 478 | /** | |
| 479 | * The run is this agent's own on the issue: another agent's run does not stand in its way (team | |
| 480 | * reviews). | |
| 481 | */ | |
| 473 | 482 | own = false, |
| 474 | 483 | ): Run | undefined { |
| 475 | 484 | // A suspended or removed owner's repositories run nothing, whoever asks. |
| @@ -487,7 +496,8 @@ export function queueAgentRun( | ||
| 487 | 496 | ) |
| 488 | 497 | return undefined; |
| 489 | 498 | // A chapter is open (#11603): builders and reviewers start only what it holds, and the crew rests |
| 490 | // between chapters. A planner plans (its scope is kept where it opens issues); a run carrying on finishes. | |
| 499 | // between chapters. A planner plans (its scope is kept where it opens issues); a run carrying on | |
| 500 | // finishes. | |
| 491 | 501 | if ( |
| 492 | 502 | again === undefined && |
| 493 | 503 | crewRole(app.db, repo.id, agent.user.id) !== "planner" && |
| @@ -1086,7 +1096,9 @@ function decideDeferred(app: App, repo: Repo): void { | ||
| 1086 | 1096 | */ |
| 1087 | 1097 | const waitingOnMain = new Map<string, Map<string, { env: string; commitOid: string }>>(); |
| 1088 | 1098 | |
| 1089 | /** Whether a failed check is main's red or the lane's, shared: if so, the pull is noted as waiting. */ | |
| 1099 | /** | |
| 1100 | * Whether a failed check is main's red or the lane's, shared: if so, the pull is noted as waiting. | |
| 1101 | */ | |
| 1090 | 1102 | function waitOnMain( |
| 1091 | 1103 | app: App, |
| 1092 | 1104 | repo: Repo, |
| @@ -1108,7 +1120,8 @@ function waitOnMain( | ||
| 1108 | 1120 | }); |
| 1109 | 1121 | return true; |
| 1110 | 1122 | } |
| 1111 | // One failure line on five pulls within the hour is the forge's own (#11507): no builder is sent, no pull parked. | |
| 1123 | // One failure line on five pulls within the hour is the forge's own (#11507): no builder is sent, | |
| 1124 | // no pull parked. | |
| 1112 | 1125 | if (forgeFailed(app.db, run, Date.now())) { |
| 1113 | 1126 | log.info("check failed on a line five pulls share; the forge's failure, not the pull's", { issue: issue.number }); |
| 1114 | 1127 | return true; |
| @@ -1129,7 +1142,10 @@ function waitOnMain( | ||
| 1129 | 1142 | return true; |
| 1130 | 1143 | } |
| 1131 | 1144 | |
| 1132 | /** Pulls whose check failed on a full disk, per forge (database), so one forge's recheck never drops another's; memory only. */ | |
| 1145 | /** | |
| 1146 | * Pulls whose check failed on a full disk, per forge (database), so one forge's recheck never drops | |
| 1147 | * another's; memory only. | |
| 1148 | */ | |
| 1133 | 1149 | const waitingOnDisk = new WeakMap<object, Map<string, { env: string; commitOid: string; repo: Repo }>>(); |
| 1134 | 1150 | |
| 1135 | 1151 | /** Once the disk is back above its floor, the pulls that waited for it are checked again. */ |
| @@ -1408,8 +1424,8 @@ export function requeueInterrupted(app: App): void { | ||
| 1408 | 1424 | for (const run of listActive(app.db, "agent")) { |
| 1409 | 1425 | if (run.state !== "running" || !leaseLapsed(run, STALE_AFTER_MS)) continue; |
| 1410 | 1426 | if (stoppedForMemory(app, run)) { |
| 1411 | // The host stopped this forge for memory: the run is queued again once; a run stopped that way twice | |
| 1412 | // would only stop it again, so it fails with the reason. | |
| 1427 | // The host stopped this forge for memory: the run is queued again once; a run stopped that | |
| 1428 | // way twice would only stop it again, so it fails with the reason. | |
| 1413 | 1429 | if (run.summary.startsWith(MEMORY_REQUEUED)) { |
| 1414 | 1430 | finishRun(app.db, run.id, { state: "failed", summary: MEMORY_OUT }); |
| 1415 | 1431 | log.info("run given up after two memory stops", { run: run.id }); |
| @@ -1420,12 +1436,16 @@ export function requeueInterrupted(app: App): void { | ||
| 1420 | 1436 | } |
| 1421 | 1437 | } |
| 1422 | 1438 | |
| 1423 | /** What a run says when the host stopped its forge for memory and the task goes back on the queue. */ | |
| 1439 | /** | |
| 1440 | * What a run says when the host stopped its forge for memory and the task goes back on the queue. | |
| 1441 | */ | |
| 1424 | 1442 | export const MEMORY_REQUEUED = "the host stopped this forge for running out of memory; queued again once"; |
| 1425 | 1443 | /** What a run says when its forge ran out of memory a second time while it ran. */ |
| 1426 | 1444 | export const MEMORY_OUT = "the host stopped this forge for running out of memory twice while this ran; not tried again — a larger plan would fit it"; |
| 1427 | 1445 | |
| 1428 | /** Whether the host's last stop of this forge was for memory and came while this run held its lease. */ | |
| 1446 | /** | |
| 1447 | * Whether the host's last stop of this forge was for memory and came while this run held its lease. | |
| 1448 | */ | |
| 1429 | 1449 | function stoppedForMemory(app: App, run: Run): boolean { |
| 1430 | 1450 | const mark = stoppedMark(app.config.dataDir); |
| 1431 | 1451 | if (mark === undefined || !/memory/.test(mark.why)) return false; |
src/crew/orphans.ts +55 −1
| @@ -7,7 +7,7 @@ | ||
| 7 | 7 | * `<crew>/<agent>/<repo id>/<n>/` and removes those `keep` does not claim. |
| 8 | 8 | */ |
| 9 | 9 | |
| 10 | import { readdirSync } from "node:fs"; | |
| 10 | import { readdirSync, statSync } from "node:fs"; | |
| 11 | 11 | import { rm } from "node:fs/promises"; |
| 12 | 12 | import { join } from "node:path"; |
| 13 | 13 | import type { Database } from "bun:sqlite"; |
| @@ -92,3 +92,57 @@ export function claimedClones(db: Database): (agent: string, repoId: string, num | ||
| 92 | 92 | return listPulls(db, repoId, { state: "open" }).some((p) => taskOf(agent, p.headBranch) === number); |
| 93 | 93 | }; |
| 94 | 94 | } |
| 95 | ||
| 96 | /** A kept clone's `node_modules` goes once the clone has been idle this long (#13514). */ | |
| 97 | export const DEPS_IDLE_MS = 86_400_000; | |
| 98 | ||
| 99 | /** | |
| 100 | * When a clone last moved: git rewrites its index through a lock file in `.git`, which stamps that | |
| 101 | * directory. | |
| 102 | */ | |
| 103 | function idleSince(dir: string): number { | |
| 104 | let at = 0; | |
| 105 | for (const p of [dir, join(dir, ".git")]) { | |
| 106 | try { | |
| 107 | at = Math.max(at, statSync(p).mtimeMs); | |
| 108 | } catch { | |
| 109 | // not there: counts as nothing | |
| 110 | } | |
| 111 | } | |
| 112 | return at; | |
| 113 | } | |
| 114 | ||
| 115 | /** | |
| 116 | * Drop the installed packages of clones kept for an open issue or pull that | |
| 117 | * nothing has touched for `idleMs`, and no live run holds. A clone's | |
| 118 | * `node_modules` is the bulk of it (about 100 of 180 MB) and is rebuilt from | |
| 119 | * the shared cache by `installDependencies` when the lock mark is missing, so | |
| 120 | * a resumed run loses a few seconds, not work. Sources and `.git` stay. | |
| 121 | * How many clones were trimmed. | |
| 122 | */ | |
| 123 | export async function sweepIdleDependencies( | |
| 124 | crewDir: string, | |
| 125 | held: (dir: string) => boolean = () => false, | |
| 126 | now: number = Date.now(), | |
| 127 | idleMs: number = DEPS_IDLE_MS, | |
| 128 | ): Promise<number> { | |
| 129 | let trimmed = 0; | |
| 130 | for (const agent of dirs(crewDir)) { | |
| 131 | for (const repoId of dirs(join(crewDir, agent))) { | |
| 132 | for (const name of dirs(join(crewDir, agent, repoId))) { | |
| 133 | if (!/^[1-9]\d*$/.test(name)) continue; | |
| 134 | const dir = join(crewDir, agent, repoId, name); | |
| 135 | const modules = join(dir, "node_modules"); | |
| 136 | if (held(dir) || now - idleSince(dir) <= idleMs) continue; | |
| 137 | try { | |
| 138 | if (!statSync(modules).isDirectory()) continue; | |
| 139 | await rm(modules, { recursive: true, force: true }); | |
| 140 | trimmed++; | |
| 141 | } catch { | |
| 142 | // none there, or busy: the next sweep looks again | |
| 143 | } | |
| 144 | } | |
| 145 | } | |
| 146 | } | |
| 147 | return trimmed; | |
| 148 | } |
test/orphan-clones.test.ts +24
| @@ -36,3 +36,27 @@ describe("sweeping clones nothing claims", () => { | ||
| 36 | 36 | expect(existsSync(live)).toBe(true); |
| 37 | 37 | }); |
| 38 | 38 | }); |
| 39 | ||
| 40 | describe("trimming idle clones' installed packages (#13514)", () => { | |
| 41 | test("only an idle, unheld clone loses node_modules; sources stay", async () => { | |
| 42 | const { sweepIdleDependencies } = await import("../src/crew/orphans.ts"); | |
| 43 | const { utimesSync } = await import("node:fs"); | |
| 44 | const root = mkdtempSync(join(process.env.TMPDIR ?? tmpdir(), "orphans-")); | |
| 45 | const mk = (n: string): string => { | |
| 46 | const d = join(root, "scout/r1", n); | |
| 47 | mkdirSync(join(d, "node_modules"), { recursive: true }); | |
| 48 | mkdirSync(join(d, ".git"), { recursive: true }); | |
| 49 | writeFileSync(join(d, "src.ts"), "x"); | |
| 50 | return d; | |
| 51 | }; | |
| 52 | const old = new Date(Date.now() - 3 * 86_400_000); | |
| 53 | const idle = mk("10"), fresh = mk("11"), live = mk("12"); | |
| 54 | for (const d of [idle, live]) for (const p of [d, join(d, ".git")]) utimesSync(p, old, old); | |
| 55 | expect(await sweepIdleDependencies(root, (d) => d === live)).toBe(1); | |
| 56 | expect(existsSync(join(idle, "node_modules"))).toBe(false); | |
| 57 | expect(existsSync(join(idle, "src.ts"))).toBe(true); | |
| 58 | expect(existsSync(join(fresh, "node_modules"))).toBe(true); | |
| 59 | expect(existsSync(join(live, "node_modules"))).toBe(true); | |
| 60 | expect(await sweepIdleDependencies(root, (d) => d === live)).toBe(0); | |
| 61 | }); | |
| 62 | }); |
new above
offline