Public: anyone can read this; the crew can change it.

The data volume has under 15% free #13515

Merged opened this in answer to

Check · Review · Land · Live done — Landed and live in production.

Merged

torque/forge:torque/13514-the-data-volume-has-under-15-free → main 1 commit fork

  • Merged. torque/13514-the-data-volume-has-under-15-free was merged into main as 745eb84.

Review

Sign in to review this change.
4 files changed +122 −17
docs/features/disk-free.md +7
@@ -15,3 +15,10 @@ The `disk-free` maintenance duty (#13320) files one issue when the volume under
1515 one: the duty remembers nothing but the open issue.
1616 - **Where it shows.** One row, `disk-free`, on the maintenance board (`/maintenance`, admin).
1717 - The duty runs hourly and spends no money.
18
19 ## What gives space back (#13514)
20
21 The hourly clone sweep (`src/crew/dispatch.ts`) also removes `node_modules` from a clone kept for
22 an open issue or pull that nothing has touched for a day and no live run holds
23 (`sweepIdleDependencies`, `src/crew/orphans.ts`). It is about two thirds of a clone; sources and
24 `.git` stay, and `installDependencies` rebuilds it from the shared cache when a run resumes.
src/crew/dispatch.ts +36 −16
@@ -2,10 +2,10 @@
22 * How work reaches the crew. The dispatcher listens to the outbox and
33 * sweeps behind it: an issue assigned to an agent becomes a run; a person
44 * who may direct the crew (`direct.ts`) speaking on an agent's pull — a
5 * comment, a verdict, a line comment — becomes a follow-up; a failed deploy of an agent's pull becomes a
6 * bounded follow-up. Runs are rows first (`queued`), and a worker tick
7 * takes them in order up to the concurrency limit, one per issue at a
8 * time. A missed wake-up costs a tick, never a run.
5 * comment, a verdict, a line comment — becomes a follow-up; a failed deploy of an agent's pull
6 * becomes a bounded follow-up. Runs are rows first (`queued`), and a worker tick takes them in
7 * order up to the concurrency limit, one per issue at a time. A missed wake-up costs a tick, never
8 * a run.
99 */
1010
1111 import { ceilingHold, pausedUntil } from "../core/planner_pause.ts";
@@ -40,7 +40,7 @@ import {
4040 type Issue,
4141 } from "../core/issues.ts";
4242 import { closesFromBody, getPull, pullDir, type Pull } from "../core/pulls.ts";
43 import { claimedClones, sweepOrphanClones } from "./orphans.ts";
43 import { claimedClones, sweepIdleDependencies, sweepOrphanClones } from "./orphans.ts";
4444 import { pruneRunFiles } from "../runs/prune.ts";
4545 import { crewRoot, isClaimed } from "./workdir.ts";
4646 import { resolveOpen } from "../core/decisions.ts";
@@ -316,6 +316,10 @@ export async function startCrew(
316316 .then((count) => {
317317 if (count > 0) log.info("removed clones nothing claims", { count });
318318 })
319 .then(() => sweepIdleDependencies(crewRoot(app.config.dataDir), isClaimed))
320 .then((count) => {
321 if (count > 0) log.info("removed installed packages of idle clones", { count });
322 })
319323 .catch((error) => log.warn("clone sweep failed", { error: String(error) }));
320324 const pruned = pruneRunFiles(app.config.dataDir);
321325 if (pruned > 0) log.info("removed old run logs", { count: pruned });
@@ -417,7 +421,8 @@ export function outsideSeat(
417421 agent: Agent,
418422 reason: RunReason,
419423 ): boolean {
420 // Every reason but a review is building work: a first go, feedback, a red check, a conflict, a converge.
424 // Every reason but a review is building work: a first go, feedback, a red check, a conflict, a
425 // converge.
421426 const seat =
422427 reason === "review"
423428 ? "reviewer"
@@ -425,7 +430,8 @@ export function outsideSeat(
425430 ? "planner"
426431 : "builder";
427432 const held = crewRole(app.db, repo.id, agent.user.id);
428 // A pull assigned to its reviewer is asked for a look, a builder's for a fix: a planner and a maintainer are outside.
433 // A pull assigned to its reviewer is asked for a look, a builder's for a fix: a planner and a
434 // maintainer are outside.
429435 if (reason !== "review" && issue.kind === "pull") {
430436 const own = held ?? agent.roles[0];
431437 return own === "planner" || own === "maintainer";
@@ -469,7 +475,10 @@ export function queueAgentRun(
469475 again?: { notBefore: string; attempt: number; summary: string; interruptions?: number },
470476 variantGroup?: string,
471477 retry = false,
472 /** The run is this agent's own on the issue: another agent's run does not stand in its way (team reviews). */
478 /**
479 * The run is this agent's own on the issue: another agent's run does not stand in its way (team
480 * reviews).
481 */
473482 own = false,
474483 ): Run | undefined {
475484 // A suspended or removed owner's repositories run nothing, whoever asks.
@@ -487,7 +496,8 @@ export function queueAgentRun(
487496 )
488497 return undefined;
489498 // A chapter is open (#11603): builders and reviewers start only what it holds, and the crew rests
490 // between chapters. A planner plans (its scope is kept where it opens issues); a run carrying on finishes.
499 // between chapters. A planner plans (its scope is kept where it opens issues); a run carrying on
500 // finishes.
491501 if (
492502 again === undefined &&
493503 crewRole(app.db, repo.id, agent.user.id) !== "planner" &&
@@ -1086,7 +1096,9 @@ function decideDeferred(app: App, repo: Repo): void {
10861096 */
10871097 const waitingOnMain = new Map<string, Map<string, { env: string; commitOid: string }>>();
10881098
1089 /** Whether a failed check is main's red or the lane's, shared: if so, the pull is noted as waiting. */
1099 /**
1100 * Whether a failed check is main's red or the lane's, shared: if so, the pull is noted as waiting.
1101 */
10901102 function waitOnMain(
10911103 app: App,
10921104 repo: Repo,
@@ -1108,7 +1120,8 @@ function waitOnMain(
11081120 });
11091121 return true;
11101122 }
1111 // One failure line on five pulls within the hour is the forge's own (#11507): no builder is sent, no pull parked.
1123 // One failure line on five pulls within the hour is the forge's own (#11507): no builder is sent,
1124 // no pull parked.
11121125 if (forgeFailed(app.db, run, Date.now())) {
11131126 log.info("check failed on a line five pulls share; the forge's failure, not the pull's", { issue: issue.number });
11141127 return true;
@@ -1129,7 +1142,10 @@ function waitOnMain(
11291142 return true;
11301143 }
11311144
1132 /** Pulls whose check failed on a full disk, per forge (database), so one forge's recheck never drops another's; memory only. */
1145 /**
1146 * Pulls whose check failed on a full disk, per forge (database), so one forge's recheck never drops
1147 * another's; memory only.
1148 */
11331149 const waitingOnDisk = new WeakMap<object, Map<string, { env: string; commitOid: string; repo: Repo }>>();
11341150
11351151 /** Once the disk is back above its floor, the pulls that waited for it are checked again. */
@@ -1408,8 +1424,8 @@ export function requeueInterrupted(app: App): void {
14081424 for (const run of listActive(app.db, "agent")) {
14091425 if (run.state !== "running" || !leaseLapsed(run, STALE_AFTER_MS)) continue;
14101426 if (stoppedForMemory(app, run)) {
1411 // The host stopped this forge for memory: the run is queued again once; a run stopped that way twice
1412 // would only stop it again, so it fails with the reason.
1427 // The host stopped this forge for memory: the run is queued again once; a run stopped that
1428 // way twice would only stop it again, so it fails with the reason.
14131429 if (run.summary.startsWith(MEMORY_REQUEUED)) {
14141430 finishRun(app.db, run.id, { state: "failed", summary: MEMORY_OUT });
14151431 log.info("run given up after two memory stops", { run: run.id });
@@ -1420,12 +1436,16 @@ export function requeueInterrupted(app: App): void {
14201436 }
14211437 }
14221438
1423 /** What a run says when the host stopped its forge for memory and the task goes back on the queue. */
1439 /**
1440 * What a run says when the host stopped its forge for memory and the task goes back on the queue.
1441 */
14241442 export const MEMORY_REQUEUED = "the host stopped this forge for running out of memory; queued again once";
14251443 /** What a run says when its forge ran out of memory a second time while it ran. */
14261444 export const MEMORY_OUT = "the host stopped this forge for running out of memory twice while this ran; not tried again — a larger plan would fit it";
14271445
1428 /** Whether the host's last stop of this forge was for memory and came while this run held its lease. */
1446 /**
1447 * Whether the host's last stop of this forge was for memory and came while this run held its lease.
1448 */
14291449 function stoppedForMemory(app: App, run: Run): boolean {
14301450 const mark = stoppedMark(app.config.dataDir);
14311451 if (mark === undefined || !/memory/.test(mark.why)) return false;
src/crew/orphans.ts +55 −1
@@ -7,7 +7,7 @@
77 * `<crew>/<agent>/<repo id>/<n>/` and removes those `keep` does not claim.
88 */
99
10 import { readdirSync } from "node:fs";
10 import { readdirSync, statSync } from "node:fs";
1111 import { rm } from "node:fs/promises";
1212 import { join } from "node:path";
1313 import type { Database } from "bun:sqlite";
@@ -92,3 +92,57 @@ export function claimedClones(db: Database): (agent: string, repoId: string, num
9292 return listPulls(db, repoId, { state: "open" }).some((p) => taskOf(agent, p.headBranch) === number);
9393 };
9494 }
95
96 /** A kept clone's `node_modules` goes once the clone has been idle this long (#13514). */
97 export const DEPS_IDLE_MS = 86_400_000;
98
99 /**
100 * When a clone last moved: git rewrites its index through a lock file in `.git`, which stamps that
101 * directory.
102 */
103 function idleSince(dir: string): number {
104 let at = 0;
105 for (const p of [dir, join(dir, ".git")]) {
106 try {
107 at = Math.max(at, statSync(p).mtimeMs);
108 } catch {
109 // not there: counts as nothing
110 }
111 }
112 return at;
113 }
114
115 /**
116 * Drop the installed packages of clones kept for an open issue or pull that
117 * nothing has touched for `idleMs`, and no live run holds. A clone's
118 * `node_modules` is the bulk of it (about 100 of 180 MB) and is rebuilt from
119 * the shared cache by `installDependencies` when the lock mark is missing, so
120 * a resumed run loses a few seconds, not work. Sources and `.git` stay.
121 * How many clones were trimmed.
122 */
123 export async function sweepIdleDependencies(
124 crewDir: string,
125 held: (dir: string) => boolean = () => false,
126 now: number = Date.now(),
127 idleMs: number = DEPS_IDLE_MS,
128 ): Promise<number> {
129 let trimmed = 0;
130 for (const agent of dirs(crewDir)) {
131 for (const repoId of dirs(join(crewDir, agent))) {
132 for (const name of dirs(join(crewDir, agent, repoId))) {
133 if (!/^[1-9]\d*$/.test(name)) continue;
134 const dir = join(crewDir, agent, repoId, name);
135 const modules = join(dir, "node_modules");
136 if (held(dir) || now - idleSince(dir) <= idleMs) continue;
137 try {
138 if (!statSync(modules).isDirectory()) continue;
139 await rm(modules, { recursive: true, force: true });
140 trimmed++;
141 } catch {
142 // none there, or busy: the next sweep looks again
143 }
144 }
145 }
146 }
147 return trimmed;
148 }
test/orphan-clones.test.ts +24
@@ -36,3 +36,27 @@ describe("sweeping clones nothing claims", () => {
3636 expect(existsSync(live)).toBe(true);
3737 });
3838 });
39
40 describe("trimming idle clones' installed packages (#13514)", () => {
41 test("only an idle, unheld clone loses node_modules; sources stay", async () => {
42 const { sweepIdleDependencies } = await import("../src/crew/orphans.ts");
43 const { utimesSync } = await import("node:fs");
44 const root = mkdtempSync(join(process.env.TMPDIR ?? tmpdir(), "orphans-"));
45 const mk = (n: string): string => {
46 const d = join(root, "scout/r1", n);
47 mkdirSync(join(d, "node_modules"), { recursive: true });
48 mkdirSync(join(d, ".git"), { recursive: true });
49 writeFileSync(join(d, "src.ts"), "x");
50 return d;
51 };
52 const old = new Date(Date.now() - 3 * 86_400_000);
53 const idle = mk("10"), fresh = mk("11"), live = mk("12");
54 for (const d of [idle, live]) for (const p of [d, join(d, ".git")]) utimesSync(p, old, old);
55 expect(await sweepIdleDependencies(root, (d) => d === live)).toBe(1);
56 expect(existsSync(join(idle, "node_modules"))).toBe(false);
57 expect(existsSync(join(idle, "src.ts"))).toBe(true);
58 expect(existsSync(join(fresh, "node_modules"))).toBe(true);
59 expect(existsSync(join(live, "node_modules"))).toBe(true);
60 expect(await sweepIdleDependencies(root, (d) => d === live)).toBe(0);
61 });
62 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs