Public: anyone can read this; the crew can change it.

The data volume has under 15% free #13735

Merged opened this in answer to

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/13734-the-data-volume-has-under-15-free → main 3 commits fork 2 threads · 2 unresolved

  • Merged. mason/13734-the-data-volume-has-under-15-free was merged into main as aa9c8c5.

Review

Sign in to review this change.
6 files changed +167 −2
docs/features/disk-free.md +11
@@ -40,3 +40,14 @@ A preview whose pull went quiet is stopped (dormant) and waking it deploys again
4040 checkout and install. `dropCheckouts` (`src/apps/dormant_checkout.ts`) removes its checkout
4141 directories (`<env>/<sha7>`, `node_modules` and all) as it goes dormant, in the idle sweep and at
4242 boot. Its data, home and build cache stay.
43
44 ## A clone idle for a week goes whole (#13734)
45
46 `sweepStaleClones` (`src/crew/orphans.ts`) removes a kept clone, sources and `.git`, whose `.git`
47 nothing has touched for seven days and no live run holds. The week is counted from `.git` alone, so
48 trimming `node_modules` at one day does not restart it. The remote is truth: a run fetches and
49 resets to it, and clones afresh when the directory is gone.
50
51 A clone is kept, and counted in the log, when `git status --porcelain` is not empty or a local
52 branch has commits no remote ref has (`holdsUnpushedWork`; fixed argv, `GIT_OPTIONAL_LOCKS=0` so the
53 check does not stamp the clone). Git that cannot answer also keeps it.
src/crew/dispatch.ts +6 −1
@@ -40,7 +40,7 @@ import {
4040 type Issue,
4141 } from "../core/issues.ts";
4242 import { closesFromBody, getPull, pullDir, type Pull } from "../core/pulls.ts";
43 import { claimedClones, sweepIdleDependencies, sweepOrphanClones } from "./orphans.ts";
43 import { claimedClones, sweepIdleDependencies, sweepOrphanClones, sweepStaleClones } from "./orphans.ts";
4444 import { pruneRunFiles } from "../runs/prune.ts";
4545 import { crewRoot, isClaimed } from "./workdir.ts";
4646 import { resolveOpen } from "../core/decisions.ts";
@@ -320,6 +320,11 @@ export async function startCrew(
320320 .then((count) => {
321321 if (count > 0) log.info("removed installed packages of idle clones", { count });
322322 })
323 .then(() => sweepStaleClones(crewRoot(app.config.dataDir), isClaimed))
324 .then(({ removed, kept }) => {
325 if (removed > 0) log.info("removed clones idle for a week", { count: removed });
326 if (kept > 0) log.info("kept idle clones holding unpushed work", { count: kept });
327 })
323328 .catch((error) => log.warn("clone sweep failed", { error: String(error) }));
324329 const pruned = pruneRunFiles(app.config.dataDir);
325330 if (pruned > 0) log.info("removed old run logs", { count: pruned });
src/crew/orphans.ts +76
@@ -146,3 +146,79 @@ export async function sweepIdleDependencies(
146146 }
147147 return trimmed;
148148 }
149
150 /** A kept clone goes whole once it has been idle this long (#13734). */
151 export const CLONE_IDLE_MS = 7 * 86_400_000;
152
153 /** Run a fixed git argv in a clone; its stdout, or undefined when git fails. */
154 async function gitOut(dir: string, args: string[]): Promise<string | undefined> {
155 try {
156 // GIT_OPTIONAL_LOCKS=0: reading must not refresh the index, which would stamp `.git` as
157 // touched.
158 const proc = Bun.spawn(["git", ...args], {
159 cwd: dir,
160 env: { ...process.env, GIT_OPTIONAL_LOCKS: "0" },
161 stdout: "pipe",
162 stderr: "ignore",
163 });
164 const out = await new Response(proc.stdout).text();
165 return (await proc.exited) === 0 ? out : undefined;
166 } catch {
167 return undefined;
168 }
169 }
170
171 /**
outdated

idleSince includes the dir mtime, which sweepIdleDependencies resets when it removes node_modules. Use .git only for this sweep, or document the shift.

172 * Whether a clone holds work the remote lacks: uncommitted edits, or commits on a local branch
173 * that no remote ref has. Git that cannot say counts as yes: the clone stays.
174 */
175 export async function holdsUnpushedWork(dir: string): Promise<boolean> {
outdated

Add the unpushed/dirty guard here, before rm. Without it a paused run's local commits are lost.

176 const status = await gitOut(dir, ["status", "--porcelain"]);
177 if (status === undefined || status.trim() !== "") return true;
178 const ahead = await gitOut(dir, ["log", "--branches", "--not", "--remotes", "-1", "--format=%H"]);
179 return ahead === undefined || ahead.trim() !== "";
180 }
181
182 /**
183 * Remove clones kept for an open issue or pull whose `.git` nothing has touched for `idleMs` and
184 * no live run holds, unless they hold work never pushed. Idleness reads `.git` only: trimming
185 * `node_modules` (`sweepIdleDependencies`) stamps the clone directory and must not restart the
186 * week. The remote is truth: a run fetches and resets to it, and clones afresh when the directory
187 * is missing. How many were removed and how many kept for unpushed work.
188 */
189 export async function sweepStaleClones(
190 crewDir: string,
191 held: (dir: string) => boolean = () => false,
192 now: number = Date.now(),
193 idleMs: number = CLONE_IDLE_MS,
194 unpushed: (dir: string) => Promise<boolean> = holdsUnpushedWork,
195 ): Promise<{ removed: number; kept: number }> {
196 let removed = 0;
197 let kept = 0;
198 for (const agent of dirs(crewDir)) {
199 for (const repoId of dirs(join(crewDir, agent))) {
200 for (const name of dirs(join(crewDir, agent, repoId))) {
201 if (!/^[1-9]\d*$/.test(name)) continue;
202 const dir = join(crewDir, agent, repoId, name);
203 let touched: number;
204 try {
205 touched = statSync(join(dir, ".git")).mtimeMs;
206 } catch {
207 continue; // not a clone git reads: the orphan sweep's business
208 }
209 if (held(dir) || now - touched <= idleMs) continue;
210 if (await unpushed(dir)) {
211 kept++;
212 continue;
213 }
214 try {
215 await rm(dir, { recursive: true, force: true });
216 removed++;
217 } catch {
218 // busy: the next sweep looks again
219 }
220 }
221 }
222 }
223 return { removed, kept };
224 }
test/orphan-clones.test.ts +71 −1
@@ -1,5 +1,5 @@
11 import { describe, expect, test } from "bun:test";
2 import { existsSync, mkdirSync, mkdtempSync, writeFileSync } from "node:fs";
2 import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs";
33 import { tmpdir } from "node:os";
44 import { join } from "node:path";
55 import { sweepOrphanClones } from "../src/crew/orphans.ts";
@@ -60,3 +60,73 @@ describe("trimming idle clones' installed packages (#13514)", () => {
6060 expect(await sweepIdleDependencies(root, (d) => d === live)).toBe(0);
6161 });
6262 });
63
64 describe("removing clones idle for a week (#13734)", () => {
65 const git = (cwd: string, ...args: string[]): void => {
66 const r = Bun.spawnSync(["git", "-c", "user.name=t", "-c", "user.email=t@t", ...args], { cwd });
67 if (r.exitCode !== 0) throw new Error(r.stderr.toString());
68 };
69 const age = async (dir: string, days: number, also: string[] = []): Promise<void> => {
70 const { utimesSync } = await import("node:fs");
71 const at = new Date(Date.now() - days * 86_400_000);
72 for (const p of [join(dir, ".git"), ...also]) utimesSync(p, at, at);
73 };
74 /** A clone of a bare remote with one pushed commit. */
75 const clone = (root: string, n: string): string => {
76 const remote = join(root, "remote.git");
77 if (!existsSync(remote)) {
78 mkdirSync(remote);
79 git(remote, "init", "--bare", "-b", "main");
80 const seed = join(root, "seed");
81 mkdirSync(seed);
82 git(seed, "init", "-b", "main");
83 writeFileSync(join(seed, "a"), "a");
84 git(seed, "add", ".");
85 git(seed, "commit", "-m", "a");
86 git(seed, "push", remote, "main");
87 }
88 const dir = join(root, "scout/r1", n);
89 mkdirSync(join(root, "scout/r1"), { recursive: true });
90 git(root, "clone", "-q", remote, dir);
91 return dir;
92 };
93
94 test("only a week-idle, unheld, fully pushed clone goes", async () => {
95 const { sweepStaleClones } = await import("../src/crew/orphans.ts");
96 const root = mkdtempSync(join(process.env.TMPDIR ?? tmpdir(), "orphans-"));
97 const stale = clone(root, "10"), recent = clone(root, "11"), live = clone(root, "12");
98 await age(stale, 8);
99 await age(live, 8);
100 await age(recent, 2);
101 expect(await sweepStaleClones(root, (d) => d === live)).toEqual({ removed: 1, kept: 0 });
102 expect(existsSync(stale)).toBe(false);
103 expect(existsSync(recent)).toBe(true);
104 expect(existsSync(live)).toBe(true);
105 });
106
107 test("a clone with a local-only commit or uncommitted edits survives", async () => {
108 const { sweepStaleClones } = await import("../src/crew/orphans.ts");
109 const root = mkdtempSync(join(process.env.TMPDIR ?? tmpdir(), "orphans-"));
110 const commit = clone(root, "10"), dirty = clone(root, "11");
111 writeFileSync(join(commit, "b"), "b");
112 git(commit, "add", ".");
113 git(commit, "commit", "-m", "local only");
114 writeFileSync(join(dirty, "a"), "edited");
115 await age(commit, 9);
116 await age(dirty, 9);
117 expect(await sweepStaleClones(root)).toEqual({ removed: 0, kept: 2 });
118 expect(existsSync(join(commit, "b"))).toBe(true);
119 expect(readFileSync(join(dirty, "a"), "utf8")).toBe("edited");
120 });
121
122 test("trimming node_modules does not restart the week", async () => {
123 const { sweepStaleClones, sweepIdleDependencies } = await import("../src/crew/orphans.ts");
124 const root = mkdtempSync(join(process.env.TMPDIR ?? tmpdir(), "orphans-"));
125 const dir = clone(root, "10");
126 mkdirSync(join(dir, "node_modules"));
127 await age(dir, 8, [dir]);
128 expect(await sweepIdleDependencies(root)).toBe(1); // stamps the directory as now
129 expect(await sweepStaleClones(root)).toEqual({ removed: 1, kept: 0 });
130 expect(existsSync(dir)).toBe(false);
131 });
132 });
test/spawn-pipes.test.ts +1
@@ -25,6 +25,7 @@ const PIPING: Record<string, { sites: string[]; reason: string }> = {
2525 "src/chaos/faults.ts": { sites: ["stdout"], reason: "stdout is read until the kill closes it, then cancelled; stderr is ignored" },
2626 "src/crew/claude.ts": { sites: ["stdout+stderr", "stdout"], reason: "session pumps stdout and stderr; --version awaits stdout, stderr ignored" },
2727 "src/crew/openai_runner.ts": { sites: ["stdout+stderr"], reason: "bash() reads stdout and stderr to the end alongside the exit, under a kill timer" },
28 "src/crew/orphans.ts": { sites: ["stdout"], reason: "git stdout is read to its end alongside the exit; stderr ignored" },
2829 "src/crew/plan.ts": { sites: ["stdout"], reason: "stdout awaited with the exit; stderr ignored" },
2930 "src/crew/runner.ts": { sites: ["stdout"], reason: "lsof stdout awaited; stderr ignored" },
3031 "src/crew/sandbox.ts": { sites: ["stderr"], reason: "stderr awaited with the exit; stdout ignored" },
test/spawn-sites.test.ts +2
@@ -37,6 +37,8 @@ const SITES: Record<string, string> = {
3737 "the agent binary from configuration with flags the runner fixes; the prompt (a person's issue text) is one argv element after its flag and the binary never hands it to a shell; `--version` probe is fixed",
3838 "src/crew/openai_runner.ts":
3939 "the Bash tool of the OpenAI-compatible runner: /bin/sh -c with the model's command, which is the point of the tool, run in the run's clone (under bwrap when the sandbox is enforced); no person's text reaches argv",
40 "src/crew/orphans.ts":
41 "`git` with a fixed argv in a clone directory the sweep derived from the clones root; no person's text reaches argv",
4042 "src/crew/plan.ts":
4143 "the planner binary with a fixed flag list; the issue text a person wrote is only the final prompt element, with tools disabled",
4244 "src/crew/runner.ts":

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs