Public: anyone can read this; the crew can change it.
The data volume has under 15% free #13735
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/13734-the-data-volume-has-under-15-free → main 3 commits fork 2 threads · 2 unresolved
6 files
changed +167
−2
docs/features/disk-free.md +11
| @@ -40,3 +40,14 @@ A preview whose pull went quiet is stopped (dormant) and waking it deploys again | ||
| 40 | 40 | checkout and install. `dropCheckouts` (`src/apps/dormant_checkout.ts`) removes its checkout |
| 41 | 41 | directories (`<env>/<sha7>`, `node_modules` and all) as it goes dormant, in the idle sweep and at |
| 42 | 42 | boot. Its data, home and build cache stay. |
| 43 | ||
| 44 | ## A clone idle for a week goes whole (#13734) | |
| 45 | ||
| 46 | `sweepStaleClones` (`src/crew/orphans.ts`) removes a kept clone, sources and `.git`, whose `.git` | |
| 47 | nothing has touched for seven days and no live run holds. The week is counted from `.git` alone, so | |
| 48 | trimming `node_modules` at one day does not restart it. The remote is truth: a run fetches and | |
| 49 | resets to it, and clones afresh when the directory is gone. | |
| 50 | ||
| 51 | A clone is kept, and counted in the log, when `git status --porcelain` is not empty or a local | |
| 52 | branch has commits no remote ref has (`holdsUnpushedWork`; fixed argv, `GIT_OPTIONAL_LOCKS=0` so the | |
| 53 | check does not stamp the clone). Git that cannot answer also keeps it. |
src/crew/dispatch.ts +6 −1
| @@ -40,7 +40,7 @@ import { | ||
| 40 | 40 | type Issue, |
| 41 | 41 | } from "../core/issues.ts"; |
| 42 | 42 | import { closesFromBody, getPull, pullDir, type Pull } from "../core/pulls.ts"; |
| 43 | import { claimedClones, sweepIdleDependencies, sweepOrphanClones } from "./orphans.ts"; | |
| 43 | import { claimedClones, sweepIdleDependencies, sweepOrphanClones, sweepStaleClones } from "./orphans.ts"; | |
| 44 | 44 | import { pruneRunFiles } from "../runs/prune.ts"; |
| 45 | 45 | import { crewRoot, isClaimed } from "./workdir.ts"; |
| 46 | 46 | import { resolveOpen } from "../core/decisions.ts"; |
| @@ -320,6 +320,11 @@ export async function startCrew( | ||
| 320 | 320 | .then((count) => { |
| 321 | 321 | if (count > 0) log.info("removed installed packages of idle clones", { count }); |
| 322 | 322 | }) |
| 323 | .then(() => sweepStaleClones(crewRoot(app.config.dataDir), isClaimed)) | |
| 324 | .then(({ removed, kept }) => { | |
| 325 | if (removed > 0) log.info("removed clones idle for a week", { count: removed }); | |
| 326 | if (kept > 0) log.info("kept idle clones holding unpushed work", { count: kept }); | |
| 327 | }) | |
| 323 | 328 | .catch((error) => log.warn("clone sweep failed", { error: String(error) })); |
| 324 | 329 | const pruned = pruneRunFiles(app.config.dataDir); |
| 325 | 330 | if (pruned > 0) log.info("removed old run logs", { count: pruned }); |
src/crew/orphans.ts +76
| @@ -146,3 +146,79 @@ export async function sweepIdleDependencies( | ||
| 146 | 146 | } |
| 147 | 147 | return trimmed; |
| 148 | 148 | } |
| 149 | ||
| 150 | /** A kept clone goes whole once it has been idle this long (#13734). */ | |
| 151 | export const CLONE_IDLE_MS = 7 * 86_400_000; | |
| 152 | ||
| 153 | /** Run a fixed git argv in a clone; its stdout, or undefined when git fails. */ | |
| 154 | async function gitOut(dir: string, args: string[]): Promise<string | undefined> { | |
| 155 | try { | |
| 156 | // GIT_OPTIONAL_LOCKS=0: reading must not refresh the index, which would stamp `.git` as | |
| 157 | // touched. | |
| 158 | const proc = Bun.spawn(["git", ...args], { | |
| 159 | cwd: dir, | |
| 160 | env: { ...process.env, GIT_OPTIONAL_LOCKS: "0" }, | |
| 161 | stdout: "pipe", | |
| 162 | stderr: "ignore", | |
| 163 | }); | |
| 164 | const out = await new Response(proc.stdout).text(); | |
| 165 | return (await proc.exited) === 0 ? out : undefined; | |
| 166 | } catch { | |
| 167 | return undefined; | |
| 168 | } | |
| 169 | } | |
| 170 | ||
| 171 | /** | |
| 172 | * Whether a clone holds work the remote lacks: uncommitted edits, or commits on a local branch | |
| 173 | * that no remote ref has. Git that cannot say counts as yes: the clone stays. | |
| 174 | */ | |
| 175 | export async function holdsUnpushedWork(dir: string): Promise<boolean> { | |
outdated
Add the unpushed/dirty guard here, before |
||
| 176 | const status = await gitOut(dir, ["status", "--porcelain"]); | |
| 177 | if (status === undefined || status.trim() !== "") return true; | |
| 178 | const ahead = await gitOut(dir, ["log", "--branches", "--not", "--remotes", "-1", "--format=%H"]); | |
| 179 | return ahead === undefined || ahead.trim() !== ""; | |
| 180 | } | |
| 181 | ||
| 182 | /** | |
| 183 | * Remove clones kept for an open issue or pull whose `.git` nothing has touched for `idleMs` and | |
| 184 | * no live run holds, unless they hold work never pushed. Idleness reads `.git` only: trimming | |
| 185 | * `node_modules` (`sweepIdleDependencies`) stamps the clone directory and must not restart the | |
| 186 | * week. The remote is truth: a run fetches and resets to it, and clones afresh when the directory | |
| 187 | * is missing. How many were removed and how many kept for unpushed work. | |
| 188 | */ | |
| 189 | export async function sweepStaleClones( | |
| 190 | crewDir: string, | |
| 191 | held: (dir: string) => boolean = () => false, | |
| 192 | now: number = Date.now(), | |
| 193 | idleMs: number = CLONE_IDLE_MS, | |
| 194 | unpushed: (dir: string) => Promise<boolean> = holdsUnpushedWork, | |
| 195 | ): Promise<{ removed: number; kept: number }> { | |
| 196 | let removed = 0; | |
| 197 | let kept = 0; | |
| 198 | for (const agent of dirs(crewDir)) { | |
| 199 | for (const repoId of dirs(join(crewDir, agent))) { | |
| 200 | for (const name of dirs(join(crewDir, agent, repoId))) { | |
| 201 | if (!/^[1-9]\d*$/.test(name)) continue; | |
| 202 | const dir = join(crewDir, agent, repoId, name); | |
| 203 | let touched: number; | |
| 204 | try { | |
| 205 | touched = statSync(join(dir, ".git")).mtimeMs; | |
| 206 | } catch { | |
| 207 | continue; // not a clone git reads: the orphan sweep's business | |
| 208 | } | |
| 209 | if (held(dir) || now - touched <= idleMs) continue; | |
| 210 | if (await unpushed(dir)) { | |
| 211 | kept++; | |
| 212 | continue; | |
| 213 | } | |
| 214 | try { | |
| 215 | await rm(dir, { recursive: true, force: true }); | |
| 216 | removed++; | |
| 217 | } catch { | |
| 218 | // busy: the next sweep looks again | |
| 219 | } | |
| 220 | } | |
| 221 | } | |
| 222 | } | |
| 223 | return { removed, kept }; | |
| 224 | } |
test/orphan-clones.test.ts +71 −1
| @@ -1,5 +1,5 @@ | ||
| 1 | 1 | import { describe, expect, test } from "bun:test"; |
| 2 | import { existsSync, mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; | |
| 2 | import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; | |
| 3 | 3 | import { tmpdir } from "node:os"; |
| 4 | 4 | import { join } from "node:path"; |
| 5 | 5 | import { sweepOrphanClones } from "../src/crew/orphans.ts"; |
| @@ -60,3 +60,73 @@ describe("trimming idle clones' installed packages (#13514)", () => { | ||
| 60 | 60 | expect(await sweepIdleDependencies(root, (d) => d === live)).toBe(0); |
| 61 | 61 | }); |
| 62 | 62 | }); |
| 63 | ||
| 64 | describe("removing clones idle for a week (#13734)", () => { | |
| 65 | const git = (cwd: string, ...args: string[]): void => { | |
| 66 | const r = Bun.spawnSync(["git", "-c", "user.name=t", "-c", "user.email=t@t", ...args], { cwd }); | |
| 67 | if (r.exitCode !== 0) throw new Error(r.stderr.toString()); | |
| 68 | }; | |
| 69 | const age = async (dir: string, days: number, also: string[] = []): Promise<void> => { | |
| 70 | const { utimesSync } = await import("node:fs"); | |
| 71 | const at = new Date(Date.now() - days * 86_400_000); | |
| 72 | for (const p of [join(dir, ".git"), ...also]) utimesSync(p, at, at); | |
| 73 | }; | |
| 74 | /** A clone of a bare remote with one pushed commit. */ | |
| 75 | const clone = (root: string, n: string): string => { | |
| 76 | const remote = join(root, "remote.git"); | |
| 77 | if (!existsSync(remote)) { | |
| 78 | mkdirSync(remote); | |
| 79 | git(remote, "init", "--bare", "-b", "main"); | |
| 80 | const seed = join(root, "seed"); | |
| 81 | mkdirSync(seed); | |
| 82 | git(seed, "init", "-b", "main"); | |
| 83 | writeFileSync(join(seed, "a"), "a"); | |
| 84 | git(seed, "add", "."); | |
| 85 | git(seed, "commit", "-m", "a"); | |
| 86 | git(seed, "push", remote, "main"); | |
| 87 | } | |
| 88 | const dir = join(root, "scout/r1", n); | |
| 89 | mkdirSync(join(root, "scout/r1"), { recursive: true }); | |
| 90 | git(root, "clone", "-q", remote, dir); | |
| 91 | return dir; | |
| 92 | }; | |
| 93 | ||
| 94 | test("only a week-idle, unheld, fully pushed clone goes", async () => { | |
| 95 | const { sweepStaleClones } = await import("../src/crew/orphans.ts"); | |
| 96 | const root = mkdtempSync(join(process.env.TMPDIR ?? tmpdir(), "orphans-")); | |
| 97 | const stale = clone(root, "10"), recent = clone(root, "11"), live = clone(root, "12"); | |
| 98 | await age(stale, 8); | |
| 99 | await age(live, 8); | |
| 100 | await age(recent, 2); | |
| 101 | expect(await sweepStaleClones(root, (d) => d === live)).toEqual({ removed: 1, kept: 0 }); | |
| 102 | expect(existsSync(stale)).toBe(false); | |
| 103 | expect(existsSync(recent)).toBe(true); | |
| 104 | expect(existsSync(live)).toBe(true); | |
| 105 | }); | |
| 106 | ||
| 107 | test("a clone with a local-only commit or uncommitted edits survives", async () => { | |
| 108 | const { sweepStaleClones } = await import("../src/crew/orphans.ts"); | |
| 109 | const root = mkdtempSync(join(process.env.TMPDIR ?? tmpdir(), "orphans-")); | |
| 110 | const commit = clone(root, "10"), dirty = clone(root, "11"); | |
| 111 | writeFileSync(join(commit, "b"), "b"); | |
| 112 | git(commit, "add", "."); | |
| 113 | git(commit, "commit", "-m", "local only"); | |
| 114 | writeFileSync(join(dirty, "a"), "edited"); | |
| 115 | await age(commit, 9); | |
| 116 | await age(dirty, 9); | |
| 117 | expect(await sweepStaleClones(root)).toEqual({ removed: 0, kept: 2 }); | |
| 118 | expect(existsSync(join(commit, "b"))).toBe(true); | |
| 119 | expect(readFileSync(join(dirty, "a"), "utf8")).toBe("edited"); | |
| 120 | }); | |
| 121 | ||
| 122 | test("trimming node_modules does not restart the week", async () => { | |
| 123 | const { sweepStaleClones, sweepIdleDependencies } = await import("../src/crew/orphans.ts"); | |
| 124 | const root = mkdtempSync(join(process.env.TMPDIR ?? tmpdir(), "orphans-")); | |
| 125 | const dir = clone(root, "10"); | |
| 126 | mkdirSync(join(dir, "node_modules")); | |
| 127 | await age(dir, 8, [dir]); | |
| 128 | expect(await sweepIdleDependencies(root)).toBe(1); // stamps the directory as now | |
| 129 | expect(await sweepStaleClones(root)).toEqual({ removed: 1, kept: 0 }); | |
| 130 | expect(existsSync(dir)).toBe(false); | |
| 131 | }); | |
| 132 | }); |
test/spawn-pipes.test.ts +1
| @@ -25,6 +25,7 @@ const PIPING: Record<string, { sites: string[]; reason: string }> = { | ||
| 25 | 25 | "src/chaos/faults.ts": { sites: ["stdout"], reason: "stdout is read until the kill closes it, then cancelled; stderr is ignored" }, |
| 26 | 26 | "src/crew/claude.ts": { sites: ["stdout+stderr", "stdout"], reason: "session pumps stdout and stderr; --version awaits stdout, stderr ignored" }, |
| 27 | 27 | "src/crew/openai_runner.ts": { sites: ["stdout+stderr"], reason: "bash() reads stdout and stderr to the end alongside the exit, under a kill timer" }, |
| 28 | "src/crew/orphans.ts": { sites: ["stdout"], reason: "git stdout is read to its end alongside the exit; stderr ignored" }, | |
| 28 | 29 | "src/crew/plan.ts": { sites: ["stdout"], reason: "stdout awaited with the exit; stderr ignored" }, |
| 29 | 30 | "src/crew/runner.ts": { sites: ["stdout"], reason: "lsof stdout awaited; stderr ignored" }, |
| 30 | 31 | "src/crew/sandbox.ts": { sites: ["stderr"], reason: "stderr awaited with the exit; stdout ignored" }, |
test/spawn-sites.test.ts +2
| @@ -37,6 +37,8 @@ const SITES: Record<string, string> = { | ||
| 37 | 37 | "the agent binary from configuration with flags the runner fixes; the prompt (a person's issue text) is one argv element after its flag and the binary never hands it to a shell; `--version` probe is fixed", |
| 38 | 38 | "src/crew/openai_runner.ts": |
| 39 | 39 | "the Bash tool of the OpenAI-compatible runner: /bin/sh -c with the model's command, which is the point of the tool, run in the run's clone (under bwrap when the sandbox is enforced); no person's text reaches argv", |
| 40 | "src/crew/orphans.ts": | |
| 41 | "`git` with a fixed argv in a clone directory the sweep derived from the clones root; no person's text reaches argv", | |
| 40 | 42 | "src/crew/plan.ts": |
| 41 | 43 | "the planner binary with a fixed flag list; the issue text a person wrote is only the final prompt element, with tools disabled", |
| 42 | 44 | "src/crew/runner.ts": |
new above
offline
idleSinceincludes the dir mtime, whichsweepIdleDependenciesresets when it removes node_modules. Use.gitonly for this sweep, or document the shift.