Public: anyone can read this; the crew can change it.

The data volume has under 15% free #13632

Merged opened this in answer to

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/13631-the-data-volume-has-under-15-free → main 2 commits fork 2 threads · 2 unresolved

  • Merged. mason/13631-the-data-volume-has-under-15-free was merged into main as 5d2d410.

Review

Sign in to review this change.
5 files changed +128 −4
docs/features/disk-free.md +7
@@ -33,3 +33,10 @@ log" for a missing one.
3333
3434 `pruneRunFiles` also keeps `runs/` under 8 GiB (`RUN_FILES_MAX_BYTES`): after the age sweep, the
3535 oldest remaining files go until the total fits.
36
37 ## A dormant preview gives back its checkouts (#13631)
38
39 A preview whose pull went quiet is stopped (dormant) and waking it deploys again with a fresh
40 checkout and install. `dropCheckouts` (`src/apps/dormant_checkout.ts`) removes its checkout
41 directories (`<env>/<sha7>`, `node_modules` and all) as it goes dormant, in the idle sweep and at
42 boot. Its data, home and build cache stay.
src/apps/dormant_checkout.ts added +44
@@ -0,0 +1,44 @@
1 /**
2 * What a dormant preview gives back (#13631). A preview whose pull went quiet is stopped, and
3 * waking it deploys again the ordinary way: a fresh checkout and install. Its checkouts (one
4 * directory named for the first seven characters of a commit, `node_modules` about 110 MB of
5 * each) are then dead weight on the data volume. They go; the environment's data, home and
6 * build cache stay.
7 */
8
9 import { readdirSync } from "node:fs";
10 import { rm } from "node:fs/promises";
11 import { join } from "node:path";
12 import { envDir, type EnvName } from "./data.ts";
13
14 const CHECKOUT = /^[0-9a-f]{7}$/;
15
16 /**
outdated

Takes no exclusion for an in-flight or current deploy sha. Accept a keep set, or run under the deploy lock.

17 * Remove every checkout directory of an environment but those of `keep` (commit ids, full or
18 * short: the current and in-flight deploys); how many went.
19 */
20 export async function dropCheckouts(
21 dataDir: string,
22 appId: string,
23 env: EnvName,
24 keep: Iterable<string> = [],
25 ): Promise<number> {
26 const spared = new Set([...keep].map((oid) => oid.slice(0, 7).toLowerCase()));
27 const root = envDir(dataDir, appId, env);
28 let names: string[];
29 try {
30 names = readdirSync(root, { withFileTypes: true }).filter((e) => e.isDirectory() && CHECKOUT.test(e.name) && !spared.has(e.name)).map((e) => e.name);
31 } catch {
32 return 0;
33 }
34 let gone = 0;
35 for (const name of names) {
36 try {
37 await rm(join(root, name), { recursive: true, force: true });
38 gone++;
39 } catch {
40 // busy: the next time it goes dormant, or the pull closing, takes it
41 }
42 }
43 return gone;
44 }
src/apps/runtime.ts +25
@@ -17,6 +17,7 @@ import type { Database } from "bun:sqlite";
1717 import type { CertificateProbe } from "./certificate.ts";
1818 import { cellHides, grantsHere } from "../spawn/hosted-mark.ts";
1919 import { VERIFYING_NOTE } from "./env_phrase.ts";
20 import { dropCheckouts } from "./dormant_checkout.ts";
2021 import { listEgressGrants } from "../core/egress_grants.ts";
2122 import { recordCarried } from "../core/carried.ts";
2223 import { commitLog } from "../git/read.ts";
@@ -2231,9 +2232,32 @@ export class AppsRuntime {
22312232 limit: 50,
22322233 }).find((r) => r.state === "ok");
22332234 state.lastRunId ??= built?.id;
2235 void this.dropDormantCheckouts(state);
outdated

Unawaited and unlocked: a wake right after boot deploys into <sha7> while this rm runs, and the rm deletes the new checkout. Await it and serialize with deploys. The swallowed catch also hides failures; log them.

22342236 log.info("dormant from boot", { app: record.slug, env });
22352237 }
22362238
2239 /**
2240 * Give back a dormant preview's checkouts on the environment's deploy chain, so a wake's
2241 * deploy starts only after the removal ends. Skipped when it was roused first; the commits
2242 * serving or coming up are spared. A failure is logged, never thrown.
2243 */
2244 dropDormantCheckouts(state: Environment): Promise<void> {
2245 const run = async (): Promise<void> => {
2246 if (state.dormant === undefined) return;
2247 const keep = [state.live, state.deploying?.run, state.queued]
2248 .map((x) => x?.commitOid)
2249 .filter((oid): oid is string => oid !== undefined && oid !== "");
2250 try {
2251 await dropCheckouts(this.app.config.dataDir, state.app.id, state.env, keep);
2252 } catch (error) {
2253 log.warn("could not drop dormant checkouts", { app: state.app.slug, env: state.env, error: String(error) });
2254 }
2255 };
2256 const done = state.chain.then(run);
2257 state.chain = done;
2258 return done;
2259 }
2260
22372261 /** A dormant preview someone asked for deploys again, the ordinary way. */
22382262 rouse(record: AppRecord, env: EnvName): void {
22392263 const state = this.envs.get(key(record.id, env));
@@ -2280,6 +2304,7 @@ export class AppsRuntime {
22802304 runId,
22812305 `asleep: nobody pushed to or commented on pull #${number} since ${idleSince}; asking for it, or Redeploy, brings it back`,
22822306 );
2307 await this.dropDormantCheckouts(state);
22832308 log.info("dormant", { app: state.app.slug, env: state.env });
22842309 }
22852310 }
test/apps-idle.test.ts +21 −4
@@ -4,6 +4,8 @@
44 * restart leaves it as it was.
55 */
66 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
7 import { existsSync, mkdirSync, readdirSync } from "node:fs";
8 import { envDir } from "../src/apps/data.ts";
79 import { mkdtemp, rm } from "node:fs/promises";
810 import { tmpdir } from "node:os";
911 import { join } from "node:path";
@@ -67,6 +69,11 @@ describe("previews of idle pulls sleep", () => {
6769 }
6870 throw new Error("did not happen in time");
6971 }
72 /** The commit checkouts (seven hex characters) an environment holds on disk. */
73 const checkouts = (n: number): string[] =>
74 existsSync(envDir(dataDir, record.id, `pull-${n}`))
75 ? readdirSync(envDir(dataDir, record.id, `pull-${n}`)).filter((f) => /^[0-9a-f]{7}$/.test(f))
76 : [];
7077 const running = (n: number) =>
7178 until(() => (envOf(n)?.state === "running" ? true : undefined));
7279
@@ -136,10 +143,13 @@ describe("previews of idle pulls sleep", () => {
136143 });
137144
138145 test("a pull touched an hour ago keeps its preview; one idle past the age loses its process, and its page says so", async () => {
139 // Invariant: the sweep stops a preview idle past the age (state kept, page says asleep), and spares a fresh one and production.
146 // Invariant: the sweep stops a preview idle past the age (state kept, page says asleep), and
147 // spares a fresh one and production.
140148 age(fresh, 3_600_000);
141149 age(idle, 4 * DAY);
142150 await until(() => (envOf(idle)?.dormant !== undefined ? true : undefined));
151 // The sweep gives the checkouts back once it marks the preview dormant.
152 await until(() => (checkouts(idle).length === 0 ? true : undefined));
143153 const asleep = envOf(idle)!;
144154 // Stopped is not failed: no error, no port, the commit and run kept.
145155 expect(asleep.state).toBe("stopped");
@@ -194,8 +204,8 @@ describe("previews of idle pulls sleep", () => {
194204 }, 120_000);
195205
196206 test("asking for a stopped preview redeploys it on the ordinary path", async () => {
197 // Invariant: a request for a dormant preview runs one "wake" deploy, and the woken preview counts as touched.
198 // The idle preview the first test put to sleep.
207 // Invariant: a request for a dormant preview runs one "wake" deploy, and the woken preview
208 // counts as touched. The idle preview the first test put to sleep.
199209 await until(() => (envOf(idle)?.dormant !== undefined ? true : undefined));
200210 const before = listRuns(app.db, repo.id, { kind: "deploy", limit: 100 }).length;
201211 const res = await fetch(`${servers.url}/`, {
@@ -203,6 +213,9 @@ describe("previews of idle pulls sleep", () => {
203213 });
204214 expect([200, 503]).toContain(res.status);
205215 await running(idle);
216 // The wake lands after the removal: the fresh checkout is there and serving.
217 expect(checkouts(idle).length).toBe(1);
218 expect((await fetch(`${servers.url}/`, { headers: { host: `hello--${idle}.${DOMAIN}` } })).status).toBe(200);
206219 expect(envOf(idle)?.dormant).toBeUndefined();
207220 const runs = listRuns(app.db, repo.id, { kind: "deploy", limit: 100 });
208221 expect(runs.length).toBe(before + 1);
@@ -213,15 +226,19 @@ describe("previews of idle pulls sleep", () => {
213226 }, 120_000);
214227
215228 test("a restart leaves a sleeping preview asleep", async () => {
216 // Invariant: a runtime booting over a dormant preview keeps it stopped and deploys nothing for it.
229 // Invariant: a runtime booting over a dormant preview keeps it stopped and deploys nothing for
230 // it.
217231 age(later, 5 * DAY);
218232 await until(() => (envOf(later)?.dormant !== undefined ? true : undefined));
219233 const before = listRuns(app.db, repo.id, { kind: "deploy", limit: 100 }).filter(
220234 (r) => r.env === `pull-${later}`,
221235 ).length;
222236 await runtime.stop();
237 // A checkout left on disk by the life before: booting dormant gives it back.
238 mkdirSync(join(envDir(dataDir, record.id, `pull-${later}`), "abcdef0", "node_modules"), { recursive: true });
223239 runtime = new AppsRuntime(app, options());
224240 await runtime.start();
241 await until(() => (checkouts(later).length === 0 ? true : undefined));
225242 expect(envOf(later)?.dormant).toBeDefined();
226243 expect(envOf(later)?.state).toBe("stopped");
227244 await sweeps(2);
test/dormant-checkout.test.ts added +31
@@ -0,0 +1,31 @@
1 import { expect, test } from "bun:test";
2 import { existsSync, mkdirSync, mkdtempSync, writeFileSync } from "node:fs";
3 import { join } from "node:path";
4 import { tmpdir } from "node:os";
5 import { envDir } from "../src/apps/data.ts";
6 import { dropCheckouts } from "../src/apps/dormant_checkout.ts";
7
8 test("a dormant preview gives back its checkouts and keeps its data", async () => {
9 const dir = mkdtempSync(join(tmpdir(), "dormant-"));
10 const env = envDir(dir, "app1", "pull-7");
11 for (const sub of ["abc1234/node_modules", "def5678/src", "data", "build-cache"]) mkdirSync(join(env, sub), { recursive: true });
12 writeFileSync(join(env, "data", "db.sqlite"), "x");
13 expect(await dropCheckouts(dir, "app1", "pull-7")).toBe(2);
14 expect(existsSync(join(env, "abc1234"))).toBe(false);
15 expect(existsSync(join(env, "def5678"))).toBe(false);
16 expect(existsSync(join(env, "data", "db.sqlite"))).toBe(true);
17 expect(existsSync(join(env, "build-cache"))).toBe(true);
18 });
19
20 test("an environment with no directory is nothing to drop", async () => {
21 expect(await dropCheckouts(mkdtempSync(join(tmpdir(), "dormant-")), "app1", "pull-7")).toBe(0);
22 });
23
24 test("the commits serving or coming up are spared", async () => {
25 const dir = mkdtempSync(join(tmpdir(), "dormant-"));
26 const env = envDir(dir, "app1", "pull-7");
27 for (const sub of ["abc1234", "def5678"]) mkdirSync(join(env, sub), { recursive: true });
28 expect(await dropCheckouts(dir, "app1", "pull-7", ["ABC1234ffffffffffffffffffffffffffffffff"])).toBe(1);
29 expect(existsSync(join(env, "abc1234"))).toBe(true);
30 expect(existsSync(join(env, "def5678"))).toBe(false);
31 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs