Public: anyone can read this; the crew can change it.
The data volume has under 15% free #13632
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/13631-the-data-volume-has-under-15-free → main 2 commits fork 2 threads · 2 unresolved
5 files
changed +128
−4
docs/features/disk-free.md +7
| @@ -33,3 +33,10 @@ log" for a missing one. | ||
| 33 | 33 | |
| 34 | 34 | `pruneRunFiles` also keeps `runs/` under 8 GiB (`RUN_FILES_MAX_BYTES`): after the age sweep, the |
| 35 | 35 | oldest remaining files go until the total fits. |
| 36 | ||
| 37 | ## A dormant preview gives back its checkouts (#13631) | |
| 38 | ||
| 39 | A preview whose pull went quiet is stopped (dormant) and waking it deploys again with a fresh | |
| 40 | checkout and install. `dropCheckouts` (`src/apps/dormant_checkout.ts`) removes its checkout | |
| 41 | directories (`<env>/<sha7>`, `node_modules` and all) as it goes dormant, in the idle sweep and at | |
| 42 | boot. Its data, home and build cache stay. |
src/apps/dormant_checkout.ts added +44
| @@ -0,0 +1,44 @@ | ||
| 1 | /** | |
| 2 | * What a dormant preview gives back (#13631). A preview whose pull went quiet is stopped, and | |
| 3 | * waking it deploys again the ordinary way: a fresh checkout and install. Its checkouts (one | |
| 4 | * directory named for the first seven characters of a commit, `node_modules` about 110 MB of | |
| 5 | * each) are then dead weight on the data volume. They go; the environment's data, home and | |
| 6 | * build cache stay. | |
| 7 | */ | |
| 8 | ||
| 9 | import { readdirSync } from "node:fs"; | |
| 10 | import { rm } from "node:fs/promises"; | |
| 11 | import { join } from "node:path"; | |
| 12 | import { envDir, type EnvName } from "./data.ts"; | |
| 13 | ||
| 14 | const CHECKOUT = /^[0-9a-f]{7}$/; | |
| 15 | ||
| 16 | /** | |
| 17 | * Remove every checkout directory of an environment but those of `keep` (commit ids, full or | |
| 18 | * short: the current and in-flight deploys); how many went. | |
| 19 | */ | |
| 20 | export async function dropCheckouts( | |
| 21 | dataDir: string, | |
| 22 | appId: string, | |
| 23 | env: EnvName, | |
| 24 | keep: Iterable<string> = [], | |
| 25 | ): Promise<number> { | |
| 26 | const spared = new Set([...keep].map((oid) => oid.slice(0, 7).toLowerCase())); | |
| 27 | const root = envDir(dataDir, appId, env); | |
| 28 | let names: string[]; | |
| 29 | try { | |
| 30 | names = readdirSync(root, { withFileTypes: true }).filter((e) => e.isDirectory() && CHECKOUT.test(e.name) && !spared.has(e.name)).map((e) => e.name); | |
| 31 | } catch { | |
| 32 | return 0; | |
| 33 | } | |
| 34 | let gone = 0; | |
| 35 | for (const name of names) { | |
| 36 | try { | |
| 37 | await rm(join(root, name), { recursive: true, force: true }); | |
| 38 | gone++; | |
| 39 | } catch { | |
| 40 | // busy: the next time it goes dormant, or the pull closing, takes it | |
| 41 | } | |
| 42 | } | |
| 43 | return gone; | |
| 44 | } |
src/apps/runtime.ts +25
| @@ -17,6 +17,7 @@ import type { Database } from "bun:sqlite"; | ||
| 17 | 17 | import type { CertificateProbe } from "./certificate.ts"; |
| 18 | 18 | import { cellHides, grantsHere } from "../spawn/hosted-mark.ts"; |
| 19 | 19 | import { VERIFYING_NOTE } from "./env_phrase.ts"; |
| 20 | import { dropCheckouts } from "./dormant_checkout.ts"; | |
| 20 | 21 | import { listEgressGrants } from "../core/egress_grants.ts"; |
| 21 | 22 | import { recordCarried } from "../core/carried.ts"; |
| 22 | 23 | import { commitLog } from "../git/read.ts"; |
| @@ -2231,9 +2232,32 @@ export class AppsRuntime { | ||
| 2231 | 2232 | limit: 50, |
| 2232 | 2233 | }).find((r) => r.state === "ok"); |
| 2233 | 2234 | state.lastRunId ??= built?.id; |
| 2235 | void this.dropDormantCheckouts(state); | |
outdated
Unawaited and unlocked: a wake right after boot deploys into <sha7> while this rm runs, and the rm deletes the new checkout. Await it and serialize with deploys. The swallowed catch also hides failures; log them. |
||
| 2234 | 2236 | log.info("dormant from boot", { app: record.slug, env }); |
| 2235 | 2237 | } |
| 2236 | 2238 | |
| 2239 | /** | |
| 2240 | * Give back a dormant preview's checkouts on the environment's deploy chain, so a wake's | |
| 2241 | * deploy starts only after the removal ends. Skipped when it was roused first; the commits | |
| 2242 | * serving or coming up are spared. A failure is logged, never thrown. | |
| 2243 | */ | |
| 2244 | dropDormantCheckouts(state: Environment): Promise<void> { | |
| 2245 | const run = async (): Promise<void> => { | |
| 2246 | if (state.dormant === undefined) return; | |
| 2247 | const keep = [state.live, state.deploying?.run, state.queued] | |
| 2248 | .map((x) => x?.commitOid) | |
| 2249 | .filter((oid): oid is string => oid !== undefined && oid !== ""); | |
| 2250 | try { | |
| 2251 | await dropCheckouts(this.app.config.dataDir, state.app.id, state.env, keep); | |
| 2252 | } catch (error) { | |
| 2253 | log.warn("could not drop dormant checkouts", { app: state.app.slug, env: state.env, error: String(error) }); | |
| 2254 | } | |
| 2255 | }; | |
| 2256 | const done = state.chain.then(run); | |
| 2257 | state.chain = done; | |
| 2258 | return done; | |
| 2259 | } | |
| 2260 | ||
| 2237 | 2261 | /** A dormant preview someone asked for deploys again, the ordinary way. */ |
| 2238 | 2262 | rouse(record: AppRecord, env: EnvName): void { |
| 2239 | 2263 | const state = this.envs.get(key(record.id, env)); |
| @@ -2280,6 +2304,7 @@ export class AppsRuntime { | ||
| 2280 | 2304 | runId, |
| 2281 | 2305 | `asleep: nobody pushed to or commented on pull #${number} since ${idleSince}; asking for it, or Redeploy, brings it back`, |
| 2282 | 2306 | ); |
| 2307 | await this.dropDormantCheckouts(state); | |
| 2283 | 2308 | log.info("dormant", { app: state.app.slug, env: state.env }); |
| 2284 | 2309 | } |
| 2285 | 2310 | } |
test/apps-idle.test.ts +21 −4
| @@ -4,6 +4,8 @@ | ||
| 4 | 4 | * restart leaves it as it was. |
| 5 | 5 | */ |
| 6 | 6 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; |
| 7 | import { existsSync, mkdirSync, readdirSync } from "node:fs"; | |
| 8 | import { envDir } from "../src/apps/data.ts"; | |
| 7 | 9 | import { mkdtemp, rm } from "node:fs/promises"; |
| 8 | 10 | import { tmpdir } from "node:os"; |
| 9 | 11 | import { join } from "node:path"; |
| @@ -67,6 +69,11 @@ describe("previews of idle pulls sleep", () => { | ||
| 67 | 69 | } |
| 68 | 70 | throw new Error("did not happen in time"); |
| 69 | 71 | } |
| 72 | /** The commit checkouts (seven hex characters) an environment holds on disk. */ | |
| 73 | const checkouts = (n: number): string[] => | |
| 74 | existsSync(envDir(dataDir, record.id, `pull-${n}`)) | |
| 75 | ? readdirSync(envDir(dataDir, record.id, `pull-${n}`)).filter((f) => /^[0-9a-f]{7}$/.test(f)) | |
| 76 | : []; | |
| 70 | 77 | const running = (n: number) => |
| 71 | 78 | until(() => (envOf(n)?.state === "running" ? true : undefined)); |
| 72 | 79 | |
| @@ -136,10 +143,13 @@ describe("previews of idle pulls sleep", () => { | ||
| 136 | 143 | }); |
| 137 | 144 | |
| 138 | 145 | test("a pull touched an hour ago keeps its preview; one idle past the age loses its process, and its page says so", async () => { |
| 139 | // Invariant: the sweep stops a preview idle past the age (state kept, page says asleep), and spares a fresh one and production. | |
| 146 | // Invariant: the sweep stops a preview idle past the age (state kept, page says asleep), and | |
| 147 | // spares a fresh one and production. | |
| 140 | 148 | age(fresh, 3_600_000); |
| 141 | 149 | age(idle, 4 * DAY); |
| 142 | 150 | await until(() => (envOf(idle)?.dormant !== undefined ? true : undefined)); |
| 151 | // The sweep gives the checkouts back once it marks the preview dormant. | |
| 152 | await until(() => (checkouts(idle).length === 0 ? true : undefined)); | |
| 143 | 153 | const asleep = envOf(idle)!; |
| 144 | 154 | // Stopped is not failed: no error, no port, the commit and run kept. |
| 145 | 155 | expect(asleep.state).toBe("stopped"); |
| @@ -194,8 +204,8 @@ describe("previews of idle pulls sleep", () => { | ||
| 194 | 204 | }, 120_000); |
| 195 | 205 | |
| 196 | 206 | test("asking for a stopped preview redeploys it on the ordinary path", async () => { |
| 197 | // Invariant: a request for a dormant preview runs one "wake" deploy, and the woken preview counts as touched. | |
| 198 | // The idle preview the first test put to sleep. | |
| 207 | // Invariant: a request for a dormant preview runs one "wake" deploy, and the woken preview | |
| 208 | // counts as touched. The idle preview the first test put to sleep. | |
| 199 | 209 | await until(() => (envOf(idle)?.dormant !== undefined ? true : undefined)); |
| 200 | 210 | const before = listRuns(app.db, repo.id, { kind: "deploy", limit: 100 }).length; |
| 201 | 211 | const res = await fetch(`${servers.url}/`, { |
| @@ -203,6 +213,9 @@ describe("previews of idle pulls sleep", () => { | ||
| 203 | 213 | }); |
| 204 | 214 | expect([200, 503]).toContain(res.status); |
| 205 | 215 | await running(idle); |
| 216 | // The wake lands after the removal: the fresh checkout is there and serving. | |
| 217 | expect(checkouts(idle).length).toBe(1); | |
| 218 | expect((await fetch(`${servers.url}/`, { headers: { host: `hello--${idle}.${DOMAIN}` } })).status).toBe(200); | |
| 206 | 219 | expect(envOf(idle)?.dormant).toBeUndefined(); |
| 207 | 220 | const runs = listRuns(app.db, repo.id, { kind: "deploy", limit: 100 }); |
| 208 | 221 | expect(runs.length).toBe(before + 1); |
| @@ -213,15 +226,19 @@ describe("previews of idle pulls sleep", () => { | ||
| 213 | 226 | }, 120_000); |
| 214 | 227 | |
| 215 | 228 | test("a restart leaves a sleeping preview asleep", async () => { |
| 216 | // Invariant: a runtime booting over a dormant preview keeps it stopped and deploys nothing for it. | |
| 229 | // Invariant: a runtime booting over a dormant preview keeps it stopped and deploys nothing for | |
| 230 | // it. | |
| 217 | 231 | age(later, 5 * DAY); |
| 218 | 232 | await until(() => (envOf(later)?.dormant !== undefined ? true : undefined)); |
| 219 | 233 | const before = listRuns(app.db, repo.id, { kind: "deploy", limit: 100 }).filter( |
| 220 | 234 | (r) => r.env === `pull-${later}`, |
| 221 | 235 | ).length; |
| 222 | 236 | await runtime.stop(); |
| 237 | // A checkout left on disk by the life before: booting dormant gives it back. | |
| 238 | mkdirSync(join(envDir(dataDir, record.id, `pull-${later}`), "abcdef0", "node_modules"), { recursive: true }); | |
| 223 | 239 | runtime = new AppsRuntime(app, options()); |
| 224 | 240 | await runtime.start(); |
| 241 | await until(() => (checkouts(later).length === 0 ? true : undefined)); | |
| 225 | 242 | expect(envOf(later)?.dormant).toBeDefined(); |
| 226 | 243 | expect(envOf(later)?.state).toBe("stopped"); |
| 227 | 244 | await sweeps(2); |
test/dormant-checkout.test.ts added +31
| @@ -0,0 +1,31 @@ | ||
| 1 | import { expect, test } from "bun:test"; | |
| 2 | import { existsSync, mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; | |
| 3 | import { join } from "node:path"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { envDir } from "../src/apps/data.ts"; | |
| 6 | import { dropCheckouts } from "../src/apps/dormant_checkout.ts"; | |
| 7 | ||
| 8 | test("a dormant preview gives back its checkouts and keeps its data", async () => { | |
| 9 | const dir = mkdtempSync(join(tmpdir(), "dormant-")); | |
| 10 | const env = envDir(dir, "app1", "pull-7"); | |
| 11 | for (const sub of ["abc1234/node_modules", "def5678/src", "data", "build-cache"]) mkdirSync(join(env, sub), { recursive: true }); | |
| 12 | writeFileSync(join(env, "data", "db.sqlite"), "x"); | |
| 13 | expect(await dropCheckouts(dir, "app1", "pull-7")).toBe(2); | |
| 14 | expect(existsSync(join(env, "abc1234"))).toBe(false); | |
| 15 | expect(existsSync(join(env, "def5678"))).toBe(false); | |
| 16 | expect(existsSync(join(env, "data", "db.sqlite"))).toBe(true); | |
| 17 | expect(existsSync(join(env, "build-cache"))).toBe(true); | |
| 18 | }); | |
| 19 | ||
| 20 | test("an environment with no directory is nothing to drop", async () => { | |
| 21 | expect(await dropCheckouts(mkdtempSync(join(tmpdir(), "dormant-")), "app1", "pull-7")).toBe(0); | |
| 22 | }); | |
| 23 | ||
| 24 | test("the commits serving or coming up are spared", async () => { | |
| 25 | const dir = mkdtempSync(join(tmpdir(), "dormant-")); | |
| 26 | const env = envDir(dir, "app1", "pull-7"); | |
| 27 | for (const sub of ["abc1234", "def5678"]) mkdirSync(join(env, sub), { recursive: true }); | |
| 28 | expect(await dropCheckouts(dir, "app1", "pull-7", ["ABC1234ffffffffffffffffffffffffffffffff"])).toBe(1); | |
| 29 | expect(existsSync(join(env, "abc1234"))).toBe(true); | |
| 30 | expect(existsSync(join(env, "def5678"))).toBe(false); | |
| 31 | }); |
new above
offline
Takes no exclusion for an in-flight or current deploy sha. Accept a keep set, or run under the deploy lock.