Public: anyone can read this; the crew can change it.
The data volume has under 15% free #13775
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/13772-the-data-volume-has-under-15-free → main 2 commits fork
5 files
changed +184
−5
docs/features/disk-free.md +8
| @@ -51,3 +51,11 @@ resets to it, and clones afresh when the directory is gone. | ||
| 51 | 51 | A clone is kept, and counted in the log, when `git status --porcelain` is not empty or a local |
| 52 | 52 | branch has commits no remote ref has (`holdsUnpushedWork`; fixed argv, `GIT_OPTIONAL_LOCKS=0` so the |
| 53 | 53 | check does not stamp the clone). Git that cannot answer also keeps it. |
| 54 | ||
| 55 | ## Checkouts nothing serves go at boot (#13772) | |
| 56 | ||
| 57 | At boot, `reconcileApp` calls `dropStrayCheckouts` (`src/apps/runtime.ts`): in each environment on | |
| 58 | disk, a checkout directory that is not the commit serving, resting, deploying or queued goes, and | |
| 59 | so does what a failed or interrupted deploy left. One touched within `STRAY_CHECKOUT_AGE_MS` (ten minutes, `src/apps/dormant_checkout.ts`) stays. | |
| 60 | This is a partial step on #13735: it gives back what strays hold but does not by itself bring free space | |
| 61 | above 15%. `test/stray-checkouts.test.ts` proves the kept commits are in state when the sweep runs. |
src/apps/dormant_checkout.ts +26 −3
| @@ -6,28 +6,51 @@ | ||
| 6 | 6 | * build cache stay. |
| 7 | 7 | */ |
| 8 | 8 | |
| 9 | import { readdirSync } from "node:fs"; | |
| 9 | import { readdirSync, statSync, type Dirent } from "node:fs"; | |
| 10 | 10 | import { rm } from "node:fs/promises"; |
| 11 | 11 | import { join } from "node:path"; |
| 12 | 12 | import { envDir, type EnvName } from "./data.ts"; |
| 13 | 13 | |
| 14 | 14 | const CHECKOUT = /^[0-9a-f]{7}$/; |
| 15 | 15 | |
| 16 | /** | |
| 17 | * How long a checkout nothing owns must lie untouched before the boot sweep takes it: ten minutes. | |
| 18 | */ | |
| 19 | export const STRAY_CHECKOUT_AGE_MS = 600_000; | |
| 20 | ||
| 21 | /** Whether nothing has touched `path` for `ms` (a checkout a deploy is still making is not). */ | |
| 22 | function untouched(path: string, ms: number, now: number): boolean { | |
| 23 | if (ms <= 0) return true; | |
| 24 | try { | |
| 25 | return now - statSync(path).mtimeMs > ms; | |
| 26 | } catch { | |
| 27 | return false; | |
| 28 | } | |
| 29 | } | |
| 30 | ||
| 16 | 31 | /** |
| 17 | 32 | * Remove every checkout directory of an environment but those of `keep` (commit ids, full or |
| 18 | * short: the current and in-flight deploys); how many went. | |
| 33 | * short: the current and in-flight deploys); how many went. With `olderThanMs`, a checkout touched | |
| 34 | * more recently stays: a deploy may be making it. | |
| 19 | 35 | */ |
| 20 | 36 | export async function dropCheckouts( |
| 21 | 37 | dataDir: string, |
| 22 | 38 | appId: string, |
| 23 | 39 | env: EnvName, |
| 24 | 40 | keep: Iterable<string> = [], |
| 41 | olderThanMs = 0, | |
| 42 | now: number = Date.now(), | |
| 25 | 43 | ): Promise<number> { |
| 26 | 44 | const spared = new Set([...keep].map((oid) => oid.slice(0, 7).toLowerCase())); |
| 27 | 45 | const root = envDir(dataDir, appId, env); |
| 46 | const droppable = (e: Dirent): boolean => | |
| 47 | e.isDirectory() && | |
| 48 | CHECKOUT.test(e.name) && | |
| 49 | !spared.has(e.name) && | |
| 50 | untouched(join(root, e.name), olderThanMs, now); | |
| 28 | 51 | let names: string[]; |
| 29 | 52 | try { |
| 30 | names = readdirSync(root, { withFileTypes: true }).filter((e) => e.isDirectory() && CHECKOUT.test(e.name) && !spared.has(e.name)).map((e) => e.name); | |
| 53 | names = readdirSync(root, { withFileTypes: true }).filter(droppable).map((e) => e.name); | |
| 31 | 54 | } catch { |
| 32 | 55 | return 0; |
| 33 | 56 | } |
src/apps/runtime.ts +28 −1
| @@ -17,7 +17,7 @@ import type { Database } from "bun:sqlite"; | ||
| 17 | 17 | import type { CertificateProbe } from "./certificate.ts"; |
| 18 | 18 | import { cellHides, grantsHere } from "../spawn/hosted-mark.ts"; |
| 19 | 19 | import { VERIFYING_NOTE } from "./env_phrase.ts"; |
| 20 | import { dropCheckouts } from "./dormant_checkout.ts"; | |
| 20 | import { dropCheckouts, STRAY_CHECKOUT_AGE_MS } from "./dormant_checkout.ts"; | |
| 21 | 21 | import { listEgressGrants } from "../core/egress_grants.ts"; |
| 22 | 22 | import { recordCarried } from "../core/carried.ts"; |
| 23 | 23 | import { commitLog } from "../git/read.ts"; |
| @@ -936,6 +936,33 @@ export class AppsRuntime { | ||
| 936 | 936 | await this.teardown(record, name); |
| 937 | 937 | } |
| 938 | 938 | } |
| 939 | if (reason === "boot") await this.dropStrayCheckouts(record); | |
| 940 | } | |
| 941 | ||
| 942 | /** | |
| 943 | * At boot, remove checkouts no deployment serves, rests on or is making: what a failed or | |
| 944 | * interrupted deploy left (#13772). Ones touched within `STRAY_CHECKOUT_AGE_MS` stay. | |
| 945 | */ | |
| 946 | async dropStrayCheckouts(record: AppRecord): Promise<void> { | |
| 947 | const root = join(appsRoot(this.app.config.dataDir), record.id); | |
| 948 | let names: string[]; | |
| 949 | try { | |
| 950 | names = readdirSync(root); | |
| 951 | } catch { | |
| 952 | return; | |
| 953 | } | |
| 954 | for (const name of names) { | |
| 955 | if (!isEnvName(name)) continue; | |
| 956 | const state = this.envs.get(key(record.id, name)); | |
| 957 | const keep = [state?.live, state?.asleep, state?.deploying?.run, state?.queued] | |
| 958 | .map((x) => x?.commitOid) | |
| 959 | .filter((oid): oid is string => oid !== undefined && oid !== ""); | |
| 960 | try { | |
| 961 | await dropCheckouts(this.app.config.dataDir, record.id, name, keep, STRAY_CHECKOUT_AGE_MS); | |
| 962 | } catch (error) { | |
| 963 | log.warn("could not drop stray checkouts", { app: record.slug, env: name, error: String(error) }); | |
| 964 | } | |
| 965 | } | |
| 939 | 966 | } |
| 940 | 967 | |
| 941 | 968 | /** |
test/dormant-checkout.test.ts +11 −1
| @@ -3,7 +3,7 @@ import { existsSync, mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; | ||
| 3 | 3 | import { join } from "node:path"; |
| 4 | 4 | import { tmpdir } from "node:os"; |
| 5 | 5 | import { envDir } from "../src/apps/data.ts"; |
| 6 | import { dropCheckouts } from "../src/apps/dormant_checkout.ts"; | |
| 6 | import { dropCheckouts, STRAY_CHECKOUT_AGE_MS } from "../src/apps/dormant_checkout.ts"; | |
| 7 | 7 | |
| 8 | 8 | test("a dormant preview gives back its checkouts and keeps its data", async () => { |
| 9 | 9 | const dir = mkdtempSync(join(tmpdir(), "dormant-")); |
| @@ -29,3 +29,13 @@ test("the commits serving or coming up are spared", async () => { | ||
| 29 | 29 | expect(existsSync(join(env, "abc1234"))).toBe(true); |
| 30 | 30 | expect(existsSync(join(env, "def5678"))).toBe(false); |
| 31 | 31 | }); |
| 32 | ||
| 33 | test("a sweep with an age leaves a checkout touched within it, and takes an older one", async () => { | |
| 34 | const dir = mkdtempSync(join(tmpdir(), "dormant-")); | |
| 35 | const env = envDir(dir, "app1", "production"); | |
| 36 | for (const sub of ["abc1234", "def5678"]) mkdirSync(join(env, sub), { recursive: true }); | |
| 37 | const now = Date.now(); | |
| 38 | expect(await dropCheckouts(dir, "app1", "production", [], STRAY_CHECKOUT_AGE_MS, now)).toBe(0); | |
| 39 | expect(await dropCheckouts(dir, "app1", "production", [], STRAY_CHECKOUT_AGE_MS, now + STRAY_CHECKOUT_AGE_MS + 1000)).toBe(2); | |
| 40 | expect(existsSync(join(env, "abc1234"))).toBe(false); | |
| 41 | }); |
test/stray-checkouts.test.ts added +111
| @@ -0,0 +1,111 @@ | ||
| 1 | /** | |
| 2 | * The boot sweep that removes checkouts nothing serves or is making (#13772): what it spares | |
| 3 | * (live, asleep, deploying, queued), what it takes, and that those commits are in state by the time | |
| 4 | * `reconcileApp` reaches it. | |
| 5 | */ | |
| 6 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 7 | import { existsSync, mkdirSync, utimesSync, writeFileSync } from "node:fs"; | |
| 8 | import { join } from "node:path"; | |
| 9 | import { newApp } from "../src/actions/apps.ts"; | |
| 10 | import { openPull } from "../src/actions/pulls.ts"; | |
| 11 | import { commitToBranch } from "../src/actions/repos.ts"; | |
| 12 | import { envDir } from "../src/apps/data.ts"; | |
| 13 | import { STRAY_CHECKOUT_AGE_MS } from "../src/apps/dormant_checkout.ts"; | |
| 14 | import { AppsRuntime } from "../src/apps/runtime.ts"; | |
| 15 | import type { AppRecord } from "../src/core/apps.ts"; | |
| 16 | import type { Repo } from "../src/core/repos.ts"; | |
| 17 | import { startAppsForge, stopAppsForge, type AppsForge } from "./helpers/apps-forge.ts"; | |
| 18 | ||
| 19 | const OLD = (Date.now() - STRAY_CHECKOUT_AGE_MS - 60_000) / 1000; | |
| 20 | ||
| 21 | describe("dropStrayCheckouts", () => { | |
| 22 | let forge: AppsForge; | |
| 23 | let record: AppRecord; | |
| 24 | let repo: Repo; | |
| 25 | beforeAll(async () => { | |
| 26 | forge = await startAppsForge({ start: false }); | |
| 27 | const made = await newApp(forge.app, forge.alice, { name: "hello", description: "x", visibility: "public" }); | |
| 28 | if (!made.ok) throw new Error(String(made.error)); | |
| 29 | record = made.value.app; | |
| 30 | repo = made.value.repo; | |
| 31 | }, 60_000); | |
| 32 | afterAll(async () => { | |
| 33 | await forge.runtime.stop(); | |
| 34 | await stopAppsForge(forge); | |
| 35 | }); | |
| 36 | ||
| 37 | const make = (env: "production" | `pull-${number}`, short: string, old = true): string => { | |
| 38 | const dir = join(envDir(forge.dataDir, record.id, env), short); | |
| 39 | mkdirSync(dir, { recursive: true }); | |
| 40 | if (old) utimesSync(dir, OLD, OLD); | |
| 41 | return dir; | |
| 42 | }; | |
| 43 | const oid = (short: string) => short + "0".repeat(33); | |
| 44 | // eslint-disable-next-line @typescript-eslint/no-explicit-any | |
| 45 | const inject = (runtime: AppsRuntime, env: string, patch: Record<string, unknown>) => | |
| 46 | Object.assign((runtime as any).envFor(record, env), patch); | |
| 47 | ||
| 48 | test("live, asleep, deploying and queued commits survive; a stray and a failed deploy's leftover go", async () => { | |
| 49 | const runtime = new AppsRuntime(forge.app, { domain: "apps.test", scheme: "http", publicPort: 1, forgeUrl: forge.servers.url }); | |
| 50 | const dirs = { | |
| 51 | live: make("production", "1111111"), | |
| 52 | asleep: make("production", "2222222"), | |
| 53 | deploying: make("production", "3333333"), | |
| 54 | queued: make("production", "4444444"), | |
| 55 | stray: make("production", "5555555"), | |
| 56 | }; | |
| 57 | inject(runtime, "production", { | |
| 58 | live: { commitOid: oid("1111111") }, | |
| 59 | asleep: { commitOid: oid("2222222") }, | |
| 60 | deploying: { run: { commitOid: oid("3333333") } }, | |
| 61 | queued: { commitOid: oid("4444444") }, | |
| 62 | }); | |
| 63 | await runtime.dropStrayCheckouts(record); | |
| 64 | for (const kept of ["live", "asleep", "deploying", "queued"] as const) expect(existsSync(dirs[kept])).toBe(true); | |
| 65 | expect(existsSync(dirs.stray)).toBe(false); | |
| 66 | }); | |
| 67 | ||
| 68 | test("a checkout touched within the age stays, though nothing owns it", async () => { | |
| 69 | const runtime = new AppsRuntime(forge.app, { domain: "apps.test", scheme: "http", publicPort: 1, forgeUrl: forge.servers.url }); | |
| 70 | const fresh = make("pull-9", "aaaaaaa", false); | |
| 71 | const old = make("pull-9", "bbbbbbb"); | |
| 72 | await runtime.dropStrayCheckouts(record); | |
| 73 | expect(existsSync(fresh)).toBe(true); | |
| 74 | expect(existsSync(old)).toBe(false); | |
| 75 | }); | |
| 76 | ||
| 77 | test("by the time reconcileApp sweeps at boot, a queued production head and an asleep preview head are in state", async () => { | |
| 78 | const runtime = new AppsRuntime(forge.app, { domain: "apps.test", scheme: "http", publicPort: 1, forgeUrl: forge.servers.url }); | |
| 79 | const branch = await commitToBranch(forge.app, forge.alice, repo, { | |
| 80 | branch: "feature/x", | |
| 81 | base: "main", | |
| 82 | message: "x", | |
| 83 | files: [{ path: "x.md", content: "x\n" }], | |
| 84 | }); | |
| 85 | if (!branch.ok) throw new Error(JSON.stringify(branch.error)); | |
| 86 | const opened = await openPull(forge.app, forge.alice, repo, { title: "x", baseBranch: "main", headBranch: "feature/x" }); | |
| 87 | if (!opened.ok) throw new Error(String(opened.error)); | |
| 88 | const env = `pull-${opened.value.number}` as const; | |
| 89 | const head = opened.value.headOid ?? (await (runtime as any).pullHead(repo, opened.value)); | |
| 90 | // The preview's head is checked out on disk, so boot settles it asleep. | |
| 91 | const headDir = make(env, head.slice(0, 7)); | |
| 92 | writeFileSync(join(headDir, "package.json"), JSON.stringify({ name: "x", scripts: { start: "true" } })); | |
| 93 | const stray = make(env, "ccccccc"); | |
| 94 | const seen: Record<string, string> = {}; | |
| 95 | const real = runtime.dropStrayCheckouts.bind(runtime); | |
| 96 | runtime.dropStrayCheckouts = async (r: AppRecord) => { | |
| 97 | for (const e of runtime.status(r)) seen[e.env] = e.commitOid; | |
| 98 | const ours = (runtime as any).envs.get(`${r.id}/${env}`) ?? [...(runtime as any).envs.values()].find((s: any) => s.env === env); | |
| 99 | seen.asleep = ours?.asleep?.commitOid ?? ""; | |
| 100 | const prod = [...(runtime as any).envs.values()].find((s: any) => s.env === "production"); | |
| 101 | seen.production = prod?.queued?.commitOid ?? prod?.deploying?.run.commitOid ?? prod?.live?.commitOid ?? ""; | |
| 102 | await real(r); | |
| 103 | }; | |
| 104 | await runtime.reconcileApp(record, repo, "boot"); | |
| 105 | expect(seen.asleep).toBe(head); | |
| 106 | expect(seen.production).toMatch(/^[0-9a-f]{40}$/); | |
| 107 | expect(existsSync(headDir)).toBe(true); | |
| 108 | expect(existsSync(stray)).toBe(false); | |
| 109 | await runtime.stop(); | |
| 110 | }, 60_000); | |
| 111 | }); |
new above
offline