Public: anyone can read this; the crew can change it.

The data volume has under 15% free #13775

Merged opened this in answer to

Check · Review · Land · Live done — Landed and live in production.

Merged

torque/forge:torque/13772-the-data-volume-has-under-15-free → main 2 commits fork

  • Merged. torque/13772-the-data-volume-has-under-15-free was merged into main as 310f789.

Review

Sign in to review this change.
5 files changed +184 −5
docs/features/disk-free.md +8
@@ -51,3 +51,11 @@ resets to it, and clones afresh when the directory is gone.
5151 A clone is kept, and counted in the log, when `git status --porcelain` is not empty or a local
5252 branch has commits no remote ref has (`holdsUnpushedWork`; fixed argv, `GIT_OPTIONAL_LOCKS=0` so the
5353 check does not stamp the clone). Git that cannot answer also keeps it.
54
55 ## Checkouts nothing serves go at boot (#13772)
56
57 At boot, `reconcileApp` calls `dropStrayCheckouts` (`src/apps/runtime.ts`): in each environment on
58 disk, a checkout directory that is not the commit serving, resting, deploying or queued goes, and
59 so does what a failed or interrupted deploy left. One touched within `STRAY_CHECKOUT_AGE_MS` (ten minutes, `src/apps/dormant_checkout.ts`) stays.
60 This is a partial step on #13735: it gives back what strays hold but does not by itself bring free space
61 above 15%. `test/stray-checkouts.test.ts` proves the kept commits are in state when the sweep runs.
src/apps/dormant_checkout.ts +26 −3
@@ -6,28 +6,51 @@
66 * build cache stay.
77 */
88
9 import { readdirSync } from "node:fs";
9 import { readdirSync, statSync, type Dirent } from "node:fs";
1010 import { rm } from "node:fs/promises";
1111 import { join } from "node:path";
1212 import { envDir, type EnvName } from "./data.ts";
1313
1414 const CHECKOUT = /^[0-9a-f]{7}$/;
1515
16 /**
17 * How long a checkout nothing owns must lie untouched before the boot sweep takes it: ten minutes.
18 */
19 export const STRAY_CHECKOUT_AGE_MS = 600_000;
20
21 /** Whether nothing has touched `path` for `ms` (a checkout a deploy is still making is not). */
22 function untouched(path: string, ms: number, now: number): boolean {
23 if (ms <= 0) return true;
24 try {
25 return now - statSync(path).mtimeMs > ms;
26 } catch {
27 return false;
28 }
29 }
30
1631 /**
1732 * Remove every checkout directory of an environment but those of `keep` (commit ids, full or
18 * short: the current and in-flight deploys); how many went.
33 * short: the current and in-flight deploys); how many went. With `olderThanMs`, a checkout touched
34 * more recently stays: a deploy may be making it.
1935 */
2036 export async function dropCheckouts(
2137 dataDir: string,
2238 appId: string,
2339 env: EnvName,
2440 keep: Iterable<string> = [],
41 olderThanMs = 0,
42 now: number = Date.now(),
2543 ): Promise<number> {
2644 const spared = new Set([...keep].map((oid) => oid.slice(0, 7).toLowerCase()));
2745 const root = envDir(dataDir, appId, env);
46 const droppable = (e: Dirent): boolean =>
47 e.isDirectory() &&
48 CHECKOUT.test(e.name) &&
49 !spared.has(e.name) &&
50 untouched(join(root, e.name), olderThanMs, now);
2851 let names: string[];
2952 try {
30 names = readdirSync(root, { withFileTypes: true }).filter((e) => e.isDirectory() && CHECKOUT.test(e.name) && !spared.has(e.name)).map((e) => e.name);
53 names = readdirSync(root, { withFileTypes: true }).filter(droppable).map((e) => e.name);
3154 } catch {
3255 return 0;
3356 }
src/apps/runtime.ts +28 −1
@@ -17,7 +17,7 @@ import type { Database } from "bun:sqlite";
1717 import type { CertificateProbe } from "./certificate.ts";
1818 import { cellHides, grantsHere } from "../spawn/hosted-mark.ts";
1919 import { VERIFYING_NOTE } from "./env_phrase.ts";
20 import { dropCheckouts } from "./dormant_checkout.ts";
20 import { dropCheckouts, STRAY_CHECKOUT_AGE_MS } from "./dormant_checkout.ts";
2121 import { listEgressGrants } from "../core/egress_grants.ts";
2222 import { recordCarried } from "../core/carried.ts";
2323 import { commitLog } from "../git/read.ts";
@@ -936,6 +936,33 @@ export class AppsRuntime {
936936 await this.teardown(record, name);
937937 }
938938 }
939 if (reason === "boot") await this.dropStrayCheckouts(record);
940 }
941
942 /**
943 * At boot, remove checkouts no deployment serves, rests on or is making: what a failed or
944 * interrupted deploy left (#13772). Ones touched within `STRAY_CHECKOUT_AGE_MS` stay.
945 */
946 async dropStrayCheckouts(record: AppRecord): Promise<void> {
947 const root = join(appsRoot(this.app.config.dataDir), record.id);
948 let names: string[];
949 try {
950 names = readdirSync(root);
951 } catch {
952 return;
953 }
954 for (const name of names) {
955 if (!isEnvName(name)) continue;
956 const state = this.envs.get(key(record.id, name));
957 const keep = [state?.live, state?.asleep, state?.deploying?.run, state?.queued]
958 .map((x) => x?.commitOid)
959 .filter((oid): oid is string => oid !== undefined && oid !== "");
960 try {
961 await dropCheckouts(this.app.config.dataDir, record.id, name, keep, STRAY_CHECKOUT_AGE_MS);
962 } catch (error) {
963 log.warn("could not drop stray checkouts", { app: record.slug, env: name, error: String(error) });
964 }
965 }
939966 }
940967
941968 /**
test/dormant-checkout.test.ts +11 −1
@@ -3,7 +3,7 @@ import { existsSync, mkdirSync, mkdtempSync, writeFileSync } from "node:fs";
33 import { join } from "node:path";
44 import { tmpdir } from "node:os";
55 import { envDir } from "../src/apps/data.ts";
6 import { dropCheckouts } from "../src/apps/dormant_checkout.ts";
6 import { dropCheckouts, STRAY_CHECKOUT_AGE_MS } from "../src/apps/dormant_checkout.ts";
77
88 test("a dormant preview gives back its checkouts and keeps its data", async () => {
99 const dir = mkdtempSync(join(tmpdir(), "dormant-"));
@@ -29,3 +29,13 @@ test("the commits serving or coming up are spared", async () => {
2929 expect(existsSync(join(env, "abc1234"))).toBe(true);
3030 expect(existsSync(join(env, "def5678"))).toBe(false);
3131 });
32
33 test("a sweep with an age leaves a checkout touched within it, and takes an older one", async () => {
34 const dir = mkdtempSync(join(tmpdir(), "dormant-"));
35 const env = envDir(dir, "app1", "production");
36 for (const sub of ["abc1234", "def5678"]) mkdirSync(join(env, sub), { recursive: true });
37 const now = Date.now();
38 expect(await dropCheckouts(dir, "app1", "production", [], STRAY_CHECKOUT_AGE_MS, now)).toBe(0);
39 expect(await dropCheckouts(dir, "app1", "production", [], STRAY_CHECKOUT_AGE_MS, now + STRAY_CHECKOUT_AGE_MS + 1000)).toBe(2);
40 expect(existsSync(join(env, "abc1234"))).toBe(false);
41 });
test/stray-checkouts.test.ts added +111
@@ -0,0 +1,111 @@
1 /**
2 * The boot sweep that removes checkouts nothing serves or is making (#13772): what it spares
3 * (live, asleep, deploying, queued), what it takes, and that those commits are in state by the time
4 * `reconcileApp` reaches it.
5 */
6 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
7 import { existsSync, mkdirSync, utimesSync, writeFileSync } from "node:fs";
8 import { join } from "node:path";
9 import { newApp } from "../src/actions/apps.ts";
10 import { openPull } from "../src/actions/pulls.ts";
11 import { commitToBranch } from "../src/actions/repos.ts";
12 import { envDir } from "../src/apps/data.ts";
13 import { STRAY_CHECKOUT_AGE_MS } from "../src/apps/dormant_checkout.ts";
14 import { AppsRuntime } from "../src/apps/runtime.ts";
15 import type { AppRecord } from "../src/core/apps.ts";
16 import type { Repo } from "../src/core/repos.ts";
17 import { startAppsForge, stopAppsForge, type AppsForge } from "./helpers/apps-forge.ts";
18
19 const OLD = (Date.now() - STRAY_CHECKOUT_AGE_MS - 60_000) / 1000;
20
21 describe("dropStrayCheckouts", () => {
22 let forge: AppsForge;
23 let record: AppRecord;
24 let repo: Repo;
25 beforeAll(async () => {
26 forge = await startAppsForge({ start: false });
27 const made = await newApp(forge.app, forge.alice, { name: "hello", description: "x", visibility: "public" });
28 if (!made.ok) throw new Error(String(made.error));
29 record = made.value.app;
30 repo = made.value.repo;
31 }, 60_000);
32 afterAll(async () => {
33 await forge.runtime.stop();
34 await stopAppsForge(forge);
35 });
36
37 const make = (env: "production" | `pull-${number}`, short: string, old = true): string => {
38 const dir = join(envDir(forge.dataDir, record.id, env), short);
39 mkdirSync(dir, { recursive: true });
40 if (old) utimesSync(dir, OLD, OLD);
41 return dir;
42 };
43 const oid = (short: string) => short + "0".repeat(33);
44 // eslint-disable-next-line @typescript-eslint/no-explicit-any
45 const inject = (runtime: AppsRuntime, env: string, patch: Record<string, unknown>) =>
46 Object.assign((runtime as any).envFor(record, env), patch);
47
48 test("live, asleep, deploying and queued commits survive; a stray and a failed deploy's leftover go", async () => {
49 const runtime = new AppsRuntime(forge.app, { domain: "apps.test", scheme: "http", publicPort: 1, forgeUrl: forge.servers.url });
50 const dirs = {
51 live: make("production", "1111111"),
52 asleep: make("production", "2222222"),
53 deploying: make("production", "3333333"),
54 queued: make("production", "4444444"),
55 stray: make("production", "5555555"),
56 };
57 inject(runtime, "production", {
58 live: { commitOid: oid("1111111") },
59 asleep: { commitOid: oid("2222222") },
60 deploying: { run: { commitOid: oid("3333333") } },
61 queued: { commitOid: oid("4444444") },
62 });
63 await runtime.dropStrayCheckouts(record);
64 for (const kept of ["live", "asleep", "deploying", "queued"] as const) expect(existsSync(dirs[kept])).toBe(true);
65 expect(existsSync(dirs.stray)).toBe(false);
66 });
67
68 test("a checkout touched within the age stays, though nothing owns it", async () => {
69 const runtime = new AppsRuntime(forge.app, { domain: "apps.test", scheme: "http", publicPort: 1, forgeUrl: forge.servers.url });
70 const fresh = make("pull-9", "aaaaaaa", false);
71 const old = make("pull-9", "bbbbbbb");
72 await runtime.dropStrayCheckouts(record);
73 expect(existsSync(fresh)).toBe(true);
74 expect(existsSync(old)).toBe(false);
75 });
76
77 test("by the time reconcileApp sweeps at boot, a queued production head and an asleep preview head are in state", async () => {
78 const runtime = new AppsRuntime(forge.app, { domain: "apps.test", scheme: "http", publicPort: 1, forgeUrl: forge.servers.url });
79 const branch = await commitToBranch(forge.app, forge.alice, repo, {
80 branch: "feature/x",
81 base: "main",
82 message: "x",
83 files: [{ path: "x.md", content: "x\n" }],
84 });
85 if (!branch.ok) throw new Error(JSON.stringify(branch.error));
86 const opened = await openPull(forge.app, forge.alice, repo, { title: "x", baseBranch: "main", headBranch: "feature/x" });
87 if (!opened.ok) throw new Error(String(opened.error));
88 const env = `pull-${opened.value.number}` as const;
89 const head = opened.value.headOid ?? (await (runtime as any).pullHead(repo, opened.value));
90 // The preview's head is checked out on disk, so boot settles it asleep.
91 const headDir = make(env, head.slice(0, 7));
92 writeFileSync(join(headDir, "package.json"), JSON.stringify({ name: "x", scripts: { start: "true" } }));
93 const stray = make(env, "ccccccc");
94 const seen: Record<string, string> = {};
95 const real = runtime.dropStrayCheckouts.bind(runtime);
96 runtime.dropStrayCheckouts = async (r: AppRecord) => {
97 for (const e of runtime.status(r)) seen[e.env] = e.commitOid;
98 const ours = (runtime as any).envs.get(`${r.id}/${env}`) ?? [...(runtime as any).envs.values()].find((s: any) => s.env === env);
99 seen.asleep = ours?.asleep?.commitOid ?? "";
100 const prod = [...(runtime as any).envs.values()].find((s: any) => s.env === "production");
101 seen.production = prod?.queued?.commitOid ?? prod?.deploying?.run.commitOid ?? prod?.live?.commitOid ?? "";
102 await real(r);
103 };
104 await runtime.reconcileApp(record, repo, "boot");
105 expect(seen.asleep).toBe(head);
106 expect(seen.production).toMatch(/^[0-9a-f]{40}$/);
107 expect(existsSync(headDir)).toBe(true);
108 expect(existsSync(stray)).toBe(false);
109 await runtime.stop();
110 }, 60_000);
111 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs