Public: anyone can read this; the crew can change it.

Template what forge-booting tests still build per test: measure first, then copy keyed by inputs #14342

opened this

Check · Review · Land · Live — Pushed since the last review. Next: reviewer, pushed since the last review.

Ready to land

mason/forge:mason/14336-template-what-forge-booting-tests-still → main 2 commits fork 2 threads · 2 unresolved

Review

Sign in to review this change.
3 files changed +302 −14
test/built-once.test.ts added +79
@@ -0,0 +1,79 @@
1 import { afterAll, expect, test } from "bun:test";
2 import { existsSync, mkdirSync, readFileSync, rmSync, statSync, utimesSync, writeFileSync } from "node:fs";
3 import { join } from "node:path";
4 import { builtOnce, copyTemplate, sweepStale } from "./helpers/built-once.ts";
5 import { familyParts } from "./helpers/moved-forge.ts";
6 import { scratchDir } from "./helpers/scratch.ts";
7
8 const root = scratchDir("built-once-");
9 afterAll(() => rmSync(root, { recursive: true, force: true }));
10
11 const build = (text: string) => async (into: string) => void writeFileSync(join(into, "seed.txt"), text);
12
13 test("the same inputs are built once and every later ask gets that directory", async () => {
14 const one = await builtOnce(root, ["seed", "a"], build("a"));
15 const two = await builtOnce(root, ["seed", "a"], build("never run"));
16 expect(two.dir).toBe(one.dir);
17 expect(one.built).toBe(true);
18 expect(two.built).toBe(false);
19 expect(readFileSync(join(one.dir, "seed.txt"), "utf8")).toBe("a");
20 });
21
22 test("a changed input builds a fresh template instead of reusing the stale one", async () => {
23 const old = await builtOnce(root, ["seed", "old"], build("old"));
24 const changed = await builtOnce(root, ["seed", "new"], build("new"));
25 expect(changed.dir).not.toBe(old.dir);
26 expect(changed.built).toBe(true);
27 expect(readFileSync(join(changed.dir, "seed.txt"), "utf8")).toBe("new");
28 expect(readFileSync(join(old.dir, "seed.txt"), "utf8")).toBe("old");
29 });
30
31 test("a changed source, template or helper file is a changed family key", () => {
32 const make = (src: string, template: string, helper: string) => {
33 const dir = scratchDir("built-once-family-");
34 mkdirSync(join(dir, "src"));
35 mkdirSync(join(dir, "templates"));
36 writeFileSync(join(dir, "src", "a.ts"), src);
37 writeFileSync(join(dir, "templates", "t.html"), template);
38 writeFileSync(join(dir, "helper.ts"), helper);
39 return { trees: [join(dir, "src"), join(dir, "templates")], files: [join(dir, "helper.ts")], dir };
40 };
41 const made = [make("a", "t", "h"), make("a", "t", "h"), make("A", "t", "h"), make("a", "T", "h"), make("a", "t", "H")];
42 try {
43 const [base, same, ...changed] = made.map((m) => familyParts("homie", m).join("/"));
44 expect(same).toBe(base!);
45 for (const key of changed) expect(key).not.toBe(base!);
46 expect(familyParts("other", made[0]!).join("/")).not.toBe(base!);
47 } finally {
48 for (const m of made) rmSync(m.dir, { recursive: true, force: true });
49 }
50 });
51
52 test("a template asked for lately survives the sweep that removes an unused old one", async () => {
53 const day = 24 * 60 * 60_000;
54 const used = await builtOnce(root, ["seed", "used"], build("u"));
55 const unused = await builtOnce(root, ["seed", "unused"], build("x"));
56 const old = new Date(Date.now() - 2 * day);
57 utimesSync(used.dir, old, old);
58 utimesSync(unused.dir, old, old);
59 await builtOnce(root, ["seed", "used"], build("never run"));
60 sweepStale(root, join(root, "none"), Date.now());
61 expect(existsSync(used.dir)).toBe(true);
62 expect(existsSync(unused.dir)).toBe(false);
63 expect(Date.now() - statSync(used.dir).mtimeMs).toBeLessThan(60_000);
64 });
65
66 test("a copy is the test's own: writing in it leaves the template as built", async () => {
67 const { dir: template } = await builtOnce(root, ["seed", "copy"], build("kept"));
68 const mine = join(root, "mine");
69 copyTemplate(template, mine);
70 writeFileSync(join(mine, "seed.txt"), "changed");
71 expect(readFileSync(join(template, "seed.txt"), "utf8")).toBe("kept");
72 expect(existsSync(join(mine, "seed.txt"))).toBe(true);
73 });
74
75 test("a build that fails leaves nothing kept, and the next ask builds again", async () => {
76 await expect(builtOnce(root, ["seed", "fails"], async () => { throw new Error("no"); })).rejects.toThrow("no");
77 const { dir } = await builtOnce(root, ["seed", "fails"], build("second"));
78 expect(readFileSync(join(dir, "seed.txt"), "utf8")).toBe("second");
79 });
test/helpers/built-once.ts added +128
@@ -0,0 +1,128 @@
1 /**
2 * A directory built once a run and copied per test, keyed by what it is
3 * built from. The way `empty-images.ts` keeps the migrated database, for
4 * setup that is a directory (a forge with its repositories, an account and
5 * a pull): the first process to need one builds it under a name made from
6 * its inputs and renames it into place whole; every test after, in any
7 * process, copies it. A changed input is another name, so a template is
8 * never reused by a build that would have made a different one. Every ask
9 * touches the template it gets, so its mtime is the last time anything
10 * wanted it; a template untouched for a day belongs to inputs since changed,
11 * and the next build removes it.
12 */
13 import { cpSync, mkdirSync, readdirSync, renameSync, rmSync, statSync, readFileSync, utimesSync } from "node:fs";
14 import { join } from "node:path";
outdated

Age by mtime of build, never refreshed on use: an in-use template gets swept. Touch on hit.

15
16 const KEPT_FOR_MS = 24 * 60 * 60_000;
17
outdated

Reads every file of src/ and templates/ synchronously; called per makeFamily. Memoize per process.

18 /** Every file under `dirs` (paths and bytes), in a fixed order, folded into `hash`. */
19 export function hashTree(hash: Bun.CryptoHasher, ...dirs: string[]): void {
20 for (const dir of dirs) {
21 const names = (readdirSync(dir, { recursive: true }) as string[]).sort();
22 for (const name of names) {
23 const path = join(dir, name);
24 if (!statSync(path).isFile()) continue;
25 hash.update(`\0${name}\0`);
26 hash.update(readFileSync(path));
27 }
28 }
29 }
30
31 /** The name `parts` are built under: this runtime and each part, text or bytes, in order. */
32 function templateKey(parts: readonly (string | Uint8Array)[]): string {
33 const hash = new Bun.CryptoHasher("sha256");
34 hash.update(Bun.version);
35 for (const part of parts) {
36 hash.update("\0part\0");
37 hash.update(part);
38 }
39 return hash.digest("hex").slice(0, 32);
40 }
41
42 const building = new Map<string, Promise<Built>>();
43
44 /** A template's directory, and whether this ask is the one that built it. */
45 export interface Built {
46 readonly dir: string;
47 readonly built: boolean;
48 }
49
50 /**
51 * The directory `build` makes for these `parts`, under `root`: built when
52 * none is kept for them, else the kept one. Read it; copy it with
53 * {@link copyTemplate}; never write in it. Each ask marks the template used,
54 * so the sweep of stale ones never takes one in daily use.
55 */
56 export async function builtOnce(root: string, parts: readonly (string | Uint8Array)[], build: (into: string) => Promise<void>): Promise<Built> {
57 const final = join(root, templateKey(parts));
58 let pending = building.get(final);
59 const first = pending === undefined;
60 if (pending === undefined) {
61 pending = make(root, final, build);
62 pending.catch(() => building.delete(final));
63 building.set(final, pending);
64 }
65 const made = await pending;
66 markUsed(final);
67 // Only the ask that started the build can have built it.
68 return { dir: final, built: first && made.built };
69 }
70
71 /** Say the template was wanted now, so its age counts from its last use. */
72 function markUsed(dir: string): void {
73 const now = new Date();
74 try {
75 utimesSync(dir, now, now);
76 } catch {
77 // Swept or never there: the copy that follows says so.
78 }
79 }
80
81 async function make(root: string, final: string, build: (into: string) => Promise<void>): Promise<Built> {
82 try {
83 if (statSync(final).isDirectory()) return { dir: final, built: false };
84 } catch {
85 // Not built yet.
86 }
87 mkdirSync(root, { recursive: true });
88 const partial = `${final}.${process.pid}.${Math.random().toString(36).slice(2)}.partial`;
89 mkdirSync(partial);
90 let built = true;
91 try {
92 await build(partial);
93 try {
94 renameSync(partial, final);
95 } catch {
96 // Another process kept one for these inputs first: the same bytes, so use it.
97 built = false;
98 rmSync(partial, { recursive: true, force: true });
99 }
100 } catch (error) {
101 rmSync(partial, { recursive: true, force: true });
102 throw error;
103 }
104 markUsed(final);
105 sweepStale(root, final, Date.now());
106 return { dir: final, built };
107 }
108
109 /**
110 * Remove what nothing has asked for in a day, except `keep`; what is in use was touched, so it
111 * stays.
112 */
113 export function sweepStale(root: string, keep: string, nowMs: number): void {
114 for (const other of readdirSync(root)) {
115 const path = join(root, other);
116 if (path === keep || other.endsWith(".partial")) continue;
117 try {
118 if (nowMs - statSync(path).mtimeMs > KEPT_FOR_MS) rmSync(path, { recursive: true, force: true });
119 } catch {
120 // Taken by another process's sweep meanwhile.
121 }
122 }
123 }
124
125 /** A private copy of a template at `to`, ready to be written in. */
126 export function copyTemplate(template: string, to: string): void {
127 cpSync(template, to, { recursive: true });
128 }
test/helpers/moved-forge.ts +95 −14
@@ -46,6 +46,8 @@ import { hostedPlacement } from "../../src/spawn/placement.ts";
4646 import type { Size } from "../../src/spawn/sizes.ts";
4747 import { gitClientEnv } from "./git-client.ts";
4848 import { promote } from "./members.ts";
49 import { builtOnce, copyTemplate, hashTree } from "./built-once.ts";
50 import { SCRATCH_ROOT } from "./scratch.ts";
4951
5052 export const PARENT_HOST = "forge.test";
5153
@@ -154,6 +156,19 @@ function fakeCells(parent: App): FakeCells {
154156 return out;
155157 }
156158
159 /** Ask the child's address through the parent's listener, as a visitor does. */
160 function askAs(servers: Servers, host: string): Family["ask"] {
161 return (path, init = {}) =>
162 fetch(`${servers.url}${path}`, {
163 redirect: "manual",
164 ...init,
165 headers: {
166 ...(init.headers as Record<string, string> | undefined),
167 host,
168 },
169 });
170 }
171
157172 async function git(args: string[], cwd: string): Promise<void> {
158173 const proc = Bun.spawn(["git", ...args], {
159174 cwd,
@@ -168,9 +183,22 @@ async function git(args: string[], cwd: string): Promise<void> {
168183 if (code !== 0) throw new Error(`git ${args[0]}: ${err}`);
169184 }
170185
171 /** A parent with one forge in its own process, full of what a move must keep. */
172 export async function makeFamily(name = "homie"): Promise<Family> {
173 const root = mkdtempSync(join(tmpdir(), "move-cell-"));
186 /**
187 * What a test needs of a built family that the directory does not hold: the secrets are random, so
188 * they are kept beside it.
189 */
190 interface Built {
191 adminToken: string;
192 cookie: string;
193 childToken: string;
194 bodyId: string;
195 }
196
197 /**
198 * Everything a test reads of a family before its first move: the data directory below is built once
199 * and copied.
200 */
201 async function populate(root: string, name: string): Promise<Built> {
174202 const dataDir = join(root, "data");
175203 const parent = createApp({ dataDir, url: `https://${PARENT_HOST}`, port: 0 });
176204 const made = await createUser(parent.db, {
@@ -247,18 +275,8 @@ export async function makeFamily(name = "homie"): Promise<Family> {
247275 )
248276 throw new Error("not a never-cell forge");
249277 const bodyId = spawned.value.bodyId;
250 const dir = childDirOf(parent, bodyId);
251 const ledgerPath = join(dataDir, "spawn", `${name}.json`);
252278 const servers = await serve(parent);
253 const ask = (path: string, init: RequestInit = {}): Promise<Response> =>
254 fetch(`${servers.url}${path}`, {
255 redirect: "manual",
256 ...init,
257 headers: {
258 ...(init.headers as Record<string, string> | undefined),
259 host,
260 },
261 });
279 const ask = askAs(servers, host);
262280
263281 // The child, open in the parent's process: an account, a token, a repository pushed over HTTP, an
264282 // issue and a pull.
@@ -324,6 +342,69 @@ export async function makeFamily(name = "homie"): Promise<Family> {
324342 const lent = await lendCredit(parent, alice, name, 25);
325343 if (!lent.ok) throw new Error(`credit: ${lent.error}`);
326344
345 servers.stop();
346 closeHostedChildren(parent);
347 parent.close();
348 return { adminToken, cookie, childToken, bodyId };
349 }
350
351 /** What a family is built from: the trees and files whose bytes decide what a build makes. */
352 export interface FamilyInputs {
353 readonly trees: readonly string[];
354 readonly files: readonly string[];
355 }
356
357 const ROOT = join(import.meta.dir, "..", "..");
358 const FAMILY_INPUTS: FamilyInputs = {
359 trees: [join(ROOT, "src"), join(ROOT, "templates")],
360 files: [import.meta.path, join(import.meta.dir, "git-client.ts")],
361 };
362
363 const digests = new Map<string, string>();
364
365 /** The digest of `inputs`, read once per process: the files do not change during a run. */
366 function inputsDigest(inputs: FamilyInputs): string {
367 const memo = JSON.stringify(inputs);
368 let digest = digests.get(memo);
369 if (digest === undefined) {
370 const hash = new Bun.CryptoHasher("sha256");
371 hashTree(hash, ...inputs.trees);
372 for (const file of inputs.files) hash.update(readFileSync(file));
373 digest = hash.digest("hex");
374 digests.set(memo, digest);
375 }
376 return digest;
377 }
378
379 /** The parts a family is keyed by: its name and everything it is built from. */
380 export function familyParts(name: string, inputs: FamilyInputs = FAMILY_INPUTS): string[] {
381 return ["family", name, inputsDigest(inputs)];
382 }
383
384 /**
385 * A parent with one forge in its own process, full of what a move must keep: a copy of one built
386 * once a run.
387 */
388 export async function makeFamily(name = "homie"): Promise<Family> {
389 const { dir: template } = await builtOnce(join(SCRATCH_ROOT, "families"), familyParts(name), async (into) => {
390 const built = await populate(into, name);
391 await Bun.write(join(into, "built.json"), JSON.stringify(built));
392 });
393 const root = mkdtempSync(join(tmpdir(), "move-cell-"));
394 rmSync(root, { recursive: true });
395 copyTemplate(template, root);
396 const { adminToken, cookie, childToken, bodyId } = JSON.parse(readFileSync(join(root, "built.json"), "utf8")) as Built;
397 const dataDir = join(root, "data");
398 const parent = createApp({ dataDir, url: `https://${PARENT_HOST}`, port: 0 });
399 const alice = getUserByName(parent.db, "alice")!;
400 const host = `${name}--${PARENT_HOST}`;
401 const dir = childDirOf(parent, bodyId);
402 const ledgerPath = join(dataDir, "spawn", `${name}.json`);
403 const servers = await serve(parent);
404 const ask = askAs(servers, host);
405 // The child, open in the parent's process.
406 const opened = await resolveHostedChild(parent, new Request(`https://${PARENT_HOST}/healthz`, { headers: { host } }));
407 if (opened === undefined || opened === "unknown" || opened === "cell") throw new Error("child not opened in process");
327408 const cells = fakeCells(parent);
328409 const ledger = (): Record<string, unknown> =>
329410 JSON.parse(readFileSync(ledgerPath, "utf8")) as Record<string, unknown>;

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs