Public: anyone can read this; the crew can change it.
Template what forge-booting tests still build per test: measure first, then copy keyed by inputs #14342
Check · Review · Land · Live — Pushed since the last review.
mason/forge:mason/14336-template-what-forge-booting-tests-still → main 2 commits fork 2 threads · 2 unresolved
3 files
changed +302
−14
test/built-once.test.ts added +79
| @@ -0,0 +1,79 @@ | ||
| 1 | import { afterAll, expect, test } from "bun:test"; | |
| 2 | import { existsSync, mkdirSync, readFileSync, rmSync, statSync, utimesSync, writeFileSync } from "node:fs"; | |
| 3 | import { join } from "node:path"; | |
| 4 | import { builtOnce, copyTemplate, sweepStale } from "./helpers/built-once.ts"; | |
| 5 | import { familyParts } from "./helpers/moved-forge.ts"; | |
| 6 | import { scratchDir } from "./helpers/scratch.ts"; | |
| 7 | ||
| 8 | const root = scratchDir("built-once-"); | |
| 9 | afterAll(() => rmSync(root, { recursive: true, force: true })); | |
| 10 | ||
| 11 | const build = (text: string) => async (into: string) => void writeFileSync(join(into, "seed.txt"), text); | |
| 12 | ||
| 13 | test("the same inputs are built once and every later ask gets that directory", async () => { | |
| 14 | const one = await builtOnce(root, ["seed", "a"], build("a")); | |
| 15 | const two = await builtOnce(root, ["seed", "a"], build("never run")); | |
| 16 | expect(two.dir).toBe(one.dir); | |
| 17 | expect(one.built).toBe(true); | |
| 18 | expect(two.built).toBe(false); | |
| 19 | expect(readFileSync(join(one.dir, "seed.txt"), "utf8")).toBe("a"); | |
| 20 | }); | |
| 21 | ||
| 22 | test("a changed input builds a fresh template instead of reusing the stale one", async () => { | |
| 23 | const old = await builtOnce(root, ["seed", "old"], build("old")); | |
| 24 | const changed = await builtOnce(root, ["seed", "new"], build("new")); | |
| 25 | expect(changed.dir).not.toBe(old.dir); | |
| 26 | expect(changed.built).toBe(true); | |
| 27 | expect(readFileSync(join(changed.dir, "seed.txt"), "utf8")).toBe("new"); | |
| 28 | expect(readFileSync(join(old.dir, "seed.txt"), "utf8")).toBe("old"); | |
| 29 | }); | |
| 30 | ||
| 31 | test("a changed source, template or helper file is a changed family key", () => { | |
| 32 | const make = (src: string, template: string, helper: string) => { | |
| 33 | const dir = scratchDir("built-once-family-"); | |
| 34 | mkdirSync(join(dir, "src")); | |
| 35 | mkdirSync(join(dir, "templates")); | |
| 36 | writeFileSync(join(dir, "src", "a.ts"), src); | |
| 37 | writeFileSync(join(dir, "templates", "t.html"), template); | |
| 38 | writeFileSync(join(dir, "helper.ts"), helper); | |
| 39 | return { trees: [join(dir, "src"), join(dir, "templates")], files: [join(dir, "helper.ts")], dir }; | |
| 40 | }; | |
| 41 | const made = [make("a", "t", "h"), make("a", "t", "h"), make("A", "t", "h"), make("a", "T", "h"), make("a", "t", "H")]; | |
| 42 | try { | |
| 43 | const [base, same, ...changed] = made.map((m) => familyParts("homie", m).join("/")); | |
| 44 | expect(same).toBe(base!); | |
| 45 | for (const key of changed) expect(key).not.toBe(base!); | |
| 46 | expect(familyParts("other", made[0]!).join("/")).not.toBe(base!); | |
| 47 | } finally { | |
| 48 | for (const m of made) rmSync(m.dir, { recursive: true, force: true }); | |
| 49 | } | |
| 50 | }); | |
| 51 | ||
| 52 | test("a template asked for lately survives the sweep that removes an unused old one", async () => { | |
| 53 | const day = 24 * 60 * 60_000; | |
| 54 | const used = await builtOnce(root, ["seed", "used"], build("u")); | |
| 55 | const unused = await builtOnce(root, ["seed", "unused"], build("x")); | |
| 56 | const old = new Date(Date.now() - 2 * day); | |
| 57 | utimesSync(used.dir, old, old); | |
| 58 | utimesSync(unused.dir, old, old); | |
| 59 | await builtOnce(root, ["seed", "used"], build("never run")); | |
| 60 | sweepStale(root, join(root, "none"), Date.now()); | |
| 61 | expect(existsSync(used.dir)).toBe(true); | |
| 62 | expect(existsSync(unused.dir)).toBe(false); | |
| 63 | expect(Date.now() - statSync(used.dir).mtimeMs).toBeLessThan(60_000); | |
| 64 | }); | |
| 65 | ||
| 66 | test("a copy is the test's own: writing in it leaves the template as built", async () => { | |
| 67 | const { dir: template } = await builtOnce(root, ["seed", "copy"], build("kept")); | |
| 68 | const mine = join(root, "mine"); | |
| 69 | copyTemplate(template, mine); | |
| 70 | writeFileSync(join(mine, "seed.txt"), "changed"); | |
| 71 | expect(readFileSync(join(template, "seed.txt"), "utf8")).toBe("kept"); | |
| 72 | expect(existsSync(join(mine, "seed.txt"))).toBe(true); | |
| 73 | }); | |
| 74 | ||
| 75 | test("a build that fails leaves nothing kept, and the next ask builds again", async () => { | |
| 76 | await expect(builtOnce(root, ["seed", "fails"], async () => { throw new Error("no"); })).rejects.toThrow("no"); | |
| 77 | const { dir } = await builtOnce(root, ["seed", "fails"], build("second")); | |
| 78 | expect(readFileSync(join(dir, "seed.txt"), "utf8")).toBe("second"); | |
| 79 | }); |
test/helpers/built-once.ts added +128
| @@ -0,0 +1,128 @@ | ||
| 1 | /** | |
| 2 | * A directory built once a run and copied per test, keyed by what it is | |
| 3 | * built from. The way `empty-images.ts` keeps the migrated database, for | |
| 4 | * setup that is a directory (a forge with its repositories, an account and | |
| 5 | * a pull): the first process to need one builds it under a name made from | |
| 6 | * its inputs and renames it into place whole; every test after, in any | |
| 7 | * process, copies it. A changed input is another name, so a template is | |
| 8 | * never reused by a build that would have made a different one. Every ask | |
| 9 | * touches the template it gets, so its mtime is the last time anything | |
| 10 | * wanted it; a template untouched for a day belongs to inputs since changed, | |
| 11 | * and the next build removes it. | |
| 12 | */ | |
| 13 | import { cpSync, mkdirSync, readdirSync, renameSync, rmSync, statSync, readFileSync, utimesSync } from "node:fs"; | |
| 14 | import { join } from "node:path"; | |
| 15 | ||
| 16 | const KEPT_FOR_MS = 24 * 60 * 60_000; | |
| 17 | ||
outdated
Reads every file of src/ and templates/ synchronously; called per makeFamily. Memoize per process. |
||
| 18 | /** Every file under `dirs` (paths and bytes), in a fixed order, folded into `hash`. */ | |
| 19 | export function hashTree(hash: Bun.CryptoHasher, ...dirs: string[]): void { | |
| 20 | for (const dir of dirs) { | |
| 21 | const names = (readdirSync(dir, { recursive: true }) as string[]).sort(); | |
| 22 | for (const name of names) { | |
| 23 | const path = join(dir, name); | |
| 24 | if (!statSync(path).isFile()) continue; | |
| 25 | hash.update(`\0${name}\0`); | |
| 26 | hash.update(readFileSync(path)); | |
| 27 | } | |
| 28 | } | |
| 29 | } | |
| 30 | ||
| 31 | /** The name `parts` are built under: this runtime and each part, text or bytes, in order. */ | |
| 32 | function templateKey(parts: readonly (string | Uint8Array)[]): string { | |
| 33 | const hash = new Bun.CryptoHasher("sha256"); | |
| 34 | hash.update(Bun.version); | |
| 35 | for (const part of parts) { | |
| 36 | hash.update("\0part\0"); | |
| 37 | hash.update(part); | |
| 38 | } | |
| 39 | return hash.digest("hex").slice(0, 32); | |
| 40 | } | |
| 41 | ||
| 42 | const building = new Map<string, Promise<Built>>(); | |
| 43 | ||
| 44 | /** A template's directory, and whether this ask is the one that built it. */ | |
| 45 | export interface Built { | |
| 46 | readonly dir: string; | |
| 47 | readonly built: boolean; | |
| 48 | } | |
| 49 | ||
| 50 | /** | |
| 51 | * The directory `build` makes for these `parts`, under `root`: built when | |
| 52 | * none is kept for them, else the kept one. Read it; copy it with | |
| 53 | * {@link copyTemplate}; never write in it. Each ask marks the template used, | |
| 54 | * so the sweep of stale ones never takes one in daily use. | |
| 55 | */ | |
| 56 | export async function builtOnce(root: string, parts: readonly (string | Uint8Array)[], build: (into: string) => Promise<void>): Promise<Built> { | |
| 57 | const final = join(root, templateKey(parts)); | |
| 58 | let pending = building.get(final); | |
| 59 | const first = pending === undefined; | |
| 60 | if (pending === undefined) { | |
| 61 | pending = make(root, final, build); | |
| 62 | pending.catch(() => building.delete(final)); | |
| 63 | building.set(final, pending); | |
| 64 | } | |
| 65 | const made = await pending; | |
| 66 | markUsed(final); | |
| 67 | // Only the ask that started the build can have built it. | |
| 68 | return { dir: final, built: first && made.built }; | |
| 69 | } | |
| 70 | ||
| 71 | /** Say the template was wanted now, so its age counts from its last use. */ | |
| 72 | function markUsed(dir: string): void { | |
| 73 | const now = new Date(); | |
| 74 | try { | |
| 75 | utimesSync(dir, now, now); | |
| 76 | } catch { | |
| 77 | // Swept or never there: the copy that follows says so. | |
| 78 | } | |
| 79 | } | |
| 80 | ||
| 81 | async function make(root: string, final: string, build: (into: string) => Promise<void>): Promise<Built> { | |
| 82 | try { | |
| 83 | if (statSync(final).isDirectory()) return { dir: final, built: false }; | |
| 84 | } catch { | |
| 85 | // Not built yet. | |
| 86 | } | |
| 87 | mkdirSync(root, { recursive: true }); | |
| 88 | const partial = `${final}.${process.pid}.${Math.random().toString(36).slice(2)}.partial`; | |
| 89 | mkdirSync(partial); | |
| 90 | let built = true; | |
| 91 | try { | |
| 92 | await build(partial); | |
| 93 | try { | |
| 94 | renameSync(partial, final); | |
| 95 | } catch { | |
| 96 | // Another process kept one for these inputs first: the same bytes, so use it. | |
| 97 | built = false; | |
| 98 | rmSync(partial, { recursive: true, force: true }); | |
| 99 | } | |
| 100 | } catch (error) { | |
| 101 | rmSync(partial, { recursive: true, force: true }); | |
| 102 | throw error; | |
| 103 | } | |
| 104 | markUsed(final); | |
| 105 | sweepStale(root, final, Date.now()); | |
| 106 | return { dir: final, built }; | |
| 107 | } | |
| 108 | ||
| 109 | /** | |
| 110 | * Remove what nothing has asked for in a day, except `keep`; what is in use was touched, so it | |
| 111 | * stays. | |
| 112 | */ | |
| 113 | export function sweepStale(root: string, keep: string, nowMs: number): void { | |
| 114 | for (const other of readdirSync(root)) { | |
| 115 | const path = join(root, other); | |
| 116 | if (path === keep || other.endsWith(".partial")) continue; | |
| 117 | try { | |
| 118 | if (nowMs - statSync(path).mtimeMs > KEPT_FOR_MS) rmSync(path, { recursive: true, force: true }); | |
| 119 | } catch { | |
| 120 | // Taken by another process's sweep meanwhile. | |
| 121 | } | |
| 122 | } | |
| 123 | } | |
| 124 | ||
| 125 | /** A private copy of a template at `to`, ready to be written in. */ | |
| 126 | export function copyTemplate(template: string, to: string): void { | |
| 127 | cpSync(template, to, { recursive: true }); | |
| 128 | } |
test/helpers/moved-forge.ts +95 −14
| @@ -46,6 +46,8 @@ import { hostedPlacement } from "../../src/spawn/placement.ts"; | ||
| 46 | 46 | import type { Size } from "../../src/spawn/sizes.ts"; |
| 47 | 47 | import { gitClientEnv } from "./git-client.ts"; |
| 48 | 48 | import { promote } from "./members.ts"; |
| 49 | import { builtOnce, copyTemplate, hashTree } from "./built-once.ts"; | |
| 50 | import { SCRATCH_ROOT } from "./scratch.ts"; | |
| 49 | 51 | |
| 50 | 52 | export const PARENT_HOST = "forge.test"; |
| 51 | 53 | |
| @@ -154,6 +156,19 @@ function fakeCells(parent: App): FakeCells { | ||
| 154 | 156 | return out; |
| 155 | 157 | } |
| 156 | 158 | |
| 159 | /** Ask the child's address through the parent's listener, as a visitor does. */ | |
| 160 | function askAs(servers: Servers, host: string): Family["ask"] { | |
| 161 | return (path, init = {}) => | |
| 162 | fetch(`${servers.url}${path}`, { | |
| 163 | redirect: "manual", | |
| 164 | ...init, | |
| 165 | headers: { | |
| 166 | ...(init.headers as Record<string, string> | undefined), | |
| 167 | host, | |
| 168 | }, | |
| 169 | }); | |
| 170 | } | |
| 171 | ||
| 157 | 172 | async function git(args: string[], cwd: string): Promise<void> { |
| 158 | 173 | const proc = Bun.spawn(["git", ...args], { |
| 159 | 174 | cwd, |
| @@ -168,9 +183,22 @@ async function git(args: string[], cwd: string): Promise<void> { | ||
| 168 | 183 | if (code !== 0) throw new Error(`git ${args[0]}: ${err}`); |
| 169 | 184 | } |
| 170 | 185 | |
| 171 | /** A parent with one forge in its own process, full of what a move must keep. */ | |
| 172 | export async function makeFamily(name = "homie"): Promise<Family> { | |
| 173 | const root = mkdtempSync(join(tmpdir(), "move-cell-")); | |
| 186 | /** | |
| 187 | * What a test needs of a built family that the directory does not hold: the secrets are random, so | |
| 188 | * they are kept beside it. | |
| 189 | */ | |
| 190 | interface Built { | |
| 191 | adminToken: string; | |
| 192 | cookie: string; | |
| 193 | childToken: string; | |
| 194 | bodyId: string; | |
| 195 | } | |
| 196 | ||
| 197 | /** | |
| 198 | * Everything a test reads of a family before its first move: the data directory below is built once | |
| 199 | * and copied. | |
| 200 | */ | |
| 201 | async function populate(root: string, name: string): Promise<Built> { | |
| 174 | 202 | const dataDir = join(root, "data"); |
| 175 | 203 | const parent = createApp({ dataDir, url: `https://${PARENT_HOST}`, port: 0 }); |
| 176 | 204 | const made = await createUser(parent.db, { |
| @@ -247,18 +275,8 @@ export async function makeFamily(name = "homie"): Promise<Family> { | ||
| 247 | 275 | ) |
| 248 | 276 | throw new Error("not a never-cell forge"); |
| 249 | 277 | const bodyId = spawned.value.bodyId; |
| 250 | const dir = childDirOf(parent, bodyId); | |
| 251 | const ledgerPath = join(dataDir, "spawn", `${name}.json`); | |
| 252 | 278 | const servers = await serve(parent); |
| 253 | const ask = (path: string, init: RequestInit = {}): Promise<Response> => | |
| 254 | fetch(`${servers.url}${path}`, { | |
| 255 | redirect: "manual", | |
| 256 | ...init, | |
| 257 | headers: { | |
| 258 | ...(init.headers as Record<string, string> | undefined), | |
| 259 | host, | |
| 260 | }, | |
| 261 | }); | |
| 279 | const ask = askAs(servers, host); | |
| 262 | 280 | |
| 263 | 281 | // The child, open in the parent's process: an account, a token, a repository pushed over HTTP, an |
| 264 | 282 | // issue and a pull. |
| @@ -324,6 +342,69 @@ export async function makeFamily(name = "homie"): Promise<Family> { | ||
| 324 | 342 | const lent = await lendCredit(parent, alice, name, 25); |
| 325 | 343 | if (!lent.ok) throw new Error(`credit: ${lent.error}`); |
| 326 | 344 | |
| 345 | servers.stop(); | |
| 346 | closeHostedChildren(parent); | |
| 347 | parent.close(); | |
| 348 | return { adminToken, cookie, childToken, bodyId }; | |
| 349 | } | |
| 350 | ||
| 351 | /** What a family is built from: the trees and files whose bytes decide what a build makes. */ | |
| 352 | export interface FamilyInputs { | |
| 353 | readonly trees: readonly string[]; | |
| 354 | readonly files: readonly string[]; | |
| 355 | } | |
| 356 | ||
| 357 | const ROOT = join(import.meta.dir, "..", ".."); | |
| 358 | const FAMILY_INPUTS: FamilyInputs = { | |
| 359 | trees: [join(ROOT, "src"), join(ROOT, "templates")], | |
| 360 | files: [import.meta.path, join(import.meta.dir, "git-client.ts")], | |
| 361 | }; | |
| 362 | ||
| 363 | const digests = new Map<string, string>(); | |
| 364 | ||
| 365 | /** The digest of `inputs`, read once per process: the files do not change during a run. */ | |
| 366 | function inputsDigest(inputs: FamilyInputs): string { | |
| 367 | const memo = JSON.stringify(inputs); | |
| 368 | let digest = digests.get(memo); | |
| 369 | if (digest === undefined) { | |
| 370 | const hash = new Bun.CryptoHasher("sha256"); | |
| 371 | hashTree(hash, ...inputs.trees); | |
| 372 | for (const file of inputs.files) hash.update(readFileSync(file)); | |
| 373 | digest = hash.digest("hex"); | |
| 374 | digests.set(memo, digest); | |
| 375 | } | |
| 376 | return digest; | |
| 377 | } | |
| 378 | ||
| 379 | /** The parts a family is keyed by: its name and everything it is built from. */ | |
| 380 | export function familyParts(name: string, inputs: FamilyInputs = FAMILY_INPUTS): string[] { | |
| 381 | return ["family", name, inputsDigest(inputs)]; | |
| 382 | } | |
| 383 | ||
| 384 | /** | |
| 385 | * A parent with one forge in its own process, full of what a move must keep: a copy of one built | |
| 386 | * once a run. | |
| 387 | */ | |
| 388 | export async function makeFamily(name = "homie"): Promise<Family> { | |
| 389 | const { dir: template } = await builtOnce(join(SCRATCH_ROOT, "families"), familyParts(name), async (into) => { | |
| 390 | const built = await populate(into, name); | |
| 391 | await Bun.write(join(into, "built.json"), JSON.stringify(built)); | |
| 392 | }); | |
| 393 | const root = mkdtempSync(join(tmpdir(), "move-cell-")); | |
| 394 | rmSync(root, { recursive: true }); | |
| 395 | copyTemplate(template, root); | |
| 396 | const { adminToken, cookie, childToken, bodyId } = JSON.parse(readFileSync(join(root, "built.json"), "utf8")) as Built; | |
| 397 | const dataDir = join(root, "data"); | |
| 398 | const parent = createApp({ dataDir, url: `https://${PARENT_HOST}`, port: 0 }); | |
| 399 | const alice = getUserByName(parent.db, "alice")!; | |
| 400 | const host = `${name}--${PARENT_HOST}`; | |
| 401 | const dir = childDirOf(parent, bodyId); | |
| 402 | const ledgerPath = join(dataDir, "spawn", `${name}.json`); | |
| 403 | const servers = await serve(parent); | |
| 404 | const ask = askAs(servers, host); | |
| 405 | // The child, open in the parent's process. | |
| 406 | const opened = await resolveHostedChild(parent, new Request(`https://${PARENT_HOST}/healthz`, { headers: { host } })); | |
| 407 | if (opened === undefined || opened === "unknown" || opened === "cell") throw new Error("child not opened in process"); | |
| 327 | 408 | const cells = fakeCells(parent); |
| 328 | 409 | const ledger = (): Record<string, unknown> => |
| 329 | 410 | JSON.parse(readFileSync(ledgerPath, "utf8")) as Record<string, unknown>; |
new above
offline
Age by mtime of build, never refreshed on use: an in-use template gets swept. Touch on hit.