Public: anyone can read this; the crew can change it.
Flaky test: a child forge spawned into a cell serves its app at <app>--<child>--<parent> #10683
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/10593-flaky-test-a-child-forge-spawned-into-a → main 4 commits fork 4 threads · 4 unresolved
2 files
changed +60
−19
test/helpers/real-spawn-e2e.ts +58 −17
| @@ -15,7 +15,7 @@ import { createRepo } from "../../src/actions/repos.ts"; | ||
| 15 | 15 | import { spawnForge } from "../../src/actions/spawn.ts"; |
| 16 | 16 | import { createApp } from "../../src/app.ts"; |
| 17 | 17 | import { cellsAvailable } from "../../src/apps/contain.ts"; |
| 18 | import { createUser } from "../../src/core/users.ts"; | |
| 18 | import { createUser, type User } from "../../src/core/users.ts"; | |
| 19 | 19 | import { repoDir } from "../../src/git/repos.ts"; |
| 20 | 20 | import { commitFiles } from "../../src/git/write.ts"; |
| 21 | 21 | import { serve } from "../../src/http/server.ts"; |
| @@ -23,6 +23,35 @@ import { cellPlacement, stopCellChildren } from "../../src/spawn/cell.ts"; | ||
| 23 | 23 | import { bootHostedChild } from "../../src/spawn/host.ts"; |
| 24 | 24 | import { memoryBodies } from "../../src/spawn/placement.ts"; |
| 25 | 25 | |
| 26 | // Two budgets, each its own: spawn+boot may take SPAWN_MS, then the poll gets | |
| 27 | // POLL_MS counted from when boot ended. A slow spawn never shortens the poll, | |
| 28 | // and a run that ran out says which phase did. The test's timeout is their sum | |
| 29 | // plus slack (see test/spawn-cell-e2e.test.ts). | |
| 30 | const SPAWN_MS = 120_000; | |
| 31 | const POLL_MS = 150_000; | |
| 32 | // A killed run (the test's timeout sends SIGTERM) still stops its cell children. | |
| 33 | let cleanup: () => Promise<void> = async () => {}; | |
| 34 | // The spawn and boot work, kept so a teardown after its budget ran out waits | |
| 35 | // for it to settle (up to SETTLE_MS) before stopping cell children: a child it | |
| 36 | // starts late is still stopped. | |
| 37 | const SETTLE_MS = 15_000; | |
| 38 | let pending: Promise<unknown> = Promise.resolve(); | |
| 39 | async function settled(): Promise<void> { | |
| 40 | await Promise.race([pending.catch(() => undefined), Bun.sleep(SETTLE_MS)]); | |
| 41 | } | |
| 42 | process.on("SIGTERM", () => void cleanup().finally(() => process.exit(143))); | |
| 43 | async function within<T>(ms: number, phase: string, work: Promise<T>): Promise<T> { | |
| 44 | let timer: ReturnType<typeof setTimeout> | undefined; | |
| 45 | const late = new Promise<never>((_, reject) => { | |
| 46 | timer = setTimeout(() => reject(new Error(`the ${phase} phase did not finish in ${ms / 1000}s`)), ms); | |
| 47 | }); | |
| 48 | try { | |
| 49 | return await Promise.race([work, late]); | |
| 50 | } finally { | |
| 51 | clearTimeout(timer); | |
| 52 | } | |
| 53 | } | |
| 54 | ||
| 26 | 55 | const cells = await cellsAvailable(); |
| 27 | 56 | if (!cells.ok) { |
| 28 | 57 | console.log(JSON.stringify({ skipped: cells.reason })); |
| @@ -34,6 +63,29 @@ const dataDir = mkdtempSync(join(tmpdir(), "real-spawn-")); | ||
| 34 | 63 | mkdirSync(dataDir, { recursive: true }); |
| 35 | 64 | const parent = createApp({ dataDir, url: "https://forge.test", port: 0 }); |
| 36 | 65 | const servers = await serve(parent); |
| 66 | cleanup = async () => { | |
| 67 | servers.stop(); | |
| 68 | await settled(); | |
| 69 | await stopCellChildren(parent); | |
| 70 | }; | |
| 71 | ||
| 72 | async function spawnAndBoot(alice: User) { | |
| 73 | const made = await spawnForge( | |
| 74 | parent, | |
| 75 | alice, | |
| 76 | { name: "kid", org: "kids", apps: ["alice/bakery"], carryBriefs: false, carrySubscriptions: false }, | |
| 77 | { catalog: [{ ref: "alice/bakery", readable: true, bytes: 10, forgeJsonBytes: 0 }], briefs: [], subscriptions: [] }, | |
outdated
The body of this IIFE is not indented. Extract it into a named function ( |
||
| 78 | { | |
| 79 | placement: cellPlacement(parent, memoryBodies(), () => 10), | |
| 80 | request: new Request("https://forge.test/x", { method: "POST", headers: { origin: "https://forge.test" } }), | |
| 81 | url: new URL("https://forge.test/x"), | |
| 82 | }, | |
| 83 | ); | |
| 84 | if (!made.ok) throw new Error(JSON.stringify(made.error)); | |
| 85 | await bootHostedChild(parent, "kid"); | |
| 86 | return made; | |
| 87 | } | |
| 88 | ||
| 37 | 89 | try { |
| 38 | 90 | const alice = await createUser(parent.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", isAdmin: true }); |
| 39 | 91 | if (!alice.ok) throw new Error("user"); |
| @@ -49,23 +101,13 @@ try { | ||
| 49 | 101 | author: { name: "a", email: "[email protected]" }, |
| 50 | 102 | }); |
| 51 | 103 | if (!committed.ok) throw new Error("commit"); |
| 52 | const made = await spawnForge( | |
| 53 | parent, | |
| 54 | alice.value, | |
| 55 | { name: "kid", org: "kids", apps: ["alice/bakery"], carryBriefs: false, carrySubscriptions: false }, | |
| 56 | { catalog: [{ ref: "alice/bakery", readable: true, bytes: 10, forgeJsonBytes: 0 }], briefs: [], subscriptions: [] }, | |
| 57 | { | |
| 58 | placement: cellPlacement(parent, memoryBodies(), () => 10), | |
| 59 | request: new Request("https://forge.test/x", { method: "POST", headers: { origin: "https://forge.test" } }), | |
| 60 | url: new URL("https://forge.test/x"), | |
| 61 | }, | |
| 62 | ); | |
| 63 | if (!made.ok) throw new Error(JSON.stringify(made.error)); | |
| 64 | await bootHostedChild(parent, "kid"); | |
| 104 | const spawning = spawnAndBoot(alice.value); | |
| 105 | pending = spawning; | |
| 106 | const made = await within(SPAWN_MS, "spawn and boot", spawning); | |
| 65 | 107 | let status = 0; |
| 66 | 108 | let body = ""; |
| 67 | 109 | let app: string | null = null; |
| 68 | const until = Date.now() + 150_000; | |
| 110 | const until = Date.now() + POLL_MS; | |
| 69 | 111 | while (Date.now() < until) { |
| 70 | 112 | const res = await fetch(`${servers.url}/`, { headers: { host: "bakery--kid--forge.test" }, redirect: "manual" }).catch(() => undefined); |
| 71 | 113 | if (res !== undefined) { |
| @@ -78,8 +120,7 @@ try { | ||
| 78 | 120 | } |
| 79 | 121 | console.log(JSON.stringify({ placement: made.value.placement, status, body: body.slice(0, 200), app })); |
| 80 | 122 | } finally { |
| 81 | servers.stop(); | |
| 82 | await stopCellChildren(parent); | |
| 123 | await cleanup(); | |
| 83 | 124 | parent.close(); |
| 84 | 125 | rmSync(dataDir, { recursive: true, force: true }); |
| 85 | 126 | } |
test/spawn-cell-e2e.test.ts +2 −2
| @@ -17,7 +17,7 @@ const cellTools = process.platform === "linux" && !portInsteadOfSocket() && !mis | ||
| 17 | 17 | test.skipIf(!cellTools)("a child forge spawned into a cell serves its app at <app>--<child>--<parent>", async () => { |
| 18 | 18 | const env: Record<string, string> = {}; |
| 19 | 19 | for (const [k, v] of Object.entries(process.env)) if (v !== undefined && k !== "FORGE_CELLS") env[k] = v; |
| 20 | const proc = Bun.spawn([process.execPath, join(import.meta.dir, "helpers", "real-spawn-e2e.ts")], { env, stdout: "pipe", stderr: "pipe" }); | |
| 20 | const proc = Bun.spawn([process.execPath, join(import.meta.dir, "helpers", "real-spawn-e2e.ts")], { env, stdout: "pipe", stderr: "pipe", timeout: 290_000 }); | |
outdated
timeout sends SIGTERM; the helper's finally/stopCellChildren will not run on it unless a handler exists, so cell children can still leak. Add a handler in the helper or do not claim cleanup. |
||
| 21 | 21 | const [out, errText] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited]); |
| 22 | 22 | const line = resultLine(out); |
| 23 | 23 | let seen: Record<string, unknown>; |
| @@ -34,4 +34,4 @@ test.skipIf(!cellTools)("a child forge spawned into a cell serves its app at <ap | ||
| 34 | 34 | // machine's disk and load are in the message, not a bare mismatch. |
| 35 | 35 | const ok = seen["placement"] === "cell" && seen["status"] === 200 && seen["app"] === "bakery" && seen["body"] === "bakery says hi"; |
| 36 | 36 | if (!ok) throw new Error(failedWith("the child forge's app did not answer through the cell", { seen, out: out.slice(-1500) }, errText)); |
| 37 | }, 200_000); | |
| 37 | }, 310_000); |
Deadline measured from helper start shrinks the poll window by however long spawn/boot took; the loaded runs that flaked can still fail. Use a poll deadline of its own and budget spawn separately.