Public: anyone can read this; the crew can change it.
freePort copies in src/snapshot/boot.ts and src/release/boot.ts race the same way as apps/process.ts did #8692
torque/forge:torque/8691-freeport-copies-in-src-snapshot-boot-ts → main 2 commits fork
6 files
changed +124
−22
src/kernel/port.ts added +36
| @@ -0,0 +1,36 @@ | ||
| 1 | /** | |
| 2 | * A loopback port for a child that binds it later. The probe releases the | |
| 3 | * port before the child starts, so another process can take it in between; | |
| 4 | * the child then dies on "address in use". `onFreePort` starts again on a | |
| 5 | * fresh port when that is how the attempt failed, and only then. | |
| 6 | */ | |
| 7 | ||
| 8 | export const PORT_TRIES = 3; | |
| 9 | ||
| 10 | /** A port the OS says is free right now. */ | |
| 11 | export async function freePort(): Promise<number> { | |
| 12 | const probe = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch: () => new Response("") }); | |
| 13 | const port = probe.port ?? 0; | |
| 14 | await probe.stop(true); | |
| 15 | return port; | |
| 16 | } | |
| 17 | ||
| 18 | /** Does what a child printed say it could not bind its port (Bun: "Is port N in use?")? */ | |
| 19 | export function portTaken(stderr: string): boolean { | |
| 20 | return /EADDRINUSE|address already in use|\bin use\b/i.test(stderr); | |
| 21 | } | |
| 22 | ||
| 23 | /** | |
| 24 | * Run `attempt` on a free port; while `lost` says the port was taken from | |
| 25 | * under it, run it again on a new one, up to `tries` times in all. The last | |
| 26 | * result is returned whatever it is. | |
| 27 | */ | |
| 28 | export async function onFreePort<T>( | |
| 29 | attempt: (port: number) => Promise<T>, | |
| 30 | lost: (result: T) => boolean, | |
| 31 | tries: number = PORT_TRIES, | |
| 32 | ): Promise<T> { | |
| 33 | let result = await attempt(await freePort()); | |
| 34 | for (let n = 1; n < tries && lost(result); n++) result = await attempt(await freePort()); | |
| 35 | return result; | |
| 36 | } |
src/release/boot.ts +10 −9
| @@ -28,6 +28,7 @@ import { tmpdir } from "node:os"; | ||
| 28 | 28 | import { join } from "node:path"; |
| 29 | 29 | import { running } from "../kernel/version.ts"; |
| 30 | 30 | import { probeHeaders } from "../kernel/probe.ts"; |
| 31 | import { onFreePort, portTaken } from "../kernel/port.ts"; | |
| 31 | 32 | |
| 32 | 33 | export const BOOT_BUDGET_MS = 120_000; |
| 33 | 34 | const POLL_MS = 250; |
| @@ -49,13 +50,6 @@ export interface BootInput { | ||
| 49 | 50 | readonly env?: Readonly<Record<string, string | undefined>>; |
| 50 | 51 | } |
| 51 | 52 | |
| 52 | async function freePort(): Promise<number> { | |
| 53 | const probe = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch: () => new Response("") }); | |
| 54 | const port = probe.port ?? 0; | |
| 55 | await probe.stop(true); | |
| 56 | return port; | |
| 57 | } | |
| 58 | ||
| 59 | 53 | /** The line of what the child printed that says why: the first error, else the last line. */ |
| 60 | 54 | export function why(stderr: string): string { |
| 61 | 55 | const lines = stderr |
| @@ -82,12 +76,19 @@ async function versionAt(port: number, probe: Record<string, string>): Promise<s | ||
| 82 | 76 | } |
| 83 | 77 | } |
| 84 | 78 | |
| 85 | /** Start the tree on `dataDir` and wait for it to answer health as `version`. */ | |
| 79 | /** Start the tree on `dataDir` and wait for it to answer health as `version`; a port taken from under it is retried on a fresh one. */ | |
| 86 | 80 | export async function bootProof(input: BootInput): Promise<BootProof> { |
| 87 | 81 | const entry = join(input.tree, "src", "main.ts"); |
| 88 | 82 | if (!existsSync(entry)) return { ok: false, line: `${input.tree} has no src/main.ts` }; |
| 83 | return onFreePort( | |
| 84 | (port) => bootOn(input, port), | |
| 85 | (r) => !r.ok && /^exited with/.test(r.line) && portTaken(r.line), | |
| 86 | ); | |
| 87 | } | |
| 88 | ||
| 89 | async function bootOn(input: BootInput, port: number): Promise<BootProof> { | |
| 90 | const entry = join(input.tree, "src", "main.ts"); | |
| 89 | 91 | const host = input.env ?? process.env; |
| 90 | const port = await freePort(); | |
| 91 | 92 | const env: Record<string, string> = { |
| 92 | 93 | PATH: host["PATH"] ?? "/usr/local/bin:/usr/bin:/bin", |
| 93 | 94 | FORGE_DATA: input.dataDir, |
src/snapshot/boot.ts +27 −13
| @@ -7,6 +7,7 @@ | ||
| 7 | 7 | */ |
| 8 | 8 | |
| 9 | 9 | import { join } from "node:path"; |
| 10 | import { onFreePort, portTaken } from "../kernel/port.ts"; | |
| 10 | 11 | |
| 11 | 12 | export interface Booted { |
| 12 | 13 | readonly ok: boolean; |
| @@ -16,18 +17,12 @@ export interface Booted { | ||
| 16 | 17 | |
| 17 | 18 | const TREE = join(import.meta.dir, "../.."); |
| 18 | 19 | |
| 19 | async function freePort(): Promise<number> { | |
| 20 | const probe = Bun.serve({ port: 0, fetch: () => new Response("") }); | |
| 21 | const port = probe.port ?? 0; | |
| 22 | await probe.stop(true); | |
| 23 | return port; | |
| 24 | } | |
| 25 | ||
| 26 | export async function bootRestored( | |
| 20 | /** One attempt; `portLost` is set when the child died because its port was taken. */ | |
| 21 | async function attemptBoot( | |
| 27 | 22 | dataDir: string, |
| 28 | 23 | budgetMs: number, |
| 29 | ): Promise<Booted> { | |
| 30 | const port = await freePort(); | |
| 24 | port: number, | |
| 25 | ): Promise<Booted & { readonly portLost?: boolean }> { | |
| 31 | 26 | const child = Bun.spawn([process.execPath, join(TREE, "src/main.ts")], { |
| 32 | 27 | cwd: TREE, |
| 33 | 28 | env: { |
| @@ -42,14 +37,25 @@ export async function bootRestored( | ||
| 42 | 37 | : {}), |
| 43 | 38 | }, |
| 44 | 39 | stdout: "ignore", |
| 45 | stderr: "ignore", | |
| 40 | stderr: "pipe", | |
| 46 | 41 | }); |
| 42 | let stderr = ""; | |
| 43 | const reading = (async () => { | |
| 44 | const decoder = new TextDecoder(); | |
| 45 | for await (const chunk of child.stderr) stderr = (stderr + decoder.decode(chunk, { stream: true })).slice(-8_192); | |
| 46 | })().catch(() => undefined); | |
| 47 | 47 | const until = performance.now() + Math.min(budgetMs, 60_000); |
| 48 | 48 | let last = "no answer"; |
| 49 | 49 | try { |
| 50 | 50 | while (performance.now() < until) { |
| 51 | if (child.exitCode !== null) | |
| 52 | return { ok: false, reason: `the restored forge exited with ${child.exitCode}` }; | |
| 51 | if (child.exitCode !== null) { | |
| 52 | await reading; | |
| 53 | return { | |
| 54 | ok: false, | |
| 55 | reason: `the restored forge exited with ${child.exitCode}`, | |
| 56 | portLost: portTaken(stderr), | |
| 57 | }; | |
| 58 | } | |
| 53 | 59 | try { |
| 54 | 60 | const res = await fetch(`http://127.0.0.1:${port}/api/v1/health`, { |
| 55 | 61 | signal: AbortSignal.timeout(2_000), |
| @@ -68,3 +74,11 @@ export async function bootRestored( | ||
| 68 | 74 | await child.exited; |
| 69 | 75 | } |
| 70 | 76 | } |
| 77 | ||
| 78 | export async function bootRestored(dataDir: string, budgetMs: number): Promise<Booted> { | |
| 79 | const { ok, reason } = await onFreePort( | |
| 80 | (port) => attemptBoot(dataDir, budgetMs, port), | |
| 81 | (r) => r.portLost === true, | |
| 82 | ); | |
| 83 | return { ok, reason }; | |
| 84 | } |
test/kernel-port.test.ts added +18
| @@ -0,0 +1,18 @@ | ||
| 1 | import { expect, test } from "bun:test"; | |
| 2 | import { onFreePort, portTaken } from "../src/kernel/port.ts"; | |
| 3 | ||
| 4 | test("portTaken reads Bun's and the OS's wording", () => { | |
| 5 | expect(portTaken("error: Failed to start server. Is port 4000 in use?")).toBe(true); | |
| 6 | expect(portTaken("listen EADDRINUSE: address already in use")).toBe(true); | |
| 7 | expect(portTaken("error: boom")).toBe(false); | |
| 8 | }); | |
| 9 | ||
| 10 | test("onFreePort retries on a fresh port while the attempt says it lost, up to the tries", async () => { | |
| 11 | const seen: number[] = []; | |
| 12 | const result = await onFreePort(async (port) => (seen.push(port), seen.length), (n) => n < 2); | |
| 13 | expect(result).toBe(2); | |
| 14 | expect(seen.length).toBe(2); | |
| 15 | const all: number[] = []; | |
| 16 | await onFreePort(async (p) => (all.push(p), 0), () => true, 3); | |
| 17 | expect(all.length).toBe(3); | |
| 18 | }); |
test/release-boot.test.ts +32
| @@ -128,3 +128,35 @@ test("the build runs the proof on the image it built, and the chart runs it on a | ||
| 128 | 128 | expect(hook).toContain("claimName: {{ .Release.Name }}-data"); |
| 129 | 129 | expect(hook).toContain("backoffLimit: 0"); |
| 130 | 130 | }); |
| 131 | ||
| 132 | test("a child that dies because its port was taken is started again on a fresh port", async () => { | |
| 133 | const dir = tree( | |
| 134 | "raced", | |
| 135 | `import { existsSync, writeFileSync } from "node:fs"; | |
| 136 | const mark = process.env.FORGE_DATA + "/tried"; | |
| 137 | if (!existsSync(mark)) { | |
| 138 | writeFileSync(mark, "1"); | |
| 139 | console.error("error: Failed to start server. Is port " + process.env.PORT + " in use?"); | |
| 140 | process.exit(1); | |
| 141 | } | |
| 142 | ` + serving("process.env.FORGE_VERSION"), | |
| 143 | ); | |
| 144 | const data = join(root, "raced-data"); | |
| 145 | mkdirSync(data); | |
| 146 | expect(await bootProof({ tree: dir, version: "1.2.3", dataDir: data, budgetMs: 30_000 })).toMatchObject({ ok: true }); | |
| 147 | }); | |
| 148 | ||
| 149 | test("a child that dies for another reason is not retried", async () => { | |
| 150 | const dir = tree( | |
| 151 | "once", | |
| 152 | `import { appendFileSync } from "node:fs"; | |
| 153 | appendFileSync(process.env.FORGE_DATA + "/runs", "x"); | |
| 154 | console.error("error: boom"); | |
| 155 | process.exit(1);`, | |
| 156 | ); | |
| 157 | const data = join(root, "once-data"); | |
| 158 | mkdirSync(data); | |
| 159 | const result = await bootProof({ tree: dir, version: "1.2.3", dataDir: data, budgetMs: 30_000 }); | |
| 160 | expect(result.ok).toBe(false); | |
| 161 | expect(await Bun.file(join(data, "runs")).text()).toBe("x"); | |
| 162 | }); |
test/spawn-pipes.test.ts +1
| @@ -32,6 +32,7 @@ const PIPING: Record<string, { sites: string[]; reason: string }> = { | ||
| 32 | 32 | "src/looks/capture.ts": { sites: ["stderr"], reason: "stderr read in a background loop; stdout ignored" }, |
| 33 | 33 | "src/release/devtools.ts": { sites: ["stderr"], reason: "stderr read for the DevTools address, then drained in a background loop; stdout ignored" }, |
| 34 | 34 | "src/release/boot.ts": { sites: ["stderr"], reason: "stderr read to its end in a background loop, awaited before the reason is read; stdout ignored" }, |
| 35 | "src/snapshot/boot.ts": { sites: ["stderr"], reason: "stderr read to its end in a background loop, awaited before the port check; stdout ignored" }, | |
| 35 | 36 | "src/release/checkout.ts": { sites: ["stdout+stderr"], reason: "stdout and stderr both read via Response.text()" }, |
| 36 | 37 | "src/checks/install.ts": { sites: ["stdout+stderr"], reason: "prefillInstallCache reads stdout and stderr to the end via Response.text() while the exit is awaited" }, |
| 37 | 38 | "src/cli/config.ts": { sites: ["stdout"], reason: "stdout awaited with the exit; stderr ignored" }, |
new above
offline