Public: anyone can read this; the crew can change it.

freePort copies in src/snapshot/boot.ts and src/release/boot.ts race the same way as apps/process.ts did #8692

Parked opened this
Closed without merging

torque/forge:torque/8691-freeport-copies-in-src-snapshot-boot-ts → main 2 commits fork

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
6 files changed +124 −22
src/kernel/port.ts added +36
@@ -0,0 +1,36 @@
1 /**
2 * A loopback port for a child that binds it later. The probe releases the
3 * port before the child starts, so another process can take it in between;
4 * the child then dies on "address in use". `onFreePort` starts again on a
5 * fresh port when that is how the attempt failed, and only then.
6 */
7
8 export const PORT_TRIES = 3;
9
10 /** A port the OS says is free right now. */
11 export async function freePort(): Promise<number> {
12 const probe = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch: () => new Response("") });
13 const port = probe.port ?? 0;
14 await probe.stop(true);
15 return port;
16 }
17
18 /** Does what a child printed say it could not bind its port (Bun: "Is port N in use?")? */
19 export function portTaken(stderr: string): boolean {
20 return /EADDRINUSE|address already in use|\bin use\b/i.test(stderr);
21 }
22
23 /**
24 * Run `attempt` on a free port; while `lost` says the port was taken from
25 * under it, run it again on a new one, up to `tries` times in all. The last
26 * result is returned whatever it is.
27 */
28 export async function onFreePort<T>(
29 attempt: (port: number) => Promise<T>,
30 lost: (result: T) => boolean,
31 tries: number = PORT_TRIES,
32 ): Promise<T> {
33 let result = await attempt(await freePort());
34 for (let n = 1; n < tries && lost(result); n++) result = await attempt(await freePort());
35 return result;
36 }
src/release/boot.ts +10 −9
@@ -28,6 +28,7 @@ import { tmpdir } from "node:os";
2828 import { join } from "node:path";
2929 import { running } from "../kernel/version.ts";
3030 import { probeHeaders } from "../kernel/probe.ts";
31 import { onFreePort, portTaken } from "../kernel/port.ts";
3132
3233 export const BOOT_BUDGET_MS = 120_000;
3334 const POLL_MS = 250;
@@ -49,13 +50,6 @@ export interface BootInput {
4950 readonly env?: Readonly<Record<string, string | undefined>>;
5051 }
5152
52 async function freePort(): Promise<number> {
53 const probe = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch: () => new Response("") });
54 const port = probe.port ?? 0;
55 await probe.stop(true);
56 return port;
57 }
58
5953 /** The line of what the child printed that says why: the first error, else the last line. */
6054 export function why(stderr: string): string {
6155 const lines = stderr
@@ -82,12 +76,19 @@ async function versionAt(port: number, probe: Record<string, string>): Promise<s
8276 }
8377 }
8478
85 /** Start the tree on `dataDir` and wait for it to answer health as `version`. */
79 /** Start the tree on `dataDir` and wait for it to answer health as `version`; a port taken from under it is retried on a fresh one. */
8680 export async function bootProof(input: BootInput): Promise<BootProof> {
8781 const entry = join(input.tree, "src", "main.ts");
8882 if (!existsSync(entry)) return { ok: false, line: `${input.tree} has no src/main.ts` };
83 return onFreePort(
84 (port) => bootOn(input, port),
85 (r) => !r.ok && /^exited with/.test(r.line) && portTaken(r.line),
86 );
87 }
88
89 async function bootOn(input: BootInput, port: number): Promise<BootProof> {
90 const entry = join(input.tree, "src", "main.ts");
8991 const host = input.env ?? process.env;
90 const port = await freePort();
9192 const env: Record<string, string> = {
9293 PATH: host["PATH"] ?? "/usr/local/bin:/usr/bin:/bin",
9394 FORGE_DATA: input.dataDir,
src/snapshot/boot.ts +27 −13
@@ -7,6 +7,7 @@
77 */
88
99 import { join } from "node:path";
10 import { onFreePort, portTaken } from "../kernel/port.ts";
1011
1112 export interface Booted {
1213 readonly ok: boolean;
@@ -16,18 +17,12 @@ export interface Booted {
1617
1718 const TREE = join(import.meta.dir, "../..");
1819
19 async function freePort(): Promise<number> {
20 const probe = Bun.serve({ port: 0, fetch: () => new Response("") });
21 const port = probe.port ?? 0;
22 await probe.stop(true);
23 return port;
24 }
25
26 export async function bootRestored(
20 /** One attempt; `portLost` is set when the child died because its port was taken. */
21 async function attemptBoot(
2722 dataDir: string,
2823 budgetMs: number,
29 ): Promise<Booted> {
30 const port = await freePort();
24 port: number,
25 ): Promise<Booted & { readonly portLost?: boolean }> {
3126 const child = Bun.spawn([process.execPath, join(TREE, "src/main.ts")], {
3227 cwd: TREE,
3328 env: {
@@ -42,14 +37,25 @@ export async function bootRestored(
4237 : {}),
4338 },
4439 stdout: "ignore",
45 stderr: "ignore",
40 stderr: "pipe",
4641 });
42 let stderr = "";
43 const reading = (async () => {
44 const decoder = new TextDecoder();
45 for await (const chunk of child.stderr) stderr = (stderr + decoder.decode(chunk, { stream: true })).slice(-8_192);
46 })().catch(() => undefined);
4747 const until = performance.now() + Math.min(budgetMs, 60_000);
4848 let last = "no answer";
4949 try {
5050 while (performance.now() < until) {
51 if (child.exitCode !== null)
52 return { ok: false, reason: `the restored forge exited with ${child.exitCode}` };
51 if (child.exitCode !== null) {
52 await reading;
53 return {
54 ok: false,
55 reason: `the restored forge exited with ${child.exitCode}`,
56 portLost: portTaken(stderr),
57 };
58 }
5359 try {
5460 const res = await fetch(`http://127.0.0.1:${port}/api/v1/health`, {
5561 signal: AbortSignal.timeout(2_000),
@@ -68,3 +74,11 @@ export async function bootRestored(
6874 await child.exited;
6975 }
7076 }
77
78 export async function bootRestored(dataDir: string, budgetMs: number): Promise<Booted> {
79 const { ok, reason } = await onFreePort(
80 (port) => attemptBoot(dataDir, budgetMs, port),
81 (r) => r.portLost === true,
82 );
83 return { ok, reason };
84 }
test/kernel-port.test.ts added +18
@@ -0,0 +1,18 @@
1 import { expect, test } from "bun:test";
2 import { onFreePort, portTaken } from "../src/kernel/port.ts";
3
4 test("portTaken reads Bun's and the OS's wording", () => {
5 expect(portTaken("error: Failed to start server. Is port 4000 in use?")).toBe(true);
6 expect(portTaken("listen EADDRINUSE: address already in use")).toBe(true);
7 expect(portTaken("error: boom")).toBe(false);
8 });
9
10 test("onFreePort retries on a fresh port while the attempt says it lost, up to the tries", async () => {
11 const seen: number[] = [];
12 const result = await onFreePort(async (port) => (seen.push(port), seen.length), (n) => n < 2);
13 expect(result).toBe(2);
14 expect(seen.length).toBe(2);
15 const all: number[] = [];
16 await onFreePort(async (p) => (all.push(p), 0), () => true, 3);
17 expect(all.length).toBe(3);
18 });
test/release-boot.test.ts +32
@@ -128,3 +128,35 @@ test("the build runs the proof on the image it built, and the chart runs it on a
128128 expect(hook).toContain("claimName: {{ .Release.Name }}-data");
129129 expect(hook).toContain("backoffLimit: 0");
130130 });
131
132 test("a child that dies because its port was taken is started again on a fresh port", async () => {
133 const dir = tree(
134 "raced",
135 `import { existsSync, writeFileSync } from "node:fs";
136 const mark = process.env.FORGE_DATA + "/tried";
137 if (!existsSync(mark)) {
138 writeFileSync(mark, "1");
139 console.error("error: Failed to start server. Is port " + process.env.PORT + " in use?");
140 process.exit(1);
141 }
142 ` + serving("process.env.FORGE_VERSION"),
143 );
144 const data = join(root, "raced-data");
145 mkdirSync(data);
146 expect(await bootProof({ tree: dir, version: "1.2.3", dataDir: data, budgetMs: 30_000 })).toMatchObject({ ok: true });
147 });
148
149 test("a child that dies for another reason is not retried", async () => {
150 const dir = tree(
151 "once",
152 `import { appendFileSync } from "node:fs";
153 appendFileSync(process.env.FORGE_DATA + "/runs", "x");
154 console.error("error: boom");
155 process.exit(1);`,
156 );
157 const data = join(root, "once-data");
158 mkdirSync(data);
159 const result = await bootProof({ tree: dir, version: "1.2.3", dataDir: data, budgetMs: 30_000 });
160 expect(result.ok).toBe(false);
161 expect(await Bun.file(join(data, "runs")).text()).toBe("x");
162 });
test/spawn-pipes.test.ts +1
@@ -32,6 +32,7 @@ const PIPING: Record<string, { sites: string[]; reason: string }> = {
3232 "src/looks/capture.ts": { sites: ["stderr"], reason: "stderr read in a background loop; stdout ignored" },
3333 "src/release/devtools.ts": { sites: ["stderr"], reason: "stderr read for the DevTools address, then drained in a background loop; stdout ignored" },
3434 "src/release/boot.ts": { sites: ["stderr"], reason: "stderr read to its end in a background loop, awaited before the reason is read; stdout ignored" },
35 "src/snapshot/boot.ts": { sites: ["stderr"], reason: "stderr read to its end in a background loop, awaited before the port check; stdout ignored" },
3536 "src/release/checkout.ts": { sites: ["stdout+stderr"], reason: "stdout and stderr both read via Response.text()" },
3637 "src/checks/install.ts": { sites: ["stdout+stderr"], reason: "prefillInstallCache reads stdout and stderr to the end via Response.text() while the exit is awaited" },
3738 "src/cli/config.ts": { sites: ["stdout"], reason: "stdout awaited with the exit; stderr ignored" },

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs