6c9f957 · 8e9ceb0 · e21a26a · e1e98b0 · dfd5524 · 9b66b73
@@ -50,7 +50,8 @@
RUN bun src/release/boot.ts
# tini reaps what an agent run leaves behind: a claude run spawns git, bun,
# the app under test, and their children.
ENTRYPOINT ["tini", "--"]
# The launcher runs src/main.ts, and when this version does not answer health
# boot.sh runs the launcher, which starts src/main.ts and then exits, leaving
# the forge as the only Bun runtime; when this version does not answer health
# within FORGE_READY_GRACE_S (150), runs the last version that did, kept on
# the volume (src/release/launch.ts, #2652).
CMD ["bun", "src/release/launch.ts"]
CMD ["bash", "src/release/boot.sh"]
@@ -15,7 +15,7 @@
One path, in the forge: **landing → on cadence the keeper takes main's tree 
## A pod restart
`release/launch.ts` is the container's first process. `keepGood` records a swapped tree with `base`, the image version it was swapped over, in `release/good.json`. When the same image restarts, the launcher starts that tree first, and a ready older image never replaces it as the last good. A different image version is a deliberate roll (a new base): it runs, and is kept over the swapped tree. A swapped tree that does not become ready in the grace gives way to the image and the rollback card says so. The swapped process is not the launcher's child, so it writes `release/serving.json` (`release/serving.ts`) and the launcher stays up while that pid lives and signals it on SIGTERM; when it dies the container exits and restarts on the newest good tree.
`release/boot.sh` is the container's first process after tini: it runs `release/launch.ts` with `FORGE_LAUNCH_HANDOFF=1`, which exits once the forge is ready and names it in `release/serving.json`, and boot.sh follows that pid (so no second Bun runtime stays resident). `keepGood` records a swapped tree with `base`, the image version it was swapped over, in `release/good.json`. When the same image restarts, the launcher starts that tree first, and a ready older image never replaces it as the last good. A different image version is a deliberate roll (a new base): it runs, and is kept over the swapped tree. A swapped tree that does not become ready in the grace gives way to the image and the rollback card says so. A swapped process is not the launcher's child, so it rewrites `release/serving.json` (`release/serving.ts`) and boot.sh keeps following the pid there, signalling it on SIGTERM; when it dies the container exits and restarts on the newest good tree.
## Two writers, apps and runs
@@ -243,7 +243,10 @@
in `src/release/`:
  `boot.ts --from /data` beside the running forge, on a copy of its database
  and top-level files. A refusal fails the upgrade while the old pod serves;
  the failed Job is kept for its log (`kubectl -n "$NS" logs job/$REL-boot-proof`).
- **After the roll.** The image starts `src/release/launch.ts`, which runs
- **After the roll.** The image starts `src/release/boot.sh`, which runs
  `src/release/launch.ts` only until the forge is ready (then it exits, so the
  container holds one Bun runtime, not two, #12348) and
  waits on the pid in `release/serving.json`. The launcher runs
  `src/main.ts` and, when that version exits or is not ready within
  `FORGE_READY_GRACE_S` (150 s, inside the startup probe), starts the last
  version that was ready, kept on the volume under `release/good/`. The forge
@@ -0,0 +1,56 @@
#!/bin/bash
# The image's first process after tini (#12348): what src/release/launch.ts
# did by staying up as a second Bun runtime (~50 MiB), as a shell.
#
# The launcher still decides which tree runs and rolls back (launch.ts), but
# in hand-off mode it exits once the forge reads ready, leaving the forge
# named in release/serving.json. This script then waits on whatever serves:
# the pid in serving.json, which a swapped release rewrites (serving.ts).
# It passes SIGTERM/SIGINT to the launcher while it runs and to the forge after.
data="${FORGE_DATA:-data}"
here="$(cd "$(dirname "$0")" && pwd)"
tail_file="${TMPDIR:-/tmp}/forge-stderr.tail"
: > "$tail_file"
# A pid left by a previous container run is not ours to signal: start with none.
rm -f "$data/release/serving.json"
# stderr is copied to a file so the launcher can say why a release did not become ready,
# and so the forge keeps its stderr after the launcher has gone. This script is the
# only one that truncates the file; the launcher reads from where it began.
exec 2> >(tee -a "$tail_file" >&2)
export FORGE_LAUNCH_HANDOFF=1 FORGE_STDERR_TAIL="$tail_file"
stopping=0
lpid=""
serving_pid() {
  # whitespace- and order-independent; only the launcher and serving.ts write the file
  grep -o '"pid"[[:space:]]*:[[:space:]]*[0-9]*' "$data/release/serving.json" 2>/dev/null | grep -o '[0-9]*$' | head -n 1
}
on_stop() {
  stopping=1
  [ -n "$lpid" ] && kill -TERM "$lpid" 2>/dev/null
  pid="$(serving_pid)"
  [ -n "$pid" ] && kill -TERM "$pid" 2>/dev/null
}
trap on_stop TERM INT
bun "$here/launch.ts" &
lpid=$!
wait "$lpid"
code=$?
while [ "$stopping" = 1 ] && kill -0 "$lpid" 2>/dev/null; do wait "$lpid"; code=$?; done
lpid=""
if [ "$code" != 0 ]; then exit "$code"; fi
# Follow whatever serves until it is gone; trim the stderr copy so it cannot grow without end.
while :; do
  pid="$(serving_pid)"
  [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null || break
  if [ "$stopping" = 1 ]; then kill -TERM "$pid" 2>/dev/null; fi
  size="$(stat -c %s "$tail_file" 2>/dev/null || echo 0)"
  [ "$size" -gt 1048576 ] && : > "$tail_file"
  sleep 1 &
  wait $!
done
[ "$stopping" = 1 ] && exit 0
echo "boot: the forge stopped; the container goes with it" >&2
exit 1
@@ -31,11 +31,14 @@
 * has to work.
 */
import { cpSync, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
import {
  closeSync, cpSync, existsSync, fstatSync, mkdirSync, openSync, readdirSync, readFileSync,
  readSync, renameSync, rmSync, statSync, writeFileSync,
} from "node:fs";
import { dirname, join } from "node:path";
import type { Subprocess } from "bun";
import { hkdfSync } from "node:crypto";
import { clearServing, follow, signalServing } from "./serving.ts";
import { clearServing, follow, signalServing, writeServing } from "./serving.ts";
export const READY_GRACE_MS = 150_000;
const POLL_MS = 500;
@@ -48,6 +51,7 @@
const VERSION = /^[0-9A-Za-z][0-9A-Za-z.+_-]{0,63}$/;
const TAIL_BYTES = 8_192;
/** How long a release that exited has to finish writing its last words. */
const LAST_WORDS_MS = 1_000;
const SETTLE_MS = 100;
/** How long the last words in the record may be. */
const SAID_MAX = 200;
@@ -205,6 +209,58 @@
function hear(stream: ReadableStream<Uint8Array>): Heard {
  };
}
/**
 * What the release said on stderr, read from a file the entrypoint
 * (`boot.sh`) tees it to, so the release keeps its stderr after this process
 * has handed over and gone. Only what is written after this call counts; the
 * file is never truncated here (boot.sh alone trims it). The tee is
 * asynchronous, so `ended` waits for the file to stop growing.
 */
function hearFile(path: string): Heard {
  const sizeOf = (): number => {
    try {
      return statSync(path).size;
    } catch {
      return 0;
    }
  };
  let from = sizeOf();
  const settle = async (): Promise<void> => {
    let last = sizeOf();
    for (let waited = 0; waited < LAST_WORDS_MS; waited += SETTLE_MS) {
      await Bun.sleep(SETTLE_MS);
      const size = sizeOf();
      if (size === last) return;
      last = size;
    }
  };
  return {
    get ended() {
      return settle();
    },
    tail: () => {
      try {
        const fd = openSync(path, "r");
        try {
          const size = fstatSync(fd).size;
          const start = size < from ? 0 : from;
          const length = Math.min(size - start, TAIL_BYTES);
          const buf = Buffer.alloc(length);
          readSync(fd, buf, 0, length, size - length);
          return buf.toString("utf8");
        } finally {
          closeSync(fd);
        }
      } catch {
        return "";
      }
    },
    forget: () => {
      from = sizeOf();
    },
  };
}
/**
 * `kernel/probe.ts`'s `probeHeaders`, repeated: the launcher imports nothing
 * of the forge's, so a release can never break the thing that rolls it back.
@@ -259,6 +315,12 @@
export interface LaunchInput {
  readonly now?: () => number;
  /** Told of each process started, so a signal reaches the one running. */
  readonly started?: (child: Subprocess) => void;
  /**
   * A file the release's stderr is also written to by whoever started us
   * (`boot.sh`). When set the release inherits our stderr instead of a pipe
   * to us, so it outlives this process; its last words are read from the file.
   */
  readonly stderrFile?: string;
  /** True once the launcher was asked to stop: a release exiting then is no failure. */
  readonly stopping?: () => boolean;
}
@@ -273,7 +335,7 @@
interface Started {
  readonly heard: Heard;
}
function start(input: LaunchInput, tree: string, version: string, commit: string | null): Started {
function spawnTree(input: LaunchInput, tree: string, version: string, commit: string | null, stderr: "pipe" | "inherit") {
  const env: Record<string, string> = {};
  for (const [k, v] of Object.entries(input.env)) if (v !== undefined) env[k] = v;
  env["FORGE_VERSION"] = version;
@@ -285,11 +347,23 @@
function start(input: LaunchInput, tree: string, version: string, commit: string
    env,
    stdin: "ignore",
    stdout: "inherit",
    // Piped, not inherited, so a release that dies says why in the record; every byte still reaches ours.
    stderr: "pipe",
    stderr,
  });
  input.started?.(child);
  return { child, heard: hear(child.stderr) };
  return child;
}
function start(input: LaunchInput, tree: string, version: string, commit: string | null): Started {
  if (input.stderrFile !== undefined) {
    // The entrypoint already copies our stderr to the file: inherit it,
    // and read the last words from there.
    const heard = hearFile(input.stderrFile);
    return { child: spawnTree(input, tree, version, commit, "inherit"), heard };
  }
  // Piped, not inherited, so a release that dies says why in the record;
  // every byte still reaches ours.
  const child = spawnTree(input, tree, version, commit, "pipe");
  return { child, heard: hear(child.stderr as ReadableStream<Uint8Array>) };
}
/** Why a release did not become ready, and the last line it wrote on stderr. */
@@ -403,6 +477,7 @@
if (import.meta.main) {
    version: VERSION.test(version) ? version : "dev",
    commit: commit === "" ? null : commit,
    env,
    ...(env["FORGE_LAUNCH_HANDOFF"] === "1" && env["FORGE_STDERR_TAIL"] ? { stderrFile: env["FORGE_STDERR_TAIL"] } : {}),
    ...(Number.isInteger(graceS) && graceS > 0 ? { graceMs: graceS * 1000 } : {}),
    started: (child) => {
      current = child;
@@ -411,6 +486,14 @@
if (import.meta.main) {
  });
  console.error(`launch: ${launched.line}`);
  if (launched.ran === "none") process.exit(stopping ? 0 : 1);
  const handoff = env["FORGE_LAUNCH_HANDOFF"] === "1";
  if (handoff) {
    // The entrypoint (boot.sh) waits on what serves from here, so this second Bun runtime ends
    // once the forge is ready: the release is named in serving.json and left running.
    writeServing(dataHere, { pid: launched.child.pid, sha: commit === "" ? version || "dev" : commit });
    launched.child.unref();
    process.exit(0);
  }
  const code = await launched.child.exited;
  // A swapped release took over and the process we started has drained: this container is that
  // process's now, and goes when it does (the pod then restarts on the newest good tree).
@@ -194,7 +194,7 @@
test("a record that is not one reads as none", () => {
test("the image starts the launcher, and the launcher imports nothing a release could break", async () => {
  const repo = join(import.meta.dir, "..");
  expect(await Bun.file(join(repo, "Dockerfile")).text()).toContain('CMD ["bun", "src/release/launch.ts"]');
  expect(await Bun.file(join(repo, "Dockerfile")).text()).toContain('CMD ["bash", "src/release/boot.sh"]');
  // serving.ts (#8042) is the launcher's own sibling, node built-ins only, so the rule holds for both.
  for (const [file, sibling] of [["launch.ts", "./serving.ts"], ["serving.ts", undefined]] as const) {
    const source = await Bun.file(join(repo, "src/release", file)).text();
@@ -202,3 +202,107 @@
test("the image starts the launcher, and the launcher imports nothing a release
    expect(imports.every((p) => p.startsWith("node:") || p === "bun" || p === sibling)).toBe(true);
  }
});
test("a release that exits says why through the stderr file the entrypoint keeps", async () => {
  const data = join(root, "data-file-tail");
  const file = join(root, "stderr-file-tail");
  writeFileSync(file, "old words from before this release\n");
  keepGood(data, tree(SERVES), "0.7.479", null);
  // The entrypoint's tee is not here: the fixture appends to the file as the tee would,
  // a beat after it starts.
  const says = `await Bun.sleep(150); require("node:fs").appendFileSync(process.env.TEE_FILE, "first\\nFORGE_X: refused\\n"); process.exit(1);\n`; // real-timer: the fixture is a child that writes after start
  const port = await freePort();
  const launched = await launch({
    tree: tree(says), dataDir: data, port, version: "0.7.481", commit: null,
    env: { PATH: process.env.PATH, PORT: String(port), TEE_FILE: file }, graceMs: 5_000,
    stderrFile: file, started: (c) => running.push(c),
  });
  expect(launched.ran).toBe("rollback");
  // Only what was written after the start counts, and the launcher does not truncate the file.
  expect(rolledBack(data)).toMatchObject({ said: "FORGE_X: refused" });
  expect(await Bun.file(file).text()).toContain("old words");
});
async function bootTree(main: string, name: string) {
  const repo = join(import.meta.dir, "..");
  const dir = join(root, `${name}-tree`);
  mkdirSync(join(dir, "src", "release"), { recursive: true });
  for (const f of ["boot.sh", "launch.ts", "serving.ts"])
    writeFileSync(join(dir, "src", "release", f), await Bun.file(join(repo, "src/release", f)).text());
  writeFileSync(join(dir, "src", "main.ts"), main);
  writeFileSync(join(dir, "package.json"), "{}");
  const data = join(root, `${name}-data`);
  mkdirSync(data, { recursive: true });
  const port = await freePort();
  const boot = () => {
    const proc = Bun.spawn(["bash", join(dir, "src/release/boot.sh")], {
      cwd: dir,
      env: { PATH: process.env.PATH, HOME: process.env.HOME, TMPDIR: root, PORT: String(port), FORGE_DATA: data, FORGE_VERSION: "9.9.9" },
      stdout: "ignore", stderr: "ignore",
    });
    running.push(proc);
    return proc;
  };
  return { dir, data, port, boot };
}
async function servingPid(data: string): Promise<number> {
  for (let i = 0; i < 100; i++) {
    await Bun.sleep(200); // real-timer: polling a child process that has no event to wait on
    try {
      return (JSON.parse(await Bun.file(join(data, "release", "serving.json")).text()) as { pid: number }).pid;
    } catch {}
  }
  return 0;
}
const isAlive = (pid: number): boolean => {
  try {
    process.kill(pid, 0);
    return true;
  } catch {
    return false;
  }
};
test("boot.sh hands over: the launcher ends once the forge is ready, and a stop reaches the forge", async () => {
  const { dir, data, port, boot: start } = await bootTree(SERVES, "boot");
  const boot = start();
  const pid = await servingPid(data);
  expect(pid).toBeGreaterThan(1);
  expect(boot.exitCode).toBeNull();
  const health = (await (await fetch(`http://127.0.0.1:${port}/api/v1/health`)).json()) as { version: string };
  expect(health.version).toBe("9.9.9");
  // The launcher is gone: nothing of ours but boot.sh and the forge itself runs launch.ts.
  const procs = (await Bun.$`ps -eo args`.text()).split("\n").filter((l) => l.includes(join(dir, "src/release/launch.ts")));
  expect(procs).toEqual([]);
  boot.kill("SIGTERM");
  expect(await boot.exited).toBe(0);
  for (let i = 0; i < 50 && isAlive(pid); i++) {
    await Bun.sleep(100); // real-timer: polls a child process
  }
  expect(isAlive(pid)).toBe(false);
}, 60_000);
test("a forge that dies after the hand-off takes boot.sh down with a failure, so the container restarts", async () => {
  const { data, boot: start } = await bootTree(SERVES, "boot-dies");
  const boot = start();
  const pid = await servingPid(data);
  expect(pid).toBeGreaterThan(1);
  process.kill(pid, "SIGKILL");
  expect(await boot.exited).toBe(1);
}, 60_000);
test("a stop before the launcher has named the forge neither hangs nor signals a pid left by an earlier run", async () => {
  const { data, boot: start } = await bootTree(HANGS, "boot-early");
  const bystander = Bun.spawn(["sleep", "30"]);
  running.push(bystander);
  mkdirSync(join(data, "release"), { recursive: true });
  writeFileSync(join(data, "release", "serving.json"), JSON.stringify({ pid: bystander.pid, sha: "stale" }));
  const boot = start();
  await Bun.sleep(300); // real-timer: let the shell start the launcher before the stop
  boot.kill("SIGTERM");
  expect(await boot.exited).toBe(0);
  expect(bystander.exitCode).toBeNull();
  expect(isAlive(bystander.pid)).toBe(true);
}, 60_000);
		@@ -50,7 +50,8 @@ RUN bun src/release/boot.ts
50	50	# tini reaps what an agent run leaves behind: a claude run spawns git, bun,
51	51	# the app under test, and their children.
52	52	ENTRYPOINT ["tini", "--"]
53		# The launcher runs src/main.ts, and when this version does not answer health
	53	# boot.sh runs the launcher, which starts src/main.ts and then exits, leaving
	54	# the forge as the only Bun runtime; when this version does not answer health
54	55	# within FORGE_READY_GRACE_S (150), runs the last version that did, kept on
55	56	# the volume (src/release/launch.ts, #2652).
56		CMD ["bun", "src/release/launch.ts"]
	57	CMD ["bash", "src/release/boot.sh"]
		@@ -15,7 +15,7 @@ One path, in the forge: **landing → on cadence the keeper takes main's tree
15	15
16	16	## A pod restart
17	17
18		`release/launch.ts` is the container's first process. `keepGood` records a swapped tree with `base`, the image version it was swapped over, in `release/good.json`. When the same image restarts, the launcher starts that tree first, and a ready older image never replaces it as the last good. A different image version is a deliberate roll (a new base): it runs, and is kept over the swapped tree. A swapped tree that does not become ready in the grace gives way to the image and the rollback card says so. The swapped process is not the launcher's child, so it writes `release/serving.json` (`release/serving.ts`) and the launcher stays up while that pid lives and signals it on SIGTERM; when it dies the container exits and restarts on the newest good tree.
	18	`release/boot.sh` is the container's first process after tini: it runs `release/launch.ts` with `FORGE_LAUNCH_HANDOFF=1`, which exits once the forge is ready and names it in `release/serving.json`, and boot.sh follows that pid (so no second Bun runtime stays resident). `keepGood` records a swapped tree with `base`, the image version it was swapped over, in `release/good.json`. When the same image restarts, the launcher starts that tree first, and a ready older image never replaces it as the last good. A different image version is a deliberate roll (a new base): it runs, and is kept over the swapped tree. A swapped tree that does not become ready in the grace gives way to the image and the rollback card says so. A swapped process is not the launcher's child, so it rewrites `release/serving.json` (`release/serving.ts`) and boot.sh keeps following the pid there, signalling it on SIGTERM; when it dies the container exits and restarts on the newest good tree.
19	19
20	20	## Two writers, apps and runs
21	21
		@@ -243,7 +243,10 @@ in `src/release/`:
243	243	`boot.ts --from /data` beside the running forge, on a copy of its database
244	244	and top-level files. A refusal fails the upgrade while the old pod serves;
245	245	the failed Job is kept for its log (`kubectl -n "$NS" logs job/$REL-boot-proof`).
246		- After the roll. The image starts `src/release/launch.ts`, which runs
	246	- After the roll. The image starts `src/release/boot.sh`, which runs
	247	`src/release/launch.ts` only until the forge is ready (then it exits, so the
	248	container holds one Bun runtime, not two, #12348) and
	249	waits on the pid in `release/serving.json`. The launcher runs
247	250	`src/main.ts` and, when that version exits or is not ready within
248	251	`FORGE_READY_GRACE_S` (150 s, inside the startup probe), starts the last
249	252	version that was ready, kept on the volume under `release/good/`. The forge
		@@ -0,0 +1,56 @@
	1	#!/bin/bash
	2	# The image's first process after tini (#12348): what src/release/launch.ts
	3	# did by staying up as a second Bun runtime (~50 MiB), as a shell.
	4	#
	5	# The launcher still decides which tree runs and rolls back (launch.ts), but
	6	# in hand-off mode it exits once the forge reads ready, leaving the forge
	7	# named in release/serving.json. This script then waits on whatever serves:
	8	# the pid in serving.json, which a swapped release rewrites (serving.ts).
	9	# It passes SIGTERM/SIGINT to the launcher while it runs and to the forge after.
	10	data="${FORGE_DATA:-data}"
	11	here="$(cd "$(dirname "$0")" && pwd)"
	12	tail_file="${TMPDIR:-/tmp}/forge-stderr.tail"
	13	: > "$tail_file"
	14	# A pid left by a previous container run is not ours to signal: start with none.
	15	rm -f "$data/release/serving.json"
	16	# stderr is copied to a file so the launcher can say why a release did not become ready,
	17	# and so the forge keeps its stderr after the launcher has gone. This script is the
	18	# only one that truncates the file; the launcher reads from where it began.
	19	exec 2> >(tee -a "$tail_file" >&2)
	20	export FORGE_LAUNCH_HANDOFF=1 FORGE_STDERR_TAIL="$tail_file"
	21
	22	stopping=0
	23	lpid=""
	24	serving_pid() {
	25	# whitespace- and order-independent; only the launcher and serving.ts write the file
	26	grep -o '"pid"[[:space:]]:[[:space:]][0-9]' "$data/release/serving.json" 2>/dev/null \| grep -o '[0-9]$' \| head -n 1
	27	}
	28	on_stop() {
	29	stopping=1
	30	[ -n "$lpid" ] && kill -TERM "$lpid" 2>/dev/null
	31	pid="$(serving_pid)"
	32	[ -n "$pid" ] && kill -TERM "$pid" 2>/dev/null
	33	}
	34	trap on_stop TERM INT
	35
	36	bun "$here/launch.ts" &
	37	lpid=$!
	38	wait "$lpid"
	39	code=$?
	40	while [ "$stopping" = 1 ] && kill -0 "$lpid" 2>/dev/null; do wait "$lpid"; code=$?; done
	41	lpid=""
	42	if [ "$code" != 0 ]; then exit "$code"; fi
	43
	44	# Follow whatever serves until it is gone; trim the stderr copy so it cannot grow without end.
	45	while :; do
	46	pid="$(serving_pid)"
	47	[ -n "$pid" ] && kill -0 "$pid" 2>/dev/null \|\| break
	48	if [ "$stopping" = 1 ]; then kill -TERM "$pid" 2>/dev/null; fi
	49	size="$(stat -c %s "$tail_file" 2>/dev/null \|\| echo 0)"
	50	[ "$size" -gt 1048576 ] && : > "$tail_file"
	51	sleep 1 &
	52	wait $!
	53	done
	54	[ "$stopping" = 1 ] && exit 0
	55	echo "boot: the forge stopped; the container goes with it" >&2
	56	exit 1
		@@ -31,11 +31,14 @@
31	31	* has to work.
32	32	*/
33	33
34		import { cpSync, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
	34	import {
	35	closeSync, cpSync, existsSync, fstatSync, mkdirSync, openSync, readdirSync, readFileSync,
	36	readSync, renameSync, rmSync, statSync, writeFileSync,
	37	} from "node:fs";
35	38	import { dirname, join } from "node:path";
36	39	import type { Subprocess } from "bun";
37	40	import { hkdfSync } from "node:crypto";
38		import { clearServing, follow, signalServing } from "./serving.ts";
	41	import { clearServing, follow, signalServing, writeServing } from "./serving.ts";
39	42
40	43	export const READY_GRACE_MS = 150_000;
41	44	const POLL_MS = 500;
		@@ -48,6 +51,7 @@ const VERSION = /^[0-9A-Za-z][0-9A-Za-z.+_-]{0,63}$/;
48	51	const TAIL_BYTES = 8_192;
49	52	/** How long a release that exited has to finish writing its last words. */
50	53	const LAST_WORDS_MS = 1_000;
	54	const SETTLE_MS = 100;
51	55	/** How long the last words in the record may be. */
52	56	const SAID_MAX = 200;
53	57
		@@ -205,6 +209,58 @@ function hear(stream: ReadableStream<Uint8Array>): Heard {
205	209	};
206	210	}
207	211
	212	/**
	213	* What the release said on stderr, read from a file the entrypoint
	214	* (`boot.sh`) tees it to, so the release keeps its stderr after this process
	215	* has handed over and gone. Only what is written after this call counts; the
	216	* file is never truncated here (boot.sh alone trims it). The tee is
	217	* asynchronous, so `ended` waits for the file to stop growing.
	218	*/
	219	function hearFile(path: string): Heard {
	220	const sizeOf = (): number => {
	221	try {
	222	return statSync(path).size;
	223	} catch {
	224	return 0;
	225	}
	226	};
	227	let from = sizeOf();
	228	const settle = async (): Promise<void> => {
	229	let last = sizeOf();
	230	for (let waited = 0; waited < LAST_WORDS_MS; waited += SETTLE_MS) {
	231	await Bun.sleep(SETTLE_MS);
	232	const size = sizeOf();
	233	if (size === last) return;
	234	last = size;
	235	}
	236	};
	237	return {
	238	get ended() {
	239	return settle();
	240	},
	241	tail: () => {
	242	try {
	243	const fd = openSync(path, "r");
	244	try {
	245	const size = fstatSync(fd).size;
	246	const start = size < from ? 0 : from;
	247	const length = Math.min(size - start, TAIL_BYTES);
	248	const buf = Buffer.alloc(length);
	249	readSync(fd, buf, 0, length, size - length);
	250	return buf.toString("utf8");
	251	} finally {
	252	closeSync(fd);
	253	}
	254	} catch {
	255	return "";
	256	}
	257	},
	258	forget: () => {
	259	from = sizeOf();
	260	},
	261	};
	262	}
	263
208	264	/**
209	265	* `kernel/probe.ts`'s `probeHeaders`, repeated: the launcher imports nothing
210	266	* of the forge's, so a release can never break the thing that rolls it back.
		@@ -259,6 +315,12 @@ export interface LaunchInput {
259	315	readonly now?: () => number;
260	316	/** Told of each process started, so a signal reaches the one running. */
261	317	readonly started?: (child: Subprocess) => void;
	318	/**
	319	* A file the release's stderr is also written to by whoever started us
	320	* (`boot.sh`). When set the release inherits our stderr instead of a pipe
	321	* to us, so it outlives this process; its last words are read from the file.
	322	*/
	323	readonly stderrFile?: string;
262	324	/** True once the launcher was asked to stop: a release exiting then is no failure. */
263	325	readonly stopping?: () => boolean;
264	326	}
		@@ -273,7 +335,7 @@ interface Started {
273	335	readonly heard: Heard;
274	336	}
275	337
276		function start(input: LaunchInput, tree: string, version: string, commit: string \| null): Started {
	338	function spawnTree(input: LaunchInput, tree: string, version: string, commit: string \| null, stderr: "pipe" \| "inherit") {
277	339	const env: Record<string, string> = {};
278	340	for (const [k, v] of Object.entries(input.env)) if (v !== undefined) env[k] = v;
279	341	env["FORGE_VERSION"] = version;
		@@ -285,11 +347,23 @@ function start(input: LaunchInput, tree: string, version: string, commit: string
285	347	env,
286	348	stdin: "ignore",
287	349	stdout: "inherit",
288		// Piped, not inherited, so a release that dies says why in the record; every byte still reaches ours.
289		stderr: "pipe",
	350	stderr,
290	351	});
291	352	input.started?.(child);
292		return { child, heard: hear(child.stderr) };
	353	return child;
	354	}
	355
	356	function start(input: LaunchInput, tree: string, version: string, commit: string \| null): Started {
	357	if (input.stderrFile !== undefined) {
	358	// The entrypoint already copies our stderr to the file: inherit it,
	359	// and read the last words from there.
	360	const heard = hearFile(input.stderrFile);
	361	return { child: spawnTree(input, tree, version, commit, "inherit"), heard };
	362	}
	363	// Piped, not inherited, so a release that dies says why in the record;
	364	// every byte still reaches ours.
	365	const child = spawnTree(input, tree, version, commit, "pipe");
	366	return { child, heard: hear(child.stderr as ReadableStream<Uint8Array>) };
293	367	}
294	368
295	369	/** Why a release did not become ready, and the last line it wrote on stderr. */
		@@ -403,6 +477,7 @@ if (import.meta.main) {
403	477	version: VERSION.test(version) ? version : "dev",
404	478	commit: commit === "" ? null : commit,
405	479	env,
	480	...(env["FORGE_LAUNCH_HANDOFF"] === "1" && env["FORGE_STDERR_TAIL"] ? { stderrFile: env["FORGE_STDERR_TAIL"] } : {}),
406	481	...(Number.isInteger(graceS) && graceS > 0 ? { graceMs: graceS * 1000 } : {}),
407	482	started: (child) => {
408	483	current = child;
		@@ -411,6 +486,14 @@ if (import.meta.main) {
411	486	});
412	487	console.error(`launch: ${launched.line}`);
413	488	if (launched.ran === "none") process.exit(stopping ? 0 : 1);
	489	const handoff = env["FORGE_LAUNCH_HANDOFF"] === "1";
	490	if (handoff) {
	491	// The entrypoint (boot.sh) waits on what serves from here, so this second Bun runtime ends
	492	// once the forge is ready: the release is named in serving.json and left running.
	493	writeServing(dataHere, { pid: launched.child.pid, sha: commit === "" ? version \|\| "dev" : commit });
	494	launched.child.unref();
	495	process.exit(0);
	496	}
414	497	const code = await launched.child.exited;
415	498	// A swapped release took over and the process we started has drained: this container is that
416	499	// process's now, and goes when it does (the pod then restarts on the newest good tree).
		@@ -194,7 +194,7 @@ test("a record that is not one reads as none", () => {
194	194
195	195	test("the image starts the launcher, and the launcher imports nothing a release could break", async () => {
196	196	const repo = join(import.meta.dir, "..");
197		expect(await Bun.file(join(repo, "Dockerfile")).text()).toContain('CMD ["bun", "src/release/launch.ts"]');
	197	expect(await Bun.file(join(repo, "Dockerfile")).text()).toContain('CMD ["bash", "src/release/boot.sh"]');
198	198	// serving.ts (#8042) is the launcher's own sibling, node built-ins only, so the rule holds for both.
199	199	for (const [file, sibling] of [["launch.ts", "./serving.ts"], ["serving.ts", undefined]] as const) {
200	200	const source = await Bun.file(join(repo, "src/release", file)).text();
		@@ -202,3 +202,107 @@ test("the image starts the launcher, and the launcher imports nothing a release
202	202	expect(imports.every((p) => p.startsWith("node:") \|\| p === "bun" \|\| p === sibling)).toBe(true);
203	203	}
204	204	});
	205
	206	test("a release that exits says why through the stderr file the entrypoint keeps", async () => {
	207	const data = join(root, "data-file-tail");
	208	const file = join(root, "stderr-file-tail");
	209	writeFileSync(file, "old words from before this release\n");
	210	keepGood(data, tree(SERVES), "0.7.479", null);
	211	// The entrypoint's tee is not here: the fixture appends to the file as the tee would,
	212	// a beat after it starts.
	213	const says = `await Bun.sleep(150); require("node:fs").appendFileSync(process.env.TEE_FILE, "first\\nFORGE_X: refused\\n"); process.exit(1);\n`; // real-timer: the fixture is a child that writes after start
	214	const port = await freePort();
	215	const launched = await launch({
	216	tree: tree(says), dataDir: data, port, version: "0.7.481", commit: null,
	217	env: { PATH: process.env.PATH, PORT: String(port), TEE_FILE: file }, graceMs: 5_000,
	218	stderrFile: file, started: (c) => running.push(c),
	219	});
	220	expect(launched.ran).toBe("rollback");
	221	// Only what was written after the start counts, and the launcher does not truncate the file.
	222	expect(rolledBack(data)).toMatchObject({ said: "FORGE_X: refused" });
	223	expect(await Bun.file(file).text()).toContain("old words");
	224	});
	225
	226	async function bootTree(main: string, name: string) {
	227	const repo = join(import.meta.dir, "..");
	228	const dir = join(root, `${name}-tree`);
	229	mkdirSync(join(dir, "src", "release"), { recursive: true });
	230	for (const f of ["boot.sh", "launch.ts", "serving.ts"])
	231	writeFileSync(join(dir, "src", "release", f), await Bun.file(join(repo, "src/release", f)).text());
	232	writeFileSync(join(dir, "src", "main.ts"), main);
	233	writeFileSync(join(dir, "package.json"), "{}");
	234	const data = join(root, `${name}-data`);
	235	mkdirSync(data, { recursive: true });
	236	const port = await freePort();
	237	const boot = () => {
	238	const proc = Bun.spawn(["bash", join(dir, "src/release/boot.sh")], {
	239	cwd: dir,
	240	env: { PATH: process.env.PATH, HOME: process.env.HOME, TMPDIR: root, PORT: String(port), FORGE_DATA: data, FORGE_VERSION: "9.9.9" },
	241	stdout: "ignore", stderr: "ignore",
	242	});
	243	running.push(proc);
	244	return proc;
	245	};
	246	return { dir, data, port, boot };
	247	}
	248
	249	async function servingPid(data: string): Promise<number> {
	250	for (let i = 0; i < 100; i++) {
	251	await Bun.sleep(200); // real-timer: polling a child process that has no event to wait on
	252	try {
	253	return (JSON.parse(await Bun.file(join(data, "release", "serving.json")).text()) as { pid: number }).pid;
	254	} catch {}
	255	}
	256	return 0;
	257	}
	258
	259	const isAlive = (pid: number): boolean => {
	260	try {
	261	process.kill(pid, 0);
	262	return true;
	263	} catch {
	264	return false;
	265	}
	266	};
	267
	268	test("boot.sh hands over: the launcher ends once the forge is ready, and a stop reaches the forge", async () => {
	269	const { dir, data, port, boot: start } = await bootTree(SERVES, "boot");
	270	const boot = start();
	271	const pid = await servingPid(data);
	272	expect(pid).toBeGreaterThan(1);
	273	expect(boot.exitCode).toBeNull();
	274	const health = (await (await fetch(`http://127.0.0.1:${port}/api/v1/health`)).json()) as { version: string };
	275	expect(health.version).toBe("9.9.9");
	276	// The launcher is gone: nothing of ours but boot.sh and the forge itself runs launch.ts.
	277	const procs = (await Bun.$`ps -eo args`.text()).split("\n").filter((l) => l.includes(join(dir, "src/release/launch.ts")));
	278	expect(procs).toEqual([]);
	279	boot.kill("SIGTERM");
	280	expect(await boot.exited).toBe(0);
	281	for (let i = 0; i < 50 && isAlive(pid); i++) {
	282	await Bun.sleep(100); // real-timer: polls a child process
	283	}
	284	expect(isAlive(pid)).toBe(false);
	285	}, 60_000);
	286
	287	test("a forge that dies after the hand-off takes boot.sh down with a failure, so the container restarts", async () => {
	288	const { data, boot: start } = await bootTree(SERVES, "boot-dies");
	289	const boot = start();
	290	const pid = await servingPid(data);
	291	expect(pid).toBeGreaterThan(1);
	292	process.kill(pid, "SIGKILL");
	293	expect(await boot.exited).toBe(1);
	294	}, 60_000);
	295
	296	test("a stop before the launcher has named the forge neither hangs nor signals a pid left by an earlier run", async () => {
	297	const { data, boot: start } = await bootTree(HANGS, "boot-early");
	298	const bystander = Bun.spawn(["sleep", "30"]);
	299	running.push(bystander);
	300	mkdirSync(join(data, "release"), { recursive: true });
	301	writeFileSync(join(data, "release", "serving.json"), JSON.stringify({ pid: bystander.pid, sha: "stale" }));
	302	const boot = start();
	303	await Bun.sleep(300); // real-timer: let the shell start the launcher before the stop
	304	boot.kill("SIGTERM");
	305	expect(await boot.exited).toBe(0);
	306	expect(bystander.exitCode).toBeNull();
	307	expect(isAlive(bystander.pid)).toBe(true);
	308	}, 60_000);