b69090e
@@ -43,6 +43,7 @@
import { RECOVERED, drainIncident, nextPage, observe, recoveredLine, type Watch
import { firedAlarms, type FiredAlarm } from "../flow/alarms.ts";
import { reachAlarms } from "../spawn/reach.ts";
import { loadClog } from "../flow/clog_tick.ts";
import { redactSecrets } from "../kernel/log.ts";
import { forgeCeiling } from "../flow/ceiling.ts";
import { askLinesFor } from "../web/studio/asks.ts";
@@ -456,7 +457,8 @@
export function startNotifier(app: App, options: NotifyOptions): Notifier {
      const url = notifyUrlOf(app.db, id);
      if (user === undefined || url === undefined || !user.isAdmin || user.kind !== "user") continue;
      reachable = true;
      if (await push(id, url, need.sentence)) delivered = true;
      // The sentence quotes the release's stderr: nothing credential-shaped leaves in a push.
      if (await push(id, url, redactSecrets(need.sentence))) delivered = true;
    }
    if (delivered || !reachable) markRollbackPaged(dataDir);
  }
@@ -13,7 +13,7 @@
import { statfsSync, statSync } from "node:fs";
import { dirname } from "node:path";
import { freemem, totalmem } from "node:os";
import { refusedInternal, type RefusedLedger } from "./internal_refused.ts";
import { recentErrors, type ErrorLedger, type ErrorLine } from "./log.ts";
import { recentErrors, redactSecrets, type ErrorLedger, type ErrorLine } from "./log.ts";
import {
  budgetFailures,
  leaseLapsed,
@@ -519,9 +519,10 @@
export function vitals(input: VitalsInput): Vitals {
  const unproven = proofNeed(input.dataDir, runningVersion().version);
  if (unproven !== undefined) need("release", unproven.sentence, unproven.since, "/board");
  // A roll the image's launcher undid is a card until a later version reads ready.
  // A roll the image's launcher undid is a card until a later version reads ready. Its sentence
  // carries the release's own last words (stderr), so anything credential-shaped is masked.
  const undone = rollbackNeed(input.dataDir);
  if (undone !== undefined) need("release:rolled-back", undone.sentence, undone.since, "/board");
  if (undone !== undefined) need("release:rolled-back", redactSecrets(undone.sentence), undone.since, "/board");
  // Production serving a version older than a release main has had over an hour is a card until they match.
  const lagging = behindStanding(now);
@@ -7,8 +7,9 @@
 * (`release/good/<version>/`: its `src/` and the two files beside it) and
 * named in `release/good.json`; the one before it is removed. A version
 * that exits first, or is still not ready when the grace runs out, is
 * stopped, `release/rolled-back.json` says what happened, and the kept tree
 * runs under its own stamp. The forge that comes up reads that record: a
 * stopped, `release/rolled-back.json` says what happened and what the
 * release last said on stderr (#10249: 0.7.481 left only "it exited with 1"),
 * and the kept tree runs under its own stamp. The forge that comes up reads that record: a
 * card while it stands (`kernel/vitals.ts`) and a page to the admins once
 * (`crew/notify.ts`). The next version that reads ready clears it.
 *
@@ -43,6 +44,12 @@
const STOP_MS = 10_000;
/** What a kept tree is: what the Dockerfile copies into the image. */
const TREE_PARTS = ["src", "package.json", "tsconfig.json"];
const VERSION = /^[0-9A-Za-z][0-9A-Za-z.+_-]{0,63}$/;
/** How much of a release's stderr is kept, to say why it did not become ready. */
const TAIL_BYTES = 8_192;
/** How long a release that exited has to finish writing its last words. */
const LAST_WORDS_MS = 1_000;
/** How long the last words in the record may be. */
const SAID_MAX = 200;
export interface Good {
  readonly version: string;
@@ -59,6 +66,8 @@
export interface RolledBack {
  readonly to: string;
  readonly at: string;
  readonly why: string;
  /** The release's last error line on stderr, when it wrote one: what a person reads to fix it. */
  readonly said: string | null;
  /** Whether the admins were told; set once by the forge that told them. */
  readonly paged: boolean;
}
@@ -130,10 +139,10 @@
export function keepGood(
export function rolledBack(dataDir: string): RolledBack | undefined {
  const r = readJson(rolledFile(dataDir));
  if (r === undefined) return undefined;
  const { from, to, at, why, paged } = r;
  const { from, to, at, why, said, paged } = r;
  if (typeof from !== "string" || typeof to !== "string" || typeof at !== "string" || typeof why !== "string")
    return undefined;
  return { from, to, at, why, paged: paged === true };
  return { from, to, at, why, said: typeof said === "string" && said !== "" ? said : null, paged: paged === true };
}
/** Record that the admins were told, so no later boot tells them again. */
@@ -146,7 +155,54 @@
export function markRollbackPaged(dataDir: string): void {
export function rollbackNeed(dataDir: string): { sentence: string; since: string } | undefined {
  const r = rolledBack(dataDir);
  if (r === undefined) return undefined;
  return { sentence: `Release ${r.from} did not become ready (${r.why}); rolled back to ${r.to}.`, since: r.at };
  const said = r.said === null ? "" : ` It said: ${r.said}`;
  return { sentence: `Release ${r.from} did not become ready (${r.why}); rolled back to ${r.to}.${said}`, since: r.at };
}
/**
 * The line of a release's stderr that says why it stopped: the last error
 * line (`error: …`, `SQLiteError: …`) when there is one, else the last line
 * that is not a stack frame, a source excerpt or Bun's version banner.
 */
export function lastWords(stderr: string): string | null {
  const lines = stderr
    .replace(/\x1b\[[0-9;]*[A-Za-z]/g, "")
    .split("\n")
    .map((line) => line.trim())
    .filter((line) => line !== "" && !/^at\s/.test(line) && !/^Bun v\d/.test(line) && !/^\d+\s*\|/.test(line) && !/^\^+$/.test(line));
  const line = lines.findLast((l) => /^(?:\w*Error|error)(?::|\s|$)/.test(l)) ?? lines.at(-1);
  if (line === undefined) return null;
  return line.length > SAID_MAX ? `${line.slice(0, SAID_MAX - 1)}…` : line;
}
/** A child's stderr, passed through to ours, with its tail kept until `forget`. */
interface Heard {
  readonly ended: Promise<void>;
  tail(): string;
  forget(): void;
}
function hear(stream: ReadableStream<Uint8Array>): Heard {
  let kept: Buffer | undefined = Buffer.alloc(0);
  const ended = (async () => {
    try {
      for await (const chunk of stream) {
        process.stderr.write(chunk);
        if (kept === undefined) continue;
        kept = Buffer.concat([kept, chunk]);
        if (kept.length > TAIL_BYTES) kept = kept.subarray(kept.length - TAIL_BYTES);
      }
    } catch {
      // the pipe closing under us is the child going: what was heard stands
    }
  })();
  return {
    ended,
    tail: () => kept?.toString("utf8") ?? "",
    forget: () => {
      kept = undefined;
    },
  };
}
/**
@@ -211,7 +267,13 @@
export type Launched =
  | { readonly ran: "release" | "rollback"; readonly child: Subprocess; readonly line: string }
  | { readonly ran: "none"; readonly line: string };
function start(input: LaunchInput, tree: string, version: string, commit: string | null): Subprocess {
/** A release started, and what it says on stderr. */
interface Started {
  readonly child: Subprocess;
  readonly heard: Heard;
}
function start(input: LaunchInput, tree: string, version: string, commit: string | null): Started {
  const env: Record<string, string> = {};
  for (const [k, v] of Object.entries(input.env)) if (v !== undefined) env[k] = v;
  env["FORGE_VERSION"] = version;
@@ -223,22 +285,35 @@
function start(input: LaunchInput, tree: string, version: string, commit: string
    env,
    stdin: "ignore",
    stdout: "inherit",
    stderr: "inherit",
    // Piped, not inherited, so a release that dies says why in the record; every byte still reaches ours.
    stderr: "pipe",
  });
  input.started?.(child);
  return child;
  return { child, heard: hear(child.stderr) };
}
/** Wait for `child` to answer health as `version`: undefined once it does, else why it did not. */
async function becomesReady(input: LaunchInput, child: Subprocess, version: string): Promise<string | undefined> {
/** Why a release did not become ready, and the last line it wrote on stderr. */
interface NotReady {
  readonly why: string;
  readonly said: string | null;
}
/** Wait for the release to answer health as `version`: undefined once it does, else why it did not. */
async function becomesReady(input: LaunchInput, { child, heard }: Started, version: string): Promise<NotReady | undefined> {
  const now = input.now ?? Date.now;
  const grace = input.graceMs ?? READY_GRACE_MS;
  const began = now();
  for (;;) {
    if (input.stopping?.()) return "asked to stop before it was ready";
    if (gone(child)) return `it exited with ${child.exitCode ?? child.signalCode} before it was ready`;
    if ((await versionAt(input.port, probeHeaders(input.dataDir))) === version) return undefined;
    if (now() - began >= grace) return `not ready within ${Math.round(grace / 1000)}s`;
    if (input.stopping?.()) return { why: "asked to stop before it was ready", said: null };
    if (gone(child)) {
      await Promise.race([heard.ended, Bun.sleep(LAST_WORDS_MS)]);
      return { why: `it exited with ${child.exitCode ?? child.signalCode} before it was ready`, said: lastWords(heard.tail()) };
    }
    if ((await versionAt(input.port, probeHeaders(input.dataDir))) === version) {
      heard.forget();
      return undefined;
    }
    if (now() - began >= grace) return { why: `not ready within ${Math.round(grace / 1000)}s`, said: lastWords(heard.tail()) };
    await Bun.sleep(POLL_MS);
  }
}
@@ -252,19 +327,20 @@
async function becomesReady(input: LaunchInput, child: Subprocess, version: stri
export async function launch(input: LaunchInput): Promise<Launched> {
  const now = input.now ?? Date.now;
  const swapped = lastGood(input.dataDir);
  let passed: { version: string; why: string } | undefined;
  let passed: ({ version: string } & NotReady) | undefined;
  if (swapped !== undefined && swapped.base === input.version && swapped.version !== input.version) {
    const first = start(input, goodTree(input.dataDir, swapped.version), swapped.version, swapped.commit);
    const failed = await becomesReady(input, first, swapped.version);
    if (failed === undefined)
      return { ran: "release", child: first, line: `${swapped.version} is ready: the swapped release kept over image ${input.version}` };
    await stop(first);
      return { ran: "release", child: first.child, line: `${swapped.version} is ready: the swapped release kept over image ${input.version}` };
    await stop(first.child);
    if (input.stopping?.()) return { ran: "none", line: "asked to stop before it was ready" };
    passed = { version: swapped.version, why: failed };
    passed = { version: swapped.version, ...failed };
  }
  const release = start(input, input.tree, input.version, input.commit);
  const why = await becomesReady(input, release, input.version);
  if (why === undefined) {
  const started = start(input, input.tree, input.version, input.commit);
  const release = started.child;
  const failed = await becomesReady(input, started, input.version);
  if (failed === undefined) {
    try {
      keepGood(input.dataDir, input.tree, input.version, input.commit, now());
    } catch (error) {
@@ -278,12 +354,14 @@
export async function launch(input: LaunchInput): Promise<Launched> {
        to: input.version,
        at: new Date(now()).toISOString(),
        why: passed.why,
        said: passed.said,
        paged: false,
      } satisfies RolledBack);
      return { ran: "rollback", child: release, line: `${passed.version}: ${passed.why}; the image ${input.version} runs instead` };
    }
    return { ran: "release", child: release, line: `${input.version} is ready and kept as the last good release` };
  }
  const { why, said } = failed;
  await stop(release);
  if (input.stopping?.()) return { ran: "none", line: "asked to stop before it was ready" };
  const good = lastGood(input.dataDir);
@@ -294,9 +372,11 @@
export async function launch(input: LaunchInput): Promise<Launched> {
    to: good.version,
    at: new Date(now()).toISOString(),
    why,
    said,
    paged: false,
  } satisfies RolledBack);
  const child = start(input, goodTree(input.dataDir, good.version), good.version, good.commit);
  const { child, heard } = start(input, goodTree(input.dataDir, good.version), good.version, good.commit);
  heard.forget();
  return { ran: "rollback", child, line: `${input.version}: ${why}; rolled back to ${good.version}` };
}
@@ -42,11 +42,11 @@
afterEach(async () => {
  await rm(dataDir, { recursive: true, force: true });
});
const record = () => {
const record = (said?: string) => {
  mkdirSync(join(dataDir, "release"), { recursive: true });
  writeFileSync(
    join(dataDir, "release", "rolled-back.json"),
    JSON.stringify({ from: "0.7.166", to: "0.7.162", at: "2026-10-19T05:17:00.000Z", why: "it exited with 1 before it was ready", paged: false }),
    JSON.stringify({ from: "0.7.166", to: "0.7.162", at: "2026-10-19T05:17:00.000Z", why: "it exited with 1 before it was ready", said, paged: false }),
  );
};
@@ -96,3 +96,11 @@
test("no record, no page", async () => {
  await boot();
  expect(posts).toEqual([]);
});
test("the release's last words reach the card and the page, with anything credential-shaped masked", async () => {
  record("error: GET https://x.test failed with Bearer abcdefghijklmnopqrstuvwxyz0123");
  const said = `${SENTENCE} It said: error: GET https://x.test failed with [redacted]`;
  expect(vitals({ db: app.db, signals: app.signals, dataDir }).degraded.find((d) => d.cause === "release:rolled-back")?.sentence).toBe(said);
  await boot();
  expect(posts).toEqual([`${said}\nhttps://forge.example.test/`]);
});
@@ -14,6 +14,7 @@
import type { Subprocess } from "bun";
import {
  keepGood,
  lastGood,
  lastWords,
  launch,
  markRollbackPaged,
  READY_GRACE_MS,
@@ -53,6 +54,8 @@
Bun.serve({ hostname: "127.0.0.1", port: Number(process.env.PORT), fetch(req) {
`;
const THROWS = `throw new Error("(test fixture) a release whose main throws at import, as 0.7.166's did");\n`;
const HANGS = `setInterval(() => {}, 1_000);\n`;
const REFUSES = `console.error("FORGE_DRAIN_LINGER_S: whole seconds from 0 to 60, not \\"x\\""); process.exit(1);\n`;
const SAYS_THEN_HANGS = `console.error("waiting on the database"); setInterval(() => {}, 1_000);\n`;
async function freePort(): Promise<number> {
  const probe = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch: () => new Response("") });
@@ -104,7 +107,8 @@
test("a version whose main throws at import is undone at once: the last good one
  expect(await healthAt(launched.port)).toMatchObject({ version: "0.7.162", commit: "a".repeat(40) });
  expect(rolledBack(data)).toMatchObject({ from: "0.7.166", to: "0.7.162", paged: false });
  expect(rollbackNeed(data)?.sentence).toBe(
    "Release 0.7.166 did not become ready (it exited with 1 before it was ready); rolled back to 0.7.162.",
    "Release 0.7.166 did not become ready (it exited with 1 before it was ready); rolled back to 0.7.162." +
      " It said: error: (test fixture) a release whose main throws at import, as 0.7.166's did",
  );
  expect(lastGood(data)?.version).toBe("0.7.162");
});
@@ -119,6 +123,43 @@
test("a version that never becomes ready is undone when the grace runs out, well
  expect(await healthAt(launched.port)).toMatchObject({ version: "0.7.162" });
});
test("a release that refuses its config and exits is recorded with the line it wrote (#10249)", async () => {
  const data = join(root, "data-refuses");
  keepGood(data, tree(SERVES), "0.7.479", null);
  expect((await run(data, tree(REFUSES), "0.7.481")).ran).toBe("rollback");
  expect(rolledBack(data)).toMatchObject({
    from: "0.7.481",
    why: "it exited with 1 before it was ready",
    said: 'FORGE_DRAIN_LINGER_S: whole seconds from 0 to 60, not "x"',
  });
});
test("a release that hangs is recorded with the last line it wrote before the grace ran out", async () => {
  const data = join(root, "data-says-hangs");
  keepGood(data, tree(SERVES), "0.7.162", null);
  await run(data, tree(SAYS_THEN_HANGS), "0.7.168", 500);
  expect(rolledBack(data)).toMatchObject({ why: "not ready within 1s", said: "waiting on the database" });
});
test("last words are the last error line, past the stack, the excerpt and the banner, and clamped", () => {
  const bun = [
    "2026-11-09T15:44:50.000Z INFO  [forge] reconciled repos=3",
    "1 | throw new Error(\"x\");",
    "          ^",
    "\x1b[31mSQLiteError\x1b[0m: database is locked",
    "      at migrate (/app/src/core/schema.ts:40:3)",
    "",
    "Bun v1.3.14 (Linux x64 baseline)",
    "",
  ].join("\n");
  expect(lastWords(bun)).toBe("SQLiteError: database is locked");
  expect(lastWords("starting\nPORT must be a port number\n")).toBe("PORT must be a port number");
  expect(lastWords("\n\n")).toBeNull();
  const long = lastWords(`error: ${"y".repeat(400)}`);
  expect(long?.length).toBe(200);
  expect(long?.endsWith("…")).toBe(true);
});
test("with nothing kept, or only itself, there is nothing to roll back to", async () => {
  expect(await run(join(root, "data-empty"), tree(THROWS), "0.7.166")).toMatchObject({
    ran: "none",
@@ -32,6 +32,7 @@
const PIPING: Record<string, { sites: string[]; reason: string }> = {
  "src/kernel/data_home.ts": { sites: ["stdout"], reason: "spawnSync buffers stdout; stderr ignored" },
  "src/looks/capture.ts": { sites: ["stderr"], reason: "stderr read in a background loop; stdout ignored" },
  "src/release/devtools.ts": { sites: ["stderr"], reason: "stderr read for the DevTools address, then drained in a background loop; stdout ignored" },
  "src/release/launch.ts": { sites: ["stderr"], reason: "hear() reads stderr to its end for the release's life, passing every chunk through to ours and keeping the tail until ready; stdout inherited" },
  "src/release/boot.ts": { sites: ["stderr"], reason: "stderr read to its end in a background loop, awaited before the reason is read; stdout ignored" },
  "src/snapshot/boot.ts": { sites: ["stderr"], reason: "stderr read to its end in a background loop, awaited before the exit is read for a lost port; stdout ignored" },
  "src/release/checkout.ts": { sites: ["stdout+stderr"], reason: "stdout and stderr both read via Response.text()" },
		@@ -43,6 +43,7 @@ import { RECOVERED, drainIncident, nextPage, observe, recoveredLine, type Watch
43	43	import { firedAlarms, type FiredAlarm } from "../flow/alarms.ts";
44	44	import { reachAlarms } from "../spawn/reach.ts";
45	45	import { loadClog } from "../flow/clog_tick.ts";
	46	import { redactSecrets } from "../kernel/log.ts";
46	47	import { forgeCeiling } from "../flow/ceiling.ts";
47	48	import { askLinesFor } from "../web/studio/asks.ts";
48	49
		@@ -456,7 +457,8 @@ export function startNotifier(app: App, options: NotifyOptions): Notifier {
456	457	const url = notifyUrlOf(app.db, id);
457	458	if (user === undefined \|\| url === undefined \|\| !user.isAdmin \|\| user.kind !== "user") continue;
458	459	reachable = true;
459		if (await push(id, url, need.sentence)) delivered = true;
	460	// The sentence quotes the release's stderr: nothing credential-shaped leaves in a push.
	461	if (await push(id, url, redactSecrets(need.sentence))) delivered = true;
460	462	}
461	463	if (delivered \|\| !reachable) markRollbackPaged(dataDir);
462	464	}
		@@ -13,7 +13,7 @@ import { statfsSync, statSync } from "node:fs";
13	13	import { dirname } from "node:path";
14	14	import { freemem, totalmem } from "node:os";
15	15	import { refusedInternal, type RefusedLedger } from "./internal_refused.ts";
16		import { recentErrors, type ErrorLedger, type ErrorLine } from "./log.ts";
	16	import { recentErrors, redactSecrets, type ErrorLedger, type ErrorLine } from "./log.ts";
17	17	import {
18	18	budgetFailures,
19	19	leaseLapsed,
		@@ -519,9 +519,10 @@ export function vitals(input: VitalsInput): Vitals {
519	519	const unproven = proofNeed(input.dataDir, runningVersion().version);
520	520	if (unproven !== undefined) need("release", unproven.sentence, unproven.since, "/board");
521	521
522		// A roll the image's launcher undid is a card until a later version reads ready.
	522	// A roll the image's launcher undid is a card until a later version reads ready. Its sentence
	523	// carries the release's own last words (stderr), so anything credential-shaped is masked.
523	524	const undone = rollbackNeed(input.dataDir);
524		if (undone !== undefined) need("release:rolled-back", undone.sentence, undone.since, "/board");
	525	if (undone !== undefined) need("release:rolled-back", redactSecrets(undone.sentence), undone.since, "/board");
525	526
526	527	// Production serving a version older than a release main has had over an hour is a card until they match.
527	528	const lagging = behindStanding(now);
		@@ -7,8 +7,9 @@
7	7	* (`release/good/<version>/`: its `src/` and the two files beside it) and
8	8	* named in `release/good.json`; the one before it is removed. A version
9	9	* that exits first, or is still not ready when the grace runs out, is
10		* stopped, `release/rolled-back.json` says what happened, and the kept tree
11		* runs under its own stamp. The forge that comes up reads that record: a
	10	* stopped, `release/rolled-back.json` says what happened and what the
	11	* release last said on stderr (#10249: 0.7.481 left only "it exited with 1"),
	12	* and the kept tree runs under its own stamp. The forge that comes up reads that record: a
12	13	* card while it stands (`kernel/vitals.ts`) and a page to the admins once
13	14	* (`crew/notify.ts`). The next version that reads ready clears it.
14	15	*
		@@ -43,6 +44,12 @@ const STOP_MS = 10_000;
43	44	/** What a kept tree is: what the Dockerfile copies into the image. */
44	45	const TREE_PARTS = ["src", "package.json", "tsconfig.json"];
45	46	const VERSION = /^[0-9A-Za-z][0-9A-Za-z.+_-]{0,63}$/;
	47	/** How much of a release's stderr is kept, to say why it did not become ready. */
	48	const TAIL_BYTES = 8_192;
	49	/** How long a release that exited has to finish writing its last words. */
	50	const LAST_WORDS_MS = 1_000;
	51	/** How long the last words in the record may be. */
	52	const SAID_MAX = 200;
46	53
47	54	export interface Good {
48	55	readonly version: string;
		@@ -59,6 +66,8 @@ export interface RolledBack {
59	66	readonly to: string;
60	67	readonly at: string;
61	68	readonly why: string;
	69	/** The release's last error line on stderr, when it wrote one: what a person reads to fix it. */
	70	readonly said: string \| null;
62	71	/** Whether the admins were told; set once by the forge that told them. */
63	72	readonly paged: boolean;
64	73	}
		@@ -130,10 +139,10 @@ export function keepGood(
130	139	export function rolledBack(dataDir: string): RolledBack \| undefined {
131	140	const r = readJson(rolledFile(dataDir));
132	141	if (r === undefined) return undefined;
133		const { from, to, at, why, paged } = r;
	142	const { from, to, at, why, said, paged } = r;
134	143	if (typeof from !== "string" \|\| typeof to !== "string" \|\| typeof at !== "string" \|\| typeof why !== "string")
135	144	return undefined;
136		return { from, to, at, why, paged: paged === true };
	145	return { from, to, at, why, said: typeof said === "string" && said !== "" ? said : null, paged: paged === true };
137	146	}
138	147
139	148	/** Record that the admins were told, so no later boot tells them again. */
		@@ -146,7 +155,54 @@ export function markRollbackPaged(dataDir: string): void {
146	155	export function rollbackNeed(dataDir: string): { sentence: string; since: string } \| undefined {
147	156	const r = rolledBack(dataDir);
148	157	if (r === undefined) return undefined;
149		return { sentence: `Release ${r.from} did not become ready (${r.why}); rolled back to ${r.to}.`, since: r.at };
	158	const said = r.said === null ? "" : ` It said: ${r.said}`;
	159	return { sentence: `Release ${r.from} did not become ready (${r.why}); rolled back to ${r.to}.${said}`, since: r.at };
	160	}
	161
	162	/**
	163	* The line of a release's stderr that says why it stopped: the last error
	164	* line (`error: …`, `SQLiteError: …`) when there is one, else the last line
	165	* that is not a stack frame, a source excerpt or Bun's version banner.
	166	*/
	167	export function lastWords(stderr: string): string \| null {
	168	const lines = stderr
	169	.replace(/\x1b\[[0-9;]*[A-Za-z]/g, "")
	170	.split("\n")
	171	.map((line) => line.trim())
	172	.filter((line) => line !== "" && !/^at\s/.test(line) && !/^Bun v\d/.test(line) && !/^\d+\s*\\|/.test(line) && !/^\^+$/.test(line));
	173	const line = lines.findLast((l) => /^(?:\w*Error\|error)(?::\|\s\|$)/.test(l)) ?? lines.at(-1);
	174	if (line === undefined) return null;
	175	return line.length > SAID_MAX ? `${line.slice(0, SAID_MAX - 1)}…` : line;
	176	}
	177
	178	/** A child's stderr, passed through to ours, with its tail kept until `forget`. */
	179	interface Heard {
	180	readonly ended: Promise<void>;
	181	tail(): string;
	182	forget(): void;
	183	}
	184
	185	function hear(stream: ReadableStream<Uint8Array>): Heard {
	186	let kept: Buffer \| undefined = Buffer.alloc(0);
	187	const ended = (async () => {
	188	try {
	189	for await (const chunk of stream) {
	190	process.stderr.write(chunk);
	191	if (kept === undefined) continue;
	192	kept = Buffer.concat([kept, chunk]);
	193	if (kept.length > TAIL_BYTES) kept = kept.subarray(kept.length - TAIL_BYTES);
	194	}
	195	} catch {
	196	// the pipe closing under us is the child going: what was heard stands
	197	}
	198	})();
	199	return {
	200	ended,
	201	tail: () => kept?.toString("utf8") ?? "",
	202	forget: () => {
	203	kept = undefined;
	204	},
	205	};
150	206	}
151	207
152	208	/**
		@@ -211,7 +267,13 @@ export type Launched =
211	267	\| { readonly ran: "release" \| "rollback"; readonly child: Subprocess; readonly line: string }
212	268	\| { readonly ran: "none"; readonly line: string };
213	269
214		function start(input: LaunchInput, tree: string, version: string, commit: string \| null): Subprocess {
	270	/** A release started, and what it says on stderr. */
	271	interface Started {
	272	readonly child: Subprocess;
	273	readonly heard: Heard;
	274	}
	275
	276	function start(input: LaunchInput, tree: string, version: string, commit: string \| null): Started {
215	277	const env: Record<string, string> = {};
216	278	for (const [k, v] of Object.entries(input.env)) if (v !== undefined) env[k] = v;
217	279	env["FORGE_VERSION"] = version;
		@@ -223,22 +285,35 @@ function start(input: LaunchInput, tree: string, version: string, commit: string
223	285	env,
224	286	stdin: "ignore",
225	287	stdout: "inherit",
226		stderr: "inherit",
	288	// Piped, not inherited, so a release that dies says why in the record; every byte still reaches ours.
	289	stderr: "pipe",
227	290	});
228	291	input.started?.(child);
229		return child;
	292	return { child, heard: hear(child.stderr) };
230	293	}
231	294
232		/** Wait for `child` to answer health as `version`: undefined once it does, else why it did not. */
233		async function becomesReady(input: LaunchInput, child: Subprocess, version: string): Promise<string \| undefined> {
	295	/** Why a release did not become ready, and the last line it wrote on stderr. */
	296	interface NotReady {
	297	readonly why: string;
	298	readonly said: string \| null;
	299	}
	300
	301	/** Wait for the release to answer health as `version`: undefined once it does, else why it did not. */
	302	async function becomesReady(input: LaunchInput, { child, heard }: Started, version: string): Promise<NotReady \| undefined> {
234	303	const now = input.now ?? Date.now;
235	304	const grace = input.graceMs ?? READY_GRACE_MS;
236	305	const began = now();
237	306	for (;;) {
238		if (input.stopping?.()) return "asked to stop before it was ready";
239		if (gone(child)) return `it exited with ${child.exitCode ?? child.signalCode} before it was ready`;
240		if ((await versionAt(input.port, probeHeaders(input.dataDir))) === version) return undefined;
241		if (now() - began >= grace) return `not ready within ${Math.round(grace / 1000)}s`;
	307	if (input.stopping?.()) return { why: "asked to stop before it was ready", said: null };
	308	if (gone(child)) {
	309	await Promise.race([heard.ended, Bun.sleep(LAST_WORDS_MS)]);
	310	return { why: `it exited with ${child.exitCode ?? child.signalCode} before it was ready`, said: lastWords(heard.tail()) };
	311	}
	312	if ((await versionAt(input.port, probeHeaders(input.dataDir))) === version) {
	313	heard.forget();
	314	return undefined;
	315	}
	316	if (now() - began >= grace) return { why: `not ready within ${Math.round(grace / 1000)}s`, said: lastWords(heard.tail()) };
242	317	await Bun.sleep(POLL_MS);
243	318	}
244	319	}
		@@ -252,19 +327,20 @@ async function becomesReady(input: LaunchInput, child: Subprocess, version: stri
252	327	export async function launch(input: LaunchInput): Promise<Launched> {
253	328	const now = input.now ?? Date.now;
254	329	const swapped = lastGood(input.dataDir);
255		let passed: { version: string; why: string } \| undefined;
	330	let passed: ({ version: string } & NotReady) \| undefined;
256	331	if (swapped !== undefined && swapped.base === input.version && swapped.version !== input.version) {
257	332	const first = start(input, goodTree(input.dataDir, swapped.version), swapped.version, swapped.commit);
258	333	const failed = await becomesReady(input, first, swapped.version);
259	334	if (failed === undefined)
260		return { ran: "release", child: first, line: `${swapped.version} is ready: the swapped release kept over image ${input.version}` };
261		await stop(first);
	335	return { ran: "release", child: first.child, line: `${swapped.version} is ready: the swapped release kept over image ${input.version}` };
	336	await stop(first.child);
262	337	if (input.stopping?.()) return { ran: "none", line: "asked to stop before it was ready" };
263		passed = { version: swapped.version, why: failed };
	338	passed = { version: swapped.version, ...failed };
264	339	}
265		const release = start(input, input.tree, input.version, input.commit);
266		const why = await becomesReady(input, release, input.version);
267		if (why === undefined) {
	340	const started = start(input, input.tree, input.version, input.commit);
	341	const release = started.child;
	342	const failed = await becomesReady(input, started, input.version);
	343	if (failed === undefined) {
268	344	try {
269	345	keepGood(input.dataDir, input.tree, input.version, input.commit, now());
270	346	} catch (error) {
		@@ -278,12 +354,14 @@ export async function launch(input: LaunchInput): Promise<Launched> {
278	354	to: input.version,
279	355	at: new Date(now()).toISOString(),
280	356	why: passed.why,
	357	said: passed.said,
281	358	paged: false,
282	359	} satisfies RolledBack);
283	360	return { ran: "rollback", child: release, line: `${passed.version}: ${passed.why}; the image ${input.version} runs instead` };
284	361	}
285	362	return { ran: "release", child: release, line: `${input.version} is ready and kept as the last good release` };
286	363	}
	364	const { why, said } = failed;
287	365	await stop(release);
288	366	if (input.stopping?.()) return { ran: "none", line: "asked to stop before it was ready" };
289	367	const good = lastGood(input.dataDir);
		@@ -294,9 +372,11 @@ export async function launch(input: LaunchInput): Promise<Launched> {
294	372	to: good.version,
295	373	at: new Date(now()).toISOString(),
296	374	why,
	375	said,
297	376	paged: false,
298	377	} satisfies RolledBack);
299		const child = start(input, goodTree(input.dataDir, good.version), good.version, good.commit);
	378	const { child, heard } = start(input, goodTree(input.dataDir, good.version), good.version, good.commit);
	379	heard.forget();
300	380	return { ran: "rollback", child, line: `${input.version}: ${why}; rolled back to ${good.version}` };
301	381	}
302	382
		@@ -42,11 +42,11 @@ afterEach(async () => {
42	42	await rm(dataDir, { recursive: true, force: true });
43	43	});
44	44
45		const record = () => {
	45	const record = (said?: string) => {
46	46	mkdirSync(join(dataDir, "release"), { recursive: true });
47	47	writeFileSync(
48	48	join(dataDir, "release", "rolled-back.json"),
49		JSON.stringify({ from: "0.7.166", to: "0.7.162", at: "2026-10-19T05:17:00.000Z", why: "it exited with 1 before it was ready", paged: false }),
	49	JSON.stringify({ from: "0.7.166", to: "0.7.162", at: "2026-10-19T05:17:00.000Z", why: "it exited with 1 before it was ready", said, paged: false }),
50	50	);
51	51	};
52	52
		@@ -96,3 +96,11 @@ test("no record, no page", async () => {
96	96	await boot();
97	97	expect(posts).toEqual([]);
98	98	});
	99
	100	test("the release's last words reach the card and the page, with anything credential-shaped masked", async () => {
	101	record("error: GET https://x.test failed with Bearer abcdefghijklmnopqrstuvwxyz0123");
	102	const said = `${SENTENCE} It said: error: GET https://x.test failed with [redacted]`;
	103	expect(vitals({ db: app.db, signals: app.signals, dataDir }).degraded.find((d) => d.cause === "release:rolled-back")?.sentence).toBe(said);
	104	await boot();
	105	expect(posts).toEqual([`${said}\nhttps://forge.example.test/`]);
	106	});
		@@ -14,6 +14,7 @@ import type { Subprocess } from "bun";
14	14	import {
15	15	keepGood,
16	16	lastGood,
	17	lastWords,
17	18	launch,
18	19	markRollbackPaged,
19	20	READY_GRACE_MS,
		@@ -53,6 +54,8 @@ Bun.serve({ hostname: "127.0.0.1", port: Number(process.env.PORT), fetch(req) {
53	54	`;
54	55	const THROWS = `throw new Error("(test fixture) a release whose main throws at import, as 0.7.166's did");\n`;
55	56	const HANGS = `setInterval(() => {}, 1_000);\n`;
	57	const REFUSES = `console.error("FORGE_DRAIN_LINGER_S: whole seconds from 0 to 60, not \\"x\\""); process.exit(1);\n`;
	58	const SAYS_THEN_HANGS = `console.error("waiting on the database"); setInterval(() => {}, 1_000);\n`;
56	59
57	60	async function freePort(): Promise<number> {
58	61	const probe = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch: () => new Response("") });
		@@ -104,7 +107,8 @@ test("a version whose main throws at import is undone at once: the last good one
104	107	expect(await healthAt(launched.port)).toMatchObject({ version: "0.7.162", commit: "a".repeat(40) });
105	108	expect(rolledBack(data)).toMatchObject({ from: "0.7.166", to: "0.7.162", paged: false });
106	109	expect(rollbackNeed(data)?.sentence).toBe(
107		"Release 0.7.166 did not become ready (it exited with 1 before it was ready); rolled back to 0.7.162.",
	110	"Release 0.7.166 did not become ready (it exited with 1 before it was ready); rolled back to 0.7.162." +
	111	" It said: error: (test fixture) a release whose main throws at import, as 0.7.166's did",
108	112	);
109	113	expect(lastGood(data)?.version).toBe("0.7.162");
110	114	});
		@@ -119,6 +123,43 @@ test("a version that never becomes ready is undone when the grace runs out, well
119	123	expect(await healthAt(launched.port)).toMatchObject({ version: "0.7.162" });
120	124	});
121	125
	126	test("a release that refuses its config and exits is recorded with the line it wrote (#10249)", async () => {
	127	const data = join(root, "data-refuses");
	128	keepGood(data, tree(SERVES), "0.7.479", null);
	129	expect((await run(data, tree(REFUSES), "0.7.481")).ran).toBe("rollback");
	130	expect(rolledBack(data)).toMatchObject({
	131	from: "0.7.481",
	132	why: "it exited with 1 before it was ready",
	133	said: 'FORGE_DRAIN_LINGER_S: whole seconds from 0 to 60, not "x"',
	134	});
	135	});
	136
	137	test("a release that hangs is recorded with the last line it wrote before the grace ran out", async () => {
	138	const data = join(root, "data-says-hangs");
	139	keepGood(data, tree(SERVES), "0.7.162", null);
	140	await run(data, tree(SAYS_THEN_HANGS), "0.7.168", 500);
	141	expect(rolledBack(data)).toMatchObject({ why: "not ready within 1s", said: "waiting on the database" });
	142	});
	143
	144	test("last words are the last error line, past the stack, the excerpt and the banner, and clamped", () => {
	145	const bun = [
	146	"2026-11-09T15:44:50.000Z INFO [forge] reconciled repos=3",
	147	"1 \| throw new Error(\"x\");",
	148	" ^",
	149	"\x1b[31mSQLiteError\x1b[0m: database is locked",
	150	" at migrate (/app/src/core/schema.ts:40:3)",
	151	"",
	152	"Bun v1.3.14 (Linux x64 baseline)",
	153	"",
	154	].join("\n");
	155	expect(lastWords(bun)).toBe("SQLiteError: database is locked");
	156	expect(lastWords("starting\nPORT must be a port number\n")).toBe("PORT must be a port number");
	157	expect(lastWords("\n\n")).toBeNull();
	158	const long = lastWords(`error: ${"y".repeat(400)}`);
	159	expect(long?.length).toBe(200);
	160	expect(long?.endsWith("…")).toBe(true);
	161	});
	162
122	163	test("with nothing kept, or only itself, there is nothing to roll back to", async () => {
123	164	expect(await run(join(root, "data-empty"), tree(THROWS), "0.7.166")).toMatchObject({
124	165	ran: "none",
		@@ -32,6 +32,7 @@ const PIPING: Record<string, { sites: string[]; reason: string }> = {
32	32	"src/kernel/data_home.ts": { sites: ["stdout"], reason: "spawnSync buffers stdout; stderr ignored" },
33	33	"src/looks/capture.ts": { sites: ["stderr"], reason: "stderr read in a background loop; stdout ignored" },
34	34	"src/release/devtools.ts": { sites: ["stderr"], reason: "stderr read for the DevTools address, then drained in a background loop; stdout ignored" },
	35	"src/release/launch.ts": { sites: ["stderr"], reason: "hear() reads stderr to its end for the release's life, passing every chunk through to ours and keeping the tail until ready; stdout inherited" },
35	36	"src/release/boot.ts": { sites: ["stderr"], reason: "stderr read to its end in a background loop, awaited before the reason is read; stdout ignored" },
36	37	"src/snapshot/boot.ts": { sites: ["stderr"], reason: "stderr read to its end in a background loop, awaited before the exit is read for a lost port; stdout ignored" },
37	38	"src/release/checkout.ts": { sites: ["stdout+stderr"], reason: "stdout and stderr both read via Response.text()" },