f7e0265 · d3af303
@@ -231,6 +231,13 @@
sees `Reduced protection: …` in one line, under the filter on the terminal's
first screen and in the web footer; nobody else does, and a fully protected
forge shows nothing.
A new version proves itself once per version: after a boot that changes
`FORGE_VERSION`, the forge waits (up to 10 minutes) for its public URL to
answer `/healthz` as that version, then runs `scripts/smoke.ts` against it and
keeps the verdict in `$FORGE_APP_DATA/release/proof.json`. A failure is a
`Release X is unproven: …` card on an admin's board and on the terminal's first
screen (the same cards), and is paged once; a later passing version clears it.
What you asked for: `Ctrl+K`, then **Asked**, is `/asked` in the terminal — your
thoughts, then the issues you can write to that are not shipped, oldest first
(`j`/`k` move, `enter` gives the issue's web address, `r` refreshes). It is read
@@ -53,10 +53,14 @@
export interface Options {
  waitMs?: number;
  /** Pause between polls, in milliseconds. */
  pollMs?: number;
  /** The fetch to use; the global one unless a caller (the forge proving itself, a test) gives its own. */
  fetch?: typeof fetch;
  /** Accept a redirect from `/board` as the sign-in it is, for a caller holding no token. */
  anonymousBoard?: boolean;
}
/** Polls `/healthz` until it reports `version`; null when it does, else a failure. */
async function waitForVersion(root: URL, version: string, waitMs: number, pollMs: number): Promise<Failure | null> {
async function waitForVersion(root: URL, version: string, waitMs: number, pollMs: number, fetch: typeof globalThis.fetch): Promise<Failure | null> {
  const url = new URL("/healthz", root).href;
  const deadline = Date.now() + waitMs;
  let seen = "none";
@@ -83,8 +87,9 @@
export async function smoke(base: string, token = "", options: Options = {}): Pr
  const root = new URL(base);
  if (root.protocol !== "http:" && root.protocol !== "https:") throw new Error("url must be http(s)");
  const failures: Failure[] = [];
  const fetch = options.fetch ?? globalThis.fetch;
  if (options.version) {
    const late = await waitForVersion(root, options.version, options.waitMs ?? 120_000, options.pollMs ?? 1_000);
    const late = await waitForVersion(root, options.version, options.waitMs ?? 120_000, options.pollMs ?? 1_000, fetch);
    if (late) return [late];
  }
      
        junipercrew
        26d ago
        
      
      Check that the get helper's fetch is the injected one. Line 66 and this line sit in nested functions; the shadowing const is declared at line 90 so it should resolve, but the redundant options.fetch ?? fetch at the waitForVersion call hides this. Pass fetch alone.
    
@@ -125,7 +130,8 @@
export async function smoke(base: string, token = "", options: Options = {}): Pr
      }
    }
    const board = await get("/board", agent, auth);
    if (board && board.res.status !== 200) fail(board.url, agent, board.res, "expected 200 (a 302 is sign-in)");
    const signIn = options.anonymousBoard && !token && board?.res.status === 302;
    if (board && board.res.status !== 200 && !signIn) fail(board.url, agent, board.res, "expected 200 (a 302 is sign-in)");
    const health = await get("/healthz", agent);
    if (health && health.res.status !== 200) fail(health.url, agent, health.res, "expected 200");
  }
@@ -48,7 +48,7 @@
export const HOLD_MS = 5 * 60_000;
/** The vitals causes this pager owns; the rest are paged elsewhere or are settings. */
export function owned(cause: string): boolean {
  return cause === "errors" || cause === "internal" || cause === "disk" || cause.startsWith("queue:");
  return cause === "errors" || cause === "internal" || cause === "disk" || cause === "release" || cause.startsWith("queue:");
}
/** One owned cause being watched. */
@@ -25,6 +25,7 @@
import { BUDGETS } from "./budgets.ts";
import { lastMirror, mirrorNeed } from "../snapshot/mirror.ts";
import { lastSnapshotOutcome, snapshotNeed } from "../snapshot/outcome.ts";
import { drillNeed, firstSeen as drillFirstSeen, lastVerdict } from "../snapshot/verdict.ts";
import { proofNeed } from "../release/prove.ts";
import { newestDrills } from "../core/domain_drills.ts";
import { DOMAINS } from "./domains.ts";
import { domainDrillNeeds } from "./domain_drill.ts";
@@ -51,7 +52,7 @@
export interface EnvCount {
/**
 * One cause of the forge being off: the `needs` sentence, a `cause` key that
 * holds for as long as its condition does (`disk`, `errors`, `internal`,
 * `queue:<kind>`, `stuck:<runId>`, `red:<repoId>`, `shared:<repoId>:<test>`), when it began by the forge's own evidence
 * `queue:<kind>`, `release`, `stuck:<runId>`, `red:<repoId>`, `shared:<repoId>:<test>`), when it began by the forge's own evidence
 * (the oldest evidence the forge holds, else the first time this process saw
 * it, held until the cause clears; `at` is never the answer to a repeat
 * call), and where to look.
@@ -393,6 +394,10 @@
export function vitals(input: VitalsInput): Vitals {
  const behind = mirrorNeed(lastMirror(input.dataDir), now);
  if (behind !== undefined) need("mirror", behind.sentence, behind.since, "/settings/backups");
  // A release the forge could not prove against its own public URL is a card until a release does.
  const unproven = proofNeed(input.dataDir);
  if (unproven !== undefined) need("release", unproven.sentence, unproven.since, "/board");
      
        junipercrew
        26d ago
        outdated
      
      Link /settings/capacity is unrelated to a release failure. Point it at where the owner can see the cause (the board or health page), and test it.
    
  const apps = { serving: 0, asleep: 0, previews: 0 };
  for (const e of input.envs ?? []) {
    if (e.env !== "production") apps.previews++;
@@ -54,6 +54,7 @@
import { configProblem, dataDirOf, preflight } from "./kernel/preflight.ts";
import { running } from "./kernel/version.ts";
import { record as recordDegraded } from "./kernel/degraded_history.ts";
import { vitals } from "./kernel/vitals.ts";
import { proveRelease } from "./release/prove.ts";
import { snapshotPlace } from "./actions/health.ts";
import { capacityDefault } from "./core/capacity.ts";
@@ -480,6 +481,13 @@
const staleTimer = setInterval(() => void tickAsks(), 120_000);
void tickAsks();
app.log.info("forge is up", { url: servers.url, data: app.config.dataDir });
// A new version proves itself against the public URL once (release/prove.ts).
void proveRelease({
  dataDir: app.config.dataDir,
  version: running().version,
  url: forgeUrl,
  token: process.env["FORGE_TOKEN"],
}).then((p) => p && app.log.info("release proof", { version: p.version, ok: String(p.ok), ...(p.failure ? { failure: p.failure } : {}) }));
// Only an agent is on a crew (the `repo_crew_agents_only` trigger refuses anyone else). A person
// put on one before that is reported here for the owner to take off — the
// forge never removes it on its own.
@@ -0,0 +1,109 @@
/**
 * Is this release so: once per version, after a boot that changed it, the
 * forge waits for its own public URL to answer `/healthz` as that version,
 * then runs the smoke checks (`scripts/smoke.ts`) against it. The verdict is
 * one small file under the data directory (`release/proof.json`), so a
 * restart of the same version never runs it again and vitals reads it
 * without the network. A failure is a `release` card, paged once by the
 * cause pager, until a later version passes. It gives up after ten minutes.
 */
import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { smoke, type Failure } from "../../scripts/smoke.ts";
import { UNSTAMPED } from "../kernel/version.ts";
/** How long the public URL has to answer as the new version. */
export const GIVE_UP_MS = 10 * 60_000;
const POLL_MS = 2_000;
export interface Proof {
  readonly version: string;
  readonly at: string;
  readonly ok: boolean;
  /** The first failing check, or the give-up, when `ok` is false. */
  readonly failure?: string;
}
const proofFile = (dataDir: string): string => join(dataDir, "release", "proof.json");
/** The last verdict kept, or undefined when none is or the file is not one. */
export function lastProof(dataDir: string): Proof | undefined {
  try {
    const p = JSON.parse(readFileSync(proofFile(dataDir), "utf8")) as Proof;
    if (typeof p.version !== "string" || typeof p.at !== "string" || typeof p.ok !== "boolean") return undefined;
    return p;
  } catch {
    return undefined;
  }
}
function keep(dataDir: string, proof: Proof): Proof {
  mkdirSync(join(dataDir, "release"), { recursive: true });
  const tmp = `${proofFile(dataDir)}.tmp`;
  writeFileSync(tmp, JSON.stringify(proof));
  renameSync(tmp, proofFile(dataDir));
  return proof;
}
/** The card an unproven release is, or undefined while the last verdict is a pass. */
export function proofNeed(dataDir: string): { sentence: string; since: string } | undefined {
  const p = lastProof(dataDir);
  if (p === undefined || p.ok) return undefined;
  return { sentence: `Release ${p.version} is unproven: ${p.failure ?? "a smoke check failed"}.`, since: p.at };
}
/** A failing check as one line: where, as whom, and why. */
export function checkLine(f: Failure): string {
  return `${f.url}${f.agent === "-" ? "" : ` as ${f.agent}`} ${f.why} (status ${f.status})`;
}
export interface ProveOptions {
  readonly dataDir: string;
  readonly version: string;
  /** The forge's public URL. */
  readonly url: string;
  /** Signs `/board` in; without it a redirect to sign-in passes. */
  readonly token?: string | undefined;
  readonly fetch?: typeof fetch;
  readonly now?: () => number;
  readonly sleep?: (ms: number) => Promise<void>;
  readonly giveUpMs?: number;
}
/**
 * Proves `version` unless it was proven (or failed) before. Returns what was
 * kept, or undefined when nothing ran. Never throws.
 */
export async function proveRelease(o: ProveOptions): Promise<Proof | undefined> {
  if (o.version === UNSTAMPED || lastProof(o.dataDir)?.version === o.version) return undefined;
  const now = o.now ?? Date.now;
  const sleep = o.sleep ?? Bun.sleep;
  const doFetch = o.fetch ?? fetch;
  const finish = (failure?: string): Proof =>
    keep(o.dataDir, {
      version: o.version,
      at: new Date(now()).toISOString(),
      ok: failure === undefined,
      ...(failure === undefined ? {} : { failure }),
    });
  try {
    const health = new URL("/healthz", o.url).href;
    const deadline = now() + (o.giveUpMs ?? GIVE_UP_MS);
    for (;;) {
      try {
        const res = await doFetch(health, { redirect: "manual", signal: AbortSignal.timeout(5_000) });
        await res.body?.cancel();
        if (res.headers.get("x-forge-version") === o.version) break;
      } catch {
        // not up yet at its public address
      }
      if (now() + POLL_MS > deadline) return finish(`the public URL never answered as ${o.version}`);
      await sleep(POLL_MS);
    }
    const failures = await smoke(o.url, o.token ?? "", { fetch: doFetch, anonymousBoard: true });
      
        junipercrew
        26d ago
        
      
      The error string is stored verbatim in the card; keep it to one short line (e.g. error.message) so the board stays readable on a 360px phone.
    
    return finish(failures[0] === undefined ? undefined : checkLine(failures[0]));
  } catch (error) {
      
        junipercrew
        26d ago
        
      
      ${checkLine(failures[0])} is a needless template wrapper; use checkLine(failures[0]).
    
    return finish(`the smoke checks could not run: ${String(error)}`);
  }
}
@@ -16,7 +16,7 @@
const MIN = 60_000;
describe("owned causes", () => {
  test("an explicit list", () => {
    for (const c of ["errors", "internal", "disk", "queue:check", "queue:run"])
    for (const c of ["errors", "internal", "disk", "release", "queue:check", "queue:run"])
      expect(owned(c)).toBe(true);
    for (const c of ["drill", "mirror", "stuck:r1", "red:r1", "snapshots", "budget:r1", "shared:r:t"])
      expect(owned(c)).toBe(false);
@@ -0,0 +1,99 @@
/**
 * A new version proves itself once against the public URL: a pass is kept,
 * a failure is a `release` card naming the first failing check, the same
 * version on restart does not run again, and ten minutes of silence is
 * "unproven" (#2574). A fake fetch plays the public URL.
 */
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { lastProof, proofNeed, proveRelease } from "../src/release/prove.ts";
import { owned } from "../src/crew/incidents.ts";
const CSS = '<html><link rel="stylesheet" href="/a.css"></html>';
function site(version: string, over: Record<string, () => Response> = {}) {
  const hits: string[] = [];
  const fake = (async (input: string | URL) => {
    const path = new URL(String(input)).pathname;
    hits.push(path);
    const made = over[path]?.();
    if (made) return made;
    if (path === "/healthz") return new Response("ok", { headers: { "x-forge-version": version } });
    if (path === "/") return new Response(CSS, { headers: { "content-type": "text/html" } });
    if (path === "/a.css") return new Response("a{}", { headers: { "content-type": "text/css" } });
    if (path === "/board") return new Response(null, { status: 302, headers: { location: "/signin" } });
    if (path === "/sw.js")
      return new Response('if (r.mode === "navigate") { e.respondWith(fetch(r)); return; }', {
        headers: { "content-type": "text/javascript" },
      });
    return new Response("no", { status: 404 });
  }) as unknown as typeof fetch;
  return { fake, hits };
}
describe("proving a release", () => {
  let dir: string;
  beforeEach(() => {
    dir = mkdtempSync(join(tmpdir(), "forge-prove-"));
  });
  afterEach(() => rmSync(dir, { recursive: true, force: true }));
  const run = (version: string, fake: typeof fetch, extra = {}) =>
    proveRelease({ dataDir: dir, version, url: "https://forge.test", fetch: fake, sleep: async () => {}, ...extra });
  test("a pass is recorded with the version and raises nothing", async () => {
    const { fake } = site("1.2.3");
    const p = await run("1.2.3", fake);
    expect(p?.ok).toBe(true);
    expect(lastProof(dir)?.version).toBe("1.2.3");
    expect(proofNeed(dir)).toBeUndefined();
  });
  test("a failure raises the cause once, naming the first failing check", async () => {
    const { fake } = site("1.2.4", { "/a.css": () => new Response("x", { status: 500 }) });
    await run("1.2.4", fake);
    const need = proofNeed(dir)!;
    expect(need.sentence).toContain("Release 1.2.4 is unproven");
    expect(need.sentence).toContain("/a.css");
    expect(owned("release")).toBe(true);
    // the same version again does not run, so the card is not re-raised
    const again = site("1.2.4");
    expect(await run("1.2.4", again.fake)).toBeUndefined();
    expect(again.hits).toEqual([]);
    expect(proofNeed(dir)?.since).toBe(need.since);
  });
  test("the same version on restart does not run again", async () => {
    const first = site("2.0.0");
    await run("2.0.0", first.fake);
    const second = site("2.0.0");
    expect(await run("2.0.0", second.fake)).toBeUndefined();
    expect(second.hits).toEqual([]);
  });
  test("a later version that passes clears the card", async () => {
    await run("3.0.0", site("3.0.0", { "/": () => new Response("x", { status: 500 }) }).fake);
    expect(proofNeed(dir)).toBeDefined();
    await run("3.0.1", site("3.0.1").fake);
    expect(proofNeed(dir)).toBeUndefined();
  });
  test("it gives up after 10 minutes as unproven", async () => {
    let t = 0;
    const old = site("0.9.0"); // the old version keeps answering
    const p = await run("1.0.0", old.fake, { now: () => t, sleep: async (ms: number) => void (t += ms) });
    expect(p?.ok).toBe(false);
    expect(t).toBeLessThanOrEqual(10 * 60_000);
    expect(t).toBeGreaterThan(9 * 60_000);
    expect(proofNeed(dir)?.sentence).toBe("Release 1.0.0 is unproven: the public URL never answered as 1.0.0.");
    expect(old.hits.every((h) => h === "/healthz")).toBe(true);
  });
  test("an unstamped forge proves nothing", async () => {
    const s = site("dev");
    expect(await run("dev", s.fake)).toBeUndefined();
    expect(s.hits).toEqual([]);
  });
});
@@ -105,3 +105,10 @@
test("the CLI takes --version and --wait", async () => {
  expect(await run("--version", "v1")).toBe(0);
  expect(await run("--version", "zz", "--wait", "0")).toBe(1);
});
test("anonymousBoard: a 302 from /board passes with no token, and only then", async () => {
  const base = stub("text/css");
  expect(await smoke(base, "", { anonymousBoard: true })).toEqual([]);
  expect((await smoke(base, "", {})).map((f) => f.status)).toEqual([302, 302]);
  expect((await smoke(base, "wrong", { anonymousBoard: true })).map((f) => f.status)).toEqual([302, 302]);
});
@@ -0,0 +1,50 @@
/**
 * An unproven release is a `release` card pointing at the board, and a
 * later pass clears it (#2574). The terminal's first screen prints the same
 * cards (`forgeNeeds`), so it shows this one too.
 */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { mkdirSync, writeFileSync } from "node:fs";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { vitals } from "../src/kernel/vitals.ts";
let dataDir: string;
let app: App;
beforeAll(async () => {
  dataDir = await mkdtemp(join(tmpdir(), "forge-vitals-release-"));
  app = createApp({ dataDir, port: 0 });
});
afterAll(async () => {
  app.db.close();
  await rm(dataDir, { recursive: true, force: true });
});
const card = () =>
  vitals({ db: app.db, signals: app.signals, dataDir }).degraded.find((d) => d.cause === "release");
const keep = (proof: object) => {
  mkdirSync(join(dataDir, "release"), { recursive: true });
  writeFileSync(join(dataDir, "release", "proof.json"), JSON.stringify(proof));
};
test("no proof, no card", () => {
  expect(card()).toBeUndefined();
});
test("a failing proof is a release card naming the check, linking the board", () => {
  keep({ version: "9.9.9", at: "2026-10-18T10:00:00.000Z", ok: false, failure: "https://f/a.css expected 200 (status 500)" });
  expect(card()).toMatchObject({
    sentence: "Release 9.9.9 is unproven: https://f/a.css expected 200 (status 500).",
    href: "/board",
    since: "2026-10-18T10:00:00.000Z",
  });
});
test("a passing proof clears it", () => {
  keep({ version: "9.9.10", at: "2026-10-18T11:00:00.000Z", ok: true });
  expect(card()).toBeUndefined();
});
		@@ -231,6 +231,13 @@ sees `Reduced protection: …` in one line, under the filter on the terminal's
231	231	first screen and in the web footer; nobody else does, and a fully protected
232	232	forge shows nothing.
233	233
	234	A new version proves itself once per version: after a boot that changes
	235	`FORGE_VERSION`, the forge waits (up to 10 minutes) for its public URL to
	236	answer `/healthz` as that version, then runs `scripts/smoke.ts` against it and
	237	keeps the verdict in `$FORGE_APP_DATA/release/proof.json`. A failure is a
	238	`Release X is unproven: …` card on an admin's board and on the terminal's first
	239	screen (the same cards), and is paged once; a later passing version clears it.
	240
234	241	What you asked for: `Ctrl+K`, then Asked, is `/asked` in the terminal — your
235	242	thoughts, then the issues you can write to that are not shipped, oldest first
236	243	(`j`/`k` move, `enter` gives the issue's web address, `r` refreshes). It is read
		@@ -53,10 +53,14 @@ export interface Options {
53	53	waitMs?: number;
54	54	/** Pause between polls, in milliseconds. */
55	55	pollMs?: number;
	56	/** The fetch to use; the global one unless a caller (the forge proving itself, a test) gives its own. */
	57	fetch?: typeof fetch;
	58	/** Accept a redirect from `/board` as the sign-in it is, for a caller holding no token. */
	59	anonymousBoard?: boolean;
56	60	}
57	61
58	62	/** Polls `/healthz` until it reports `version`; null when it does, else a failure. */
59		async function waitForVersion(root: URL, version: string, waitMs: number, pollMs: number): Promise<Failure \| null> {
	63	async function waitForVersion(root: URL, version: string, waitMs: number, pollMs: number, fetch: typeof globalThis.fetch): Promise<Failure \| null> {
60	64	const url = new URL("/healthz", root).href;
61	65	const deadline = Date.now() + waitMs;
62	66	let seen = "none";
		@@ -83,8 +87,9 @@ export async function smoke(base: string, token = "", options: Options = {}): Pr
83	87	const root = new URL(base);
84	88	if (root.protocol !== "http:" && root.protocol !== "https:") throw new Error("url must be http(s)");
85	89	const failures: Failure[] = [];
	90	const fetch = options.fetch ?? globalThis.fetch;
86	91	if (options.version) {
87		const late = await waitForVersion(root, options.version, options.waitMs ?? 120_000, options.pollMs ?? 1_000);
	92	const late = await waitForVersion(root, options.version, options.waitMs ?? 120_000, options.pollMs ?? 1_000, fetch);
88	93	if (late) return [late];
89	94	}
		junipercrew 26d ago Check that the `get` helper's `fetch` is the injected one. Line 66 and this line sit in nested functions; the shadowing const is declared at line 90 so it should resolve, but the redundant `options.fetch ?? fetch` at the waitForVersion call hides this. Pass `fetch` alone.
90	95
		@@ -125,7 +130,8 @@ export async function smoke(base: string, token = "", options: Options = {}): Pr
125	130	}
126	131	}
127	132	const board = await get("/board", agent, auth);
128		if (board && board.res.status !== 200) fail(board.url, agent, board.res, "expected 200 (a 302 is sign-in)");
	133	const signIn = options.anonymousBoard && !token && board?.res.status === 302;
	134	if (board && board.res.status !== 200 && !signIn) fail(board.url, agent, board.res, "expected 200 (a 302 is sign-in)");
129	135	const health = await get("/healthz", agent);
130	136	if (health && health.res.status !== 200) fail(health.url, agent, health.res, "expected 200");
131	137	}
		@@ -48,7 +48,7 @@ export const HOLD_MS = 5 * 60_000;
48	48
49	49	/** The vitals causes this pager owns; the rest are paged elsewhere or are settings. */
50	50	export function owned(cause: string): boolean {
51		return cause === "errors" \|\| cause === "internal" \|\| cause === "disk" \|\| cause.startsWith("queue:");
	51	return cause === "errors" \|\| cause === "internal" \|\| cause === "disk" \|\| cause === "release" \|\| cause.startsWith("queue:");
52	52	}
53	53
54	54	/** One owned cause being watched. */
		@@ -25,6 +25,7 @@ import { BUDGETS } from "./budgets.ts";
25	25	import { lastMirror, mirrorNeed } from "../snapshot/mirror.ts";
26	26	import { lastSnapshotOutcome, snapshotNeed } from "../snapshot/outcome.ts";
27	27	import { drillNeed, firstSeen as drillFirstSeen, lastVerdict } from "../snapshot/verdict.ts";
	28	import { proofNeed } from "../release/prove.ts";
28	29	import { newestDrills } from "../core/domain_drills.ts";
29	30	import { DOMAINS } from "./domains.ts";
30	31	import { domainDrillNeeds } from "./domain_drill.ts";
		@@ -51,7 +52,7 @@ export interface EnvCount {
51	52	/**
52	53	* One cause of the forge being off: the `needs` sentence, a `cause` key that
53	54	* holds for as long as its condition does (`disk`, `errors`, `internal`,
54		* `queue:<kind>`, `stuck:<runId>`, `red:<repoId>`, `shared:<repoId>:<test>`), when it began by the forge's own evidence
	55	* `queue:<kind>`, `release`, `stuck:<runId>`, `red:<repoId>`, `shared:<repoId>:<test>`), when it began by the forge's own evidence
55	56	* (the oldest evidence the forge holds, else the first time this process saw
56	57	* it, held until the cause clears; `at` is never the answer to a repeat
57	58	* call), and where to look.
		@@ -393,6 +394,10 @@ export function vitals(input: VitalsInput): Vitals {
393	394	const behind = mirrorNeed(lastMirror(input.dataDir), now);
394	395	if (behind !== undefined) need("mirror", behind.sentence, behind.since, "/settings/backups");
395	396
	397	// A release the forge could not prove against its own public URL is a card until a release does.
	398	const unproven = proofNeed(input.dataDir);
	399	if (unproven !== undefined) need("release", unproven.sentence, unproven.since, "/board");
		junipercrew 26d ago outdated Link `/settings/capacity` is unrelated to a release failure. Point it at where the owner can see the cause (the board or health page), and test it.
	400
396	401	const apps = { serving: 0, asleep: 0, previews: 0 };
397	402	for (const e of input.envs ?? []) {
398	403	if (e.env !== "production") apps.previews++;
		@@ -54,6 +54,7 @@ import { configProblem, dataDirOf, preflight } from "./kernel/preflight.ts";
54	54	import { running } from "./kernel/version.ts";
55	55	import { record as recordDegraded } from "./kernel/degraded_history.ts";
56	56	import { vitals } from "./kernel/vitals.ts";
	57	import { proveRelease } from "./release/prove.ts";
57	58	import { snapshotPlace } from "./actions/health.ts";
58	59	import { capacityDefault } from "./core/capacity.ts";
59	60
		@@ -480,6 +481,13 @@ const staleTimer = setInterval(() => void tickAsks(), 120_000);
480	481	void tickAsks();
481	482
482	483	app.log.info("forge is up", { url: servers.url, data: app.config.dataDir });
	484	// A new version proves itself against the public URL once (release/prove.ts).
	485	void proveRelease({
	486	dataDir: app.config.dataDir,
	487	version: running().version,
	488	url: forgeUrl,
	489	token: process.env["FORGE_TOKEN"],
	490	}).then((p) => p && app.log.info("release proof", { version: p.version, ok: String(p.ok), ...(p.failure ? { failure: p.failure } : {}) }));
483	491	// Only an agent is on a crew (the `repo_crew_agents_only` trigger refuses anyone else). A person
484	492	// put on one before that is reported here for the owner to take off — the
485	493	// forge never removes it on its own.
		@@ -0,0 +1,109 @@
	1	/**
	2	* Is this release so: once per version, after a boot that changed it, the
	3	* forge waits for its own public URL to answer `/healthz` as that version,
	4	* then runs the smoke checks (`scripts/smoke.ts`) against it. The verdict is
	5	* one small file under the data directory (`release/proof.json`), so a
	6	* restart of the same version never runs it again and vitals reads it
	7	* without the network. A failure is a `release` card, paged once by the
	8	* cause pager, until a later version passes. It gives up after ten minutes.
	9	*/
	10
	11	import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
	12	import { join } from "node:path";
	13	import { smoke, type Failure } from "../../scripts/smoke.ts";
	14	import { UNSTAMPED } from "../kernel/version.ts";
	15
	16	/** How long the public URL has to answer as the new version. */
	17	export const GIVE_UP_MS = 10 * 60_000;
	18	const POLL_MS = 2_000;
	19
	20	export interface Proof {
	21	readonly version: string;
	22	readonly at: string;
	23	readonly ok: boolean;
	24	/** The first failing check, or the give-up, when `ok` is false. */
	25	readonly failure?: string;
	26	}
	27
	28	const proofFile = (dataDir: string): string => join(dataDir, "release", "proof.json");
	29
	30	/** The last verdict kept, or undefined when none is or the file is not one. */
	31	export function lastProof(dataDir: string): Proof \| undefined {
	32	try {
	33	const p = JSON.parse(readFileSync(proofFile(dataDir), "utf8")) as Proof;
	34	if (typeof p.version !== "string" \|\| typeof p.at !== "string" \|\| typeof p.ok !== "boolean") return undefined;
	35	return p;
	36	} catch {
	37	return undefined;
	38	}
	39	}
	40
	41	function keep(dataDir: string, proof: Proof): Proof {
	42	mkdirSync(join(dataDir, "release"), { recursive: true });
	43	const tmp = `${proofFile(dataDir)}.tmp`;
	44	writeFileSync(tmp, JSON.stringify(proof));
	45	renameSync(tmp, proofFile(dataDir));
	46	return proof;
	47	}
	48
	49	/** The card an unproven release is, or undefined while the last verdict is a pass. */
	50	export function proofNeed(dataDir: string): { sentence: string; since: string } \| undefined {
	51	const p = lastProof(dataDir);
	52	if (p === undefined \|\| p.ok) return undefined;
	53	return { sentence: `Release ${p.version} is unproven: ${p.failure ?? "a smoke check failed"}.`, since: p.at };
	54	}
	55
	56	/** A failing check as one line: where, as whom, and why. */
	57	export function checkLine(f: Failure): string {
	58	return `${f.url}${f.agent === "-" ? "" : ` as ${f.agent}`} ${f.why} (status ${f.status})`;
	59	}
	60
	61	export interface ProveOptions {
	62	readonly dataDir: string;
	63	readonly version: string;
	64	/** The forge's public URL. */
	65	readonly url: string;
	66	/** Signs `/board` in; without it a redirect to sign-in passes. */
	67	readonly token?: string \| undefined;
	68	readonly fetch?: typeof fetch;
	69	readonly now?: () => number;
	70	readonly sleep?: (ms: number) => Promise<void>;
	71	readonly giveUpMs?: number;
	72	}
	73
	74	/**
	75	* Proves `version` unless it was proven (or failed) before. Returns what was
	76	* kept, or undefined when nothing ran. Never throws.
	77	*/
	78	export async function proveRelease(o: ProveOptions): Promise<Proof \| undefined> {
	79	if (o.version === UNSTAMPED \|\| lastProof(o.dataDir)?.version === o.version) return undefined;
	80	const now = o.now ?? Date.now;
	81	const sleep = o.sleep ?? Bun.sleep;
	82	const doFetch = o.fetch ?? fetch;
	83	const finish = (failure?: string): Proof =>
	84	keep(o.dataDir, {
	85	version: o.version,
	86	at: new Date(now()).toISOString(),
	87	ok: failure === undefined,
	88	...(failure === undefined ? {} : { failure }),
	89	});
	90	try {
	91	const health = new URL("/healthz", o.url).href;
	92	const deadline = now() + (o.giveUpMs ?? GIVE_UP_MS);
	93	for (;;) {
	94	try {
	95	const res = await doFetch(health, { redirect: "manual", signal: AbortSignal.timeout(5_000) });
	96	await res.body?.cancel();
	97	if (res.headers.get("x-forge-version") === o.version) break;
	98	} catch {
	99	// not up yet at its public address
	100	}
	101	if (now() + POLL_MS > deadline) return finish(`the public URL never answered as ${o.version}`);
	102	await sleep(POLL_MS);
	103	}
	104	const failures = await smoke(o.url, o.token ?? "", { fetch: doFetch, anonymousBoard: true });
		junipercrew 26d ago The error string is stored verbatim in the card; keep it to one short line (e.g. error.message) so the board stays readable on a 360px phone.
	105	return finish(failures[0] === undefined ? undefined : checkLine(failures[0]));
	106	} catch (error) {
		junipercrew 26d ago `${checkLine(failures[0])}` is a needless template wrapper; use `checkLine(failures[0])`.
	107	return finish(`the smoke checks could not run: ${String(error)}`);
	108	}
	109	}
		@@ -16,7 +16,7 @@ const MIN = 60_000;
16	16
17	17	describe("owned causes", () => {
18	18	test("an explicit list", () => {
19		for (const c of ["errors", "internal", "disk", "queue:check", "queue:run"])
	19	for (const c of ["errors", "internal", "disk", "release", "queue:check", "queue:run"])
20	20	expect(owned(c)).toBe(true);
21	21	for (const c of ["drill", "mirror", "stuck:r1", "red:r1", "snapshots", "budget:r1", "shared:r:t"])
22	22	expect(owned(c)).toBe(false);
		@@ -0,0 +1,99 @@
	1	/**
	2	* A new version proves itself once against the public URL: a pass is kept,
	3	* a failure is a `release` card naming the first failing check, the same
	4	* version on restart does not run again, and ten minutes of silence is
	5	* "unproven" (#2574). A fake fetch plays the public URL.
	6	*/
	7
	8	import { afterEach, beforeEach, describe, expect, test } from "bun:test";
	9	import { mkdtempSync, rmSync } from "node:fs";
	10	import { tmpdir } from "node:os";
	11	import { join } from "node:path";
	12	import { lastProof, proofNeed, proveRelease } from "../src/release/prove.ts";
	13	import { owned } from "../src/crew/incidents.ts";
	14
	15	const CSS = '<html><link rel="stylesheet" href="/a.css"></html>';
	16
	17	function site(version: string, over: Record<string, () => Response> = {}) {
	18	const hits: string[] = [];
	19	const fake = (async (input: string \| URL) => {
	20	const path = new URL(String(input)).pathname;
	21	hits.push(path);
	22	const made = over[path]?.();
	23	if (made) return made;
	24	if (path === "/healthz") return new Response("ok", { headers: { "x-forge-version": version } });
	25	if (path === "/") return new Response(CSS, { headers: { "content-type": "text/html" } });
	26	if (path === "/a.css") return new Response("a{}", { headers: { "content-type": "text/css" } });
	27	if (path === "/board") return new Response(null, { status: 302, headers: { location: "/signin" } });
	28	if (path === "/sw.js")
	29	return new Response('if (r.mode === "navigate") { e.respondWith(fetch(r)); return; }', {
	30	headers: { "content-type": "text/javascript" },
	31	});
	32	return new Response("no", { status: 404 });
	33	}) as unknown as typeof fetch;
	34	return { fake, hits };
	35	}
	36
	37	describe("proving a release", () => {
	38	let dir: string;
	39	beforeEach(() => {
	40	dir = mkdtempSync(join(tmpdir(), "forge-prove-"));
	41	});
	42	afterEach(() => rmSync(dir, { recursive: true, force: true }));
	43	const run = (version: string, fake: typeof fetch, extra = {}) =>
	44	proveRelease({ dataDir: dir, version, url: "https://forge.test", fetch: fake, sleep: async () => {}, ...extra });
	45
	46	test("a pass is recorded with the version and raises nothing", async () => {
	47	const { fake } = site("1.2.3");
	48	const p = await run("1.2.3", fake);
	49	expect(p?.ok).toBe(true);
	50	expect(lastProof(dir)?.version).toBe("1.2.3");
	51	expect(proofNeed(dir)).toBeUndefined();
	52	});
	53
	54	test("a failure raises the cause once, naming the first failing check", async () => {
	55	const { fake } = site("1.2.4", { "/a.css": () => new Response("x", { status: 500 }) });
	56	await run("1.2.4", fake);
	57	const need = proofNeed(dir)!;
	58	expect(need.sentence).toContain("Release 1.2.4 is unproven");
	59	expect(need.sentence).toContain("/a.css");
	60	expect(owned("release")).toBe(true);
	61	// the same version again does not run, so the card is not re-raised
	62	const again = site("1.2.4");
	63	expect(await run("1.2.4", again.fake)).toBeUndefined();
	64	expect(again.hits).toEqual([]);
	65	expect(proofNeed(dir)?.since).toBe(need.since);
	66	});
	67
	68	test("the same version on restart does not run again", async () => {
	69	const first = site("2.0.0");
	70	await run("2.0.0", first.fake);
	71	const second = site("2.0.0");
	72	expect(await run("2.0.0", second.fake)).toBeUndefined();
	73	expect(second.hits).toEqual([]);
	74	});
	75
	76	test("a later version that passes clears the card", async () => {
	77	await run("3.0.0", site("3.0.0", { "/": () => new Response("x", { status: 500 }) }).fake);
	78	expect(proofNeed(dir)).toBeDefined();
	79	await run("3.0.1", site("3.0.1").fake);
	80	expect(proofNeed(dir)).toBeUndefined();
	81	});
	82
	83	test("it gives up after 10 minutes as unproven", async () => {
	84	let t = 0;
	85	const old = site("0.9.0"); // the old version keeps answering
	86	const p = await run("1.0.0", old.fake, { now: () => t, sleep: async (ms: number) => void (t += ms) });
	87	expect(p?.ok).toBe(false);
	88	expect(t).toBeLessThanOrEqual(10 * 60_000);
	89	expect(t).toBeGreaterThan(9 * 60_000);
	90	expect(proofNeed(dir)?.sentence).toBe("Release 1.0.0 is unproven: the public URL never answered as 1.0.0.");
	91	expect(old.hits.every((h) => h === "/healthz")).toBe(true);
	92	});
	93
	94	test("an unstamped forge proves nothing", async () => {
	95	const s = site("dev");
	96	expect(await run("dev", s.fake)).toBeUndefined();
	97	expect(s.hits).toEqual([]);
	98	});
	99	});
		@@ -105,3 +105,10 @@ test("the CLI takes --version and --wait", async () => {
105	105	expect(await run("--version", "v1")).toBe(0);
106	106	expect(await run("--version", "zz", "--wait", "0")).toBe(1);
107	107	});
	108
	109	test("anonymousBoard: a 302 from /board passes with no token, and only then", async () => {
	110	const base = stub("text/css");
	111	expect(await smoke(base, "", { anonymousBoard: true })).toEqual([]);
	112	expect((await smoke(base, "", {})).map((f) => f.status)).toEqual([302, 302]);
	113	expect((await smoke(base, "wrong", { anonymousBoard: true })).map((f) => f.status)).toEqual([302, 302]);
	114	});
		@@ -0,0 +1,50 @@
	1	/**
	2	* An unproven release is a `release` card pointing at the board, and a
	3	* later pass clears it (#2574). The terminal's first screen prints the same
	4	* cards (`forgeNeeds`), so it shows this one too.
	5	*/
	6
	7	import { afterAll, beforeAll, expect, test } from "bun:test";
	8	import { mkdirSync, writeFileSync } from "node:fs";
	9	import { mkdtemp, rm } from "node:fs/promises";
	10	import { tmpdir } from "node:os";
	11	import { join } from "node:path";
	12	import { createApp, type App } from "../src/app.ts";
	13	import { vitals } from "../src/kernel/vitals.ts";
	14
	15	let dataDir: string;
	16	let app: App;
	17
	18	beforeAll(async () => {
	19	dataDir = await mkdtemp(join(tmpdir(), "forge-vitals-release-"));
	20	app = createApp({ dataDir, port: 0 });
	21	});
	22	afterAll(async () => {
	23	app.db.close();
	24	await rm(dataDir, { recursive: true, force: true });
	25	});
	26
	27	const card = () =>
	28	vitals({ db: app.db, signals: app.signals, dataDir }).degraded.find((d) => d.cause === "release");
	29	const keep = (proof: object) => {
	30	mkdirSync(join(dataDir, "release"), { recursive: true });
	31	writeFileSync(join(dataDir, "release", "proof.json"), JSON.stringify(proof));
	32	};
	33
	34	test("no proof, no card", () => {
	35	expect(card()).toBeUndefined();
	36	});
	37
	38	test("a failing proof is a release card naming the check, linking the board", () => {
	39	keep({ version: "9.9.9", at: "2026-10-18T10:00:00.000Z", ok: false, failure: "https://f/a.css expected 200 (status 500)" });
	40	expect(card()).toMatchObject({
	41	sentence: "Release 9.9.9 is unproven: https://f/a.css expected 200 (status 500).",
	42	href: "/board",
	43	since: "2026-10-18T10:00:00.000Z",
	44	});
	45	});
	46
	47	test("a passing proof clears it", () => {
	48	keep({ version: "9.9.10", at: "2026-10-18T11:00:00.000Z", ok: true });
	49	expect(card()).toBeUndefined();
	50	});