2c41687 · ea08e17 · 4d26a19 · 4a7d589 · 475e53f · 60658ca
@@ -14,8 +14,8 @@
RUN apt-get update \
WORKDIR /app
COPY package.json tsconfig.json ./
COPY src ./src
# src/ imports from scripts/ (checks/service.ts, release/prove.ts): without it the image cannot boot (0.7.166).
COPY scripts ./scripts
# src/ alone: nothing under it imports scripts/ or test/ (test/src-imports.test.ts),
# and the boot proof below fails the build if that ever stops being so (#2652).
# What the running forge says it is (src/kernel/version.ts): the release
# passes the tag it is building as VERSION and the commit as COMMIT, and the
@@ -29,7 +29,15 @@
ENV FORGE_VERSION=$VERSION FORGE_COMMIT=$COMMIT
ENV FORGE_DATA=/data PORT=8080 FORGE_SSH_PORT=2222
EXPOSE 8080 2222
USER bun
# The build boots what it built (#2652): `src/main.ts` on an empty data
# directory must answer /api/v1/health as $VERSION, or the build fails and
# nothing is pushed. 0.7.166 passed its check from the repository and crashed
# at import from this image.
RUN bun src/release/boot.ts
# tini reaps what an agent run leaves behind: a claude run spawns git, bun,
# the app under test, and their children.
ENTRYPOINT ["tini", "--"]
CMD ["bun", "src/main.ts"]
# The launcher runs src/main.ts, and when this version does not answer health
# within FORGE_READY_GRACE_S (150), runs the last version that did, kept on
# the volume (src/release/launch.ts, #2652).
CMD ["bun", "src/release/launch.ts"]
@@ -243,7 +243,7 @@
forge shows nothing.
A new version proves itself once per version: after a boot that changes
`FORGE_VERSION`, the forge waits (up to 10 minutes) for its public URL to
answer `/healthz` as that version, then runs `scripts/smoke.ts` against it and
answer `/healthz` as that version, then runs the smoke checks (`src/release/smoke.ts`) against it and
keeps the verdict in `$FORGE_APP_DATA/release/proof.json`. A failure is a
`Release X is unproven: …` card on an admin's board and on the terminal's first
screen (the same cards), and is paged once; a later passing version clears it.
@@ -0,0 +1,65 @@
{{/*
The promote preflight (#2652): before an upgrade touches the Deployment, the
new image boots beside the running forge on a copy of its data
(`src/release/boot.ts --from /data`) and must answer health as its version.
A refusal fails the upgrade, so the old pod is never taken down for a
release that cannot start. The volume is local-path, bound to its node, so
this pod schedules where the forge is; it reads the volume, writes only its
own scratch directory on it, and removes that.
*/}}
apiVersion: batch/v1
kind: Job
metadata:
  name: {{ .Release.Name }}-boot-proof
  labels:
    app: {{ .Release.Name }}-boot-proof
  annotations:
    helm.sh/hook: pre-upgrade
    helm.sh/hook-weight: "0"
    # A failed proof is kept for its log until the next upgrade replaces it.
    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
spec:
  backoffLimit: 0
  activeDeadlineSeconds: 300
  template:
    metadata:
      labels:
        app: {{ .Release.Name }}-boot-proof
    spec:
      restartPolicy: Never
      {{- with .Values.imagePullSecrets }}
      imagePullSecrets:
        {{- toYaml . | nindent 8 }}
      {{- end }}
      securityContext:
        runAsNonRoot: true
        runAsUser: 1000
        runAsGroup: 1000
        fsGroup: 1000
        fsGroupChangePolicy: OnRootMismatch
        seccompProfile:
          type: RuntimeDefault
      containers:
        - name: boot-proof
          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
          command: ["bun", "src/release/boot.ts", "--from", "/data", "--budget", "180"]
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop: [ALL]
          volumeMounts:
            - name: data
              mountPath: /data
          # A forge with no crew, no apps and no SSH: a fraction of the
          # running one, so it fits beside it where a surge pod may not.
          resources:
            requests:
              cpu: 100m
              memory: 256Mi
            limits:
              cpu: "1"
              memory: 1Gi
      volumes:
        - name: data
          persistentVolumeClaim:
            claimName: {{ .Release.Name }}-data
@@ -141,3 +141,25 @@
request routed to it in that beat is refused.
Release as the release workflow does. Nothing in this recipe changes the
workflow or the chart.
## A release that does not boot (#2652)
Three guards stand between a release that cannot start and production, all
in `src/release/`:
- **At build.** The Dockerfile runs `bun src/release/boot.ts` on the image it
  just built: `src/main.ts` on an empty data directory must answer
  `/api/v1/health` as `$VERSION` within two minutes, or the build fails and
  nothing is pushed. The image ships `src/` alone; `test/src-imports.test.ts`
  refuses any import from `scripts/` or `test/`.
- **Before the roll.** The chart's pre-upgrade hook
  (`charts/app/templates/boot-proof.yaml`) runs the new image's
  `boot.ts --from /data` beside the running forge, on a copy of its database
  and top-level files. A refusal fails the upgrade while the old pod serves;
  the failed Job is kept for its log (`kubectl -n "$NS" logs job/$REL-boot-proof`).
- **After the roll.** The image starts `src/release/launch.ts`, which runs
  `src/main.ts` and, when that version exits or is not ready within
  `FORGE_READY_GRACE_S` (150 s, inside the startup probe), starts the last
  version that was ready, kept on the volume under `release/good/`. The forge
  that comes up shows a `release:rolled-back` card and pages the admins once.
  The next version that reads ready clears it.
@@ -24,25 +24,10 @@
export const SHARDS_MAX = 32;
 */
export const WORKER_MEMORY_BYTES = 512 * 1024 * 1024;
/**
 * The share of a check cell's declared CPUs the workers are sized to. The
 * cell's watchdog (`src/apps/contain.ts`) stops a check whose tree averages
 * more than `check.limits.cpus` cores over 30 s, and a worker is more than
 * one core at its peaks (Bun's own threads, git children, a booted forge):
 * one worker per declared CPU averaged 16.1 of 16 and stopped main (#2161).
 */
export const CELL_CPU_SHARE = 3 / 4;
/**
 * The CPUs the gate sizes its workers to: the machine's, held under
 * {@link CELL_CPU_SHARE} of the cell's CPU limit (`FORGE_CELL_CPUS`) when
 * there is one, at least one. `availableParallelism()` sees the host, not
 * the cell: the cell is a watchdog, not a CPU quota the process can read.
 */
export function workerCpus(machine: number, declared: number | undefined): number {
  if (declared === undefined || !Number.isFinite(declared) || declared <= 0) return machine;
  return Math.max(1, Math.min(machine, Math.floor(declared * CELL_CPU_SHARE)));
}
// Lives in src/ because the running forge sizes its checks with it, and the
// image ships src/ alone (#2652).
export { CELL_CPU_SHARE, workerCpus } from "../src/core/capacity.ts";
import { workerCpus } from "../src/core/capacity.ts";
/**
 * How many shards: one per CPU the process may use, no more than its
@@ -11,144 +11,8 @@
 * never answers with `Response.error()`.
 */
export const AGENTS: Record<string, string> = {
  phone:
    "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1",
  desktop:
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
};
export interface Failure {
  url: string;
  agent: string;
  status: number | string;
  contentType: string;
  why: string;
}
/** The first stylesheet href in a page, or null. */
export function stylesheetHref(html: string): string | null {
  for (const tag of html.match(/<link\b[^>]*>/gi) ?? []) {
    if (!/\brel\s*=\s*["']?stylesheet/i.test(tag)) continue;
    const href = /\bhref\s*=\s*(?:"([^"]*)"|'([^']*)')/i.exec(tag);
    const value = href?.[1] ?? href?.[2];
    if (value) return value.replaceAll("&amp;", "&");
  }
  return null;
}
/** The part of a service worker that answers navigations, comments removed. */
export function navigationBranch(source: string): string | null {
  const at = source.search(/mode\s*===?\s*["']navigate["']/);
  if (at < 0) return null;
  const rest = source.slice(at);
  const end = rest.search(/\breturn\s*;/);
  return (end < 0 ? rest : rest.slice(0, end)).replace(/\/\*[\s\S]*?\*\//g, "").replace(/\/\/[^\n]*/g, "");
}
export interface Options {
  /** Wait for `/healthz` to report this `X-Forge-Version`. */
  version?: string | undefined;
  /** Longest wait for it, in milliseconds. */
  waitMs?: number;
  /** Pause between polls, in milliseconds. */
  pollMs?: number;
  /** The fetch to use; the global one unless a caller (the forge proving itself, a test) gives its own. */
  fetch?: typeof fetch;
  /** Accept a redirect from `/board` as the sign-in it is, for a caller holding no token. */
  anonymousBoard?: boolean;
}
/** Polls `/healthz` until it reports `version`; null when it does, else a failure. */
async function waitForVersion(root: URL, version: string, waitMs: number, pollMs: number, fetch: typeof globalThis.fetch): Promise<Failure | null> {
  const url = new URL("/healthz", root).href;
  const deadline = Date.now() + waitMs;
  let seen = "none";
  let status: number | string = "error";
  for (;;) {
    try {
      const res = await fetch(url, { redirect: "manual", signal: AbortSignal.timeout(5_000) });
      await res.body?.cancel();
      status = res.status;
      seen = res.headers.get("x-forge-version") ?? "none";
      if (seen === version) return null;
    } catch (error) {
      status = "error";
      seen = `none (${error})`;
    }
    if (Date.now() + pollMs > deadline) break;
    await Bun.sleep(pollMs);
  }
  return { url, agent: "-", status, contentType: "", why: `wanted X-Forge-Version ${version}, last saw ${seen}` };
}
/** Runs every check against `base`; returns the failures (empty is green). */
export async function smoke(base: string, token = "", options: Options = {}): Promise<Failure[]> {
  const root = new URL(base);
  if (root.protocol !== "http:" && root.protocol !== "https:") throw new Error("url must be http(s)");
  const failures: Failure[] = [];
  const fetch = options.fetch ?? globalThis.fetch;
  if (options.version) {
    const late = await waitForVersion(root, options.version, options.waitMs ?? 120_000, options.pollMs ?? 1_000, fetch);
    if (late) return [late];
  }
  const get = async (path: string, agent: string, headers: Record<string, string> = {}) => {
    const url = new URL(path, root).href;
    try {
      const res = await fetch(url, {
        redirect: "manual",
        signal: AbortSignal.timeout(10_000),
        headers: { "user-agent": AGENTS[agent]!, ...headers },
      });
      return { url, res, body: await res.text() };
    } catch (error) {
      failures.push({ url, agent, status: "error", contentType: "", why: String(error) });
      return null;
    }
  };
  const fail = (url: string, agent: string, res: Response, why: string) =>
    failures.push({ url, agent, status: res.status, contentType: res.headers.get("content-type") ?? "", why });
  for (const agent of Object.keys(AGENTS)) {
    const auth: Record<string, string> = token ? { authorization: `Bearer ${token}` } : {};
    const home = await get("/", agent);
    if (home) {
      if (home.res.status !== 200) fail(home.url, agent, home.res, "expected 200");
      else {
        const href = stylesheetHref(home.body);
        if (!href) failures.push({ url: home.url, agent, status: 200, contentType: "", why: "no stylesheet linked" });
        else {
          const css = await get(href, agent);
          if (css) {
            const type = css.res.headers.get("content-type") ?? "";
            if (css.res.status !== 200) fail(css.url, agent, css.res, "expected 200");
            else if (!/^text\/css\b/i.test(type)) fail(css.url, agent, css.res, "expected text/css");
            else if (!css.body.trim()) fail(css.url, agent, css.res, "empty stylesheet");
          }
        }
      }
    }
    const board = await get("/board", agent, auth);
    const signIn = options.anonymousBoard && !token && board?.res.status === 302;
    if (board && board.res.status !== 200 && !signIn) fail(board.url, agent, board.res, "expected 200 (a 302 is sign-in)");
    const health = await get("/healthz", agent);
    if (health && health.res.status !== 200) fail(health.url, agent, health.res, "expected 200");
  }
  const sw = await get("/sw.js", "desktop");
  if (sw) {
    const type = sw.res.headers.get("content-type") ?? "";
    if (sw.res.status !== 200) fail(sw.url, "desktop", sw.res, "expected 200");
    else if (!/^(text|application)\/(x-)?javascript\b/i.test(type)) fail(sw.url, "desktop", sw.res, "expected javascript");
    else {
      const nav = navigationBranch(sw.body);
      if (nav === null) fail(sw.url, "desktop", sw.res, "no navigation branch");
      else if (/Response\s*\.\s*error\s*\(/.test(nav)) fail(sw.url, "desktop", sw.res, "a navigation can be answered with Response.error()");
    }
  }
  return failures;
}
import { smoke } from "../src/release/smoke.ts";
export * from "../src/release/smoke.ts";
if (import.meta.main) {
  const args = process.argv.slice(2);
@@ -61,7 +61,7 @@
import { appendRunLog } from "../runs/log.ts";
import { runningLimitSentence, runningRuns } from "../settings/quotas.ts";
import { exportTree, readFileAt } from "../apps/checkout.ts";
import { parseCheck } from "../apps/manifest.ts";
import { workerCpus } from "../../scripts/shards.ts";
import { workerCpus } from "../core/capacity.ts";
import { accountActive } from "../core/accounts.ts";
import { childEnv, runStep } from "../apps/process.ts";
import { noteReached } from "../apps/egress_report.ts";
@@ -176,6 +176,26 @@
export function cpuLimit(file = "/sys/fs/cgroup/cpu.max"): number {
  return machine;
}
/**
 * The share of a check cell's declared CPUs the workers are sized to. The
 * cell's watchdog (`src/apps/contain.ts`) stops a check whose tree averages
 * more than `check.limits.cpus` cores over 30 s, and a worker is more than
 * one core at its peaks (Bun's own threads, git children, a booted forge):
 * one worker per declared CPU averaged 16.1 of 16 and stopped main (#2161).
 */
export const CELL_CPU_SHARE = 3 / 4;
/**
 * The CPUs the gate sizes its workers to: the machine's, held under
 * {@link CELL_CPU_SHARE} of the cell's CPU limit (`FORGE_CELL_CPUS`) when
 * there is one, at least one. `availableParallelism()` sees the host, not
 * the cell: the cell is a watchdog, not a CPU quota the process can read.
 */
export function workerCpus(machine: number, declared: number | undefined): number {
  if (declared === undefined || !Number.isFinite(declared) || declared <= 0) return machine;
  return Math.max(1, Math.min(machine, Math.floor(declared * CELL_CPU_SHARE)));
}
const GIB = 1024 ** 3;
const MIB = 1024 ** 2;
@@ -31,6 +31,7 @@
import { getUserById, type User } from "../core/users.ts";
import { landedLines } from "../core/watches.ts";
import { err, ok, type Result } from "../kernel/result.ts";
import { lastMirror, mirrorNeed } from "../snapshot/mirror.ts";
import { markRollbackPaged, rollbackNeed, rolledBack } from "../release/launch.ts";
import { drillNeed, firstSeen, lastVerdict } from "../snapshot/verdict.ts";
import { health } from "../http/health.ts";
import { forgeVitals } from "../actions/health.ts";
@@ -309,6 +310,28 @@
export function startNotifier(app: App, options: NotifyOptions): Notifier {
    }
  }
  /**
   * A roll the image's launcher undid (`release/launch.ts`) is news once,
   * ever: it happened before this process began, so no baseline would page
   * it. Marked on disk once it reached an admin, or when no admin can be
   * reached; a page the backoff held is tried again on the pager's tick.
   */
  async function pageRollback(): Promise<void> {
    const dataDir = app.config.dataDir;
    const need = rollbackNeed(dataDir);
    if (need === undefined || rolledBack(dataDir)?.paged !== false) return;
    let reachable = false;
    let delivered = false;
    for (const id of people()) {
      const user = getUserById(app.db, id);
      const url = notifyUrlOf(app.db, id);
      if (user === undefined || url === undefined || !user.isAdmin || user.kind !== "user") continue;
      reachable = true;
      if (await push(id, url, need.sentence)) delivered = true;
    }
    if (delivered || !reachable) markRollbackPaged(dataDir);
  }
  async function pageDrain(userId: string, url: string, baseline: boolean): Promise<void> {
    const current = drainIncident(drainStart(), now());
    if (baseline || !paged.has(userId)) {
@@ -409,6 +432,7 @@
export function startNotifier(app: App, options: NotifyOptions): Notifier {
      await tell(id, true).catch(() => undefined);
      await page(id, true).catch(() => undefined);
    }
    await pageRollback().catch(() => undefined);
  });
  // Eight days of silence is no event: the clock wakes the notifier too.
@@ -428,6 +452,7 @@
export function startNotifier(app: App, options: NotifyOptions): Notifier {
    queue = queue.then(async () => {
      try {
        for (const id of people()) await page(id, false);
        await pageRollback();
      } catch (error) {
        app.log.error("notify failed", { error: String(error) });
      }
@@ -26,6 +26,7 @@
import { lastMirror, mirrorNeed } from "../snapshot/mirror.ts";
import { lastSnapshotOutcome, snapshotNeed } from "../snapshot/outcome.ts";
import { drillNeed, firstSeen as drillFirstSeen, lastVerdict } from "../snapshot/verdict.ts";
import { proofNeed } from "../release/prove.ts";
import { rollbackNeed } from "../release/launch.ts";
import { behindStanding } from "../release/behind.ts";
import { newestDrills } from "../core/domain_drills.ts";
import { DOMAINS } from "./domains.ts";
@@ -399,6 +400,10 @@
export function vitals(input: VitalsInput): Vitals {
  const unproven = proofNeed(input.dataDir);
  if (unproven !== undefined) need("release", unproven.sentence, unproven.since, "/board");
  // A roll the image's launcher undid is a card until a later version reads ready.
  const undone = rollbackNeed(input.dataDir);
  if (undone !== undefined) need("release:rolled-back", undone.sentence, undone.since, "/board");
  // Production serving a version older than a release main has had over an hour is a card until they match.
  const lagging = behindStanding(now);
  if (lagging !== undefined) need("release:behind", `${lagging.sentence}.`, lagging.since, "/board");
@@ -0,0 +1,231 @@
/**
 * Does this tree boot? `bun src/release/boot.ts` starts `src/main.ts` on a
 * scratch data directory and a free loopback port, asks `/api/v1/health`
 * until it answers as the version the image is stamped with, and stops it
 * whatever the answer. One line out, exit 0 or 1 (2 for a bad argument).
 *
 *   bun src/release/boot.ts                    an empty data directory: the
 *                                              release build (Dockerfile)
 *   bun src/release/boot.ts --from /data       a copy of a live one: the
 *                                              promote preflight (chart hook)
 *
 * 0.7.166 passed the full check from the repository and crashed at boot from
 * its image (`Cannot find module '../../scripts/smoke.ts'`); nothing had
 * started the image it was (#2652). This is that start, at build and before
 * the roll. With `--from`, the copy is the database (`VACUUM INTO`, so a
 * forge writing it beside this is read whole) and the data directory's
 * top-level files (the secret key); repositories and app data stay where
 * they are. It lives in a directory of its own under the data directory and
 * is removed in a `finally`.
 *
 * It imports nothing a release could break but the stamp it reads: the child
 * is the thing on trial, not this.
 */
import { Database } from "bun:sqlite";
import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync, statSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { running } from "../kernel/version.ts";
export const BOOT_BUDGET_MS = 120_000;
const POLL_MS = 250;
/** A scratch directory older than this was left by a killed run. */
const STALE_MS = 60 * 60 * 1000;
const SCRATCH = "boot-proof-";
export type BootProof = { ok: true; ms: number } | { ok: false; line: string };
export interface BootInput {
  /** The directory holding `src/main.ts`. */
  readonly tree: string;
  /** What `/api/v1/health` must say it is. */
  readonly version: string;
  /** The data directory the forge boots on; a scratch one, never a live one. */
  readonly dataDir: string;
  readonly budgetMs?: number;
  /** Where PATH, HOME and TMPDIR come from; the process's own unless a test says. */
  readonly env?: Readonly<Record<string, string | undefined>>;
}
async function freePort(): Promise<number> {
  const probe = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch: () => new Response("") });
  const port = probe.port ?? 0;
  await probe.stop(true);
  return port;
}
/** The line of what the child printed that says why: the first error, else the last line. */
export function why(stderr: string): string {
  const lines = stderr
    .split("\n")
    .map((l) => l.trim())
    .filter((l) => l !== "");
  const line = lines.find((l) => /\berror\b/i.test(l)) ?? lines.at(-1) ?? "";
  // eslint-disable-next-line no-control-regex
  return line.replace(/[\u0000-\u001f\u007f]+/g, " ").slice(0, 300);
}
async function versionAt(port: number): Promise<string | undefined> {
  try {
    const res = await fetch(`http://127.0.0.1:${port}/api/v1/health`, {
      headers: { connection: "close" },
      signal: AbortSignal.timeout(2_000),
    });
    if (!res.ok) return undefined;
    const body = (await res.json()) as { version?: unknown };
    return typeof body.version === "string" ? body.version : undefined;
  } catch {
    return undefined;
  }
}
/** Start the tree on `dataDir` and wait for it to answer health as `version`. */
export async function bootProof(input: BootInput): Promise<BootProof> {
  const entry = join(input.tree, "src", "main.ts");
  if (!existsSync(entry)) return { ok: false, line: `${input.tree} has no src/main.ts` };
  const host = input.env ?? process.env;
  const port = await freePort();
  const env: Record<string, string> = {
    PATH: host["PATH"] ?? "/usr/local/bin:/usr/bin:/bin",
    FORGE_DATA: input.dataDir,
    PORT: String(port),
    FORGE_SSH_PORT: "0",
    // The crew stays off a copy of production's queue.
    FORGE_APP_ENV: "preview",
    FORGE_VERSION: input.version,
  };
  for (const key of ["HOME", "TMPDIR", "FORGE_TEST_SOCKET"]) {
    const value = host[key];
    if (value !== undefined && value !== "") env[key] = value;
  }
  const started = performance.now();
  const budget = input.budgetMs ?? BOOT_BUDGET_MS;
  const child = Bun.spawn([process.execPath, entry], {
    cwd: input.tree,
    env,
    stdin: "ignore",
    stdout: "ignore",
    stderr: "pipe",
  });
  let stderr = "";
  const reading = (async () => {
    const decoder = new TextDecoder();
    for await (const chunk of child.stderr) stderr = (stderr + decoder.decode(chunk, { stream: true })).slice(-8_192);
  })().catch(() => undefined);
  let last = "no answer";
  try {
    while (performance.now() - started < budget) {
      if (child.exitCode !== null) {
        await reading;
        const said = why(stderr);
        return { ok: false, line: `exited with ${child.exitCode} before it answered health${said ? `: ${said}` : ""}` };
      }
      const seen = await versionAt(port);
      if (seen === input.version) return { ok: true, ms: Math.round(performance.now() - started) };
      if (seen !== undefined) last = `health answered as ${seen}`;
      await Bun.sleep(POLL_MS);
    }
    const said = why(stderr);
    return {
      ok: false,
      line: `did not answer health as ${input.version} within ${Math.round(budget / 1000)}s (${last})${said ? `: ${said}` : ""}`,
    };
  } finally {
    child.kill("SIGKILL");
    await child.exited;
  }
}
/**
 * Copy what a forge boots on from `from` into `to`: the database, whole, as
 * of one moment, and the data directory's other top-level files.
 */
export function copyData(from: string, to: string): void {
  mkdirSync(to, { recursive: true });
  for (const entry of readdirSync(from, { withFileTypes: true })) {
    if (!entry.isFile() || entry.name.startsWith("forge.db")) continue;
    copyFileSync(join(from, entry.name), join(to, entry.name));
  }
  const live = join(from, "forge.db");
  if (!existsSync(live)) return;
  const db = new Database(live, { readonly: true });
  try {
    db.run("VACUUM INTO ?", [join(to, "forge.db")]);
  } finally {
    db.close();
  }
}
function sweepStale(dir: string): void {
  for (const name of readdirSync(dir)) {
    if (!name.startsWith(SCRATCH)) continue;
    const path = join(dir, name);
    try {
      if (Date.now() - statSync(path).mtimeMs > STALE_MS) rmSync(path, { recursive: true, force: true });
    } catch {
      // Gone already, or not ours: leave it.
    }
  }
}
export interface ProveBootInput {
  readonly tree: string;
  readonly version: string;
  /** A live data directory to boot a copy of; an empty one when absent. */
  readonly from?: string | undefined;
  readonly budgetMs?: number;
  readonly env?: Readonly<Record<string, string | undefined>>;
}
/** The whole proof: a scratch directory, the copy when asked, the boot, the clean-up. */
export async function proveBoot(input: ProveBootInput): Promise<BootProof> {
  let root = input.env?.["TMPDIR"] || tmpdir();
  if (input.from !== undefined) {
    if (!existsSync(join(input.from, "forge.db")))
      return { ok: false, line: `${input.from} holds no forge.db to copy` };
    root = input.from;
    sweepStale(root);
  }
  const scratch = mkdtempSync(join(root, SCRATCH));
  try {
    const dataDir = join(scratch, "data");
    if (input.from !== undefined) copyData(input.from, dataDir);
    else mkdirSync(dataDir);
    return await bootProof({ ...input, dataDir });
  } catch (error) {
    return { ok: false, line: `could not prepare the scratch data directory: ${String(error).slice(0, 200)}` };
  } finally {
    rmSync(scratch, { recursive: true, force: true });
  }
}
/** `--from <dir>` and `--budget <seconds>`, or why not. */
export function parseArgs(argv: readonly string[]): { from?: string; budgetMs: number } | string {
  let from: string | undefined;
  let budgetMs = BOOT_BUDGET_MS;
  for (let i = 0; i < argv.length; i++) {
    const flag = argv[i];
    const value = argv[i + 1];
    if (flag === "--from" && value !== undefined && value !== "") from = value;
    else if (flag === "--budget" && value !== undefined && /^[1-9][0-9]{0,3}$/.test(value)) budgetMs = Number(value) * 1000;
    else return "usage: bun src/release/boot.ts [--from <data dir>] [--budget <seconds, 1 to 9999>]";
    i++;
  }
  return from === undefined ? { budgetMs } : { from, budgetMs };
}
if (import.meta.main) {
  const args = parseArgs(process.argv.slice(2));
  if (typeof args === "string") {
    console.error(args);
    process.exit(2);
  }
  const { version } = running();
  const tree = join(import.meta.dir, "..", "..");
  const result = await proveBoot({ tree, version, ...args });
  const what = args.from === undefined ? "an empty data directory" : `a copy of ${args.from}`;
  if (result.ok) console.log(`boot ok: ${version} answered health on ${what} in ${result.ms}ms`);
  else console.error(`boot failed: ${version} on ${what} ${result.line}`);
  process.exit(result.ok ? 0 : 1);
}
@@ -0,0 +1,262 @@
/**
 * The image's first process (#2652): start this image's forge, and if it
 * does not answer health as its own version within the grace, start the
 * last version that did instead.
 *
 * A version that reads ready is copied onto the volume
 * (`release/good/<version>/`: its `src/` and the two files beside it) and
 * named in `release/good.json`; the one before it is removed. A version
 * that exits first, or is still not ready when the grace runs out, is
 * stopped, `release/rolled-back.json` says what happened, and the kept tree
 * runs under its own stamp. The forge that comes up reads that record: a
 * card while it stands (`kernel/vitals.ts`) and a page to the admins once
 * (`crew/notify.ts`). The next version that reads ready clears it.
 *
 * 0.7.166 crash-looped for two hours because the old pod was gone and
 * nothing undid the roll; with this, the same image serves the old version
 * seconds after its crash, and a hang costs the grace (150 s, inside the
 * chart's three-minute startup probe).
 *
 * Node built-ins only, on purpose: it is the one part of a bad release that
 * has to work.
 */
import { cpSync, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import type { Subprocess } from "bun";
export const READY_GRACE_MS = 150_000;
const POLL_MS = 500;
/** How long a stopped release has to exit before it is killed. */
const STOP_MS = 10_000;
/** What a kept tree is: what the Dockerfile copies into the image. */
const TREE_PARTS = ["src", "package.json", "tsconfig.json"];
const VERSION = /^[0-9A-Za-z][0-9A-Za-z.+_-]{0,63}$/;
export interface Good {
  readonly version: string;
  readonly commit: string | null;
  readonly at: string;
}
export interface RolledBack {
  /** The version that did not become ready. */
  readonly from: string;
  /** The version running in its place. */
  readonly to: string;
  readonly at: string;
  readonly why: string;
  /** Whether the admins were told; set once by the forge that told them. */
  readonly paged: boolean;
}
const releaseDir = (dataDir: string): string => join(dataDir, "release");
const goodFile = (dataDir: string): string => join(releaseDir(dataDir), "good.json");
const rolledFile = (dataDir: string): string => join(releaseDir(dataDir), "rolled-back.json");
export const goodTree = (dataDir: string, version: string): string => join(releaseDir(dataDir), "good", version);
function readJson(path: string): Record<string, unknown> | undefined {
  try {
    const value = JSON.parse(readFileSync(path, "utf8")) as unknown;
    return typeof value === "object" && value !== null ? (value as Record<string, unknown>) : undefined;
  } catch {
    return undefined;
  }
}
function writeJson(path: string, value: object): void {
  mkdirSync(dirname(path), { recursive: true });
  const tmp = `${path}.tmp`;
  writeFileSync(tmp, JSON.stringify(value));
  renameSync(tmp, path);
}
/** The last version that read ready, while its tree is kept. */
export function lastGood(dataDir: string): Good | undefined {
  const g = readJson(goodFile(dataDir));
  const version = g?.["version"];
  if (typeof version !== "string" || !VERSION.test(version) || typeof g?.["at"] !== "string") return undefined;
  if (!existsSync(join(goodTree(dataDir, version), "src", "main.ts"))) return undefined;
  return { version, commit: typeof g["commit"] === "string" ? g["commit"] : null, at: g["at"] };
}
/** Keep `tree` as the last good release; drop the one it replaces and any standing rollback. */
export function keepGood(dataDir: string, tree: string, version: string, commit: string | null, now = Date.now()): void {
  if (!VERSION.test(version)) return;
  if (lastGood(dataDir)?.version !== version) {
    const target = goodTree(dataDir, version);
    const staging = `${target}.partial-${process.pid}`;
    rmSync(staging, { recursive: true, force: true });
    mkdirSync(staging, { recursive: true });
    for (const part of TREE_PARTS)
      if (existsSync(join(tree, part))) cpSync(join(tree, part), join(staging, part), { recursive: true });
    rmSync(target, { recursive: true, force: true });
    renameSync(staging, target);
    writeJson(goodFile(dataDir), { version, commit, at: new Date(now).toISOString() });
  }
  for (const name of readdirSync(join(releaseDir(dataDir), "good")))
    if (name !== version) rmSync(join(releaseDir(dataDir), "good", name), { recursive: true, force: true });
  rmSync(rolledFile(dataDir), { force: true });
}
/** The rollback the launcher made, while it stands. */
export function rolledBack(dataDir: string): RolledBack | undefined {
  const r = readJson(rolledFile(dataDir));
  if (r === undefined) return undefined;
  const { from, to, at, why, paged } = r;
  if (typeof from !== "string" || typeof to !== "string" || typeof at !== "string" || typeof why !== "string")
    return undefined;
  return { from, to, at, why, paged: paged === true };
}
/** Record that the admins were told, so no later boot tells them again. */
export function markRollbackPaged(dataDir: string): void {
  const r = rolledBack(dataDir);
  if (r !== undefined && !r.paged) writeJson(rolledFile(dataDir), { ...r, paged: true });
}
/** The card a standing rollback is. */
export function rollbackNeed(dataDir: string): { sentence: string; since: string } | undefined {
  const r = rolledBack(dataDir);
  if (r === undefined) return undefined;
  return { sentence: `Release ${r.from} did not become ready (${r.why}); rolled back to ${r.to}.`, since: r.at };
}
async function versionAt(port: number): Promise<string | undefined> {
  try {
    const res = await fetch(`http://127.0.0.1:${port}/api/v1/health`, {
      headers: { connection: "close" },
      signal: AbortSignal.timeout(2_000),
    });
    if (!res.ok) return undefined;
    const body = (await res.json()) as { version?: unknown };
    return typeof body.version === "string" ? body.version : undefined;
  } catch {
    return undefined;
  }
}
const gone = (child: Subprocess): boolean => child.exitCode !== null || child.signalCode !== null;
async function stop(child: Subprocess): Promise<void> {
  if (gone(child)) return;
  child.kill("SIGTERM");
  const timer = setTimeout(() => child.kill("SIGKILL"), STOP_MS);
  await child.exited;
  clearTimeout(timer);
}
export interface LaunchInput {
  /** This image's tree: the directory holding `src/main.ts`. */
  readonly tree: string;
  readonly dataDir: string;
  readonly port: number;
  /** What this image is stamped as, and what its health must answer. */
  readonly version: string;
  readonly commit: string | null;
  /** The environment either forge starts with. */
  readonly env: Readonly<Record<string, string | undefined>>;
  readonly graceMs?: number;
  readonly now?: () => number;
  /** Told of each process started, so a signal reaches the one running. */
  readonly started?: (child: Subprocess) => void;
  /** True once the launcher was asked to stop: a release exiting then is no failure. */
  readonly stopping?: () => boolean;
}
export type Launched =
  | { readonly ran: "release" | "rollback"; readonly child: Subprocess; readonly line: string }
  | { readonly ran: "none"; readonly line: string };
function start(input: LaunchInput, tree: string, version: string, commit: string | null): Subprocess {
  const env: Record<string, string> = {};
  for (const [k, v] of Object.entries(input.env)) if (v !== undefined) env[k] = v;
  env["FORGE_VERSION"] = version;
  env["FORGE_COMMIT"] = commit ?? "";
  const child = Bun.spawn([process.execPath, join(tree, "src", "main.ts")], {
    cwd: tree,
    env,
    stdin: "ignore",
    stdout: "inherit",
    stderr: "inherit",
  });
  input.started?.(child);
  return child;
}
/**
 * Start this image's forge and answer once it is ready (`release`), was
 * replaced by the last good one (`rollback`), or neither runs (`none`).
 * The child returned is the one to wait on.
 */
export async function launch(input: LaunchInput): Promise<Launched> {
  const now = input.now ?? Date.now;
  const grace = input.graceMs ?? READY_GRACE_MS;
  const release = start(input, input.tree, input.version, input.commit);
  const began = now();
  let why: string;
  for (;;) {
    if (input.stopping?.()) return { ran: "none", line: "asked to stop before it was ready" };
    if (gone(release)) {
      why = `it exited with ${release.exitCode ?? release.signalCode} before it was ready`;
      break;
    }
    if ((await versionAt(input.port)) === input.version) {
      try {
        keepGood(input.dataDir, input.tree, input.version, input.commit, now());
      } catch (error) {
        // Serving matters more than keeping: the next version falls back to the one before.
        return { ran: "release", child: release, line: `${input.version} is ready; not kept: ${String(error).slice(0, 200)}` };
      }
      return { ran: "release", child: release, line: `${input.version} is ready and kept as the last good release` };
    }
    if (now() - began >= grace) {
      why = `not ready within ${Math.round(grace / 1000)}s`;
      break;
    }
    await Bun.sleep(POLL_MS);
  }
  await stop(release);
  if (input.stopping?.()) return { ran: "none", line: "asked to stop before it was ready" };
  const good = lastGood(input.dataDir);
  if (good === undefined || good.version === input.version)
    return { ran: "none", line: `${input.version}: ${why}; no earlier release is kept to roll back to` };
  writeJson(rolledFile(input.dataDir), {
    from: input.version,
    to: good.version,
    at: new Date(now()).toISOString(),
    why,
    paged: false,
  } satisfies RolledBack);
  const child = start(input, goodTree(input.dataDir, good.version), good.version, good.commit);
  return { ran: "rollback", child, line: `${input.version}: ${why}; rolled back to ${good.version}` };
}
if (import.meta.main) {
  const env = process.env;
  const version = (env["FORGE_VERSION"] ?? "").trim().replace(/^v(?=\d)/, "");
  const commit = (env["FORGE_COMMIT"] ?? "").trim();
  const graceS = Number(env["FORGE_READY_GRACE_S"] ?? "");
  let current: Subprocess | undefined;
  let stopping = false;
  for (const signal of ["SIGTERM", "SIGINT"] as const)
    process.on(signal, () => {
      stopping = true;
      current?.kill(signal);
    });
  const launched = await launch({
    tree: join(import.meta.dir, "..", ".."),
    dataDir: env["FORGE_DATA"] || "data",
    port: Number(env["PORT"] ?? 3000),
    version: VERSION.test(version) ? version : "dev",
    commit: commit === "" ? null : commit,
    env,
    ...(Number.isInteger(graceS) && graceS > 0 ? { graceMs: graceS * 1000 } : {}),
    started: (child) => {
      current = child;
    },
    stopping: () => stopping,
  });
  console.error(`launch: ${launched.line}`);
  process.exit(launched.ran === "none" ? (stopping ? 0 : 1) : await launched.child.exited);
}
@@ -1,7 +1,7 @@
/**
 * Is this release so: once per version, after a boot that changed it, the
 * forge waits for its own public URL to answer `/healthz` as that version,
 * then runs the smoke checks (`scripts/smoke.ts`) against it. The verdict is
 * then runs the smoke checks (`release/smoke.ts`) against it. The verdict is
 * one small file under the data directory (`release/proof.json`), so a
 * restart of the same version never runs it again and vitals reads it
 * without the network. A failure is a `release` card, paged once by the
@@ -10,7 +10,7 @@
import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { smoke, type Failure } from "../../scripts/smoke.ts";
import { smoke, type Failure } from "./smoke.ts";
import { UNSTAMPED } from "../kernel/version.ts";
/** How long the public URL has to answer as the new version. */
@@ -0,0 +1,151 @@
/**
 * Is the forge answering, as a phone and as a desktop? Each of `/`, `/board`
 * (signed in with a token) and `/healthz` is fetched once per user agent and
 * must answer 200; the stylesheet linked from `/` must be a non-empty
 * `text/css` 200; `/sw.js` must be a 200 JavaScript response whose navigation
 * branch never answers with `Response.error()`. With `version`, `/healthz` is
 * first polled until `X-Forge-Version` is that version.
 *
 * The running forge proves each new version with it (`release/prove.ts`), so
 * it lives here: the image ships src/ and nothing else (#2652).
 * `bun scripts/smoke.ts <url>` is the same checks from a shell.
 */
export const AGENTS: Record<string, string> = {
  phone:
    "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1",
  desktop:
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
};
export interface Failure {
  url: string;
  agent: string;
  status: number | string;
  contentType: string;
  why: string;
}
/** The first stylesheet href in a page, or null. */
export function stylesheetHref(html: string): string | null {
  for (const tag of html.match(/<link\b[^>]*>/gi) ?? []) {
    if (!/\brel\s*=\s*["']?stylesheet/i.test(tag)) continue;
    const href = /\bhref\s*=\s*(?:"([^"]*)"|'([^']*)')/i.exec(tag);
    const value = href?.[1] ?? href?.[2];
    if (value) return value.replaceAll("&amp;", "&");
  }
  return null;
}
/** The part of a service worker that answers navigations, comments removed. */
export function navigationBranch(source: string): string | null {
  const at = source.search(/mode\s*===?\s*["']navigate["']/);
  if (at < 0) return null;
  const rest = source.slice(at);
  const end = rest.search(/\breturn\s*;/);
  return (end < 0 ? rest : rest.slice(0, end)).replace(/\/\*[\s\S]*?\*\//g, "").replace(/\/\/[^\n]*/g, "");
}
export interface Options {
  /** Wait for `/healthz` to report this `X-Forge-Version`. */
  version?: string | undefined;
  /** Longest wait for it, in milliseconds. */
  waitMs?: number;
  /** Pause between polls, in milliseconds. */
  pollMs?: number;
  /** The fetch to use; the global one unless a caller (the forge proving itself, a test) gives its own. */
  fetch?: typeof fetch;
  /** Accept a redirect from `/board` as the sign-in it is, for a caller holding no token. */
  anonymousBoard?: boolean;
}
/** Polls `/healthz` until it reports `version`; null when it does, else a failure. */
async function waitForVersion(root: URL, version: string, waitMs: number, pollMs: number, fetch: typeof globalThis.fetch): Promise<Failure | null> {
  const url = new URL("/healthz", root).href;
  const deadline = Date.now() + waitMs;
  let seen = "none";
  let status: number | string = "error";
  for (;;) {
    try {
      const res = await fetch(url, { redirect: "manual", signal: AbortSignal.timeout(5_000) });
      await res.body?.cancel();
      status = res.status;
      seen = res.headers.get("x-forge-version") ?? "none";
      if (seen === version) return null;
    } catch (error) {
      status = "error";
      seen = `none (${error})`;
    }
    if (Date.now() + pollMs > deadline) break;
    await Bun.sleep(pollMs);
  }
  return { url, agent: "-", status, contentType: "", why: `wanted X-Forge-Version ${version}, last saw ${seen}` };
}
/** Runs every check against `base`; returns the failures (empty is green). */
export async function smoke(base: string, token = "", options: Options = {}): Promise<Failure[]> {
  const root = new URL(base);
  if (root.protocol !== "http:" && root.protocol !== "https:") throw new Error("url must be http(s)");
  const failures: Failure[] = [];
  const fetch = options.fetch ?? globalThis.fetch;
  if (options.version) {
    const late = await waitForVersion(root, options.version, options.waitMs ?? 120_000, options.pollMs ?? 1_000, fetch);
    if (late) return [late];
  }
  const get = async (path: string, agent: string, headers: Record<string, string> = {}) => {
    const url = new URL(path, root).href;
    try {
      const res = await fetch(url, {
        redirect: "manual",
        signal: AbortSignal.timeout(10_000),
        headers: { "user-agent": AGENTS[agent]!, ...headers },
      });
      return { url, res, body: await res.text() };
    } catch (error) {
      failures.push({ url, agent, status: "error", contentType: "", why: String(error) });
      return null;
    }
  };
  const fail = (url: string, agent: string, res: Response, why: string) =>
    failures.push({ url, agent, status: res.status, contentType: res.headers.get("content-type") ?? "", why });
  for (const agent of Object.keys(AGENTS)) {
    const auth: Record<string, string> = token ? { authorization: `Bearer ${token}` } : {};
    const home = await get("/", agent);
    if (home) {
      if (home.res.status !== 200) fail(home.url, agent, home.res, "expected 200");
      else {
        const href = stylesheetHref(home.body);
        if (!href) failures.push({ url: home.url, agent, status: 200, contentType: "", why: "no stylesheet linked" });
        else {
          const css = await get(href, agent);
          if (css) {
            const type = css.res.headers.get("content-type") ?? "";
            if (css.res.status !== 200) fail(css.url, agent, css.res, "expected 200");
            else if (!/^text\/css\b/i.test(type)) fail(css.url, agent, css.res, "expected text/css");
            else if (!css.body.trim()) fail(css.url, agent, css.res, "empty stylesheet");
          }
        }
      }
    }
    const board = await get("/board", agent, auth);
    const signIn = options.anonymousBoard && !token && board?.res.status === 302;
    if (board && board.res.status !== 200 && !signIn) fail(board.url, agent, board.res, "expected 200 (a 302 is sign-in)");
    const health = await get("/healthz", agent);
    if (health && health.res.status !== 200) fail(health.url, agent, health.res, "expected 200");
  }
  const sw = await get("/sw.js", "desktop");
  if (sw) {
    const type = sw.res.headers.get("content-type") ?? "";
    if (sw.res.status !== 200) fail(sw.url, "desktop", sw.res, "expected 200");
    else if (!/^(text|application)\/(x-)?javascript\b/i.test(type)) fail(sw.url, "desktop", sw.res, "expected javascript");
    else {
      const nav = navigationBranch(sw.body);
      if (nav === null) fail(sw.url, "desktop", sw.res, "no navigation branch");
      else if (/Response\s*\.\s*error\s*\(/.test(nav)) fail(sw.url, "desktop", sw.res, "a navigation can be answered with Response.error()");
    }
  }
  return failures;
}
@@ -0,0 +1,98 @@
/**
 * A roll the image's launcher undid pages the admins once (#2652). It
 * happened before the forge that serves began, so the boot baseline would
 * swallow it; the record on disk says whether it was told, and a second
 * boot does not tell it again.
 */
import { afterEach, beforeEach, expect, test } from "bun:test";
import { mkdirSync, writeFileSync } from "node:fs";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { createUser } from "../src/core/users.ts";
import { setNotifyUrl, startNotifier } from "../src/crew/notify.ts";
import { vitals } from "../src/kernel/vitals.ts";
import { rolledBack } from "../src/release/launch.ts";
let dataDir: string;
let app: App;
const posts: string[] = [];
let deliver = true;
beforeEach(async () => {
  dataDir = await mkdtemp(join(tmpdir(), "forge-notify-rollback-"));
  app = createApp({ dataDir, port: 0 });
  const made = await createUser(app.db, {
    origin: "seed",
    trusted: true,
    username: "alice",
    email: "[email protected]",
    password: "a-good-passphrase",
    adminIfFirst: true,
  });
  if (!made.ok) throw new Error(made.error);
  setNotifyUrl(app.db, made.value.id, "https://ntfy.example.test/root-topic");
  posts.length = 0;
  deliver = true;
});
afterEach(async () => {
  app.db.close();
  await rm(dataDir, { recursive: true, force: true });
});
const record = () => {
  mkdirSync(join(dataDir, "release"), { recursive: true });
  writeFileSync(
    join(dataDir, "release", "rolled-back.json"),
    JSON.stringify({ from: "0.7.166", to: "0.7.162", at: "2026-10-19T05:17:00.000Z", why: "it exited with 1 before it was ready", paged: false }),
  );
};
async function boot(): Promise<void> {
  const notifier = startNotifier(app, {
    forgeUrl: "https://forge.example.test",
    drainingSince: () => null,
    causes: () => [],
    send: async (_u, body) => {
      if (deliver) posts.push(body);
      return deliver;
    },
  });
  await notifier.settled();
  await notifier.tick();
  await notifier.close();
}
const SENTENCE = "Release 0.7.166 did not become ready (it exited with 1 before it was ready); rolled back to 0.7.162.";
test("a rollback is a card linking the board, and pages once across boots", async () => {
  record();
  expect(vitals({ db: app.db, signals: app.signals, dataDir }).degraded.find((d) => d.cause === "release:rolled-back")).toMatchObject({
    sentence: SENTENCE,
    href: "/board",
    since: "2026-10-19T05:17:00.000Z",
  });
  await boot();
  expect(posts).toEqual([`${SENTENCE}\nhttps://forge.example.test/`]);
  expect(rolledBack(dataDir)?.paged).toBe(true);
  await boot();
  expect(posts).toHaveLength(1);
});
test("a page that was not delivered is not marked, and goes out on a later try", async () => {
  record();
  deliver = false;
  await boot();
  expect(rolledBack(dataDir)?.paged).toBe(false);
  deliver = true;
  await boot();
  expect(posts).toHaveLength(1);
  expect(rolledBack(dataDir)?.paged).toBe(true);
});
test("no record, no page", async () => {
  await boot();
  expect(posts).toEqual([]);
});
@@ -0,0 +1,130 @@
/**
 * A release that does not boot is refused before it ships (#2652): the
 * build and the promote preflight both run `src/release/boot.ts`, which
 * starts the tree on a scratch data directory and wants `/api/v1/health` to
 * answer as the stamped version. 0.7.166's image crashed at import; a tree
 * that does the same is refused here, naming the error.
 */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { Database } from "bun:sqlite";
import { cpSync, existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { bootProof, parseArgs, proveBoot, why } from "../src/release/boot.ts";
let root: string;
beforeAll(() => {
  root = mkdtempSync(join(tmpdir(), "forge-release-boot-"));
});
afterAll(() => rmSync(root, { recursive: true, force: true }));
/** A tree whose `src/main.ts` is `main`. */
function tree(name: string, main: string): string {
  const dir = join(root, name);
  mkdirSync(join(dir, "src"), { recursive: true });
  writeFileSync(join(dir, "src", "main.ts"), main);
  return dir;
}
/** A forge in miniature: health answers with whatever `version()` says. */
const serving = (version: string) => `
Bun.serve({ hostname: "127.0.0.1", port: Number(process.env.PORT), fetch(req) {
  if (new URL(req.url).pathname !== "/api/v1/health") return new Response("", { status: 404 });
  return Response.json({ ok: true, version: ${version} });
} });
`;
test("a tree whose main import throws is refused, naming the module it could not find", async () => {
  const dir = tree("broken", `import { smoke } from "../../scripts/smoke.ts";\nconsole.log(smoke);\n`);
  const result = await proveBoot({ tree: dir, version: "0.7.166", budgetMs: 30_000 });
  expect(result.ok).toBe(false);
  if (!result.ok) {
    expect(result.line).toStartWith("exited with 1 before it answered health: ");
    expect(result.line).toContain("Cannot find module '../../scripts/smoke.ts'");
  }
});
test("a tree that answers as its version passes; the scratch directory is gone after", async () => {
  const dir = tree("well", serving("process.env.FORGE_VERSION"));
  const before = readdirSync(tmpdir()).filter((n) => n.startsWith("boot-proof-")).length;
  const result = await proveBoot({ tree: dir, version: "1.2.3", budgetMs: 30_000 });
  expect(result).toMatchObject({ ok: true });
  expect(readdirSync(tmpdir()).filter((n) => n.startsWith("boot-proof-")).length).toBe(before);
});
test("an answer as another version is not a pass: the budget runs out and says what it saw", async () => {
  const dir = tree("stale", serving(`"0.0.1"`));
  const result = await bootProof({ tree: dir, version: "1.2.3", dataDir: join(root, "stale-data"), budgetMs: 2_000 });
  expect(result).toEqual({ ok: false, line: "did not answer health as 1.2.3 within 2s (health answered as 0.0.1)" });
});
test("--from boots a copy of the live data: the database as it is, the top-level files, never the live directory", async () => {
  const live = join(root, "live");
  mkdirSync(join(live, "repos", "a"), { recursive: true });
  writeFileSync(join(live, "secret.key"), "k", { mode: 0o600 });
  const db = new Database(join(live, "forge.db"), { create: true });
  db.run("PRAGMA journal_mode=WAL");
  db.run("CREATE TABLE stamp (v TEXT)");
  db.run("INSERT INTO stamp VALUES ('4.5.6')");
  // Held open, as the running forge holds it: the row is in the log, not yet the file.
  const dir = tree(
    "copied",
    `import { Database } from "bun:sqlite";
import { existsSync, readFileSync } from "node:fs";
const data = process.env.FORGE_DATA!;
if (readFileSync(data + "/secret.key", "utf8") !== "k" || existsSync(data + "/repos")) process.exit(3);
const v = new Database(data + "/forge.db").query("SELECT v FROM stamp").get().v;
${serving("v")}`,
  );
  try {
    expect(await proveBoot({ tree: dir, version: "4.5.6", from: live, budgetMs: 30_000 })).toMatchObject({ ok: true });
  } finally {
    db.close();
  }
  expect(readdirSync(live).sort()).toEqual(["forge.db", "repos", "secret.key"]);
  expect(await proveBoot({ tree: dir, version: "4.5.6", from: join(root, "nowhere") })).toEqual({
    ok: false,
    line: `${join(root, "nowhere")} holds no forge.db to copy`,
  });
});
test("this tree, shipped as the image ships it (src/ alone), boots", async () => {
  const image = join(root, "image");
  const repo = join(import.meta.dir, "..");
  cpSync(join(repo, "src"), join(image, "src"), { recursive: true });
  for (const file of ["package.json", "tsconfig.json"]) cpSync(join(repo, file), join(image, file));
  expect(existsSync(join(image, "scripts"))).toBe(false);
  const result = await proveBoot({ tree: image, version: "9.9.9", budgetMs: 60_000 });
  expect(result).toMatchObject({ ok: true });
}, 90_000);
test("the reason is the error line, one line, bounded", () => {
  expect(why("\n1 | import x\n    ^\nerror: Cannot find module 'x'\n    at y\n\nBun v1.3\n")).toBe("error: Cannot find module 'x'");
  expect(why("just died\n")).toBe("just died");
  expect(why("")).toBe("");
  expect(why("x".repeat(500)).length).toBe(300);
});
test("arguments: --from and --budget, nothing else", () => {
  expect(parseArgs([])).toEqual({ budgetMs: 120_000 });
  expect(parseArgs(["--from", "/data", "--budget", "60"])).toEqual({ from: "/data", budgetMs: 60_000 });
  for (const bad of [["--from"], ["--budget", "0"], ["--budget", "1e3"], ["--nope"], ["/data"]])
    expect(typeof parseArgs(bad)).toBe("string");
});
test("the build runs the proof on the image it built, and the chart runs it on a copy before an upgrade", async () => {
  const repo = join(import.meta.dir, "..");
  const docker = await Bun.file(join(repo, "Dockerfile")).text();
  const lines = docker.split("\n");
  const copied = lines.findIndex((l) => l.startsWith("COPY src "));
  const proved = lines.indexOf("RUN bun src/release/boot.ts");
  expect(copied).toBeGreaterThan(-1);
  expect(proved).toBeGreaterThan(copied);
  expect(proved).toBeGreaterThan(lines.findIndex((l) => l.startsWith("ENV FORGE_VERSION=")));
  const hook = await Bun.file(join(repo, "charts/app/templates/boot-proof.yaml")).text();
  expect(hook).toContain("helm.sh/hook: pre-upgrade");
  expect(hook).toContain('command: ["bun", "src/release/boot.ts", "--from", "/data", "--budget", "180"]');
  expect(hook).toContain("claimName: {{ .Release.Name }}-data");
  expect(hook).toContain("backoffLimit: 0");
});
@@ -0,0 +1,160 @@
/**
 * A roll to a version that never becomes ready undoes itself (#2652): the
 * image's launcher (`src/release/launch.ts`) starts its forge, and when that
 * forge exits or is not ready within the grace, it starts the last version
 * that was, from the tree kept on the volume, and records the rollback for
 * the forge to show and page once.
 */
import { afterAll, afterEach, beforeAll, expect, test } from "bun:test";
import { existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { Subprocess } from "bun";
import {
  keepGood,
  lastGood,
  launch,
  markRollbackPaged,
  READY_GRACE_MS,
  rollbackNeed,
  rolledBack,
  type Launched,
} from "../src/release/launch.ts";
let root: string;
let n = 0;
const running: Subprocess[] = [];
beforeAll(() => {
  root = mkdtempSync(join(tmpdir(), "forge-release-launch-"));
});
const reap = async () => {
  for (const child of running.splice(0)) {
    child.kill("SIGKILL");
    await child.exited;
  }
};
afterEach(reap);
afterAll(() => rmSync(root, { recursive: true, force: true }));
function tree(main: string): string {
  const dir = join(root, `tree-${++n}`);
  mkdirSync(join(dir, "src"), { recursive: true });
  writeFileSync(join(dir, "src", "main.ts"), main);
  writeFileSync(join(dir, "package.json"), "{}");
  return dir;
}
const SERVES = `
Bun.serve({ hostname: "127.0.0.1", port: Number(process.env.PORT), fetch(req) {
  if (new URL(req.url).pathname !== "/api/v1/health") return new Response("", { status: 404 });
  return Response.json({ ok: true, version: process.env.FORGE_VERSION, commit: process.env.FORGE_COMMIT || null });
} });
`;
const THROWS = `throw new Error("(test fixture) a release whose main throws at import, as 0.7.166's did");\n`;
const HANGS = `setInterval(() => {}, 1_000);\n`;
async function freePort(): Promise<number> {
  const probe = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch: () => new Response("") });
  const port = probe.port ?? 0;
  await probe.stop(true);
  return port;
}
async function run(dataDir: string, dir: string, version: string, graceMs = 30_000): Promise<Launched & { port: number }> {
  const port = await freePort();
  const launched = await launch({
    tree: dir,
    dataDir,
    port,
    version,
    commit: null,
    env: { PATH: process.env.PATH, PORT: String(port) },
    graceMs,
    started: (child) => running.push(child),
  });
  return { ...launched, port };
}
async function healthAt(port: number): Promise<{ version: string; commit: string | null } | undefined> {
  for (let i = 0; i < 100; i++) {
    try {
      return (await (await fetch(`http://127.0.0.1:${port}/api/v1/health`)).json()) as { version: string; commit: string | null };
    } catch {
      await Bun.sleep(100);
    }
  }
  return undefined;
}
test("a version that reads ready is kept as the last good release", async () => {
  const data = join(root, "data-ready");
  expect((await run(data, tree(SERVES), "0.7.162")).ran).toBe("release");
  expect(lastGood(data)).toMatchObject({ version: "0.7.162" });
  expect(existsSync(join(data, "release", "good", "0.7.162", "src", "main.ts"))).toBe(true);
  expect(existsSync(join(data, "release", "good", "0.7.162", "package.json"))).toBe(true);
});
test("a version whose main throws at import is undone at once: the last good one serves under its own stamp", async () => {
  const data = join(root, "data-throws");
  keepGood(data, tree(SERVES), "0.7.162", "a".repeat(40));
  const launched = await run(data, tree(THROWS), "0.7.166");
  expect(launched.ran).toBe("rollback");
  expect(launched.line).toBe("0.7.166: it exited with 1 before it was ready; rolled back to 0.7.162");
  expect(await healthAt(launched.port)).toMatchObject({ version: "0.7.162", commit: "a".repeat(40) });
  expect(rolledBack(data)).toMatchObject({ from: "0.7.166", to: "0.7.162", paged: false });
  expect(rollbackNeed(data)?.sentence).toBe(
    "Release 0.7.166 did not become ready (it exited with 1 before it was ready); rolled back to 0.7.162.",
  );
  expect(lastGood(data)?.version).toBe("0.7.162");
});
test("a version that never becomes ready is undone when the grace runs out, well inside five minutes", async () => {
  expect(READY_GRACE_MS).toBeLessThan(5 * 60_000);
  const data = join(root, "data-hangs");
  keepGood(data, tree(SERVES), "0.7.162", null);
  const launched = await run(data, tree(HANGS), "0.7.167", 2_000);
  expect(launched.ran).toBe("rollback");
  expect(launched.line).toBe("0.7.167: not ready within 2s; rolled back to 0.7.162");
  expect(await healthAt(launched.port)).toMatchObject({ version: "0.7.162" });
});
test("with nothing kept, or only itself, there is nothing to roll back to", async () => {
  expect(await run(join(root, "data-empty"), tree(THROWS), "0.7.166")).toMatchObject({
    ran: "none",
    line: "0.7.166: it exited with 1 before it was ready; no earlier release is kept to roll back to",
  });
  const self = join(root, "data-self");
  keepGood(self, tree(SERVES), "0.7.166", null);
  expect((await run(self, tree(THROWS), "0.7.166")).ran).toBe("none");
  expect(rolledBack(self)).toBeUndefined();
});
test("the next version that reads ready replaces the kept tree and clears the rollback", async () => {
  const data = join(root, "data-next");
  keepGood(data, tree(SERVES), "0.7.162", null);
  await run(data, tree(THROWS), "0.7.166");
  await reap();
  markRollbackPaged(data);
  expect(rolledBack(data)?.paged).toBe(true);
  expect((await run(data, tree(SERVES), "0.7.167")).ran).toBe("release");
  expect(readdirSync(join(data, "release", "good"))).toEqual(["0.7.167"]);
  expect(rolledBack(data)).toBeUndefined();
});
test("a record that is not one reads as none", () => {
  const data = join(root, "data-garbage");
  mkdirSync(join(data, "release"), { recursive: true });
  writeFileSync(join(data, "release", "rolled-back.json"), "{nope");
  writeFileSync(join(data, "release", "good.json"), JSON.stringify({ version: "../x", at: "t" }));
  expect(rolledBack(data)).toBeUndefined();
  expect(lastGood(data)).toBeUndefined();
});
test("the image starts the launcher, and the launcher imports nothing a release could break", async () => {
  const repo = join(import.meta.dir, "..");
  expect(await Bun.file(join(repo, "Dockerfile")).text()).toContain('CMD ["bun", "src/release/launch.ts"]');
  const source = await Bun.file(join(repo, "src/release/launch.ts")).text();
  const imports = new Bun.Transpiler({ loader: "ts" }).scanImports(source).map((i) => i.path);
  expect(imports.every((p) => p.startsWith("node:") || p === "bun")).toBe(true);
});
@@ -28,6 +28,7 @@
const PIPING: Record<string, { sites: string[]; reason: string }> = {
  "src/git/smart-ssh.ts": { sites: ["stdout+stderr"], reason: "pump() drains stdout and stderr into the channel" },
  "src/kernel/data_home.ts": { sites: ["stdout"], reason: "spawnSync buffers stdout; stderr ignored" },
  "src/looks/capture.ts": { sites: ["stderr"], reason: "stderr read in a background loop; stdout ignored" },
  "src/release/boot.ts": { sites: ["stderr"], reason: "stderr read to its end in a background loop, awaited before the reason is read; stdout ignored" },
  "src/release/checkout.ts": { sites: ["stdout+stderr"], reason: "stdout and stderr both read via Response.text()" },
  "src/cli/config.ts": { sites: ["stdout"], reason: "stdout awaited with the exit; stderr ignored" },
};
@@ -0,0 +1,70 @@
/**
 * The image ships `src/` and nothing else (Dockerfile): a module under src/
 * that imports from scripts/ or test/ boots from the repository and crashes
 * in production. 0.7.166 did, on `src/release/prove.ts` importing
 * `scripts/smoke.ts`, and was down two hours (#2652). Every relative import
 * a src/ file makes, static or dynamic, must resolve inside src/.
 */
import { expect, test } from "bun:test";
import { mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { dirname, join, relative, resolve, sep } from "node:path";
const ROOT = resolve(import.meta.dir, "..");
const SRC = join(ROOT, "src");
function sources(dir: string): string[] {
  const out: string[] = [];
  for (const entry of readdirSync(dir, { withFileTypes: true })) {
    const path = join(dir, entry.name);
    if (entry.isDirectory()) out.push(...sources(path));
    else if (/\.(ts|tsx|js|mjs)$/.test(entry.name) && !entry.name.endsWith(".d.ts")) out.push(path);
  }
  return out;
}
/** Each relative import in `files` that resolves outside `src`, as `file -> specifier`. */
export function escapes(files: readonly string[], src: string): string[] {
  const scanners = { ts: new Bun.Transpiler({ loader: "ts" }), tsx: new Bun.Transpiler({ loader: "tsx" }) };
  const bad: string[] = [];
  for (const file of files) {
    const scanner = file.endsWith(".tsx") ? scanners.tsx : scanners.ts;
    for (const { path } of scanner.scanImports(readFileSync(file, "utf8").replace(/^#!.*/, ""))) {
      if (!path.startsWith(".")) continue;
      const target = resolve(dirname(file), path);
      if (target !== src && !target.startsWith(src + sep)) bad.push(`${relative(ROOT, file)} -> ${path}`);
    }
  }
  return bad;
}
test("nothing under src/ imports from outside src/ (scripts/, test/, …)", () => {
  expect(escapes(sources(SRC), SRC)).toEqual([]);
});
test("the walk sees a static import, a re-export and a dynamic import that leave src/", () => {
  const dir = join(process.env.TMPDIR ?? "/tmp", `src-imports-${process.pid}`);
  const src = join(dir, "src");
  const file = join(src, "release", "x.ts");
  mkdirSync(dirname(file), { recursive: true });
  writeFileSync(
    file,
    [
      'import { smoke } from "../../scripts/smoke.ts";',
      'export { h } from "../../test/helpers/h.ts";',
      'const m = await import("../../scripts/shards.ts");',
      'import { ok } from "../kernel/result.ts";',
      'const text = `import { lint } from "../../scripts/lint-design.ts";`;',
      "console.log(smoke, m, ok, text);",
    ].join("\n"),
  );
  try {
    expect(escapes([file], src).map((l) => l.split(" -> ")[1])).toEqual([
      "../../scripts/smoke.ts",
      "../../test/helpers/h.ts",
      "../../scripts/shards.ts",
    ]);
  } finally {
    rmSync(dir, { recursive: true, force: true });
  }
});
		@@ -14,8 +14,8 @@ RUN apt-get update \
14	14	WORKDIR /app
15	15	COPY package.json tsconfig.json ./
16	16	COPY src ./src
17		# src/ imports from scripts/ (checks/service.ts, release/prove.ts): without it the image cannot boot (0.7.166).
18		COPY scripts ./scripts
	17	# src/ alone: nothing under it imports scripts/ or test/ (test/src-imports.test.ts),
	18	# and the boot proof below fails the build if that ever stops being so (#2652).
19	19
20	20	# What the running forge says it is (src/kernel/version.ts): the release
21	21	# passes the tag it is building as VERSION and the commit as COMMIT, and the
		@@ -29,7 +29,15 @@ ENV FORGE_VERSION=$VERSION FORGE_COMMIT=$COMMIT
29	29	ENV FORGE_DATA=/data PORT=8080 FORGE_SSH_PORT=2222
30	30	EXPOSE 8080 2222
31	31	USER bun
	32	# The build boots what it built (#2652): `src/main.ts` on an empty data
	33	# directory must answer /api/v1/health as $VERSION, or the build fails and
	34	# nothing is pushed. 0.7.166 passed its check from the repository and crashed
	35	# at import from this image.
	36	RUN bun src/release/boot.ts
32	37	# tini reaps what an agent run leaves behind: a claude run spawns git, bun,
33	38	# the app under test, and their children.
34	39	ENTRYPOINT ["tini", "--"]
35		CMD ["bun", "src/main.ts"]
	40	# The launcher runs src/main.ts, and when this version does not answer health
	41	# within FORGE_READY_GRACE_S (150), runs the last version that did, kept on
	42	# the volume (src/release/launch.ts, #2652).
	43	CMD ["bun", "src/release/launch.ts"]
		@@ -243,7 +243,7 @@ forge shows nothing.
243	243
244	244	A new version proves itself once per version: after a boot that changes
245	245	`FORGE_VERSION`, the forge waits (up to 10 minutes) for its public URL to
246		answer `/healthz` as that version, then runs `scripts/smoke.ts` against it and
	246	answer `/healthz` as that version, then runs the smoke checks (`src/release/smoke.ts`) against it and
247	247	keeps the verdict in `$FORGE_APP_DATA/release/proof.json`. A failure is a
248	248	`Release X is unproven: …` card on an admin's board and on the terminal's first
249	249	screen (the same cards), and is paged once; a later passing version clears it.
		@@ -0,0 +1,65 @@
	1	{{/*
	2	The promote preflight (#2652): before an upgrade touches the Deployment, the
	3	new image boots beside the running forge on a copy of its data
	4	(`src/release/boot.ts --from /data`) and must answer health as its version.
	5	A refusal fails the upgrade, so the old pod is never taken down for a
	6	release that cannot start. The volume is local-path, bound to its node, so
	7	this pod schedules where the forge is; it reads the volume, writes only its
	8	own scratch directory on it, and removes that.
	9	*/}}
	10	apiVersion: batch/v1
	11	kind: Job
	12	metadata:
	13	name: {{ .Release.Name }}-boot-proof
	14	labels:
	15	app: {{ .Release.Name }}-boot-proof
	16	annotations:
	17	helm.sh/hook: pre-upgrade
	18	helm.sh/hook-weight: "0"
	19	# A failed proof is kept for its log until the next upgrade replaces it.
	20	helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
	21	spec:
	22	backoffLimit: 0
	23	activeDeadlineSeconds: 300
	24	template:
	25	metadata:
	26	labels:
	27	app: {{ .Release.Name }}-boot-proof
	28	spec:
	29	restartPolicy: Never
	30	{{- with .Values.imagePullSecrets }}
	31	imagePullSecrets:
	32	{{- toYaml . \| nindent 8 }}
	33	{{- end }}
	34	securityContext:
	35	runAsNonRoot: true
	36	runAsUser: 1000
	37	runAsGroup: 1000
	38	fsGroup: 1000
	39	fsGroupChangePolicy: OnRootMismatch
	40	seccompProfile:
	41	type: RuntimeDefault
	42	containers:
	43	- name: boot-proof
	44	image: "{{ .Values.image.repository }}:{{ .Values.image.tag \| default .Chart.AppVersion }}"
	45	command: ["bun", "src/release/boot.ts", "--from", "/data", "--budget", "180"]
	46	securityContext:
	47	allowPrivilegeEscalation: false
	48	capabilities:
	49	drop: [ALL]
	50	volumeMounts:
	51	- name: data
	52	mountPath: /data
	53	# A forge with no crew, no apps and no SSH: a fraction of the
	54	# running one, so it fits beside it where a surge pod may not.
	55	resources:
	56	requests:
	57	cpu: 100m
	58	memory: 256Mi
	59	limits:
	60	cpu: "1"
	61	memory: 1Gi
	62	volumes:
	63	- name: data
	64	persistentVolumeClaim:
	65	claimName: {{ .Release.Name }}-data
		@@ -141,3 +141,25 @@ request routed to it in that beat is refused.
141	141
142	142	Release as the release workflow does. Nothing in this recipe changes the
143	143	workflow or the chart.
	144
	145	## A release that does not boot (#2652)
	146
	147	Three guards stand between a release that cannot start and production, all
	148	in `src/release/`:
	149
	150	- At build. The Dockerfile runs `bun src/release/boot.ts` on the image it
	151	just built: `src/main.ts` on an empty data directory must answer
	152	`/api/v1/health` as `$VERSION` within two minutes, or the build fails and
	153	nothing is pushed. The image ships `src/` alone; `test/src-imports.test.ts`
	154	refuses any import from `scripts/` or `test/`.
	155	- Before the roll. The chart's pre-upgrade hook
	156	(`charts/app/templates/boot-proof.yaml`) runs the new image's
	157	`boot.ts --from /data` beside the running forge, on a copy of its database
	158	and top-level files. A refusal fails the upgrade while the old pod serves;
	159	the failed Job is kept for its log (`kubectl -n "$NS" logs job/$REL-boot-proof`).
	160	- After the roll. The image starts `src/release/launch.ts`, which runs
	161	`src/main.ts` and, when that version exits or is not ready within
	162	`FORGE_READY_GRACE_S` (150 s, inside the startup probe), starts the last
	163	version that was ready, kept on the volume under `release/good/`. The forge
	164	that comes up shows a `release:rolled-back` card and pages the admins once.
	165	The next version that reads ready clears it.
		@@ -24,25 +24,10 @@ export const SHARDS_MAX = 32;
24	24	*/
25	25	export const WORKER_MEMORY_BYTES = 512 * 1024 * 1024;
26	26
27		/**
28		* The share of a check cell's declared CPUs the workers are sized to. The
29		* cell's watchdog (`src/apps/contain.ts`) stops a check whose tree averages
30		* more than `check.limits.cpus` cores over 30 s, and a worker is more than
31		* one core at its peaks (Bun's own threads, git children, a booted forge):
32		* one worker per declared CPU averaged 16.1 of 16 and stopped main (#2161).
33		*/
34		export const CELL_CPU_SHARE = 3 / 4;
35
36		/**
37		* The CPUs the gate sizes its workers to: the machine's, held under
38		* {@link CELL_CPU_SHARE} of the cell's CPU limit (`FORGE_CELL_CPUS`) when
39		* there is one, at least one. `availableParallelism()` sees the host, not
40		* the cell: the cell is a watchdog, not a CPU quota the process can read.
41		*/
42		export function workerCpus(machine: number, declared: number \| undefined): number {
43		if (declared === undefined \|\| !Number.isFinite(declared) \|\| declared <= 0) return machine;
44		return Math.max(1, Math.min(machine, Math.floor(declared * CELL_CPU_SHARE)));
45		}
	27	// Lives in src/ because the running forge sizes its checks with it, and the
	28	// image ships src/ alone (#2652).
	29	export { CELL_CPU_SHARE, workerCpus } from "../src/core/capacity.ts";
	30	import { workerCpus } from "../src/core/capacity.ts";
46	31
47	32	/**
48	33	* How many shards: one per CPU the process may use, no more than its
		@@ -11,144 +11,8 @@
11	11	* never answers with `Response.error()`.
12	12	*/
13	13
14		export const AGENTS: Record<string, string> = {
15		phone:
16		"Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1",
17		desktop:
18		"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
19		};
20
21		export interface Failure {
22		url: string;
23		agent: string;
24		status: number \| string;
25		contentType: string;
26		why: string;
27		}
28
29		/** The first stylesheet href in a page, or null. */
30		export function stylesheetHref(html: string): string \| null {
31		for (const tag of html.match(/<link\b[^>]*>/gi) ?? []) {
32		if (!/\brel\s=\s["']?stylesheet/i.test(tag)) continue;
33		const href = /\bhref\s=\s(?:"([^"])"\|'([^'])')/i.exec(tag);
34		const value = href?.[1] ?? href?.[2];
35		if (value) return value.replaceAll("&", "&");
36		}
37		return null;
38		}
39
40		/** The part of a service worker that answers navigations, comments removed. */
41		export function navigationBranch(source: string): string \| null {
42		const at = source.search(/mode\s===?\s["']navigate["']/);
43		if (at < 0) return null;
44		const rest = source.slice(at);
45		const end = rest.search(/\breturn\s*;/);
46		return (end < 0 ? rest : rest.slice(0, end)).replace(/\/\[\s\S]?\\//g, "").replace(/\/\/[^\n]/g, "");
47		}
48
49		export interface Options {
50		/** Wait for `/healthz` to report this `X-Forge-Version`. */
51		version?: string \| undefined;
52		/** Longest wait for it, in milliseconds. */
53		waitMs?: number;
54		/** Pause between polls, in milliseconds. */
55		pollMs?: number;
56		/** The fetch to use; the global one unless a caller (the forge proving itself, a test) gives its own. */
57		fetch?: typeof fetch;
58		/** Accept a redirect from `/board` as the sign-in it is, for a caller holding no token. */
59		anonymousBoard?: boolean;
60		}
61
62		/** Polls `/healthz` until it reports `version`; null when it does, else a failure. */
63		async function waitForVersion(root: URL, version: string, waitMs: number, pollMs: number, fetch: typeof globalThis.fetch): Promise<Failure \| null> {
64		const url = new URL("/healthz", root).href;
65		const deadline = Date.now() + waitMs;
66		let seen = "none";
67		let status: number \| string = "error";
68		for (;;) {
69		try {
70		const res = await fetch(url, { redirect: "manual", signal: AbortSignal.timeout(5_000) });
71		await res.body?.cancel();
72		status = res.status;
73		seen = res.headers.get("x-forge-version") ?? "none";
74		if (seen === version) return null;
75		} catch (error) {
76		status = "error";
77		seen = `none (${error})`;
78		}
79		if (Date.now() + pollMs > deadline) break;
80		await Bun.sleep(pollMs);
81		}
82		return { url, agent: "-", status, contentType: "", why: `wanted X-Forge-Version ${version}, last saw ${seen}` };
83		}
84
85		/** Runs every check against `base`; returns the failures (empty is green). */
86		export async function smoke(base: string, token = "", options: Options = {}): Promise<Failure[]> {
87		const root = new URL(base);
88		if (root.protocol !== "http:" && root.protocol !== "https:") throw new Error("url must be http(s)");
89		const failures: Failure[] = [];
90		const fetch = options.fetch ?? globalThis.fetch;
91		if (options.version) {
92		const late = await waitForVersion(root, options.version, options.waitMs ?? 120_000, options.pollMs ?? 1_000, fetch);
93		if (late) return [late];
94		}
95
96		const get = async (path: string, agent: string, headers: Record<string, string> = {}) => {
97		const url = new URL(path, root).href;
98		try {
99		const res = await fetch(url, {
100		redirect: "manual",
101		signal: AbortSignal.timeout(10_000),
102		headers: { "user-agent": AGENTS[agent]!, ...headers },
103		});
104		return { url, res, body: await res.text() };
105		} catch (error) {
106		failures.push({ url, agent, status: "error", contentType: "", why: String(error) });
107		return null;
108		}
109		};
110		const fail = (url: string, agent: string, res: Response, why: string) =>
111		failures.push({ url, agent, status: res.status, contentType: res.headers.get("content-type") ?? "", why });
112
113		for (const agent of Object.keys(AGENTS)) {
114		const auth: Record<string, string> = token ? { authorization: `Bearer ${token}` } : {};
115		const home = await get("/", agent);
116		if (home) {
117		if (home.res.status !== 200) fail(home.url, agent, home.res, "expected 200");
118		else {
119		const href = stylesheetHref(home.body);
120		if (!href) failures.push({ url: home.url, agent, status: 200, contentType: "", why: "no stylesheet linked" });
121		else {
122		const css = await get(href, agent);
123		if (css) {
124		const type = css.res.headers.get("content-type") ?? "";
125		if (css.res.status !== 200) fail(css.url, agent, css.res, "expected 200");
126		else if (!/^text\/css\b/i.test(type)) fail(css.url, agent, css.res, "expected text/css");
127		else if (!css.body.trim()) fail(css.url, agent, css.res, "empty stylesheet");
128		}
129		}
130		}
131		}
132		const board = await get("/board", agent, auth);
133		const signIn = options.anonymousBoard && !token && board?.res.status === 302;
134		if (board && board.res.status !== 200 && !signIn) fail(board.url, agent, board.res, "expected 200 (a 302 is sign-in)");
135		const health = await get("/healthz", agent);
136		if (health && health.res.status !== 200) fail(health.url, agent, health.res, "expected 200");
137		}
138
139		const sw = await get("/sw.js", "desktop");
140		if (sw) {
141		const type = sw.res.headers.get("content-type") ?? "";
142		if (sw.res.status !== 200) fail(sw.url, "desktop", sw.res, "expected 200");
143		else if (!/^(text\|application)\/(x-)?javascript\b/i.test(type)) fail(sw.url, "desktop", sw.res, "expected javascript");
144		else {
145		const nav = navigationBranch(sw.body);
146		if (nav === null) fail(sw.url, "desktop", sw.res, "no navigation branch");
147		else if (/Response\s\.\serror\s*\(/.test(nav)) fail(sw.url, "desktop", sw.res, "a navigation can be answered with Response.error()");
148		}
149		}
150		return failures;
151		}
	14	import { smoke } from "../src/release/smoke.ts";
	15	export * from "../src/release/smoke.ts";
152	16
153	17	if (import.meta.main) {
154	18	const args = process.argv.slice(2);
		@@ -61,7 +61,7 @@ import { appendRunLog } from "../runs/log.ts";
61	61	import { runningLimitSentence, runningRuns } from "../settings/quotas.ts";
62	62	import { exportTree, readFileAt } from "../apps/checkout.ts";
63	63	import { parseCheck } from "../apps/manifest.ts";
64		import { workerCpus } from "../../scripts/shards.ts";
	64	import { workerCpus } from "../core/capacity.ts";
65	65	import { accountActive } from "../core/accounts.ts";
66	66	import { childEnv, runStep } from "../apps/process.ts";
67	67	import { noteReached } from "../apps/egress_report.ts";
		@@ -176,6 +176,26 @@ export function cpuLimit(file = "/sys/fs/cgroup/cpu.max"): number {
176	176	return machine;
177	177	}
178	178
	179	/**
	180	* The share of a check cell's declared CPUs the workers are sized to. The
	181	* cell's watchdog (`src/apps/contain.ts`) stops a check whose tree averages
	182	* more than `check.limits.cpus` cores over 30 s, and a worker is more than
	183	* one core at its peaks (Bun's own threads, git children, a booted forge):
	184	* one worker per declared CPU averaged 16.1 of 16 and stopped main (#2161).
	185	*/
	186	export const CELL_CPU_SHARE = 3 / 4;
	187
	188	/**
	189	* The CPUs the gate sizes its workers to: the machine's, held under
	190	* {@link CELL_CPU_SHARE} of the cell's CPU limit (`FORGE_CELL_CPUS`) when
	191	* there is one, at least one. `availableParallelism()` sees the host, not
	192	* the cell: the cell is a watchdog, not a CPU quota the process can read.
	193	*/
	194	export function workerCpus(machine: number, declared: number \| undefined): number {
	195	if (declared === undefined \|\| !Number.isFinite(declared) \|\| declared <= 0) return machine;
	196	return Math.max(1, Math.min(machine, Math.floor(declared * CELL_CPU_SHARE)));
	197	}
	198
179	199	const GIB = 1024 ** 3;
180	200	const MIB = 1024 ** 2;
181	201
		@@ -31,6 +31,7 @@ import { getUserById, type User } from "../core/users.ts";
31	31	import { landedLines } from "../core/watches.ts";
32	32	import { err, ok, type Result } from "../kernel/result.ts";
33	33	import { lastMirror, mirrorNeed } from "../snapshot/mirror.ts";
	34	import { markRollbackPaged, rollbackNeed, rolledBack } from "../release/launch.ts";
34	35	import { drillNeed, firstSeen, lastVerdict } from "../snapshot/verdict.ts";
35	36	import { health } from "../http/health.ts";
36	37	import { forgeVitals } from "../actions/health.ts";
		@@ -309,6 +310,28 @@ export function startNotifier(app: App, options: NotifyOptions): Notifier {
309	310	}
310	311	}
311	312
	313	/**
	314	* A roll the image's launcher undid (`release/launch.ts`) is news once,
	315	* ever: it happened before this process began, so no baseline would page
	316	* it. Marked on disk once it reached an admin, or when no admin can be
	317	* reached; a page the backoff held is tried again on the pager's tick.
	318	*/
	319	async function pageRollback(): Promise<void> {
	320	const dataDir = app.config.dataDir;
	321	const need = rollbackNeed(dataDir);
	322	if (need === undefined \|\| rolledBack(dataDir)?.paged !== false) return;
	323	let reachable = false;
	324	let delivered = false;
	325	for (const id of people()) {
	326	const user = getUserById(app.db, id);
	327	const url = notifyUrlOf(app.db, id);
	328	if (user === undefined \|\| url === undefined \|\| !user.isAdmin \|\| user.kind !== "user") continue;
	329	reachable = true;
	330	if (await push(id, url, need.sentence)) delivered = true;
	331	}
	332	if (delivered \|\| !reachable) markRollbackPaged(dataDir);
	333	}
	334
312	335	async function pageDrain(userId: string, url: string, baseline: boolean): Promise<void> {
313	336	const current = drainIncident(drainStart(), now());
314	337	if (baseline \|\| !paged.has(userId)) {
		@@ -409,6 +432,7 @@ export function startNotifier(app: App, options: NotifyOptions): Notifier {
409	432	await tell(id, true).catch(() => undefined);
410	433	await page(id, true).catch(() => undefined);
411	434	}
	435	await pageRollback().catch(() => undefined);
412	436	});
413	437
414	438	// Eight days of silence is no event: the clock wakes the notifier too.
		@@ -428,6 +452,7 @@ export function startNotifier(app: App, options: NotifyOptions): Notifier {
428	452	queue = queue.then(async () => {
429	453	try {
430	454	for (const id of people()) await page(id, false);
	455	await pageRollback();
431	456	} catch (error) {
432	457	app.log.error("notify failed", { error: String(error) });
433	458	}
		@@ -26,6 +26,7 @@ import { lastMirror, mirrorNeed } from "../snapshot/mirror.ts";
26	26	import { lastSnapshotOutcome, snapshotNeed } from "../snapshot/outcome.ts";
27	27	import { drillNeed, firstSeen as drillFirstSeen, lastVerdict } from "../snapshot/verdict.ts";
28	28	import { proofNeed } from "../release/prove.ts";
	29	import { rollbackNeed } from "../release/launch.ts";
29	30	import { behindStanding } from "../release/behind.ts";
30	31	import { newestDrills } from "../core/domain_drills.ts";
31	32	import { DOMAINS } from "./domains.ts";
		@@ -399,6 +400,10 @@ export function vitals(input: VitalsInput): Vitals {
399	400	const unproven = proofNeed(input.dataDir);
400	401	if (unproven !== undefined) need("release", unproven.sentence, unproven.since, "/board");
401	402
	403	// A roll the image's launcher undid is a card until a later version reads ready.
	404	const undone = rollbackNeed(input.dataDir);
	405	if (undone !== undefined) need("release:rolled-back", undone.sentence, undone.since, "/board");
	406
402	407	// Production serving a version older than a release main has had over an hour is a card until they match.
403	408	const lagging = behindStanding(now);
404	409	if (lagging !== undefined) need("release:behind", `${lagging.sentence}.`, lagging.since, "/board");