49d261c · 1ef2d65 · 55e6221
@@ -1555,7 +1555,7 @@
work here.
  `forge/boards`, a repo the forge owns and makes on the first boot with an admin; every save is a commit
  by the admin who made it, and a file that does not validate is not shown. Make, rename,
  duplicate, delete, add/edit/remove/reorder panels, or pin the query on `/boards/data`.
  The first boot also writes six starting boards, **Usage**, **Crew**, **Releases**, **Apps**, **Security** and **Throughput**. Apps' per-app panels show repo ids until a source exposes a safe name; Security (sign-ins, failed sign-ins, tokens by scope, admin actions, from the admin-only `signins`, `tokens` and `adminActions` sources and `usage`) shows no token text, hash, payload or IP; Throughput is items entering each stage from issue to live, per hour, from the `stages` source. Each is written once: `.offered` in that repo records them, so an edit stays and a deletion sticks.
  The first boot also writes seven starting boards, **Usage**, **Crew**, **Releases**, **Apps**, **Security**, **Throughput** and **Forges**. Apps' per-app panels show repo ids until a source exposes a safe name; Security (sign-ins, failed sign-ins, tokens by scope, admin actions, from the admin-only `signins`, `tokens` and `adminActions` sources and `usage`) shows no token text, hash, payload or IP; Throughput is items entering each stage from issue to live, per hour, from the `stages` source. Each is written once: `.offered` in that repo records them, so an edit stays and a deletion sticks.
  A live board (any `refresh` but `off`; the form offers off or on, and `10s` and `1m` are still accepted and mean the same) redraws its grid when a run, pull, check, issue or usage event arrives, through the live-region swap
  (`/boards/:slug/live`, etag'd, one fetch per burst). The same doors are in the API: `GET /api/v1/boards`, `GET|PUT|DELETE /api/v1/boards/:slug`
  (an agent or anyone but an admin is 403; a panel whose query cannot run is refused with 422). The same four doors are MCP tools (`boards`, `board_write`, `board_delete`; `boards` with a `slug` reads one) and, for the same person, a **Boards** screen in the terminal that lists boards and shows a board's panels as text. No dependency was added.
@@ -1201,6 +1201,19 @@
leaves its series absent, never zero, and `/boards/data?source=machine`
names each one. Its rows belong to no repo, so only an every-repo scope
reads them; the sources list carries each column's unit in `docs`.
The `spawns` and `spawnTime` sources (admin only) read the one-click path's
events: `forge.spawn.started`, `forge.spawn.live` (`ms` since started, when the
child's `/healthz` first answers; `spawn/live.ts`) and `forge.spawn.failed`
(`step`: place, copy or health). A payload `drill: 1` makes it the `drill`
series, so the nightly drill never mixes into a person's numbers. The Forges
starting board shows forges made per day, p50/p95 time to live against the
3-minute budget, and failures by step. The watcher that waits for `/healthz`
lives in the process: a restart mid-spawn leaves a `started` with no outcome,
and that spawn is simply never counted live or failed. A retry of an
unfinished child appends its own `started`, so every `failed` has one. The
watchers poll every second, then every five after 30 seconds, stop when the
forge closes, and at most 20 run at once; a spawn past that is made, not timed.
The `stages` source counts items moving from issue to live: one row per
item entering a stage, with `stage` one of `1 issue` (opened), `2 run` (a
builder first started on the issue; a retry is not a second item), `3 pull`
@@ -94,6 +94,8 @@
export interface SpawnDeps {
  readonly modelKey?: string;
  /** Anthropic-compatible API base URL sealed with the starter key (OpenRouter, etc.). */
  readonly modelEndpoint?: string;
  /** The nightly drill's own forge: its spawn events carry `drill`, so the boards keep it as its own series. */
  readonly drill?: boolean;
}
interface Ledger {
@@ -123,6 +125,12 @@
const SAFE_ID = /^[A-Za-z0-9_-]{1,80}$/;
export const STARTER_KEY = "starter";
const inFlight = new Set<string>();
/** A failure the boards count: the step that broke (a fixed few), never its words. */
function failed(app: App, deps: SpawnDeps, step: "place" | "copy" | "health", message: string) {
  app.bus.append({ kind: "forge.spawn.failed", payload: { step, ...(deps.drill === true ? { drill: 1 } : {}) } });
  return fail(message);
}
const fail = (message: string) => err<SpawnError>({ kind: "failed", message });
export const childDirOf = (app: App, bodyId: string): string =>
@@ -228,12 +236,14 @@
async function make(
      ledger.beatKey ??= secret();
      await writeLedger(app, ledger);
    }
    // A retry of an unfinished child is a new attempt: its failure has a start.
    app.bus.append({ kind: "forge.spawn.started", payload: { ...(deps.drill === true ? { drill: 1 } : {}) } });
  } else {
    const placed = await deps.placement.place(plan);
    if (!placed.ok) return err({ kind: "refused", ...placed.error });
    if (!SAFE_ID.test(placed.value.bodyId)) {
      await deps.placement.retire(placed.value.bodyId);
      return fail("placement returned an unusable body id");
      return failed(app, deps, "place", "placement returned an unusable body id");
    }
    ledger = {
      name: plan.name,
@@ -248,10 +258,11 @@
async function make(
      beatKey: secret(),
    };
    await writeLedger(app, ledger);
    app.bus.append({ kind: "forge.spawn.started", payload: { ...(deps.drill === true ? { drill: 1 } : {}) } });
  }
  const copied = await copyInto(app, owner, plan, ledger, deps.modelKey, deps.modelEndpoint);
  if (!copied.ok) return fail(copied.error);
  if (!copied.ok) return failed(app, deps, "copy", copied.error);
  const claimUrl = claimLink(liveChildUrl(app, plan.name), copied.value);
  ledger.done = true;
@@ -204,6 +204,8 @@
export interface App {
  readonly hookAddress: { current: string };
  /** Per-boot secret every internal request must carry (`FORGE_HOOK_SECRET` in a hook's env). */
  readonly hookSecret: string;
  /** Aborted by `close()`: background pollers stop on it and write nothing after. */
  readonly closing: AbortController;
  /** What `vitals` turns into needs: this forge's own, not the process's. */
  readonly signals: { errors: ErrorLedger; refused: RefusedLedger };
  close(): void;
@@ -228,6 +230,7 @@
export function createApp(config: ForgeConfig): App {
  // Idempotent — a forge without them changes nothing.
  dropLegacyNewAccountGuards(db);
  const bus = new EventBus(db);
  const closing = new AbortController();
  // Every event durably enqueues webhook deliveries; the worker (main.ts)
  // sends them. Listener throws are swallowed by the bus, so this can never
@@ -251,7 +254,9 @@
export function createApp(config: ForgeConfig): App {
    socketPath,
    hookAddress: { current: socketPath },
    hookSecret: randomBytes(32).toString("hex"),
    closing,
    close: () => {
      closing.abort();
      // The pooled `cat-file` readers for this forge's repos are processes,
      // and processes outlive a closed database unless somebody says
      // otherwise. One data dir is one forge, so closing the forge is where
@@ -43,6 +43,23 @@
const percentile = (name: string, column: string, p: number) =>
  `COUNT(*) OVER (PARTITION BY route, day) AS n FROM page_vital WHERE ${column} IS NOT NULL) ` +
  `WHERE rn = (n * ${p} + 99) / 100`;
/** The budget a spawned forge has to answer its `/healthz`: three minutes (#2914). */
export const SPAWN_BUDGET_MS = 180_000;
/** A spawn event's payload fields, as a drill or a person's forge, read in place of the `event` table. */
const SPAWN_SERIES = "CASE WHEN json_extract(payload, '$.drill') = 1 THEN 'drill' ELSE 'forge' END";
const SPAWN_KINDS = "kind IN ('forge.spawn.started', 'forge.spawn.live', 'forge.spawn.failed')";
const SPAWN_DAY = "substr(created_at, 1, 10) || 'T00:00:00.000Z'";
/** Nearest-rank percentile `p` of the ms to live, per series per day, as rows of the `spawnTime` source. */
const spawnPercentile = (name: string, p: number) =>
  `SELECT '${name}' AS metric, series, at, ms AS value FROM (` +
  `SELECT ${SPAWN_SERIES} AS series, ${SPAWN_DAY} AS at, json_extract(payload, '$.ms') AS ms, ` +
  `ROW_NUMBER() OVER (PARTITION BY ${SPAWN_SERIES}, ${SPAWN_DAY} ORDER BY json_extract(payload, '$.ms')) AS rn, ` +
  `COUNT(*) OVER (PARTITION BY ${SPAWN_SERIES}, ${SPAWN_DAY}) AS n ` +
  `FROM event WHERE kind = 'forge.spawn.live' AND json_extract(payload, '$.ms') IS NOT NULL) ` +
  `WHERE rn = (n * ${p} + 99) / 100`;
export const SOURCES: Readonly<Record<string, Source>> = {
  runs: {
    table: "run",
@@ -232,6 +249,44 @@
export const SOURCES: Readonly<Record<string, Source>> = {
      value: "the count, or the percentile (nearest rank) of that route that day",
    },
  },
  // Forges made by the one-click path (#2914): one row per spawn event
  // (`actions/spawn.ts`, `spawn/live.ts`). The forge's name and the failure's
  // words are not listed; the step is one of a fixed few.
  spawns: {
    table: "spawn",
    owner: "admin",
    select:
      `SELECT substr(kind, 13) AS outcome, ${SPAWN_SERIES} AS series, json_extract(payload, '$.step') AS step, ` +
      `json_extract(payload, '$.ms') AS ms, created_at AS at FROM event WHERE ${SPAWN_KINDS}`,
    time: "at",
    columns: { outcome: "text", series: "text", step: "text", ms: "number", at: "time" },
    docs: {
      outcome: "started, live (its /healthz first answered) or failed",
      series: "forge (a person's) or drill (the nightly drill's own forge)",
      step: "failed only: place, copy or health",
      ms: "live only: milliseconds from started to the first /healthz answer",
    },
  },
  // Time to live per day: p50 and p95 of the spawns that went live, and the
  // budget as a third row so a line chart shows the margin.
  spawnTime: {
    table: "spawn_time",
    owner: "admin",
    select: [
      spawnPercentile("p50", 50),
      spawnPercentile("p95", 95),
      `SELECT 'budget', series, at, ${SPAWN_BUDGET_MS} FROM (SELECT DISTINCT ${SPAWN_SERIES} AS series, ${SPAWN_DAY} AS at ` +
        `FROM event WHERE ${SPAWN_KINDS})`,
    ].join(" UNION ALL "),
    time: "at",
    columns: { metric: "text", series: "text", at: "time", value: "number" },
    docs: {
      metric: "p50 or p95 (nearest rank) of the ms to live that day, or budget",
      series: "forge (a person's) or drill (the nightly drill's own forge)",
      at: "the day, UTC",
      value: "milliseconds; the budget is 180000 (three minutes)",
    },
  },
  // Items entering each stage from issue to live, one row per entry, read
  // from issue, run and pull as `flow/throughput.ts` reads them: an issue
  // opens, a builder first starts on it (a retry is not a second item), its
@@ -1,5 +1,5 @@
/**
 * The boards the forge offers a new install: Usage, Crew, Releases, Apps, Security and Throughput. They are plain
 * The boards the forge offers a new install: Usage, Crew, Releases, Apps, Security, Throughput and Forges. They are plain
 * boards in `forge/boards` once written, so a person edits or deletes them
 * like any other; `.offered` (a JSON list of slugs, committed with the
 * boards) records what was offered so a deletion sticks and an edit is
@@ -17,6 +17,13 @@
const panel = (title: string, chart: Panel["chart"], query: Record<string, unkno
});
const MINUTE = { bucket: "minute" };
const HOUR = { bucket: "hour" };
const DAY = { bucket: "day" };
const SPAWNS = { source: "spawns", aggregate: "count" };
const started = { column: "outcome", op: "eq", value: "started" };
const timeToLive = (series: string) => ({
  source: "spawnTime", aggregate: "max", column: "value", groupBy: DAY, series: "metric",
  filter: [{ column: "series", op: "eq", value: series }],
});
const STAGES = { source: "stages", aggregate: "count" };
const machine = (op: "eq" | "starts", value: string, column: string) => ({
  source: "machine", aggregate: "avg", column, groupBy: MINUTE, filter: [{ column: "target", op, value }],
@@ -105,6 +112,19 @@
export const STARTING_BOARDS: Readonly<Record<string, Board>> = {
      panel("Stages by hour", "heat", { ...STAGES, groupBy: HOUR, series: "stage" }, 2),
    ],
  },
  forges: {
    title: "Forges",
    range: "30d",
    refresh: "1m",
    panels: [
      panel("Forges made per day", "bar", { ...SPAWNS, groupBy: DAY, series: "series", filter: [started] }, 2),
      panel("Forges live per day", "bar", { ...SPAWNS, groupBy: DAY, series: "series", filter: [{ column: "outcome", op: "eq", value: "live" }] }, 2),
      panel("Time to live (ms): p50, p95, budget 3 min", "line", timeToLive("forge"), 2),
      panel("The drill's time to live (ms): p50, p95, budget 3 min", "line", timeToLive("drill"), 2),
      panel("Failures by step", "bar", { ...SPAWNS, groupBy: "step", series: "series", filter: [{ column: "outcome", op: "eq", value: "failed" }] }, 2),
      panel("Failures per day", "bar", { ...SPAWNS, groupBy: DAY, series: "step", filter: [{ column: "outcome", op: "eq", value: "failed" }] }, 2),
    ],
  },
};
/** The slugs already offered, from the repo; none when it has no such file yet. */
@@ -6,6 +6,7 @@
 * the caller says `false` — returns the plan and changes nothing.
 */
import { startWatch, probeOf, type Probe } from "./live.ts";
import { listChildren, reissueClaim, spawnForge, type SpawnError } from "../actions/spawn.ts";
import type { App } from "../app.ts";
import { maxChildren } from "../core/capacity.ts";
@@ -37,6 +38,12 @@
export interface Door {
const MAX_CREW = 100;
const placements = new WeakMap<App, Placement>();
const probes = new WeakMap<App, (url: string) => Probe>();
/** Tests swap how a new child is asked whether it is live. */
export function useProbe(app: App, probe: (url: string) => Probe): void {
  probes.set(app, probe);
}
/** Tests (and an external placement, one day) swap where children live. */
export function usePlacement(app: App, placement: Placement): void {
@@ -264,6 +271,7 @@
export async function spawnDoor(
        },
      };
  }
  const startedAt = Date.now();
  const made = await spawnForge(ctx.app, user, input, source, {
    placement: placementFor(ctx.app),
    request: ctx.request,
@@ -274,7 +282,10 @@
export async function spawnDoor(
  if (!made.ok) return spawnWords(made.error);
  if (anon && !made.value.existed) noteAnonSpawn(ctx.request);
  // Its apps come up now, so the first visit after a claim finds them awake.
  if (!made.value.existed) void bootHostedChild(ctx.app, made.value.name);
  if (!made.value.existed) {
    void bootHostedChild(ctx.app, made.value.name);
    startWatch(ctx.app, { name: made.value.name, startedAt, probe: (probes.get(ctx.app) ?? probeOf)(made.value.url) });
      
        junipercrew
        25d ago
        outdated
      
      This is void watchLive(...) with no handle. It polls for up to 10 minutes, and nothing cancels it when the app closes. app.bus.append on a closed db then throws inside an unawaited promise, which is an unhandled rejection. Scout says the existing spawn-api tests now hit the real probe, so this already happens in the suite. Fix: keep the watchers in a per-app set, stop them on app.close(), and have the door tests use useProbe so no test touches the network.
    
  }
  return {
    status: made.value.existed ? 200 : 201,
    body: {
@@ -0,0 +1,94 @@
/**
 * Time to live (#2932): the moment a spawned child's `/health` first
 * answers, recorded on the outbox as `forge.spawn.live` with the ms since
 * the spawn began, or `forge.spawn.failed` at step `health` when it never
 * does. The boards read both (`boards/sources.ts`). One watcher per child,
 * polling its public address; the probe is injected so a test needs no network.
 * The watcher lives in this process: a restart mid-spawn leaves a `started`
 * with no `live` or `failed`, and the boards show that gap as unfinished.
 */
import type { App } from "../app.ts";
export const POLL_MS = 1000;
/** After this long without an answer, poll every `SLOW_POLL_MS` instead. */
export const SLOW_AFTER_MS = 30_000;
export const SLOW_POLL_MS = 5000;
/** At most this many watchers at once per forge; a spawn past it is made but not timed. */
export const MAX_WATCHERS = 20;
const open = new WeakMap<App, number>();
/** Give up after this long: well past the 3-minute budget, so a slow forge still counts as live, late. */
export const GIVE_UP_MS = 10 * 60_000;
export type Probe = () => Promise<boolean>;
/** A probe of `<url>/healthz`: true on a 200, false on anything else. Never throws. */
export function probeOf(url: string): Probe {
  return async () => {
    try {
      const res = await fetch(`${url.replace(/\/+$/, "")}/healthz`, {
        signal: AbortSignal.timeout(2000),
        redirect: "manual",
      });
      await res.body?.cancel();
      return res.status === 200;
    } catch {
      return false;
    }
  };
}
export interface Watch {
  readonly name: string;
  /** When the spawn began, ms since the epoch. */
  readonly startedAt: number;
  readonly probe: Probe;
  readonly drill?: boolean;
  readonly pollMs?: number;
  readonly giveUpMs?: number;
  readonly now?: () => number;
}
/** Start a watcher unless this forge already runs `MAX_WATCHERS`; true when started. */
export function startWatch(app: App, w: Watch): boolean {
      
        junipercrew
        25d ago
        outdated
      
      The loop has no abort signal and no cap on concurrent watchers. Each new spawn, anonymous ones included, adds a poller that does a fetch every second for 10 minutes. Take an AbortSignal (or check a closed flag) in the loop. Back off the poll interval, for example 1s then 5s after 30s. Cap the open watchers. Add a test that closing the app stops the watcher without appending an event.
    
  const n = open.get(app) ?? 0;
  if (n >= MAX_WATCHERS) return false;
  open.set(app, n + 1);
  void watchLive(app, w).finally(() => open.set(app, (open.get(app) ?? 1) - 1));
  return true;
}
/**
 * Poll until the child answers or time runs out; appends exactly one event,
 * or none when the forge closes first. Never throws.
 */
export async function watchLive(app: App, w: Watch): Promise<"live" | "failed" | "stopped"> {
  const now = w.now ?? Date.now;
  const poll = w.pollMs ?? POLL_MS;
  const until = w.startedAt + (w.giveUpMs ?? GIVE_UP_MS);
  const drill = w.drill === true ? { drill: 1 } : {};
  const sleep = (ms: number) =>
    new Promise<void>((done) => {
      const t = setTimeout(done, ms);
      app.closing.signal.addEventListener("abort", () => { clearTimeout(t); done(); }, { once: true });
    });
  while (true) {
    if (app.closing.signal.aborted) return "stopped";
    let up = false;
    try {
      up = await w.probe();
    } catch {
      up = false;
    }
    if (app.closing.signal.aborted) return "stopped";
    if (up) {
      const ms = Math.max(0, Math.round(now() - w.startedAt));
      app.bus.append({ kind: "forge.spawn.live", payload: { ms, ...drill } });
      return "live";
    }
    const t = now();
    if (t >= until) break;
    await sleep(t - w.startedAt >= SLOW_AFTER_MS ? Math.max(poll, SLOW_POLL_MS) : poll);
  }
  app.bus.append({ kind: "forge.spawn.failed", payload: { step: "health", ...drill } });
  return "failed";
}
@@ -1,5 +1,5 @@
/**
 * The forge offers the six starting boards on boot: every panel runs, an
 * The forge offers the seven starting boards on boot: every panel runs, an
 * edit survives the next boot, and a deletion is not undone.
 */
import { afterAll, beforeAll, expect, test } from "bun:test";
@@ -36,10 +36,10 @@
test("a forge with no person to found the repo writes nothing", async () => {
  expect(await offerStartingBoards(bare)).toEqual([]);
});
test("a fresh boot writes the six starting boards, and every panel runs", async () => {
  expect((await offerStartingBoards(app)).sort()).toEqual(["apps", "crew", "releases", "security", "throughput", "usage"]);
test("a fresh boot writes the seven starting boards, and every panel runs", async () => {
  expect((await offerStartingBoards(app)).sort()).toEqual(["apps", "crew", "forges", "releases", "security", "throughput", "usage"]);
  const boards = await listBoards(app.reposRoot);
  expect(boards.map((b) => b.slug)).toEqual(["apps", "crew", "releases", "security", "throughput", "usage"]);
  expect(boards.map((b) => b.slug)).toEqual(["apps", "crew", "forges", "releases", "security", "throughput", "usage"]);
  let panels = 0;
  for (const { board } of boards) {
    for (const p of board.panels) {
@@ -16,7 +16,7 @@
import { createUser } from "../src/core/users.ts";
import { commitFiles } from "../src/git/write.ts";
import { repoDir } from "../src/git/repos.ts";
import { serve, type Servers } from "../src/http/server.ts";
import { usePlacement } from "../src/spawn/door.ts";
import { usePlacement, useProbe } from "../src/spawn/door.ts";
import { childrenForMemory, maxChildren, setCapacity } from "../src/core/capacity.ts";
import { hostedPlacement, memoryBodies } from "../src/spawn/placement.ts";
import { quietLogs } from "./helpers/globals.ts";
@@ -302,6 +302,7 @@
describe("+ New forge", () => {
  });
  test("a forge at its depth limit shows no door, and its page says why", async () => {
    useProbe(app, () => async () => true);
    usePlacement(app, hostedPlacement({ host: "forge.test", depth: 2 }));
    expect(await (await get(admin, "/")).text()).not.toContain('href="/forges/new"');
    const html = await (await get(admin, "/forges/new")).text();
@@ -312,6 +313,7 @@
describe("+ New forge", () => {
  test("at the room limit the door stays, disabled, with the sentence; raising the setting reopens it", async () => {
    const store = memoryBodies();
    store.add({ bodyId: "b1", name: "one", url: "https://one--forge.test", depth: 1, agents: 1, ssh: false });
    useProbe(app, () => async () => true);
    usePlacement(app, hostedPlacement({ host: "forge.test", store, maxChildren: () => maxChildren(app.db) }));
    setCapacity(app.db, "children.max", "1", "system");
    const sentence = "1 of 1 forges sent on this node — retire one, or raise the room in settings";
@@ -10,7 +10,7 @@
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { commitToBranch, createRepo } from "../src/actions/repos.ts";
import { usePlacement } from "../src/spawn/door.ts";
import { usePlacement, useProbe } from "../src/spawn/door.ts";
import { resetAnonSpawns } from "../src/spawn/rate.ts";
import { createToken } from "../src/core/tokens.ts";
import { createUser } from "../src/core/users.ts";
@@ -69,6 +69,7 @@
describe("anonymous spawn", () => {
    });
    const priv = await createRepo(app, a.value, { name: "diary", visibility: "private" });
    if (!priv.ok) throw new Error("private repo");
    useProbe(app, () => async () => true);
    usePlacement(app, {
      place: async (plan) => ok({ url: `https://${plan.name}.fake.test`, bodyId: `fake-${++placed}` }),
      retire: async () => ok(),
@@ -10,7 +10,7 @@
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { createAgent } from "../src/actions/crew.ts";
import { commitToBranch, createRepo } from "../src/actions/repos.ts";
import { spawnWords, usePlacement } from "../src/spawn/door.ts";
import { spawnWords, usePlacement, useProbe } from "../src/spawn/door.ts";
import { createToken } from "../src/core/tokens.ts";
import { createUser } from "../src/core/users.ts";
import { serve, type Servers } from "../src/http/server.ts";
@@ -61,6 +61,7 @@
describe("spawn over the API and MCP", () => {
    await commitToBranch(app, a.value, repo.value, { message: "seed", files: [{ path: "a.txt", content: "a" }] });
    const agent = await createAgent(app, a.value, { username: "scout", brief: "ship small" });
    if (!agent.ok) throw new Error(`agent ${JSON.stringify(agent.error)}`);
    useProbe(app, () => async () => true);
    usePlacement(app, {
      place: async (plan) => ok({ url: `https://${plan.name}.fake.test`, bodyId: `fake-${++placed}` }),
      retire: async () => ok(),
@@ -0,0 +1,130 @@
/**
 * The events spawn emits (#2932), driven through `spawnForge` and the door
 * with a fake placement: removing an append in either fails a test here.
 */
import { afterEach, beforeEach, expect, test } from "bun:test";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createRepo } from "../src/actions/repos.ts";
import { spawnForge, type SpawnDeps } from "../src/actions/spawn.ts";
import { createApp, type App } from "../src/app.ts";
import { createUser, type User } from "../src/core/users.ts";
import { repoDir } from "../src/git/repos.ts";
import { commitFiles } from "../src/git/write.ts";
import { ok } from "../src/kernel/result.ts";
import { spawnDoor, usePlacement, useProbe } from "../src/spawn/door.ts";
import type { Placement } from "../src/spawn/placement.ts";
import type { SpawnSource } from "../src/spawn/plan.ts";
import type { Ctx } from "../src/http/context.ts";
let dir: string;
let app: App;
let alice: User;
let source: SpawnSource;
let retired: string[];
let bodyId: string;
const input = { name: "kid", org: "kids", apps: ["alice/notes"], carryBriefs: false, carrySubscriptions: false };
const placement = (): Placement => ({
  place: async (plan) => ok({ url: `https://${plan.name}.fake.test`, bodyId }),
  retire: async (id) => { retired.push(id); return ok(); },
});
const deps = (extra: Partial<SpawnDeps> = {}): SpawnDeps => ({
  placement: placement(),
  request: new Request("https://forge.test/spawn", { method: "POST", headers: { origin: "https://forge.test" } }),
  url: new URL("https://forge.test/spawn"),
  ...extra,
});
const events = () =>
  app.db.query("SELECT kind, payload FROM event WHERE kind LIKE 'forge.spawn.%' ORDER BY seq").all()
    .map((r) => ({ kind: (r as any).kind as string, payload: JSON.parse((r as any).payload) }));
beforeEach(async () => {
  dir = await mkdtemp(join(tmpdir(), "forge-spawn-events-"));
  app = createApp({ dataDir: dir, url: "https://forge.test" });
  retired = [];
  bodyId = "body-1";
  const made = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", isAdmin: true });
  if (!made.ok) throw new Error("user");
  alice = made.value;
  const repo = await createRepo(app, alice, { name: "notes" });
  if (!repo.ok) throw new Error("repo");
  const c = await commitFiles(repoDir(app.reposRoot, "alice", "notes"), {
    branch: "main", message: "one", files: [{ path: "a.txt", content: "a" }], author: { name: "a", email: "[email protected]" },
  });
  if (!c.ok) throw new Error("commit");
  source = { catalog: [{ ref: "alice/notes", readable: true, bytes: 10, forgeJsonBytes: 0 }], briefs: [], subscriptions: [] };
});
afterEach(async () => {
  app.close();
  await rm(dir, { recursive: true, force: true });
});
test("a good spawn appends exactly one started, and a retry of it none", async () => {
  expect((await spawnForge(app, alice, input, source, deps())).ok).toBe(true);
  expect(events()).toEqual([{ kind: "forge.spawn.started", payload: {} }]);
  const again = await spawnForge(app, alice, input, source, deps());
  expect(again.ok && again.value.existed).toBe(true);
  expect(events().length).toBe(1);
});
test("a retry after a failed copy starts again, so each failed has a started", async () => {
  const gone = repoDir(app.reposRoot, "alice", "notes");
  await Bun.$`mv ${gone} ${gone}.away`.quiet();
  await spawnForge(app, alice, input, source, deps());
  await spawnForge(app, alice, input, source, deps());
  expect(events().map((e) => e.kind)).toEqual([
    "forge.spawn.started", "forge.spawn.failed", "forge.spawn.started", "forge.spawn.failed",
  ]);
  await Bun.$`mv ${gone}.away ${gone}`.quiet();
  expect((await spawnForge(app, alice, input, source, deps())).ok).toBe(true);
  expect(events().map((e) => e.kind).slice(4)).toEqual(["forge.spawn.started"]);
});
test("an unsafe body id fails at place and retires the body", async () => {
  bodyId = "../escape";
  const r = await spawnForge(app, alice, input, source, deps());
  expect(r.ok).toBe(false);
  expect(retired).toEqual(["../escape"]);
  expect(events()).toEqual([{ kind: "forge.spawn.failed", payload: { step: "place" } }]);
});
test("a copy that breaks fails at copy, after started, with no words of the failure", async () => {
  await rm(repoDir(app.reposRoot, "alice", "notes"), { recursive: true, force: true });
  const r = await spawnForge(app, alice, input, source, deps());
  expect(r.ok).toBe(false);
  expect(events()).toEqual([
    { kind: "forge.spawn.started", payload: {} },
    { kind: "forge.spawn.failed", payload: { step: "copy" } },
  ]);
});
test("deps.drill marks started and failed as the drill", async () => {
  bodyId = "../escape";
  await spawnForge(app, alice, input, source, deps({ drill: true }));
  expect(events()).toEqual([{ kind: "forge.spawn.failed", payload: { step: "place", drill: 1 } }]);
  bodyId = "body-2";
  await spawnForge(app, alice, { ...input, name: "kid2", org: "kids2" }, source, deps({ drill: true }));
  expect(events()[1]).toEqual({ kind: "forge.spawn.started", payload: { drill: 1 } });
});
test("the door starts a watcher for a new child and not for an existing one", async () => {
  usePlacement(app, placement());
  const probed: string[] = [];
  useProbe(app, (url) => async () => { probed.push(url); return true; });
  const ctx = { app, request: deps().request, url: deps().url } as unknown as Ctx;
  const raw = { name: "kid", org: "kids", apps: ["alice/notes"], dryRun: false };
  const live = () => events().filter((e) => e.kind === "forge.spawn.live").length;
  const until = async (n: number) => { for (let i = 0; i < 100 && live() < n; i++) await Bun.sleep(20); };
  expect((await spawnDoor(ctx, alice, raw)).status).toBe(201);
  await until(1);
  expect(live()).toBe(1);
  expect(probed.length).toBe(1);
  expect((await spawnDoor(ctx, alice, raw)).status).toBe(200);
  await Bun.sleep(100);
  expect(probed.length).toBe(1);
  expect(live()).toBe(1);
});
@@ -0,0 +1,114 @@
/**
 * Forges made, time to live and failures (#2932): the spawn events land on
 * the outbox, and the Forges board reads them, the drill as its own series.
 */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp } from "../src/app.ts";
import { runQuery } from "../src/boards/query.ts";
import { STARTING_BOARDS } from "../src/boards/starting.ts";
import { MAX_WATCHERS, startWatch, watchLive } from "../src/spawn/live.ts";
let dir: string;
let app: ReturnType<typeof createApp>;
const all = { repoIds: "all" } as const;
const kinds = () => app.db.query("SELECT kind FROM event WHERE kind LIKE 'forge.spawn.%' ORDER BY seq").all().map((r) => (r as { kind: string }).kind);
beforeAll(async () => {
  dir = await mkdtemp(join(tmpdir(), "forge-spawn-live-"));
  app = createApp({ dataDir: dir, port: 0 });
});
afterAll(async () => {
  await rm(dir, { recursive: true, force: true });
});
test("a child that answers is live, with the ms since the spawn began", async () => {
  let calls = 0;
  let t = 1000;
  const got = await watchLive(app, {
    name: "x", startedAt: 1000, pollMs: 1, now: () => (t += 500),
    probe: async () => ++calls === 3,
  });
  expect(got).toBe("live");
  const row = app.db.query("SELECT payload FROM event WHERE kind = 'forge.spawn.live'").get() as { payload: string };
  expect(JSON.parse(row.payload)).toEqual({ ms: 1500 });
});
test("a child that never answers fails at step health, once", async () => {
  let t = 0;
  const got = await watchLive(app, { name: "y", startedAt: 0, pollMs: 1, giveUpMs: 1000, now: () => (t += 400), probe: async () => false, drill: true });
  expect(got).toBe("failed");
  const row = app.db.query("SELECT payload FROM event WHERE kind = 'forge.spawn.failed'").get() as { payload: string };
  expect(JSON.parse(row.payload)).toEqual({ step: "health", drill: 1 });
});
test("a probe that throws counts as not yet", async () => {
  let calls = 0;
  const got = await watchLive(app, { name: "z", startedAt: Date.now(), pollMs: 1, probe: async () => { if (++calls < 2) throw new Error("boom"); return true; } });
  expect(got).toBe("live");
});
test("a day with only failures still has a budget line", async () => {
  const fresh = createApp({ dataDir: await mkdtemp(join(tmpdir(), "forge-spawn-fail-")), port: 0 });
  fresh.bus.append({ kind: "forge.spawn.failed", payload: { step: "copy" } });
  const r = runQuery(fresh.db, { source: "spawnTime", aggregate: "max", column: "value", groupBy: "metric" }, all);
  expect(r).toEqual({ ok: true, rows: [{ group: "budget", value: 180000 }] });
  fresh.close();
});
test("the spawns source separates the drill, and the time source gives p50, p95 and the budget", () => {
  for (const [ms, drill] of [[60000, 0], [90000, 0], [120000, 0], [150000, 0], [30000, 1]] as const)
    app.bus.append({ kind: "forge.spawn.live", payload: { ms, ...(drill ? { drill: 1 } : {}) } });
  app.bus.append({ kind: "forge.spawn.started", payload: {} });
  const made = runQuery(app.db, { source: "spawns", aggregate: "count", groupBy: "series", filter: [{ column: "outcome", op: "eq", value: "live" }] }, all);
  expect(made).toEqual({ ok: true, rows: [{ group: "drill", value: 1 }, { group: "forge", value: 6 }] });
  const failures = runQuery(app.db, { source: "spawns", aggregate: "count", groupBy: "step", series: "series", filter: [{ column: "outcome", op: "eq", value: "failed" }] }, all);
  expect(failures).toEqual({ ok: true, rows: [{ group: "health", value: 1, series: "drill" }] });
  const time = runQuery(app.db, { source: "spawnTime", aggregate: "max", column: "value", groupBy: "metric", filter: [{ column: "series", op: "eq", value: "drill" }] }, all);
  expect(time.ok && time.rows.map((r) => r.group)).toEqual(["budget", "p50", "p95"]);
  const forge = runQuery(app.db, { source: "spawnTime", aggregate: "max", column: "value", groupBy: "metric", filter: [{ column: "series", op: "eq", value: "forge" }] }, all);
  expect(forge.ok && forge.rows.map((r) => r.value)).toEqual([180000, expect.any(Number), expect.any(Number)]);
  expect(kinds()).toEqual([
    "forge.spawn.live", "forge.spawn.failed", "forge.spawn.live",
    "forge.spawn.live", "forge.spawn.live", "forge.spawn.live", "forge.spawn.live", "forge.spawn.live",
    "forge.spawn.started",
  ]);
});
test("only a scope of every repo reads the spawn sources", () => {
  expect(runQuery(app.db, { source: "spawns", aggregate: "count" }, { repoIds: ["r"] })).toEqual({ ok: true, rows: [] });
});
test("the Forges board is offered, with a panel for made, time to live and failures", () => {
  const b = STARTING_BOARDS["forges"]!;
  expect(b.panels.map((p) => p.title).join("|")).toMatch(/made.*Time to live.*Failures by step/);
  for (const p of b.panels) expect(runQuery(app.db, p.query, all).ok).toBe(true);
});
test("closing the app stops a watcher, and it appends nothing", async () => {
  const mine = createApp({ dataDir: await mkdtemp(join(tmpdir(), "forge-spawn-close-")), port: 0 });
  const pending = watchLive(mine, { name: "c", startedAt: Date.now(), pollMs: 10_000, probe: async () => false });
  await Bun.sleep(20);
  mine.close();
  expect(await pending).toBe("stopped");
});
test("at most MAX_WATCHERS run at once per forge", async () => {
  const mine = createApp({ dataDir: await mkdtemp(join(tmpdir(), "forge-spawn-cap-")), port: 0 });
  const w = { name: "c", startedAt: Date.now(), pollMs: 10_000, probe: async () => false };
  const started = Array.from({ length: MAX_WATCHERS + 3 }, () => startWatch(mine, w));
  expect(started.filter(Boolean).length).toBe(MAX_WATCHERS);
  mine.close();
});
test("after 30 seconds the poll slows to five", async () => {
  const mine = createApp({ dataDir: await mkdtemp(join(tmpdir(), "forge-spawn-slow-")), port: 0 });
  let calls = 0;
  const pending = watchLive(mine, { name: "s", startedAt: 0, pollMs: 5, now: () => 31_000, giveUpMs: 10 ** 9, probe: async () => { calls++; return false; } });
  await Bun.sleep(100);
  mine.close();
  await pending;
  expect(calls).toBe(1);
});
@@ -9,7 +9,7 @@
import { join } from "node:path";
import { createRepo } from "../src/actions/repos.ts";
import { createApp, type App } from "../src/app.ts";
import { createUser, type User } from "../src/core/users.ts";
import { usePlacement } from "../src/spawn/door.ts";
import { usePlacement, useProbe } from "../src/spawn/door.ts";
import { hostedPlacement } from "../src/spawn/placement.ts";
import { createForgeApp, forgeCommands, Session } from "../src/tui/forge.ts";
import { newForgeScreen } from "../src/tui/screens/newforge.ts";
@@ -57,6 +57,7 @@
beforeAll(async () => {
  bob = await mk("bob", false);
  const repo = await createRepo(app, admin, { name: "bakery", visibility: "public" });
  if (!repo.ok) throw new Error("repo");
  useProbe(app, () => async () => true);
  usePlacement(app, hostedPlacement({ host: "forge.test" }));
});
afterAll(async () => {
		@@ -1555,7 +1555,7 @@ work here.
1555	1555	`forge/boards`, a repo the forge owns and makes on the first boot with an admin; every save is a commit
1556	1556	by the admin who made it, and a file that does not validate is not shown. Make, rename,
1557	1557	duplicate, delete, add/edit/remove/reorder panels, or pin the query on `/boards/data`.
1558		The first boot also writes six starting boards, Usage, Crew, Releases, Apps, Security and Throughput. Apps' per-app panels show repo ids until a source exposes a safe name; Security (sign-ins, failed sign-ins, tokens by scope, admin actions, from the admin-only `signins`, `tokens` and `adminActions` sources and `usage`) shows no token text, hash, payload or IP; Throughput is items entering each stage from issue to live, per hour, from the `stages` source. Each is written once: `.offered` in that repo records them, so an edit stays and a deletion sticks.
	1558	The first boot also writes seven starting boards, Usage, Crew, Releases, Apps, Security, Throughput and Forges. Apps' per-app panels show repo ids until a source exposes a safe name; Security (sign-ins, failed sign-ins, tokens by scope, admin actions, from the admin-only `signins`, `tokens` and `adminActions` sources and `usage`) shows no token text, hash, payload or IP; Throughput is items entering each stage from issue to live, per hour, from the `stages` source. Each is written once: `.offered` in that repo records them, so an edit stays and a deletion sticks.
1559	1559	A live board (any `refresh` but `off`; the form offers off or on, and `10s` and `1m` are still accepted and mean the same) redraws its grid when a run, pull, check, issue or usage event arrives, through the live-region swap
1560	1560	(`/boards/:slug/live`, etag'd, one fetch per burst). The same doors are in the API: `GET /api/v1/boards`, `GET\|PUT\|DELETE /api/v1/boards/:slug`
1561	1561	(an agent or anyone but an admin is 403; a panel whose query cannot run is refused with 422). The same four doors are MCP tools (`boards`, `board_write`, `board_delete`; `boards` with a `slug` reads one) and, for the same person, a Boards screen in the terminal that lists boards and shows a board's panels as text. No dependency was added.
		@@ -1201,6 +1201,19 @@ leaves its series absent, never zero, and `/boards/data?source=machine`
1201	1201	names each one. Its rows belong to no repo, so only an every-repo scope
1202	1202	reads them; the sources list carries each column's unit in `docs`.
1203	1203
	1204	The `spawns` and `spawnTime` sources (admin only) read the one-click path's
	1205	events: `forge.spawn.started`, `forge.spawn.live` (`ms` since started, when the
	1206	child's `/healthz` first answers; `spawn/live.ts`) and `forge.spawn.failed`
	1207	(`step`: place, copy or health). A payload `drill: 1` makes it the `drill`
	1208	series, so the nightly drill never mixes into a person's numbers. The Forges
	1209	starting board shows forges made per day, p50/p95 time to live against the
	1210	3-minute budget, and failures by step. The watcher that waits for `/healthz`
	1211	lives in the process: a restart mid-spawn leaves a `started` with no outcome,
	1212	and that spawn is simply never counted live or failed. A retry of an
	1213	unfinished child appends its own `started`, so every `failed` has one. The
	1214	watchers poll every second, then every five after 30 seconds, stop when the
	1215	forge closes, and at most 20 run at once; a spawn past that is made, not timed.
	1216
1204	1217	The `stages` source counts items moving from issue to live: one row per
1205	1218	item entering a stage, with `stage` one of `1 issue` (opened), `2 run` (a
1206	1219	builder first started on the issue; a retry is not a second item), `3 pull`
		@@ -94,6 +94,8 @@ export interface SpawnDeps {
94	94	readonly modelKey?: string;
95	95	/** Anthropic-compatible API base URL sealed with the starter key (OpenRouter, etc.). */
96	96	readonly modelEndpoint?: string;
	97	/** The nightly drill's own forge: its spawn events carry `drill`, so the boards keep it as its own series. */
	98	readonly drill?: boolean;
97	99	}
98	100
99	101	interface Ledger {
		@@ -123,6 +125,12 @@ const SAFE_ID = /^[A-Za-z0-9_-]{1,80}$/;
123	125	export const STARTER_KEY = "starter";
124	126	const inFlight = new Set<string>();
125	127
	128	/** A failure the boards count: the step that broke (a fixed few), never its words. */
	129	function failed(app: App, deps: SpawnDeps, step: "place" \| "copy" \| "health", message: string) {
	130	app.bus.append({ kind: "forge.spawn.failed", payload: { step, ...(deps.drill === true ? { drill: 1 } : {}) } });
	131	return fail(message);
	132	}
	133
126	134	const fail = (message: string) => err<SpawnError>({ kind: "failed", message });
127	135
128	136	export const childDirOf = (app: App, bodyId: string): string =>
		@@ -228,12 +236,14 @@ async function make(
228	236	ledger.beatKey ??= secret();
229	237	await writeLedger(app, ledger);
230	238	}
	239	// A retry of an unfinished child is a new attempt: its failure has a start.
	240	app.bus.append({ kind: "forge.spawn.started", payload: { ...(deps.drill === true ? { drill: 1 } : {}) } });
231	241	} else {
232	242	const placed = await deps.placement.place(plan);
233	243	if (!placed.ok) return err({ kind: "refused", ...placed.error });
234	244	if (!SAFE_ID.test(placed.value.bodyId)) {
235	245	await deps.placement.retire(placed.value.bodyId);
236		return fail("placement returned an unusable body id");
	246	return failed(app, deps, "place", "placement returned an unusable body id");
237	247	}
238	248	ledger = {
239	249	name: plan.name,
		@@ -248,10 +258,11 @@ async function make(
248	258	beatKey: secret(),
249	259	};
250	260	await writeLedger(app, ledger);
	261	app.bus.append({ kind: "forge.spawn.started", payload: { ...(deps.drill === true ? { drill: 1 } : {}) } });
251	262	}
252	263
253	264	const copied = await copyInto(app, owner, plan, ledger, deps.modelKey, deps.modelEndpoint);
254		if (!copied.ok) return fail(copied.error);
	265	if (!copied.ok) return failed(app, deps, "copy", copied.error);
255	266	const claimUrl = claimLink(liveChildUrl(app, plan.name), copied.value);
256	267
257	268	ledger.done = true;
		@@ -204,6 +204,8 @@ export interface App {
204	204	readonly hookAddress: { current: string };
205	205	/** Per-boot secret every internal request must carry (`FORGE_HOOK_SECRET` in a hook's env). */
206	206	readonly hookSecret: string;
	207	/** Aborted by `close()`: background pollers stop on it and write nothing after. */
	208	readonly closing: AbortController;
207	209	/** What `vitals` turns into needs: this forge's own, not the process's. */
208	210	readonly signals: { errors: ErrorLedger; refused: RefusedLedger };
209	211	close(): void;
		@@ -228,6 +230,7 @@ export function createApp(config: ForgeConfig): App {
228	230	// Idempotent — a forge without them changes nothing.
229	231	dropLegacyNewAccountGuards(db);
230	232	const bus = new EventBus(db);
	233	const closing = new AbortController();
231	234
232	235	// Every event durably enqueues webhook deliveries; the worker (main.ts)
233	236	// sends them. Listener throws are swallowed by the bus, so this can never
		@@ -251,7 +254,9 @@ export function createApp(config: ForgeConfig): App {
251	254	socketPath,
252	255	hookAddress: { current: socketPath },
253	256	hookSecret: randomBytes(32).toString("hex"),
	257	closing,
254	258	close: () => {
	259	closing.abort();
255	260	// The pooled `cat-file` readers for this forge's repos are processes,
256	261	// and processes outlive a closed database unless somebody says
257	262	// otherwise. One data dir is one forge, so closing the forge is where
		@@ -43,6 +43,23 @@ const percentile = (name: string, column: string, p: number) =>
43	43	`COUNT(*) OVER (PARTITION BY route, day) AS n FROM page_vital WHERE ${column} IS NOT NULL) ` +
44	44	`WHERE rn = (n * ${p} + 99) / 100`;
45	45
	46	/** The budget a spawned forge has to answer its `/healthz`: three minutes (#2914). */
	47	export const SPAWN_BUDGET_MS = 180_000;
	48
	49	/** A spawn event's payload fields, as a drill or a person's forge, read in place of the `event` table. */
	50	const SPAWN_SERIES = "CASE WHEN json_extract(payload, '$.drill') = 1 THEN 'drill' ELSE 'forge' END";
	51	const SPAWN_KINDS = "kind IN ('forge.spawn.started', 'forge.spawn.live', 'forge.spawn.failed')";
	52	const SPAWN_DAY = "substr(created_at, 1, 10) \|\| 'T00:00:00.000Z'";
	53
	54	/** Nearest-rank percentile `p` of the ms to live, per series per day, as rows of the `spawnTime` source. */
	55	const spawnPercentile = (name: string, p: number) =>
	56	`SELECT '${name}' AS metric, series, at, ms AS value FROM (` +
	57	`SELECT ${SPAWN_SERIES} AS series, ${SPAWN_DAY} AS at, json_extract(payload, '$.ms') AS ms, ` +
	58	`ROW_NUMBER() OVER (PARTITION BY ${SPAWN_SERIES}, ${SPAWN_DAY} ORDER BY json_extract(payload, '$.ms')) AS rn, ` +
	59	`COUNT(*) OVER (PARTITION BY ${SPAWN_SERIES}, ${SPAWN_DAY}) AS n ` +
	60	`FROM event WHERE kind = 'forge.spawn.live' AND json_extract(payload, '$.ms') IS NOT NULL) ` +
	61	`WHERE rn = (n * ${p} + 99) / 100`;
	62
46	63	export const SOURCES: Readonly<Record<string, Source>> = {
47	64	runs: {
48	65	table: "run",
		@@ -232,6 +249,44 @@ export const SOURCES: Readonly<Record<string, Source>> = {
232	249	value: "the count, or the percentile (nearest rank) of that route that day",
233	250	},
234	251	},
	252	// Forges made by the one-click path (#2914): one row per spawn event
	253	// (`actions/spawn.ts`, `spawn/live.ts`). The forge's name and the failure's
	254	// words are not listed; the step is one of a fixed few.
	255	spawns: {
	256	table: "spawn",
	257	owner: "admin",
	258	select:
	259	`SELECT substr(kind, 13) AS outcome, ${SPAWN_SERIES} AS series, json_extract(payload, '$.step') AS step, ` +
	260	`json_extract(payload, '$.ms') AS ms, created_at AS at FROM event WHERE ${SPAWN_KINDS}`,
	261	time: "at",
	262	columns: { outcome: "text", series: "text", step: "text", ms: "number", at: "time" },
	263	docs: {
	264	outcome: "started, live (its /healthz first answered) or failed",
	265	series: "forge (a person's) or drill (the nightly drill's own forge)",
	266	step: "failed only: place, copy or health",
	267	ms: "live only: milliseconds from started to the first /healthz answer",
	268	},
	269	},
	270	// Time to live per day: p50 and p95 of the spawns that went live, and the
	271	// budget as a third row so a line chart shows the margin.
	272	spawnTime: {
	273	table: "spawn_time",
	274	owner: "admin",
	275	select: [
	276	spawnPercentile("p50", 50),
	277	spawnPercentile("p95", 95),
	278	`SELECT 'budget', series, at, ${SPAWN_BUDGET_MS} FROM (SELECT DISTINCT ${SPAWN_SERIES} AS series, ${SPAWN_DAY} AS at ` +
	279	`FROM event WHERE ${SPAWN_KINDS})`,
	280	].join(" UNION ALL "),
	281	time: "at",
	282	columns: { metric: "text", series: "text", at: "time", value: "number" },
	283	docs: {
	284	metric: "p50 or p95 (nearest rank) of the ms to live that day, or budget",
	285	series: "forge (a person's) or drill (the nightly drill's own forge)",
	286	at: "the day, UTC",
	287	value: "milliseconds; the budget is 180000 (three minutes)",
	288	},
	289	},
235	290	// Items entering each stage from issue to live, one row per entry, read
236	291	// from issue, run and pull as `flow/throughput.ts` reads them: an issue
237	292	// opens, a builder first starts on it (a retry is not a second item), its
		@@ -1,5 +1,5 @@
1	1	/**
2		* The boards the forge offers a new install: Usage, Crew, Releases, Apps, Security and Throughput. They are plain
	2	* The boards the forge offers a new install: Usage, Crew, Releases, Apps, Security, Throughput and Forges. They are plain
3	3	* boards in `forge/boards` once written, so a person edits or deletes them
4	4	* like any other; `.offered` (a JSON list of slugs, committed with the
5	5	* boards) records what was offered so a deletion sticks and an edit is
		@@ -17,6 +17,13 @@ const panel = (title: string, chart: Panel["chart"], query: Record<string, unkno
17	17	});
18	18	const MINUTE = { bucket: "minute" };
19	19	const HOUR = { bucket: "hour" };
	20	const DAY = { bucket: "day" };
	21	const SPAWNS = { source: "spawns", aggregate: "count" };
	22	const started = { column: "outcome", op: "eq", value: "started" };
	23	const timeToLive = (series: string) => ({
	24	source: "spawnTime", aggregate: "max", column: "value", groupBy: DAY, series: "metric",
	25	filter: [{ column: "series", op: "eq", value: series }],
	26	});
20	27	const STAGES = { source: "stages", aggregate: "count" };
21	28	const machine = (op: "eq" \| "starts", value: string, column: string) => ({
22	29	source: "machine", aggregate: "avg", column, groupBy: MINUTE, filter: [{ column: "target", op, value }],
		@@ -105,6 +112,19 @@ export const STARTING_BOARDS: Readonly<Record<string, Board>> = {
105	112	panel("Stages by hour", "heat", { ...STAGES, groupBy: HOUR, series: "stage" }, 2),
106	113	],
107	114	},
	115	forges: {
	116	title: "Forges",
	117	range: "30d",
	118	refresh: "1m",
	119	panels: [
	120	panel("Forges made per day", "bar", { ...SPAWNS, groupBy: DAY, series: "series", filter: [started] }, 2),
	121	panel("Forges live per day", "bar", { ...SPAWNS, groupBy: DAY, series: "series", filter: [{ column: "outcome", op: "eq", value: "live" }] }, 2),
	122	panel("Time to live (ms): p50, p95, budget 3 min", "line", timeToLive("forge"), 2),
	123	panel("The drill's time to live (ms): p50, p95, budget 3 min", "line", timeToLive("drill"), 2),
	124	panel("Failures by step", "bar", { ...SPAWNS, groupBy: "step", series: "series", filter: [{ column: "outcome", op: "eq", value: "failed" }] }, 2),
	125	panel("Failures per day", "bar", { ...SPAWNS, groupBy: DAY, series: "step", filter: [{ column: "outcome", op: "eq", value: "failed" }] }, 2),
	126	],
	127	},
108	128	};
109	129
110	130	/** The slugs already offered, from the repo; none when it has no such file yet. */
		@@ -6,6 +6,7 @@
6	6	* the caller says `false` — returns the plan and changes nothing.
7	7	*/
8	8
	9	import { startWatch, probeOf, type Probe } from "./live.ts";
9	10	import { listChildren, reissueClaim, spawnForge, type SpawnError } from "../actions/spawn.ts";
10	11	import type { App } from "../app.ts";
11	12	import { maxChildren } from "../core/capacity.ts";
		@@ -37,6 +38,12 @@ export interface Door {
37	38
38	39	const MAX_CREW = 100;
39	40	const placements = new WeakMap<App, Placement>();
	41	const probes = new WeakMap<App, (url: string) => Probe>();
	42
	43	/** Tests swap how a new child is asked whether it is live. */
	44	export function useProbe(app: App, probe: (url: string) => Probe): void {
	45	probes.set(app, probe);
	46	}
40	47
41	48	/** Tests (and an external placement, one day) swap where children live. */
42	49	export function usePlacement(app: App, placement: Placement): void {
		@@ -264,6 +271,7 @@ export async function spawnDoor(
264	271	},
265	272	};
266	273	}
	274	const startedAt = Date.now();
267	275	const made = await spawnForge(ctx.app, user, input, source, {
268	276	placement: placementFor(ctx.app),
269	277	request: ctx.request,
		@@ -274,7 +282,10 @@ export async function spawnDoor(
274	282	if (!made.ok) return spawnWords(made.error);
275	283	if (anon && !made.value.existed) noteAnonSpawn(ctx.request);
276	284	// Its apps come up now, so the first visit after a claim finds them awake.
277		if (!made.value.existed) void bootHostedChild(ctx.app, made.value.name);
	285	if (!made.value.existed) {
	286	void bootHostedChild(ctx.app, made.value.name);
	287	startWatch(ctx.app, { name: made.value.name, startedAt, probe: (probes.get(ctx.app) ?? probeOf)(made.value.url) });
		junipercrew 25d ago outdated This is `void watchLive(...)` with no handle. It polls for up to 10 minutes, and nothing cancels it when the app closes. `app.bus.append` on a closed db then throws inside an unawaited promise, which is an unhandled rejection. Scout says the existing spawn-api tests now hit the real probe, so this already happens in the suite. Fix: keep the watchers in a per-app set, stop them on `app.close()`, and have the door tests use `useProbe` so no test touches the network.
	288	}
278	289	return {
279	290	status: made.value.existed ? 200 : 201,
280	291	body: {
		@@ -0,0 +1,94 @@
	1	/**
	2	* Time to live (#2932): the moment a spawned child's `/health` first
	3	* answers, recorded on the outbox as `forge.spawn.live` with the ms since
	4	* the spawn began, or `forge.spawn.failed` at step `health` when it never
	5	* does. The boards read both (`boards/sources.ts`). One watcher per child,
	6	* polling its public address; the probe is injected so a test needs no network.
	7	* The watcher lives in this process: a restart mid-spawn leaves a `started`
	8	* with no `live` or `failed`, and the boards show that gap as unfinished.
	9	*/
	10	import type { App } from "../app.ts";
	11
	12	export const POLL_MS = 1000;
	13	/** After this long without an answer, poll every `SLOW_POLL_MS` instead. */
	14	export const SLOW_AFTER_MS = 30_000;
	15	export const SLOW_POLL_MS = 5000;
	16	/** At most this many watchers at once per forge; a spawn past it is made but not timed. */
	17	export const MAX_WATCHERS = 20;
	18	const open = new WeakMap<App, number>();
	19	/** Give up after this long: well past the 3-minute budget, so a slow forge still counts as live, late. */
	20	export const GIVE_UP_MS = 10 * 60_000;
	21
	22	export type Probe = () => Promise<boolean>;
	23
	24	/** A probe of `<url>/healthz`: true on a 200, false on anything else. Never throws. */
	25	export function probeOf(url: string): Probe {
	26	return async () => {
	27	try {
	28	const res = await fetch(`${url.replace(/\/+$/, "")}/healthz`, {
	29	signal: AbortSignal.timeout(2000),
	30	redirect: "manual",
	31	});
	32	await res.body?.cancel();
	33	return res.status === 200;
	34	} catch {
	35	return false;
	36	}
	37	};
	38	}
	39
	40	export interface Watch {
	41	readonly name: string;
	42	/** When the spawn began, ms since the epoch. */
	43	readonly startedAt: number;
	44	readonly probe: Probe;
	45	readonly drill?: boolean;
	46	readonly pollMs?: number;
	47	readonly giveUpMs?: number;
	48	readonly now?: () => number;
	49	}
	50
	51	/** Start a watcher unless this forge already runs `MAX_WATCHERS`; true when started. */
	52	export function startWatch(app: App, w: Watch): boolean {
		junipercrew 25d ago outdated The loop has no abort signal and no cap on concurrent watchers. Each new spawn, anonymous ones included, adds a poller that does a fetch every second for 10 minutes. Take an `AbortSignal` (or check a `closed` flag) in the loop. Back off the poll interval, for example 1s then 5s after 30s. Cap the open watchers. Add a test that closing the app stops the watcher without appending an event.
	53	const n = open.get(app) ?? 0;
	54	if (n >= MAX_WATCHERS) return false;
	55	open.set(app, n + 1);
	56	void watchLive(app, w).finally(() => open.set(app, (open.get(app) ?? 1) - 1));
	57	return true;
	58	}
	59
	60	/**
	61	* Poll until the child answers or time runs out; appends exactly one event,
	62	* or none when the forge closes first. Never throws.
	63	*/
	64	export async function watchLive(app: App, w: Watch): Promise<"live" \| "failed" \| "stopped"> {
	65	const now = w.now ?? Date.now;
	66	const poll = w.pollMs ?? POLL_MS;
	67	const until = w.startedAt + (w.giveUpMs ?? GIVE_UP_MS);
	68	const drill = w.drill === true ? { drill: 1 } : {};
	69	const sleep = (ms: number) =>
	70	new Promise<void>((done) => {
	71	const t = setTimeout(done, ms);
	72	app.closing.signal.addEventListener("abort", () => { clearTimeout(t); done(); }, { once: true });
	73	});
	74	while (true) {
	75	if (app.closing.signal.aborted) return "stopped";
	76	let up = false;
	77	try {
	78	up = await w.probe();
	79	} catch {
	80	up = false;
	81	}
	82	if (app.closing.signal.aborted) return "stopped";
	83	if (up) {
	84	const ms = Math.max(0, Math.round(now() - w.startedAt));
	85	app.bus.append({ kind: "forge.spawn.live", payload: { ms, ...drill } });
	86	return "live";
	87	}
	88	const t = now();
	89	if (t >= until) break;
	90	await sleep(t - w.startedAt >= SLOW_AFTER_MS ? Math.max(poll, SLOW_POLL_MS) : poll);
	91	}
	92	app.bus.append({ kind: "forge.spawn.failed", payload: { step: "health", ...drill } });
	93	return "failed";
	94	}
		@@ -1,5 +1,5 @@
1	1	/**
2		* The forge offers the six starting boards on boot: every panel runs, an
	2	* The forge offers the seven starting boards on boot: every panel runs, an
3	3	* edit survives the next boot, and a deletion is not undone.
4	4	*/
5	5	import { afterAll, beforeAll, expect, test } from "bun:test";
		@@ -36,10 +36,10 @@ test("a forge with no person to found the repo writes nothing", async () => {
36	36	expect(await offerStartingBoards(bare)).toEqual([]);
37	37	});
38	38
39		test("a fresh boot writes the six starting boards, and every panel runs", async () => {
40		expect((await offerStartingBoards(app)).sort()).toEqual(["apps", "crew", "releases", "security", "throughput", "usage"]);
	39	test("a fresh boot writes the seven starting boards, and every panel runs", async () => {
	40	expect((await offerStartingBoards(app)).sort()).toEqual(["apps", "crew", "forges", "releases", "security", "throughput", "usage"]);
41	41	const boards = await listBoards(app.reposRoot);
42		expect(boards.map((b) => b.slug)).toEqual(["apps", "crew", "releases", "security", "throughput", "usage"]);
	42	expect(boards.map((b) => b.slug)).toEqual(["apps", "crew", "forges", "releases", "security", "throughput", "usage"]);
43	43	let panels = 0;
44	44	for (const { board } of boards) {
45	45	for (const p of board.panels) {
		@@ -16,7 +16,7 @@ import { createUser } from "../src/core/users.ts";
16	16	import { commitFiles } from "../src/git/write.ts";
17	17	import { repoDir } from "../src/git/repos.ts";
18	18	import { serve, type Servers } from "../src/http/server.ts";
19		import { usePlacement } from "../src/spawn/door.ts";
	19	import { usePlacement, useProbe } from "../src/spawn/door.ts";
20	20	import { childrenForMemory, maxChildren, setCapacity } from "../src/core/capacity.ts";
21	21	import { hostedPlacement, memoryBodies } from "../src/spawn/placement.ts";
22	22	import { quietLogs } from "./helpers/globals.ts";
		@@ -302,6 +302,7 @@ describe("+ New forge", () => {
302	302	});
303	303
304	304	test("a forge at its depth limit shows no door, and its page says why", async () => {
	305	useProbe(app, () => async () => true);
305	306	usePlacement(app, hostedPlacement({ host: "forge.test", depth: 2 }));
306	307	expect(await (await get(admin, "/")).text()).not.toContain('href="/forges/new"');
307	308	const html = await (await get(admin, "/forges/new")).text();
		@@ -312,6 +313,7 @@ describe("+ New forge", () => {
312	313	test("at the room limit the door stays, disabled, with the sentence; raising the setting reopens it", async () => {
313	314	const store = memoryBodies();
314	315	store.add({ bodyId: "b1", name: "one", url: "https://one--forge.test", depth: 1, agents: 1, ssh: false });
	316	useProbe(app, () => async () => true);
315	317	usePlacement(app, hostedPlacement({ host: "forge.test", store, maxChildren: () => maxChildren(app.db) }));
316	318	setCapacity(app.db, "children.max", "1", "system");
317	319	const sentence = "1 of 1 forges sent on this node — retire one, or raise the room in settings";