ded0ba8
@@ -0,0 +1,38 @@
name: outside-gate
# Nightly: every file in EXCLUDED (scripts/gate.ts), one at a time with the
# gate's 60 s timeout, so a suite outside the gate that breaks is seen the
# day it breaks. A newly red file opens one `Red: test/….test.ts` issue
# naming its first failing test; a file green again closes it
# (scripts/flake-hunt.ts outside). The list is EXCLUDED itself: a file moved
# into the gate stops being run here.
#
# Needs secret FORGE_TOKEN and variable FORGE_URL, as flake-hunt does.
on:
  schedule:
    - cron: "47 3 * * *"
  workflow_dispatch: {}
jobs:
  outside:
    runs-on: isolated
    timeout-minutes: 30
    env:
      REGISTRY: forgejo-http.forgejo.svc.cluster.local:3000
    steps:
      - name: clone main
        env:
          TOKEN: ${{ github.token }}
        run: |
          set -eu
          git clone -q "http://x-access-token:$TOKEN@$REGISTRY/$GITHUB_REPOSITORY.git" src
          cd src
          bun install --frozen-lockfile
      - name: run the excluded files
        working-directory: src
        env:
          FORGE_API: ${{ vars.FORGE_URL }}/api/v1
          FORGE_REPO: ${{ github.repository }}
          FORGE_TOKEN: ${{ secrets.FORGE_TOKEN }}
          FORGE_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_number }}
        run: bun scripts/flake-hunt.ts outside --file-issues
@@ -254,9 +254,12 @@
flag exists only in the stdio dev runner and is never reachable over SSH.
  whose default export is the SQL — never the next number: `0001`–`0091`
  are the numbered history, and a shared next number is what parallel pulls
  collided on (#1837). Files run in name order, once each, recognised by a
  hash of their text in `schema_migration` (`kernel/migrate.ts`), so two
  pulls' migrations land in either order. Never edit an existing migration
  — an edited one is a new one and runs again. It must only add
  hash of their text in `schema_migration` (`kernel/migrate.ts`, which also
  records each name), so two pulls' migrations land in either order, and a
  database with one this build does not know boots and names it. Never edit
  an existing migration — an edited one is a new one and runs again; once
  pinned in `core/migration-pins.json` (`bun scripts/pin-migrations.ts`, run
  by the release agent, never by a pull) the schema test fails on an edit. It must only add
  (`test/schema-additive.test.ts`); the release agent lists its id in
  `core/releases.json`.
- **New git read:** put it in `git/read.ts`; reuse the long-lived `cat-file`
@@ -1114,7 +1114,7 @@
a link to the milestone (`docs/PLANNING.md`); its progress never does.
| `FORGE_APP_DOMAIN`        | the host in `FORGE_URL`/`_HOST` | what app hostnames attach to (`example` or `--x`) |
| `FORGE_APPS_CONCURRENCY`  | `2`                             | deploys running their heavy steps at once         |
| `FORGE_PREVIEW_IDLE_DAYS` | `3`                             | days a pull may sit untouched before its preview sleeps; `0` = never |
| `FORGE_CREW_CONCURRENCY`  | `2`                             | agent runs at once (live: Settings → Capacity)    |
| `FORGE_CREW_CONCURRENCY`  | 1 per 3 GiB above 2 GiB, 2–16   | agent runs at once (live: Settings → Capacity)    |
| `FORGE_CHECKS_CONCURRENCY` | `1`                            | checks at once (live: Settings → Capacity)        |
| `FORGE_CHECKS_PLACEMENT`  | `process`                       | `job` runs a check's commands as a cluster Job (see Placement above) |
| `FORGE_CHECKS_DRAIN_S`       | `300`                          | seconds a roll lets running checks finish first   |
@@ -176,8 +176,9 @@
NOT NULL column without a default, and `test/schema-additive.test.ts`
refuses one in any migration not shipped by both of the two newest releases
recorded in `src/core/releases.json` (by id: a count, and the stamped ids),
which the release agent bumps when it cuts one.
An old process that restarts mid-roll refuses a database ahead of it
(`user_version`), as it always has.
An old process that restarts mid-roll boots on a database ahead of it and
logs, by name, the migrations it does not know (`schema_migration` records
each name, #783) — they only add, so it serves as it did.
State transitions use compare-and-swap (`UPDATE … WHERE state = ?`), and every
transition writes its event row in the same transaction. On boot, a
@@ -35,13 +35,14 @@
two devices backed by the same physical disk, are not.
## Running a drill
`scripts/drills/run.ts` walks you through one. It refuses a domain whose drill
is none, prints the fault to cause, and waits for Enter. It then polls
is none, prints the fault to cause, and waits for Enter (a line on stdin, so piping one works; closing stdin aborts). It then polls
`/api/v1/health` and `/` once a second, keeping each distinct line the forge
says while unhealthy; when the forge is healthy again it posts what it saw and
the seconds it took to `POST /api/v1/drills` and prints the verdict. If the
forge is still down after an hour it posts nothing, says so, and exits 0. A
refused post exits non-zero with the forge's message. `FORGE_TOKEN` must be a
site admin's token.
site admin's token; without it the script still watches, prints the result as
JSON and says that nothing was posted.
```
FORGE_TOKEN=… bun scripts/drills/run.ts runner --forge https://forge.open-platform.sh
@@ -28,6 +28,7 @@
for a tunnel and a forge that sleeps.
| tool           | does                                                                                              | writes |
| -------------- | ------------------------------------------------------------------------------------------------- | ------ |
| `apps`         | the apps you can see: live or not, where served, how each runs itself, what is working            |        |
| `repo`         | `{ repo }`: one repository's description, visibility, autonomy and `audience { level, sentence }`, the same as `GET /api/v1/repos/:owner/:name` |        |
| `tell`         | a sentence becomes an issue, a builder is picked, a run starts on the crew's next tick            | yes    |
| `say`          | speak on an issue or a pull; a working builder hears it as its next step                          | yes    |
| `pull`         | `{ repo, number }`: a pull's title, state, head and base, and for an open pull whether it merges into the base's current head (`mergeable`) and the paths that stop it (`conflicts`), and `check: { state, baseOid }`: the latest check's state and the base tip it was merged with (empty when it merged nothing); `mergeable` and `conflicts` are the same fields as `GET …/pulls/:number`, `check` is `state` and `run.baseOid` from `GET …/pulls/:number/checks` |        |
@@ -39,7 +40,7 @@
for a tunnel and a forge that sleeps.
| `audit`        | who can do what, per app, read from its contract.json in git: yes, own or no per door, and whether outsiders may direct its crew (`crewOutsiders`) |        |
| `outsiders`    | `{ repo, on?, dryRun? }`: whether people who cannot write may direct the crew; with `on`, planned first (`dryRun` is true unless `false`); set only by the app's admin with an admin token, never an agent | yes    |
| `decisions_by` | `{ repo, by?, dryRun? }`: the standing rule for a repository's hand-overs (`me`, `decider`, `decider_tell`); with `by`, planned first (`dryRun` is true unless `false`); set only by the app's admin with an admin token, never an agent | yes    |
| `archive`      | `{ repo, archived?, dryRun? }`: archive (true) or wake (false) a repository, by name; planned first (`dryRun` is true unless `false`); only the repository's admin with an admin token, never an agent | yes    |
| `archive`      | `{ repo | repos, archived?, dryRun? }`: archive (true) or wake (false) a repository, by name; planned first (`dryRun` is true unless `false`); only the repository's admin with an admin token, never an agent. `repos` (at most 50) runs the fleet `archive` operation over them, answers each one's outcome and the `operation` id; `limb` with `action: "undo"`, that `operation` and any one of the repositories as `app` reverts it for a day | yes    |
| `probe`        | knock on a running app's doors as nobody, a user and an admin; the ones that disobey its contract |        |
| `following`    | where a fork stands against what it follows, and what the crew is doing about it                  |        |
| `spawn`        | `{ name, apps?, crew?, dryRun? }`: a child forge, planned first (`dryRun` is true unless `false`); an admin's, and an `ro` token may plan but not make | yes    |
@@ -16,8 +16,8 @@
Sundays 09:00 UTC the forge drills a restore of the newest snapshot, boots it
on a side port and reads `/api/v1/health`. One comment lands on the recovery
issue (`FORGE_RECOVERY_ISSUE` in `FORGE_RECOVERY_REPO`): snapshot, restore
time, health, counts, pass or fail. A failed drill is an inbox card naming
the step (fetch, restore, open, count, unseal, health) until one passes. No
comment this week means the backups are unproven.
the step (fetch, restore, open, count, unseal, health), run again at boot and
every three hours until one passes. No comment this week: backups unproven.
## What a restore needs
@@ -8,7 +8,23 @@
empty data directory. `src/snapshot/`.
## What a snapshot holds
- `forge.db`, copied consistently while the forge runs (`VACUUM INTO`,
  never a raw copy of a file in WAL mode).
  never a raw copy of a file in WAL mode), then zstd-compressed and cut in
  16 MB parts: `forge.db.zst.0000`, `forge.db.zst.0001`, … Each part is its
  own put with its own deadline and retries, so a slow uplink carries it
  and a dropped connection costs one part, not the database. The manifest's
  `packed` says what the parts make and how big that is; a restore joins
  them, unpacks, and refuses a size that does not match. The log line
  `database packed` gives each night's ratio. Snapshots from before this
  hold a plain `forge.db` and still restore. The parts go up through the
  snapshot's upload queue like every other object: at most eight objects
  or 64 MB in flight, so four 16 MB parts at a time.
  Packing and restoring both hold the database in memory: the copy, its
  compressed form (the parts are views of it, not copies), and on restore
  the joined parts and the unpacked file, about three times the database
  at the peak (a 334 MB database at about 1 GB). A database that outgrows
  the app's memory limit needs a streaming pack; until then that limit,
  not the uplink, is the ceiling.
- The key the database's sealed values need, **only wrapped**:
  `secret.key.wrapped`, AES-256-GCM under a key scrypt-derived from the
  operator's passphrase. The raw `secret.key` is never written to the store,
@@ -24,8 +40,8 @@
empty data directory. `src/snapshot/`.
  stay out.
- The crew's run logs (`runs/`).
- `manifest.json`: the name, when, the forge's version, counts, every file
  with its size, and `key`: `wrapped` or `none` (older snapshots: `raw`). A
  restore checks each size.
  with its size, `packed` (the files that went up in parts), and `key`:
  `wrapped` or `none` (older snapshots: `raw`). A restore checks each size.
What stays out is what a boot or a deploy rebuilds: clones, checks,
pictures, previews.
@@ -86,7 +102,24 @@
mirror is copied from the primary store, not written in parallel, so a snapshot
that never landed on the primary is not seen by it.
The store is `Bun.S3Client` or a directory behind one small interface
(`src/snapshot/store.ts`): put, get, list, delete. Nothing else.
(`src/snapshot/store.ts`): put, get, list, delete. Nothing else. Every call
is tried five times; an attempt that has not answered by its deadline counts
as failed. A put's deadline follows its size: its bytes at a floor of
5 Mbit/s, plus 60 s. A fixed 180 s once failed every attempt at a 334 MB
database behind a 15 Mbit/s uplink (#2536).
## When a snapshot fails
Its outcome is kept in `snapshot/state.json` under the data directory, so a
restart forgets nothing. A failure shows straight away as a card on the board
(`snapshot-failed`, pointing at Backups), in the terminal's dashboard among
the forge's needs, and as a red line on the Backups page with the reason and
since when. The forge tries again within the hour, not the next night. The
next snapshot that lands, nightly or on demand, clears both.
A forge given nowhere to put a snapshot has not failed: nothing is kept, no
card is raised, and nothing is retried sooner than the nightly. The Backups
page says there is no destination.
## The restore drill
@@ -96,12 +129,26 @@
scratch directory under the data directory, opens the database read-only,
counts repositories and issues, opens one sealed value with the restored key,
and removes the scratch whatever happened. The page waits until it finishes,
for up to 10 minutes; a step that hangs past that is red and names the step.
The restore reads several objects at a time, under the same limits as the
upload (`IN_FLIGHT` gets, `IN_FLIGHT_BYTES` held): read one after another, a
snapshot of thousands of objects spent the drill's whole 10 minutes in the
restore step (#2539).
Repositories come back four clones at a time, not one after another (#2543).
One drill runs at a time: a second click, or a retry after a proxy timeout,
joins the drill already running rather than starting another restore. Every
run is a row in the `drill` table, and the last one shows as a line on the
Backups page: when, which snapshot, how long, the counts, and green or the
step that failed.
The drill runs on its own every Sunday 09:00 UTC, and a forge that was down
then catches up five minutes after boot. A red drill does not keep the
week's slot: it is owed again `RED_RETRY_MS` (three hours) after it ran,
and at a boot once it is `BOOT_RED_FLOOR_MS` (30 minutes) old, until one
passes (`src/snapshot/due.ts`). The floor keeps a crash loop with a broken
restore from running a restore on every boot. Kept for the
week, a drill red at 09:11 stood as a card long after the fix deployed at
10:18 (#2541).
## Restore
The printable card, seven lines, is [RESTORE.md](RESTORE.md); a test runs it.
@@ -4,11 +4,13 @@
 * polls `/api/v1/health` and `/` once a second keeping each distinct line the
 * forge says while unhealthy, and when the forge is healthy again posts what
 * it saw and how long it took to `POST /api/v1/drills` (`FORGE_TOKEN` from
 * the environment). If the forge never comes back within the ceiling it posts
 * nothing. Zero dependencies; it never shells out.
 * the environment). Without a token it prints the result and says nothing was
 * posted. If the forge never comes back within the ceiling it posts nothing.
 * Zero dependencies; it never shells out.
 */
import { DOMAINS } from "../../src/kernel/domains.ts";
import { UNREACHABLE, clean, linesOf, watchPolls, type Poll } from "./watch.ts";
export interface DrillRun {
  domain: string;
@@ -27,50 +29,17 @@
export interface DrillRun {
const SEEN_MAX = 8_000;
const DEFAULT_CEILING_MS = 3_600_000;
/** C0 and C1 control characters out (tab stays): the forge's words never drive the terminal. */
export const clean = (s: string): string => s.replace(/[\x00-\x08\x0a-\x1f\x7f-\x9f]/g, "");
export { clean };
/** Every string in a JSON value, or the text's own lines when it is not JSON. */
function linesOf(text: string, contentType: string): string[] {
  return rawLinesOf(text, contentType).map(clean).filter((l) => l.trim() !== "");
}
function rawLinesOf(text: string, contentType: string): string[] {
  if (contentType.includes("json")) {
    try {
      const out: string[] = [];
      const walk = (v: unknown): void => {
        if (typeof v === "string") out.push(v);
        else if (Array.isArray(v)) v.forEach(walk);
        else if (v && typeof v === "object") Object.values(v).forEach(walk);
      };
      walk(JSON.parse(text));
      return out;
    } catch {
      /* fall through to plain lines */
    }
  }
  return text
    .replace(/<(script|style)[\s\S]*?<\/\1>/gi, " ")
    .replace(/<[^>]*>/g, "\n")
    .replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">")
    .replace(/&quot;/g, '"').replace(/&#0?39;/g, "'")
    .split("\n")
    .map((l) => l.trim())
    .filter((l) => l !== "");
}
interface Look {
  healthy: boolean;
  health: string[];
  board: string[];
}
async function look(forge: string, token: string): Promise<Look> {
/**
 * One look at the forge: healthy only when `/api/v1/health` reports ok and `/`
 * answers; what each said is kept under `health` and `board`.
 */
export function forgePoll(forge: string, token: string, fetchIt: typeof fetch = fetch): () => Promise<Poll> {
  const get = async (path: string): Promise<{ ok: boolean; lines: string[]; json: unknown }> => {
    try {
      const res = await fetch(`${forge}${path}`, {
        headers: { authorization: `Bearer ${token}` },
      const res = await fetchIt(`${forge}${path}`, {
        headers: token === "" ? {} : { authorization: `Bearer ${token}` },
        redirect: "manual",
        signal: AbortSignal.timeout(5_000),
      });
@@ -82,17 +51,16 @@
async function look(forge: string, token: string): Promise<Look> {
      }
      return { ok: res.status < 400, lines: linesOf(text, type), json };
    } catch {
      return { ok: false, lines: [], json: null };
      return { ok: false, lines: [UNREACHABLE], json: null };
    }
  };
  const [h, b] = await Promise.all([get("/api/v1/health"), get("/")]);
  const reported = (h.json as { ok?: unknown } | null)?.ok === true;
  return { healthy: h.ok && b.ok && reported, health: h.lines, board: b.lines };
  return async () => {
    const [h, b] = await Promise.all([get("/api/v1/health"), get("/")]);
    const reported = (h.json as { ok?: unknown } | null)?.ok === true;
    return { healthy: h.ok && b.ok && reported, said: { health: h.lines, board: b.lines } };
  };
}
/** A line with its numbers blanked, so a clock or a counter is not a new line. */
const shape = (l: string): string => l.replace(/\d+/g, "#");
const cap = (lines: Iterable<string>): string => [...lines].join("\n").slice(0, SEEN_MAX);
/** Runs the drill; resolves to the process exit code. */
@@ -116,29 +84,29 @@
export async function runDrill(run: DrillRun): Promise<number> {
  run.say(`The forge should be back within ${domain.backMinutes} minutes. Press Enter when you have caused it.`);
  await run.enter();
  const started = now();
  const health = new Map<string, string>();
  const board = new Map<string, string>();
  let sawDown = false;
  for (;;) {
    const seen = await look(run.forge, run.token);
    if (seen.healthy && sawDown) break;
    if (!seen.healthy) {
      sawDown = true;
      for (const l of seen.health) if (!health.has(shape(l))) { health.set(shape(l), l); run.say(`health: ${l}`); }
      for (const l of seen.board) if (!board.has(shape(l))) { board.set(shape(l), l); run.say(`board: ${l}`); }
    }
    if (now() - started >= ceilingMs) {
      const ceilingS = Math.round(ceilingMs / 1000);
      run.say(`The forge did not come back within the ceiling of ${ceilingS}s; nothing was posted.`);
      run.say(JSON.stringify({ domain: domain.key, backSeconds: null, posted: false, ceilingSeconds: ceilingS }));
      return 0;
    }
    await sleep(intervalMs);
  const watched = await watchPolls({
    poll: forgePoll(run.forge, run.token),
    sleep,
    limit: Infinity,
    ceilingMs,
    intervalMs,
    now,
    onLine: (l, source) => run.say(`${source}: ${l}`),
  });
  if (watched.timeout) {
    const ceilingS = Math.round(ceilingMs / 1000);
    run.say(`The forge did not come back within the ceiling of ${ceilingS}s; nothing was posted.`);
    run.say(JSON.stringify({ domain: domain.key, backSeconds: null, posted: false, ceilingSeconds: ceilingS }));
    return 0;
  }
  const { health = [], board = [] } = watched.said;
  const backSeconds = Math.max(0, Math.round(watched.healthyAt / 1000));
  const body = { domain: domain.key, seen: { health: cap(health), board: cap(board) }, backSeconds };
  if (run.token === "") {
    run.say("FORGE_TOKEN is not set; nothing was posted. The result:");
    run.say(JSON.stringify({ domain: domain.key, backSeconds, posted: false, body }));
    return 0;
  }
  const backSeconds = Math.max(0, Math.round((now() - started) / 1000));
  const body = { domain: domain.key, seen: { health: cap(health.values()), board: cap(board.values()) }, backSeconds };
  let res: Response;
  try {
    res = await fetch(`${run.forge}/api/v1/drills`, {
@@ -172,14 +140,13 @@
export interface DrillArgs {
  token: string;
}
const USAGE = "usage: FORGE_TOKEN=… bun scripts/drills/run.ts <domain> --forge <url> [--insecure]";
const USAGE = "usage: [FORGE_TOKEN=…] bun scripts/drills/run.ts <domain> --forge <url> [--insecure]";
const LOOPBACK = new Set(["localhost", "127.0.0.1", "[::1]", "::1"]);
/** The options, or the one plain line to print before exiting 2. */
export function checkArgs(
  argv: string[],
  env: Record<string, string | undefined>,
  isTTY: boolean,
): { ok: true; args: DrillArgs } | { ok: false; error: string } {
  const at = argv.indexOf("--forge");
  const forge = at >= 0 ? argv[at + 1] : undefined;
@@ -193,16 +160,14 @@
export function checkArgs(
    return { ok: false, error: `--forge must be an http(s) URL; ${USAGE}` };
  }
  const token = env.FORGE_TOKEN ?? "";
  if (token === "") return { ok: false, error: `FORGE_TOKEN is empty; ${USAGE}` };
  if (!isTTY) return { ok: false, error: "stdin is not a terminal; run the drill from an interactive shell" };
  if (url.protocol === "http:" && !LOOPBACK.has(url.hostname) && !insecure) {
  if (token !== "" && url.protocol === "http:" && !LOOPBACK.has(url.hostname) && !insecure) {
    return { ok: false, error: `refusing to send the admin token over plain http to ${url.hostname}; use https, or pass --insecure` };
  }
  return { ok: true, args: { domain, forge: forge.replace(/\/+$/, ""), token } };
}
if (import.meta.main) {
  const checked = checkArgs(process.argv.slice(2), process.env, process.stdin.isTTY === true);
  const checked = checkArgs(process.argv.slice(2), process.env);
  if (!checked.ok) {
    console.error(checked.error);
    process.exit(2);
@@ -211,7 +176,8 @@
if (import.meta.main) {
    ...checked.args,
    say: (l) => console.log(clean(l)),
    enter: async () => {
      for await (const _ of console) break;
      for await (const _ of console) return;
      throw new Error("stdin closed before Enter; nothing was caused or watched");
    },
  });
  process.exit(code);
@@ -0,0 +1,140 @@
/**
 * `watchDrill`: polls one URL until the forge is healthy again after having
 * been unhealthy, and returns each distinct line it said meanwhile. Pure: the
 * fetch and the sleep come in as arguments, nothing is read, posted or global.
 * Times are milliseconds: the clock you pass, or else polls made times the
 * interval. `watchPolls` is the loop for any poll that says healthy or not and
 * what each of its sources said; `watchDrill` is it over one URL.
 */
/** C0 and C1 control characters out (tab stays): the forge's words never drive the terminal. */
export const clean = (s: string): string => s.replace(/[\x00-\x08\x0a-\x1f\x7f-\x9f]/g, "");
/** Every string in a JSON value, or the text's own lines when it is not JSON. */
export function linesOf(text: string, contentType: string): string[] {
  return rawLinesOf(text, contentType).map(clean).filter((l) => l.trim() !== "");
}
function rawLinesOf(text: string, contentType: string): string[] {
  if (contentType.includes("json")) {
    try {
      const out: string[] = [];
      const walk = (v: unknown): void => {
        if (typeof v === "string") out.push(v);
        else if (Array.isArray(v)) v.forEach(walk);
        else if (v && typeof v === "object") Object.values(v).forEach(walk);
      };
      walk(JSON.parse(text));
      return out;
    } catch {
      /* fall through to plain lines */
    }
  }
  return text
    .replace(/<(script|style)[\s\S]*?<\/\1>/gi, " ")
    .replace(/<[^>]*>/g, "\n")
    .replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">")
    .replace(/&quot;/g, '"').replace(/&#0?39;/g, "'")
    .split("\n")
    .map((l) => l.trim())
    .filter((l) => l !== "");
}
/** A line with its numbers blanked, so a clock or a counter is not a new line. */
const shape = (l: string): string => l.replace(/\d+/g, "#");
/** The part of a Response that a watch reads. */
export interface Reply {
  status: number;
  text: () => Promise<string>;
  headers: { get: (name: string) => string | null };
}
/** What a fetch that threw is recorded as saying. */
export const UNREACHABLE = "unreachable";
/** One look at the forge: is it healthy, and what each source said (named by source). */
export interface Poll {
  healthy: boolean;
  said: Record<string, string[]>;
}
export interface Watch {
  poll: () => Promise<Poll>;
  /** Waits one interval. */
  sleep: (ms: number) => Promise<unknown>;
  /** The most polls to make before giving up. */
  limit: number;
  intervalMs?: number;
  /** Milliseconds on any clock; without it, time is the polls made times `intervalMs`. */
  now?: () => number;
  /** Gives up once this many milliseconds have passed, however few polls that took. */
  ceilingMs?: number;
  /** Called once with each new distinct line and its source, as it is first said. */
  onLine?: (line: string, source: string) => void;
}
export type Watched =
  | { timeout: false; startedAt: number; healthyAt: number; said: Record<string, string[]> }
  | { timeout: true; startedAt: number | null; said: Record<string, string[]> };
/** Polls until healthy again after having been unhealthy; keeps each distinct line said while unhealthy. */
export async function watchPolls(w: Watch): Promise<Watched> {
  const intervalMs = w.intervalMs ?? 1_000;
  const seen = new Map<string, Map<string, string>>();
  const said = (): Record<string, string[]> =>
    Object.fromEntries([...seen].map(([source, m]) => [source, [...m.values()]]));
  const began = w.now?.() ?? 0;
  let startedAt: number | null = null;
  for (let n = 0; n < w.limit; n++) {
    const look = await w.poll();
    const at = w.now ? w.now() - began : n * intervalMs;
    if (look.healthy && startedAt !== null) return { timeout: false, startedAt, healthyAt: at, said: said() };
    if (!look.healthy) {
      startedAt ??= at;
      for (const [source, lines] of Object.entries(look.said)) {
        const m = seen.get(source) ?? new Map<string, string>();
        seen.set(source, m);
        for (const l of lines) if (!m.has(shape(l))) { m.set(shape(l), l); w.onLine?.(l, source); }
      }
    }
    if (w.ceilingMs !== undefined && at >= w.ceilingMs) break;
    await w.sleep(intervalMs);
  }
  return { timeout: true, startedAt, said: said() };
}
export interface WatchDrill extends Omit<Watch, "poll" | "onLine"> {
  url: string;
  fetch: (url: string) => Promise<Reply>;
  onLine?: (line: string) => void;
}
export type WatchedDrill =
  | { timeout: false; startedAt: number; healthyAt: number; lines: string[] }
  | { timeout: true; startedAt: number | null; lines: string[] };
/** `watchPolls` over one URL: healthy is a status under 400, and the lines are its body's. */
export async function watchDrill(w: WatchDrill): Promise<WatchedDrill> {
  const poll = async (): Promise<Poll> => {
    try {
      const res = await w.fetch(w.url);
      if (res.status < 400) return { healthy: true, said: {} };
      let lines: string[] = [];
      try {
        lines = linesOf(await res.text(), res.headers.get("content-type") ?? "");
      } catch {
        /* an unreadable body is unhealthy and says nothing */
      }
      return { healthy: false, said: { "": lines } };
    } catch {
      return { healthy: false, said: { "": [UNREACHABLE] } };
    }
  };
  const { url: _url, fetch: _fetch, onLine, ...rest } = w;
  const r = await watchPolls({ ...rest, poll, ...(onLine && { onLine: (l: string) => onLine(l) }) });
  const lines = r.said[""] ?? [];
  return r.timeout
    ? { timeout: true, startedAt: r.startedAt, lines }
    : { timeout: false, startedAt: r.startedAt, healthyAt: r.healthyAt, lines };
}
@@ -9,6 +9,12 @@
 *       `--file-issues` each gets an issue carrying the failure text, the
 *       seed and the run (FORGE_API, FORGE_REPO, FORGE_TOKEN; FORGE_RUN_URL
 *       names the run). Exits 1 when it found one.
 *   bun scripts/flake-hunt.ts outside [--file-issues]
 *       Every file in EXCLUDED once, one at a time, with the gate's timeout:
 *       the schedule for what the gate does not run (#812). With
 *       `--file-issues` a newly red file opens one `Red: test/….test.ts`
 *       issue naming its first failing test, a file red on another test
 *       updates it, and a file green again closes it.
 *   bun scripts/flake-hunt.ts quarantine
 *       Puts every file an open `Flaky: test/….test.ts` issue the hunt filed
 *       (author is the token's user, not untrusted) names into
@@ -30,12 +36,16 @@
import {
  huntedFile,
  HUNT_RUNS,
  issueFor,
  redActions,
  redFile,
  redIssueFor,
  type OpenRed,
  quarantineFlaky,
  readRun,
  shuffled,
  type FileRun,
} from "./flake.ts";
import { gateFiles, presentTests } from "./gate.ts";
import { gateFiles, outsideFiles, presentTests } from "./gate.ts";
import { parseShardSpec, sliceFiles } from "./shards.ts";
const GATE_FILE = new URL("./gate.ts", import.meta.url).pathname;
@@ -66,7 +76,7 @@
function runsOf(fallback: number): number {
const freshSeed = (): number => crypto.getRandomValues(new Uint32Array(1))[0]!;
/** One file, one seed, one `bun test`: the gate's own unit. */
/** One file, one `bun test`: the gate's own unit (random order when seeded). */
async function runFile(file: string, seed: number): Promise<FileRun> {
  const env = { ...process.env };
  // The gate hands its children none of these (scripts/test-shards.ts).
@@ -84,8 +94,7 @@
async function runFile(file: string, seed: number): Promise<FileRun> {
      "test",
      "--timeout",
      "60000",
      "--randomize",
      `--seed=${seed}`,
      ...(seed === 0 ? [] : ["--randomize", `--seed=${seed}`]),
      `./${file}`,
    ],
    { stdin: "ignore", stdout: "pipe", stderr: "pipe", env },
@@ -171,6 +180,102 @@
async function openFlakyIssues(): Promise<Map<string, number>> {
  return found;
}
async function forgeJson(
  method: string,
  path: string,
  body: unknown,
): Promise<Response> {
  const { api, repo, token } = forgeApi();
  return fetch(`${api}/repos/${repo}${path}`, {
    method,
    headers: {
      authorization: `Bearer ${token}`,
      "content-type": "application/json",
    },
    body: JSON.stringify(body),
  });
}
/** Open `Red:` issues the scheduled run filed: file → issue. */
async function openRedIssues(): Promise<Map<string, OpenRed>> {
  const { api, repo, token } = forgeApi();
  const self = await huntUserId(api, token);
  const found = new Map<string, OpenRed>();
  let cursor: string | number | null = "";
  for (let page = 0; page < 100 && cursor !== null; page++) {
    const query = cursor === "" ? "" : `&cursor=${encodeURIComponent(cursor)}`;
    const res = await fetch(
      `${api}/repos/${repo}/issues?state=open&limit=100${query}`,
      { headers: { authorization: `Bearer ${token}` } },
    );
    if (!res.ok) fail(`listing issues: ${res.status}`);
    const body = (await res.json()) as {
      items?: {
        number?: unknown;
        title?: unknown;
        body?: unknown;
        authorId?: unknown;
        untrusted?: unknown;
      }[];
      next_cursor?: string | number | null;
    };
    for (const issue of body.items ?? []) {
      if (issue.authorId !== self || (issue.untrusted ?? false) !== false)
        continue;
      const file =
        typeof issue.title === "string" ? redFile(issue.title) : undefined;
      if (file !== undefined && Number.isInteger(issue.number))
        found.set(file, {
          number: issue.number as number,
          ...(typeof issue.body === "string" ? { body: issue.body } : {}),
        });
    }
    cursor = body.next_cursor ?? null;
  }
  return found;
}
/** The files outside the gate, one at a time; red ones reported to the forge. */
async function outside(): Promise<number> {
  const files = outsideFiles(presentTests());
  const fileIssues = rest.includes("--file-issues");
  const open = fileIssues ? await openRedIssues() : new Map<string, OpenRed>();
  const runs: FileRun[] = [];
  for (const file of files) {
    const run = await runFile(file, 0);
    console.log(`${run.green ? "green" : "red  "}: ${file}`);
    runs.push(run);
  }
  const where = process.env["FORGE_RUN_URL"] ?? "";
  for (const action of redActions(runs, open)) {
    if (!fileIssues) continue;
    if (action.kind === "open") {
      const res = await forgeJson("POST", "/issues", redIssueFor(action.run, where));
      const made = (await res.json().catch(() => ({}))) as { number?: unknown };
      if (!res.ok) fail(`filing the issue for ${action.run.file}: ${res.status}`);
      console.log(`  filed #${made.number} for ${action.run.file}`);
    } else if (action.kind === "update") {
      const { body } = redIssueFor(action.run, where);
      const res = await forgeJson("PATCH", `/issues/${action.number}`, { body });
      if (!res.ok) {
        const note = await forgeJson("POST", `/issues/${action.number}/comments`, { body });
        if (!note.ok) fail(`updating #${action.number}: ${note.status}`);
      }
      console.log(`  updated #${action.number} for ${action.run.file}`);
    } else {
      await forgeJson("POST", `/issues/${action.number}/comments`, {
        body: `\`${action.file}\` passed in the scheduled run${where ? ` (${where})` : ""}.`,
      });
      const res = await forgeJson("PATCH", `/issues/${action.number}`, { state: "closed" });
      if (!res.ok) fail(`closing #${action.number}: ${res.status}`);
      console.log(`  closed #${action.number} for ${action.file}`);
    }
  }
  const red = runs.filter((run) => !run.green).length;
  console.log(`${files.length} files outside the gate, ${red} red`);
  return 0;
}
async function hunt(): Promise<number> {
  const runs = runsOf(HUNT_RUNS);
  const spec = parseShardSpec(valueOf("--shard"));
@@ -263,10 +368,11 @@
async function admit(): Promise<number> {
  return admitted(results) ? 0 : 1;
}
if (mode === "hunt") process.exit(await hunt());
if (mode === "outside") process.exit(await outside());
else if (mode === "hunt") process.exit(await hunt());
else if (mode === "quarantine") process.exit(await quarantineAll());
else if (mode === "admit") process.exit(await admit());
else
  fail(
    "usage: flake-hunt.ts hunt [--runs N] [--shard N/M] [--file-issues] | quarantine | admit test/x.test.ts [--runs N]",
    "usage: flake-hunt.ts hunt [--runs N] [--shard N/M] [--file-issues] | outside [--file-issues] | quarantine | admit test/x.test.ts [--runs N]",
  );
@@ -21,6 +21,9 @@
export const HUNT_RUNS = 5;
/** The title prefix of the issue the hunt files; quarantine reads it back. */
export const FLAKY_TITLE = "Flaky: ";
/** The title prefix of the issue the scheduled run files for a red file. */
export const RED_TITLE = "Red: ";
/**
 * The least a test may bound a request or a spawn by, in wall-clock ms. A
 * loopback fetch on a loaded shared runner takes tens of milliseconds; a
@@ -321,3 +324,72 @@
export function shuffled(files: readonly string[], seed: number): string[] {
  }
  return out;
}
/** The first failing test's name as the issue words it. */
export function firstFailing(run: FileRun): string {
  return run.failing[0] !== undefined
    ? `\`${run.failing[0]}\``
    : "none named; the file failed outside a test";
}
/** The issue for a file outside the gate that is red. */
export function redIssueFor(
  run: FileRun,
  where: string,
): { title: string; body: string } {
  const fence = "```";
  return {
    title: `${RED_TITLE}${run.file}`,
    body: [
      `\`${run.file}\` is outside the gate and was red in the scheduled run. This issue closes itself when the file passes again.`,
      "",
      `- first failing test: ${firstFailing(run)}`,
      `- run: ${where || "(a local run)"}`,
      `- reproduce: \`bun test --timeout 60000 ./${run.file}\``,
      "",
      fence,
      run.text || "(no failure text captured)",
      fence,
    ].join("\n"),
  };
}
/** The file an issue titled `Red: …` names, or undefined. */
export function redFile(title: string): string | undefined {
  if (!title.startsWith(RED_TITLE)) return undefined;
  return flakyFile(`${FLAKY_TITLE}${title.slice(RED_TITLE.length)}`);
}
/** An open `Red:` issue as the scheduled run sees it. */
export type OpenRed = { number: number; body?: string };
export type RedAction =
  | { kind: "open"; run: FileRun }
  | { kind: "update"; run: FileRun; number: number }
  | { kind: "close"; file: string; number: number };
/**
 * What the forge must be told after a scheduled run: a newly red file opens
 * one issue; a file still red whose first failing test is not in its issue
 * updates that issue; a file green again closes it. A file that stays red
 * on the same test says nothing, so the asks show it once.
 */
export function redActions(
  runs: readonly FileRun[],
  open: ReadonlyMap<string, OpenRed>,
): RedAction[] {
  const actions: RedAction[] = [];
  for (const run of runs) {
    const issue = open.get(run.file);
    if (run.green) {
      if (issue)
        actions.push({ kind: "close", file: run.file, number: issue.number });
    } else if (!issue) actions.push({ kind: "open", run });
    else if (
      typeof issue.body === "string" &&
      !issue.body.includes(firstFailing(run))
    )
      actions.push({ kind: "update", run, number: issue.number });
  }
  return actions;
}
@@ -15,6 +15,9 @@
 * order under load (`bun scripts/flake-hunt.ts admit <file>`). A flaky file
 * leaves it only through `bun scripts/flake-hunt.ts quarantine`, under the
 * issue the nightly hunt filed (`.forgejo/workflows/flake-hunt.yml`).
 *
 * The excluded files are not unseen: `.forgejo/workflows/outside-gate.yml`
 * runs `outsideFiles` nightly and files a `Red:` issue for each that fails (#812).
 */
import { readdirSync } from "node:fs";
@@ -135,6 +138,20 @@
export function gateFiles(
  return present.filter((file) => !out.has(file));
}
/**
 * What the scheduled run runs (`bun scripts/flake-hunt.ts outside`): the
 * present files `EXCLUDED` names, in name order. Derived from the same list
 * as `gateFiles`, so a file is in exactly one of the two: moving it into
 * the gate stops the schedule running it.
 */
export function outsideFiles(
  present: readonly string[] = presentTests(),
  excluded: readonly Exclusion[] = EXCLUDED,
): string[] {
  const gated = new Set(gateFiles(present, excluded));
  return present.filter((file) => !gated.has(file));
}
/**
 * The gate a checked-out tree's own `scripts/gate.ts` names, whichever shape
 * it has: a tree from before #2071 lists it in `GATE`; a later one derives
@@ -0,0 +1,33 @@
/**
 * `bun scripts/pin-migrations.ts`: pin every migration in
 * `src/core/migrations/` that `src/core/migration-pins.json` does not
 * name yet, by its id (`migrationId`). An existing pin is never rewritten:
 * once pinned, a migration's text is history, and
 * `test/schema-migrations.test.ts` fails if it changes (#783).
 *
 * The release agent runs it when it cuts a release, beside bumping
 * `src/core/releases.json`, so pulls never touch the file and two of them
 * adding a migration each never meet in it.
 */
import { readFileSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { migrationFiles } from "../src/core/schema.ts";
import { type MigrationPins, pinBreaks, pinned } from "../src/kernel/migrate.ts";
const file = join(import.meta.dir, "../src/core/migration-pins.json");
const pins = JSON.parse(readFileSync(file, "utf8")) as MigrationPins;
const breaks = pinBreaks(migrationFiles, pins);
if (breaks.length > 0) {
  for (const line of breaks) console.error(`pin-migrations: ${line}`);
  process.exit(1);
}
const next = pinned(migrationFiles, pins);
const added = Object.keys(next).filter((name) => !(name in pins));
writeFileSync(file, `${JSON.stringify(next, null, 2)}\n`);
// CLI answer for the person at the terminal.
console.log(
  added.length === 0
    ? "pin-migrations: every migration is already pinned"
    : `pin-migrations: pinned ${added.join(", ")}`,
);
@@ -495,3 +495,37 @@
export function besideOthers(
  const line = `fails only beside others: ${cleared.join(", ")} (shard-mates: ${mates})`;
  return { red: stillRed, besideOthers: cleared, line };
}
const HEADER = /^(\S+\.test\.[cm]?[jt]sx?):$/;
const FAILED = /^\(fail\) /;
/**
 * The files whose tests failed in one `bun test` report, in the order the
 * report reached them: bun prints `path:` before a file's tests and
 * `(fail) name` for each one that failed. A report that names no failing
 * test (a crash, a file that would not load) yields none.
 */
export function failingFiles(output: string): string[] {
  const found: string[] = [];
  let current: string | undefined;
  for (const line of output.split("\n")) {
    const header = HEADER.exec(line);
    if (header) current = header[1]!;
    else if (FAILED.test(line) && current !== undefined && !found.includes(current))
      found.push(current);
  }
  return found;
}
/**
 * Whether every red in one `bun test` report is a failed test: the report
 * reached its summary, counts no errors, and has no unhandled error between
 * tests. A file that threw while loading, an error outside any test, or a
 * crash before the summary is red for a reason a second run alone does not
 * explain away, so nothing in such a report may be cleared.
 */
export function onlyTestFailures(output: string): boolean {
  if (!/^Ran \d+ tests? across \d+ files?/m.test(output)) return false;
  if (/^# Unhandled error/m.test(output)) return false;
  return !/^\s*[1-9][0-9]* errors?\s*$/m.test(output);
}
@@ -4,6 +4,11 @@
 * `/healthz` is fetched once per user agent and must answer 200; the
 * stylesheet linked from `/` must be a non-empty `text/css` 200. Exit 1 and
 * one line per failure (url, user agent, status, content type) otherwise.
 *
 * `--version <v>` first polls `/healthz` until `X-Forge-Version` is `<v>`, for
 * at most `--wait <seconds>` (default 120), and fails naming the last version
 * seen. `/sw.js` must be a 200 JavaScript response whose navigation branch
 * never answers with `Response.error()`.
 */
export const AGENTS: Record<string, string> = {
@@ -32,11 +37,61 @@
export function stylesheetHref(html: string): string | null {
  return null;
}
/** The part of a service worker that answers navigations, comments removed. */
export function navigationBranch(source: string): string | null {
  const at = source.search(/mode\s*===?\s*["']navigate["']/);
  if (at < 0) return null;
  const rest = source.slice(at);
  const end = rest.search(/\breturn\s*;/);
  return (end < 0 ? rest : rest.slice(0, end)).replace(/\/\*[\s\S]*?\*\//g, "").replace(/\/\/[^\n]*/g, "");
}
export interface Options {
  /** Wait for `/healthz` to report this `X-Forge-Version`. */
  version?: string | undefined;
  /** Longest wait for it, in milliseconds. */
  waitMs?: number;
  /** Pause between polls, in milliseconds. */
  pollMs?: number;
  /** The fetch to use; the global one unless a caller (the forge proving itself, a test) gives its own. */
  fetch?: typeof fetch;
  /** Accept a redirect from `/board` as the sign-in it is, for a caller holding no token. */
  anonymousBoard?: boolean;
}
/** Polls `/healthz` until it reports `version`; null when it does, else a failure. */
async function waitForVersion(root: URL, version: string, waitMs: number, pollMs: number, fetch: typeof globalThis.fetch): Promise<Failure | null> {
  const url = new URL("/healthz", root).href;
  const deadline = Date.now() + waitMs;
  let seen = "none";
  let status: number | string = "error";
  for (;;) {
    try {
      const res = await fetch(url, { redirect: "manual", signal: AbortSignal.timeout(5_000) });
      await res.body?.cancel();
      status = res.status;
      seen = res.headers.get("x-forge-version") ?? "none";
      if (seen === version) return null;
    } catch (error) {
      status = "error";
      seen = `none (${error})`;
    }
    if (Date.now() + pollMs > deadline) break;
    await Bun.sleep(pollMs);
  }
  return { url, agent: "-", status, contentType: "", why: `wanted X-Forge-Version ${version}, last saw ${seen}` };
}
/** Runs every check against `base`; returns the failures (empty is green). */
export async function smoke(base: string, token = ""): Promise<Failure[]> {
export async function smoke(base: string, token = "", options: Options = {}): Promise<Failure[]> {
  const root = new URL(base);
  if (root.protocol !== "http:" && root.protocol !== "https:") throw new Error("url must be http(s)");
  const failures: Failure[] = [];
  const fetch = options.fetch ?? globalThis.fetch;
  if (options.version) {
    const late = await waitForVersion(root, options.version, options.waitMs ?? 120_000, options.pollMs ?? 1_000, fetch);
    if (late) return [late];
  }
  const get = async (path: string, agent: string, headers: Record<string, string> = {}) => {
    const url = new URL(path, root).href;
@@ -75,20 +130,42 @@
export async function smoke(base: string, token = ""): Promise<Failure[]> {
      }
    }
    const board = await get("/board", agent, auth);
    if (board && board.res.status !== 200) fail(board.url, agent, board.res, "expected 200 (a 302 is sign-in)");
    const signIn = options.anonymousBoard && !token && board?.res.status === 302;
    if (board && board.res.status !== 200 && !signIn) fail(board.url, agent, board.res, "expected 200 (a 302 is sign-in)");
    const health = await get("/healthz", agent);
    if (health && health.res.status !== 200) fail(health.url, agent, health.res, "expected 200");
  }
  const sw = await get("/sw.js", "desktop");
  if (sw) {
    const type = sw.res.headers.get("content-type") ?? "";
    if (sw.res.status !== 200) fail(sw.url, "desktop", sw.res, "expected 200");
    else if (!/^(text|application)\/(x-)?javascript\b/i.test(type)) fail(sw.url, "desktop", sw.res, "expected javascript");
    else {
      const nav = navigationBranch(sw.body);
      if (nav === null) fail(sw.url, "desktop", sw.res, "no navigation branch");
      else if (/Response\s*\.\s*error\s*\(/.test(nav)) fail(sw.url, "desktop", sw.res, "a navigation can be answered with Response.error()");
    }
  }
  return failures;
}
if (import.meta.main) {
  const base = process.argv[2];
  if (!base) {
    console.error("usage: bun scripts/smoke.ts <url>");
  const args = process.argv.slice(2);
  const flag = (name: string) => {
    const i = args.indexOf(name);
    if (i < 0) return undefined;
    return args.splice(i, 2)[1];
  };
  const version = flag("--version");
  const wait = flag("--wait");
  const base = args[0];
  const seconds = wait === undefined ? 120 : Number(wait);
  if (!base || !Number.isFinite(seconds) || seconds < 0 || args.length > 1) {
    console.error("usage: bun scripts/smoke.ts <url> [--version <v>] [--wait <seconds>]");
    process.exit(2);
  }
  const failures = await smoke(base, process.env["FORGE_TOKEN"] ?? "");
  const failures = await smoke(base, process.env["FORGE_TOKEN"] ?? "", { version, waitMs: seconds * 1000 });
  for (const f of failures)
    console.error(`FAIL ${f.url} agent=${f.agent} status=${f.status} content-type=${f.contentType || "-"} (${f.why})`);
  if (failures.length) process.exit(1);
@@ -58,6 +58,9 @@
import {
  splitCapped,
  tally,
  weigher,
  besideOthers,
  failingFiles,
  onlyTestFailures,
  type ShardResult,
} from "./shards.ts";
import { gateFiles, gateRefusal, presentTests } from "./gate.ts";
@@ -425,6 +428,12 @@
async function drain(
const measured = new Map<string, string[]>();
const unmeasured = new Set<string>();
// Failing files a red shard re-runs alone.
const RETRY_CAP = 3;
// Set once any shard has gone red for good: no retries then.
let halted = false;
const stopping = (): boolean => halted;
type Timed = ShardResult & { seconds: number };
/**
@@ -463,29 +472,60 @@
async function runShard(
  const cover = covDir
    ? ["--coverage", "--coverage-reporter=lcov", `--coverage-dir=${covDir}`]
    : [];
  const child = Bun.spawn(["bun", "test", ...flags, ...cover, ...paths], {
    stdin: "ignore",
    stdout: "pipe",
    stderr: "pipe",
    // Bun keeps shell-inherited vars after `delete process.env.X`; pass a
    // cleaned env so capacityDefault does not see the gate's shard count.
    env: childEnv,
  });
  children.add(child);
  // One buffer, in arrival order: `bun test` splits its report across
  // stdout and stderr.
  const output: string[] = [];
  const run = async (
    targets: string[],
    extra: string[],
    through?: (text: string) => void,
  ): Promise<{ code: number; output: string[] }> => {
    const child = Bun.spawn(["bun", "test", ...flags, ...extra, ...targets], {
      stdin: "ignore",
      stdout: "pipe",
      stderr: "pipe",
      // Bun keeps shell-inherited vars after `delete process.env.X`; pass a
      // cleaned env so capacityDefault does not see the gate's shard count.
      env: childEnv,
    });
    children.add(child);
    // One buffer, in arrival order: `bun test` splits its report across
    // stdout and stderr.
    const output: string[] = [];
    await Promise.all([
      drain(child.stdout, output, through),
      drain(child.stderr, output, through),
    ]);
    const code = await child.exited;
    children.delete(child);
    return { code, output };
  };
  const through = live
    ? (text: string): void => {
        process.stdout.write(text);
      }
    : undefined;
  await Promise.all([
    drain(child.stdout, output, through),
    drain(child.stderr, output, through),
  ]);
  const code = await child.exited;
  children.delete(child);
  const first = await run(paths, cover, through);
  let code = first.code;
  const output = first.output;
  // A sliced shard that went red: each failing file (the first few) is run
  // once alone, same env and flags. One that passes alone was red only
  // beside others, and does not fail the check; the log names it.
  if (code !== 0 && slice.length > 1 && !covDir && !stopping()) {
    const report = output.join("");
    // A file that errored (not a failed test) keeps the whole shard red.
    const red = onlyTestFailures(report) ? failingFiles(report) : [];
    const alone = new Map<string, boolean>();
    for (const file of red.slice(0, RETRY_CAP)) {
      if (stopping()) break;
      const again = await run([file.startsWith("/") || file.startsWith("./") ? file : `./${file}`], []);
      alone.set(file, again.code === 0);
    }
    const verdict = besideOthers(red, alone, RETRY_CAP);
    if (verdict.line !== null) {
      const note = `${output.join("").endsWith("\n") ? "" : "\n"}test-shards: ${verdict.line}\n`;
      output.push(note);
      through?.(note);
      if (verdict.red.length === 0) code = 0;
    }
  }
  const text = output.join("");
  const seconds = ((performance.now() - started) / 1000).toFixed(1);
  process.stdout.write(
@@ -524,6 +564,7 @@
async function pool(width: number): Promise<Timed[]> {
      results[index] = await runShard(slices[index]!, index, width === 1);
      if (results[index]!.code !== 0) {
        stop = true;
        halted = true;
        for (const child of children) child.kill("SIGTERM");
      }
    }
@@ -18,6 +18,7 @@
import { appsRuntime } from "../apps/runtime.ts";
import {
  dropCredentials,
  listAccounts as readAccounts,
  originOf,
  scrubAccount,
  setSuspended,
  type AccountRow,
@@ -331,7 +332,7 @@
export function newAccountLines(
      const id = `account:${a.username}`;
      return {
        id,
        line: `${a.username} joined the forge${a.via === null ? "" : ` (${a.via})`}`,
        line: `${a.username} joined the forge${originOf(a) === "" ? "" : ` (${originOf(a)})`}`,
        items: [id],
      };
    });
@@ -21,6 +21,7 @@
import {
  deleteToken as revokeToken,
  type TokenScope,
} from "../core/tokens.ts";
import { inviteSpentBy } from "../core/invites.ts";
import {
  changePassword as rehash,
  cleanDisplayName,
@@ -65,6 +66,10 @@
export function signupClosed(app: App, invite?: string): boolean {
  return newAccountsClosed(app.db, invite);
}
/** What a person with an invite link that no longer opens the door is told. */
export const INVITE_SPENT_MESSAGE =
  "This invite link has been used, revoked or has expired. Ask whoever sent it for a new one.";
/** What anyone is told at a door that still offers a password-only sign-up. */
export const PASSKEY_REQUIRED_MESSAGE =
  "New accounts sign up with a passkey and an email address. Open the sign-up page in a browser and choose \u201cCreate account with a passkey\u201d.";
@@ -122,6 +127,7 @@
export async function signUp(
      ...(from.address === undefined ? {} : { address: from.address }),
    },
  });
  auditInvite(app, created.value.id);
  // The address is proven by the link it is sent; until then nothing it
  // does may create or cost.
  await welcome(app, created.value);
@@ -178,10 +184,18 @@
export async function signUpWithPasskey(
      ...(from.address === undefined ? {} : { address: from.address }),
    },
  });
  auditInvite(app, created.value.id);
  await welcome(app, created.value);
  return created;
}
/** Using an invite link is an event: the invite's id and who it let in, never its token. */
function auditInvite(app: App, userId: string): void {
  const inviteId = inviteSpentBy(app.db, userId);
  if (inviteId !== undefined)
    app.bus.append({ kind: "invite.used", actorId: userId, payload: { id: inviteId } });
}
/**
 * Mint a personal access token. Callers decide what credential may mint one
 * (a session or the password, never another token — no privilege
@@ -193,9 +207,11 @@
export function createToken(
  name: string,
  scope: TokenScope,
  options: { expiresAt?: string } = {},
): Result<{ id: string; token: string }, "unauthorized"> {
): Result<{ id: string; token: string }, "unauthorized" | "forbidden"> {
  const user = requireUser(actor);
  if (!user.ok) return user;
  // A token that acts as admin is for an admin to hold; ro and rw are anyone's.
  if (scope === "admin" && !user.value.isAdmin) return err("forbidden");
  const minted = mintToken(app.db, user.value.id, name, scope, options);
  app.bus.append({
    kind: "token.created",
@@ -9,13 +9,14 @@
import type { Database } from "bun:sqlite";
import type { Decider } from "./crew/decide.ts";
import type { VerifyHuman } from "./core/turnstile.ts";
import { randomBytes } from "node:crypto";
import { mkdirSync } from "node:fs";
import { existsSync, mkdirSync } from "node:fs";
import { join, resolve } from "node:path";
import { migrations } from "./core/schema.ts";
import { migrationFiles } from "./core/schema.ts";
import { dropLegacyNewAccountGuards } from "./core/schema.ts";
import { EventBus } from "./core/events.ts";
import { follow } from "./core/following.ts";
import { enqueueForEvent } from "./core/webhooks.ts";
import { seedSignup } from "./settings/signup.ts";
import { openDb } from "./kernel/db.ts";
import { id } from "./kernel/id.ts";
import {
@@ -31,6 +32,12 @@
import { closeReadersUnder } from "./git/read.ts";
export interface ForgeConfig {
  /** Everything lives here: index db, bare repos, the internal socket. */
  dataDir: string;
  /**
   * A database made by this call starts with signup = invite (`seedSignup`);
   * one that already exists is left as it is. Off by default so a test's
   * forge reads the setting's `open`; `main.ts` turns it on.
   */
  seedSignup?: boolean;
  /** The disk probe vitals reads; the volume under `dataDir` when unset. Tests give it a reading. */
  disk?: (dir: string) => { freeBytes: number; totalBytes: number };
  /** @default 3000 */
@@ -212,7 +219,10 @@
export function createApp(config: ForgeConfig): App {
  mkdirSync(reposRoot, { recursive: true });
  const socketPath = join(dataDir, `forge-${id()}.sock`);
  const db = openDb(join(dataDir, "forge.db"), migrations);
  const dbPath = join(dataDir, "forge.db");
  const fresh = !existsSync(dbPath);
  const db = openDb(dbPath, migrationFiles);
  if (config.seedSignup === true) seedSignup(db, fresh);
  // Hand-installed new-account guards become the `guard_new_accounts`
  // setting (default off): drop the legacy triggers once, here, on boot.
  // Idempotent — a forge without them changes nothing.
@@ -4,12 +4,12 @@
 */
import { openDb } from "../kernel/db.ts";
import { migrations } from "../core/schema.ts";
import { migrationFiles } from "../core/schema.ts";
import { claimRun, createRun, heartbeatRuns } from "../core/runs.ts";
const [dbPath, repoId] = process.argv.slice(2);
if (dbPath === undefined || repoId === undefined) process.exit(2);
const db = openDb(dbPath, migrations);
const db = openDb(dbPath, migrationFiles);
console.log("ready");
for (;;) {
  const run = createRun(db, { repoId, kind: "check" });
@@ -8,6 +8,8 @@
 */
import type { CheckSummary } from "../core/runs.ts";
import { checkSteps, stepsWords } from "./summary.ts";
import { parseTestsRow, testsRowLine } from "./tests-row.ts";
import { readRunLog } from "../runs/log.ts";
export type CheckProgress = {
@@ -90,3 +92,22 @@
export function runningProgress(dataDir: string, check: CheckSummary): string {
    return "";
  return progressWords(parseShardProgress(readRunLog(dataDir, run.id).lines));
}
/**
 * The run page's three step rows as one line, for the terminal: the same
 * `checkSteps` over the same log, and nothing at all when the run is not a
 * check or when the viewer may not read its log (`mayReadRunLog`), as the
 * page withholds the rows then.
 */
export function stepsLine(
  dataDir: string,
  check: CheckSummary,
  mayReadLog: boolean,
): string {
  const run = check.run;
  if (!mayReadLog || run === null || run.kind !== "check") return "";
  const { lines, truncated } = readRunLog(dataDir, run.id);
  const steps = stepsWords(checkSteps({ ...run, state: check.state === "none" ? run.state : check.state }, lines, truncated));
  const tests = testsRowLine(parseTestsRow(lines));
  return tests === "" ? steps : `${steps} · ${tests}`;
}
@@ -61,6 +61,7 @@
import { appendRunLog } from "../runs/log.ts";
import { runningLimitSentence, runningRuns } from "../settings/quotas.ts";
import { exportTree, readFileAt } from "../apps/checkout.ts";
import { parseCheck } from "../apps/manifest.ts";
import { workerCpus } from "../../scripts/shards.ts";
import { accountActive } from "../core/accounts.ts";
import { childEnv, runStep } from "../apps/process.ts";
import { noteReached } from "../apps/egress_report.ts";
@@ -68,6 +69,7 @@
import {
  capacityCount,
  noteBooted,
  capacityEnv,
  cpuLimit,
  egressSetting,
} from "../core/capacity.ts";
import {
@@ -245,9 +247,10 @@
export function provideChecksForTest(app: App, handle: ChecksHandle): void {
export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
  // Read on every pick (`core/capacity.ts`): an admin's change is honoured
  // by the next check, not the next boot. `concurrency` is the boot default.
  noteBooted(app.db, "checks.concurrency", String(options.concurrency ?? 1));
  const booted = options.concurrency ?? 1;
  noteBooted(app.db, "checks.concurrency", String(booted));
  const capacity = (): number =>
    capacityCount(app.db, "checks.concurrency", options.concurrency ?? 1);
    capacityCount(app.db, "checks.concurrency", booted);
  const gate = options.gate ?? new Gate(capacity());
  const placement = options.placement ?? placementFromEnv();
  /** Full gates on land in flight: not slots in the queue, but stopped with it. */
@@ -792,9 +795,11 @@
export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
        }
      : undefined;
    try {
      const cellCpus = resolveLimits("step", manifest.limits).cpus;
      const env = childEnv({
        // The shard count an admin set (`core/capacity.ts`), never the pod's.
        ...capacityEnv(app.db),
        // The shard count an admin set (`core/capacity.ts`), else the
        // default sized to what this check's cell holds, never the pod's.
        ...capacityEnv(app.db, workerCpus(cpuLimit(), cellCpus)),
        CI: "1",
        FORGE_URL: options.forgeUrl,
        FORGE_CHECK: run.id,
@@ -802,7 +807,7 @@
export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
        FORGE_CHECK_IMPACT: IMPACT_FILE,
        // The cell's CPU limit, which `availableParallelism()` may not
        // report inside it; the shard count follows this (#2257).
        FORGE_CELL_CPUS: String(resolveLimits("step", manifest.limits).cpus),
        FORGE_CELL_CPUS: String(cellCpus),
        ...(base === undefined
          ? {}
          : {
@@ -111,3 +111,91 @@
export function checkWords(
  }
  return undefined;
}
export type StepName = "install" | "typecheck" | "tests";
export type StepState =
  | "pending"
  | "running"
  | "ok"
  | "failed"
  | "stopped"
  | "unknown";
export interface CheckStep {
  readonly name: StepName;
  readonly state: StepState;
}
/** `$ <command>` is what `runStep` and the job runner echo before each command. */
const ECHO = /^\$ /;
/** The package manager's own install, never a script that merely has the word in its name. */
const INSTALL_ECHO = /^\$ (?:bun|npm|pnpm|yarn)(?: -\S+)* (?:install|ci)\b/;
/** The shard runner's banner and shard headers: the tests have begun. */
const TESTS_BEGUN = /^test-shards: |^── shard /;
/**
 * The three rows of a check's run page — install, typecheck, tests — read
 * from the run's state and the lines the check wrote, never from a new
 * record. The log carries no step boundaries of its own, so each is
 * inferred from the furthest sign of progress: an echoed command, then the
 * shard runner's banner (typecheck comes before it in the check command).
 * A repository without an install command has its install row `ok` once
 * the check begins. A job-placed check echoes every command up front, so
 * its install is only done once output follows the echoes. Always three
 * rows, so the page never shifts when a state changes. `lines` may be only
 * the tail of a long log (`truncated`): the echoes are then gone, so unless
 * the tests are seen to have begun, or the run passed, the rows are
 * `unknown` rather than read wrong.
 */
export function checkSteps(
  run: Pick<Run, "state" | "placement">,
  lines: readonly string[],
  truncated = false,
): readonly CheckStep[] {
  const echoes = lines.flatMap((line, i) => (ECHO.test(line) ? [i] : []));
  const hasInstall = lines.some((line) => INSTALL_ECHO.test(line));
  const needed = hasInstall ? 2 : 1;
  const lastEcho = echoes.at(-1);
  const output = lastEcho !== undefined && lastEcho < lines.length - 1;
  const checkBegan =
    echoes.length >= needed && (run.placement !== "job" || output);
  const testsBegan = lines.some((line) => TESTS_BEGUN.test(line));
  if (truncated && run.state !== "ok" && !testsBegan)
    return (["install", "typecheck", "tests"] as const).map((name) => ({
      name,
      state: "unknown",
    }));
  // A tail that begins after the echoes still shows the tests by their own banner.
  const live = run.state === "running";
  const dead = run.state === "failed" || run.state === "cancelled";
  const after = run.state === "cancelled" ? "stopped" : "failed";
  // The first step not yet done is where the run is, or where it ended.
  const reached = testsBegan ? 2 : checkBegan ? 1 : 0;
  const names: StepName[] = ["install", "typecheck", "tests"];
  return names.map((name, i): CheckStep => {
    if (run.state === "ok") return { name, state: "ok" };
    if (i < reached) return { name, state: "ok" };
    if (i === reached) {
      if (live) return { name, state: "running" };
      if (dead) return { name, state: after };
    }
    return { name, state: "pending" };
  });
}
const STEP_WORDS: Record<StepState, string> = {
  pending: "waiting",
  running: "running",
  ok: "passed",
  failed: "failed",
  stopped: "stopped",
  unknown: "unknown",
};
/** `install passed · typecheck passed · tests running`: the page's rows, in a line. */
export const stepWord = (state: StepState): string => STEP_WORDS[state];
export function stepsWords(steps: readonly CheckStep[]): string {
  return steps.map((s) => `${s.name} ${stepWord(s.state)}`).join(" · ");
}
@@ -0,0 +1,99 @@
/**
 * What the tests row of a running check's run page adds to its name: how
 * many files the shard runner has started out of the run's count, the file
 * (or files) in the shard running now, and every failure so far with the
 * first lines `bun test` printed for it. Read only from the shard runner's
 * own lines and `bun test`'s `(fail)` markers; a figure the log does not
 * give is left out, never guessed.
 */
import { parseShardProgress, type CheckProgress } from "./progress.ts";
export type PinnedFailure = {
  readonly name: string;
  /** The first lines of what was printed above the `(fail)` marker. */
  readonly lines: readonly string[];
};
export type TestsRow = CheckProgress & {
  /** Files in shards that have begun, finished or not. */
  started?: number;
  /** The files of the live shard that has not finished, if one is open. */
  running?: readonly string[];
  failures: readonly PinnedFailure[];
};
const SHARD_LIVE_OPEN = /^── shard \d+\/\d+: (.+)$/;
const SHARD_BATCHED = /^── shard \d+\/\d+ \(exit \d+, [0-9.]+s\): (.+)$/;
const SHARD_DONE = /^── shard \d+\/\d+ done \(exit \d+, [0-9.]+s\)$/;
const FAIL_LINE = /^\(fail\) (.+?)(?: \[[0-9.]+m?s\])?(?: \[shard [0-9]+\])?$/;
/** Lines that end one test's output and so bound the next one's. */
const BOUNDARY = /^(?:\(pass\) |\(skip\) |\(todo\) |── |test-shards: |\$ )|\.test\.[cm]?[jt]sx?:$/;
export const MAX_PINNED = 5;
export const PIN_LINES = 3;
const LINE_MAX = 200;
const names = (text: string): string[] => text.split(" ").filter((n) => n !== "");
const bound = (text: string): string =>
  text.length > LINE_MAX ? `${text.slice(0, LINE_MAX - 1)}…` : text;
export function parseTestsRow(lines: readonly string[]): TestsRow {
  const base = parseShardProgress(lines);
  let started: number | undefined;
  let running: string[] | undefined;
  const failures: PinnedFailure[] = [];
  // Lines since the last boundary: the failing test's own output.
  let block: string[] = [];
  for (const raw of lines) {
    const line = raw.trimEnd();
    const live = SHARD_LIVE_OPEN.exec(line);
    const batched = SHARD_BATCHED.exec(line);
    if (live) {
      running = names(live[1]!);
      started = (started ?? 0) + running.length;
    } else if (batched) {
      started = (started ?? 0) + names(batched[1]!).length;
    } else if (SHARD_DONE.test(line)) {
      running = undefined;
    }
    const fail = FAIL_LINE.exec(line);
    if (fail) {
      const name = bound(fail[1]!);
      if (failures.length < MAX_PINNED && !failures.some((f) => f.name === name))
        failures.push({ name, lines: block.slice(0, PIN_LINES).map(bound) });
      block = [];
      continue;
    }
    if (BOUNDARY.test(line)) {
      block = [];
      continue;
    }
    if (line.trim() !== "") block.push(line.trim());
  }
  const row: TestsRow = { ...base, failures };
  if (started !== undefined) row.started = started;
  if (running !== undefined) row.running = running;
  return row;
}
/** `3 of 9 files started`, `812 passed, 1 failed`: the parts that are known. */
export function testsRowWords(row: TestsRow): string {
  const parts: string[] = [];
  if (row.started !== undefined && row.of !== undefined && row.started <= row.of)
    parts.push(`${row.started} of ${row.of} files started`);
  if (row.passed !== undefined && row.failed !== undefined)
    parts.push(`${row.passed} passed, ${row.failed} failed`);
  return parts.join(" · ");
}
/**
 * The terminal's words for the same row: `tests 2 of 9 files started · 812
 * passed, 1 failed · failing: a, b`, empty when the log gave nothing.
 */
export function testsRowLine(row: TestsRow): string {
  const words = testsRowWords(row);
  const pins = row.failures.map((f) => f.name).join(", ");
  const parts = [words, pins === "" ? "" : `failing: ${pins}`].filter((p) => p !== "");
  return parts.length === 0 ? "" : `tests ${parts.join(" · ")}`;
}
@@ -21,6 +21,8 @@
export interface AccountRow {
  readonly createdAt: string;
  /** How the account came: "signup", "edge", or null for one older than the record. */
  readonly via: string | null;
  /** The linked sign-in method it came through ("open-platform.sh" for an edge-made account), else null. */
  readonly signsInThrough: string | null;
  /** The address it signed up from, when one was recorded. */
  readonly address: string | null;
  readonly repos: number;
@@ -33,6 +35,15 @@
export interface AccountRow {
  readonly verified: boolean;
}
/** What the platform edge's sign-in is called to the owner. */
export const EDGE_METHOD = "open-platform.sh";
/** How an account came, in words: its sign-in method, else its origin, else "crew" for an agent, else "". */
export function originOf(a: Pick<AccountRow, "kind" | "via" | "signsInThrough">): string {
  if (a.signsInThrough !== null) return `signs in through ${a.signsInThrough}`;
  return a.via ?? (a.kind === "agent" ? "crew" : "");
}
/** Every person and agent, newest first. Organizations are not accounts. */
export function listAccounts(db: Database): AccountRow[] {
  const rows = db
@@ -78,6 +89,7 @@
export function listAccounts(db: Database): AccountRow[] {
      isAdmin: r.is_admin === 1,
      createdAt: r.created_at,
      via: origin.via,
      signsInThrough: origin.via === "edge" ? EDGE_METHOD : null,
      address: origin.address,
      repos: r.repos,
      apps: r.apps,
@@ -0,0 +1,223 @@
/**
 * Who can see a repository or an app, in one sentence.
 *
 * `audienceOf` is pure: plain facts in, a level and a sentence out, so every
 * wording is a test with no database. `audienceFacts` reads the facts, and
 * asks the same two rules the forge enforces — `REPO_VISIBLE` and
 * `REPO_WRITABLE` — once per candidate, so the sentence cannot drift from
 * what the forge lets through. Site admins are not counted: their reach is
 * the forge's, not the repository's. Nor are suspended or removed accounts.
 */
import type { Database } from "bun:sqlite";
import type { Repo } from "./repos.ts";
import type { User } from "./users.ts";
import { REPO_VISIBLE, REPO_WRITABLE } from "./visibility.ts";
export interface AudienceFacts {
  readonly visibility: "public" | "private";
  /** Who is looking: nobody signed in, a person, or a site admin. */
  readonly viewer: "signed-out" | "person" | "admin";
  readonly ownerKind: "user" | "org";
  readonly ownerName: string;
  /** People (not the crew) other than the viewer who may read it. */
  readonly readers: number;
  /** People (not the crew) who may change it, the viewer among them. */
  readonly writers: number;
  readonly viewerWrites: boolean;
  /** Whether any of the crew works here. */
  readonly crew: boolean;
  /**
   * The app's access, when the repository has an app. People are counted
   * once, never the viewer: `writers` may change the repository, `named` are
   * named by the app and cannot.
   */
  readonly app?: {
    readonly access: "public" | "private";
    readonly writers: number;
    readonly named: number;
    readonly viewerNamed: boolean;
  };
}
export interface Audience {
  /** `split`: public code, private app. */
  readonly level: "public" | "private" | "split";
  readonly sentence: string;
}
const people = (n: number): string => (n === 1 ? "1 person" : `${n} people`);
/** "a", "a and b", "a, b, and c". */
function list(parts: readonly string[]): string {
  if (parts.length <= 2) return parts.join(" and ");
  return `${parts.slice(0, -1).join(", ")}, and ${parts[parts.length - 1]}`;
}
export function audienceOf(facts: AudienceFacts): Audience {
  const crew = facts.crew ? ["the crew"] : [];
  // "you" is only said to someone signed in.
  const you = facts.viewer === "signed-out" ? [] : ["you"];
  if (facts.visibility === "public") {
    if (facts.app?.access === "private") {
      // Only someone who can change it, or is named, is told "you".
      const youToo =
        you.length > 0 && (facts.viewerWrites || facts.app.viewerNamed)
          ? you
          : [];
      const parts = [
        ...youToo,
        ...(facts.app.writers > 0
          ? [`${people(facts.app.writers)} who can change it`]
          : []),
        ...(facts.app.named > 0 ? [`${people(facts.app.named)} it names`] : []),
        ...crew,
      ];
      return {
        level: "split",
        sentence: `The code is public; the app answers only to ${
          parts.length === 0 ? "nobody" : list(parts)
        }.`,
      };
    }
    const changers = [
      ...(facts.writers > 0 ? [people(facts.writers)] : []),
      ...crew,
    ];
    return {
      level: "public",
      sentence: `Public: anyone can read this; ${
        changers.length === 0 ? "nobody" : list(changers)
      } can change it.`,
    };
  }
  const parts = [...you];
  if (facts.readers > 0) {
    parts.push(
      facts.ownerKind === "org"
        ? `${people(facts.readers)} in ${facts.ownerName}`
        : facts.ownerName,
    );
  }
  const alone = parts.length === 1 && you.length === 1;
  parts.push(...crew);
  return {
    level: "private",
    sentence: `Private: ${alone ? "only " : ""}${
      parts.length === 0 ? "nobody" : list(parts)
    }.`,
  };
}
/** How many active, non-admin people satisfy `rule` (over `?1` = repo id). */
function countPeople(
  db: Database,
  repoId: string,
  rule: string,
  exceptId: string | undefined,
): number {
  // The rules take the person as `?1`; here each candidate is that person.
  const asCandidate = rule.replaceAll("?1", "cand.id");
  return db
    .query<{ n: number }, [string, string]>(
      `SELECT COUNT(*) AS n FROM user AS cand
        WHERE cand.kind = 'user' AND cand.is_agent = 0 AND cand.is_admin = 0
          AND cand.suspended_at IS NULL AND cand.removed_at IS NULL
          AND cand.id != ?2
          AND EXISTS (SELECT 1 FROM repo WHERE repo.id = ?1 AND ${asCandidate})`,
    )
    .get(repoId, exceptId ?? "")!.n;
}
/**
 * `REPO_WRITABLE` without its site-admin clause: an admin's reach is the
 * forge's, so only membership of the repository itself earns them "you".
 * Throws if the rule is reshaped, so a drift cannot go unnoticed.
 */
const MEMBER_WRITABLE = REPO_WRITABLE.replace(
  /\n\s*OR EXISTS \(SELECT 1 FROM user AS me WHERE me\.id = \?1 AND me\.is_admin = 1\)/,
  "",
);
if (MEMBER_WRITABLE === REPO_WRITABLE) {
  throw new Error("audience: REPO_WRITABLE lost its admin clause; update audience.ts");
}
export function audienceFacts(
  db: Database,
  repo: Repo,
  viewer: User | undefined,
): AudienceFacts {
  const owner = db
    .query<{ kind: "user" | "org" }, [string]>(
      "SELECT kind FROM user WHERE id = ?1",
    )
    .get(repo.ownerId);
  const viewerId = viewer?.id;
  const writes =
    viewerId !== undefined &&
    db
      .query<{ n: number }, [string, string]>(
        `SELECT 1 AS n FROM repo WHERE repo.id = ?1 AND ${MEMBER_WRITABLE.replaceAll("?1", "?2")}`,
      )
      .get(repo.id, viewerId) !== null;
  const app = db
    .query<{ id: string; access: "public" | "private" }, [string]>(
      "SELECT id, access FROM app WHERE repo_id = ?1",
    )
    .get(repo.id);
  const namedRule = `EXISTS (SELECT 1 FROM app_member AS m WHERE m.app_id = ?1 AND m.user_id = cand.id)`;
  const activePerson = `cand.kind = 'user' AND cand.is_agent = 0 AND cand.is_admin = 0
          AND cand.suspended_at IS NULL AND cand.removed_at IS NULL`;
  const named = app
    ? db
        .query<{ n: number }, [string, string, string]>(
          `SELECT COUNT(*) AS n FROM user AS cand
            WHERE ${activePerson} AND cand.id != ?3 AND ${namedRule}
              AND NOT EXISTS (SELECT 1 FROM repo WHERE repo.id = ?2 AND ${REPO_WRITABLE.replaceAll("?1", "cand.id")})`,
        )
        .get(app.id, repo.id, viewerId ?? "")!.n
    : 0;
  const viewerNamed =
    app !== null &&
    viewerId !== undefined &&
    db
      .query<{ n: number }, [string, string]>(
        "SELECT 1 AS n FROM app_member WHERE app_id = ?1 AND user_id = ?2",
      )
      .get(app.id, viewerId) !== null;
  return {
    visibility: repo.visibility,
    viewer:
      viewer === undefined ? "signed-out" : viewer.isAdmin ? "admin" : "person",
    ownerKind: owner?.kind ?? "user",
    ownerName: repo.ownerName,
    readers: countPeople(db, repo.id, REPO_VISIBLE, viewerId),
    writers: countPeople(db, repo.id, REPO_WRITABLE, undefined),
    viewerWrites: writes,
    crew:
      db
        .query<{ n: number }, [string]>(
          "SELECT 1 AS n FROM repo_crew WHERE repo_id = ?1",
        )
        .get(repo.id) !== null,
    ...(app
      ? {
          app: {
            access: app.access,
            writers: countPeople(db, repo.id, REPO_WRITABLE, viewerId),
            named,
            viewerNamed,
          },
        }
      : {}),
  };
}
/** The audience of `repo` for `viewer`: the one call the API and the MCP share. */
export function audienceFor(
  db: Database,
  repo: Repo,
  viewer: User | undefined,
): Audience {
  return audienceOf(audienceFacts(db, repo, viewer));
}
@@ -13,7 +13,7 @@
import type { Database } from "bun:sqlite";
import { readFileSync } from "node:fs";
import { totalmem } from "node:os";
import { availableParallelism, totalmem } from "node:os";
export type CapacityKey =
  | "crew.concurrency"
@@ -34,6 +34,8 @@
interface Definition {
    | { readonly type: "choice"; readonly values: readonly string[] };
  /** What it is when neither the store nor the environment says. */
  readonly fallback: string;
  /** The default from the pod's CPUs, in place of `fallback` (#2549). */
  readonly sized?: (cpus: number) => number;
  readonly help: string;
}
@@ -43,21 +45,23 @@
export const CAPACITY: Record<CapacityKey, Definition> = {
    env: "FORGE_CREW_CONCURRENCY",
    kind: { type: "count", min: 1, max: 16 },
    fallback: "2",
    help: "How many agent runs go at once. Lowering it lets running ones finish; the next one waits.",
    help: "How many agent runs go at once. Empty: one per 3 GiB of memory above the forge's own 2 GiB, from the pod's limit, at least 2 and at most 16. Lowering it lets running ones finish; the next one waits.",
  },
  "checks.concurrency": {
    label: "Checks at once",
    env: "FORGE_CHECKS_CONCURRENCY",
    kind: { type: "count", min: 1, max: 16 },
    fallback: "1",
    help: "How many repository checks run at once. Lowering it lets running ones finish; the next one waits.",
    sized: (cpus) => checksForCpus(cpus),
    help: "How many repository checks run at once. Lowering it lets running ones finish; the next one waits. Empty: the pod's CPUs over three per shard (shards × 3), from 1 to 16 — 80 CPUs give 6.",
  },
  "checks.shards": {
    label: "Test shards per check",
    env: "FORGE_CHECK_SHARDS",
    kind: { type: "count", min: 1, max: 32 },
    fallback: "",
    help: "How many parallel shards a check's test run uses (handed to checks and crew shells as FORGE_CHECK_SHARDS). Empty: as many as the machine has cores.",
    fallback: "1",
    sized: (cpus) => shardsForCpus(cpus),
    help: "How many parallel shards a check's test run uses (handed to checks and crew shells as FORGE_CHECK_SHARDS). Empty: the pod's CPUs, at most 4 — and, in a check, no more than its cell holds.",
  },
  "children.max": {
    label: "Forges this one may send",
@@ -106,10 +110,70 @@
export function validCapacity(key: CapacityKey, value: string): boolean {
}
/** The boot default: the environment, when it holds a valid value. */
export function capacityDefault(key: CapacityKey): string {
export function capacityDefault(key: CapacityKey, cpus?: number): string {
  const env = explicitDefault(key);
  if (env !== undefined) return env;
  const sized = CAPACITY[key].sized;
  if (sized !== undefined) return String(sized(cpus ?? cpuLimit()));
  if (key === "children.max") return String(childrenForMemory(memoryLimitBytes()));
  if (key === "crew.concurrency") return String(crewForMemory(memoryLimitBytes()));
  return CAPACITY[key].fallback;
}
/** The environment's value for `key`, when it holds a valid one. */
function explicitDefault(key: CapacityKey): string | undefined {
  const env = process.env[CAPACITY[key].env]?.trim() ?? "";
  if (env !== "" && validCapacity(key, env)) return env;
  return key === "children.max" ? String(childrenForMemory(memoryLimitBytes())) : CAPACITY[key].fallback;
  return env !== "" && validCapacity(key, env) ? env : undefined;
}
/** Most shards a check takes by default: past four, a test run waits on git, not CPU. */
const SHARDS_DEFAULT_MAX = 4;
/** CPUs a shard is given by default: its bun test, the git children it starts, and the forge it boots. */
const CPUS_PER_SHARD = 3;
/** Shards per check by default: `min(4, cpus)`, at least one. 80 CPUs give 4; 2 give 2. */
export function shardsForCpus(cpus: number): number {
  const n = Math.floor(cpus);
  return Number.isFinite(n) ? Math.max(1, Math.min(SHARDS_DEFAULT_MAX, n)) : 1;
}
/**
 * Checks at once by default: `floor(cpus / (shards × 3))`, held to 1–16, so
 * every check's shards have three CPUs each. 80 CPUs: 4 shards, 80 / 12 = 6
 * checks; 16 CPUs: 1; 4 CPUs: 4 shards, 4 / 12 → 1 (#2549).
 */
export function checksForCpus(cpus: number, shards = shardsForCpus(cpus)): number {
  const n = Math.floor(cpus / (shards * CPUS_PER_SHARD));
  return Number.isFinite(n) ? Math.max(1, Math.min(16, n)) : 1;
}
let cpusForTest: number | undefined;
/**
 * A test's CPU count in place of the pod's, so a default sized from it is
 * the same on a laptop and an 80-core runner; `undefined` reads the pod again.
 */
export function provideCpusForTest(cpus: number | undefined): void {
  cpusForTest = cpus;
}
/**
 * The pod's CPU limit: cgroup v2 `cpu.max` ("quota period", in µs), rounded
 * down and never below one, else `os.availableParallelism()` — "max" (no
 * quota), an unreadable file or anything not two numbers falls back. Never
 * more than the machine has.
 */
export function cpuLimit(file = "/sys/fs/cgroup/cpu.max"): number {
  if (cpusForTest !== undefined) return cpusForTest;
  const machine = availableParallelism();
  try {
    const [quota, period] = readFileSync(file, "utf8").trim().split(/\s+/).map(Number);
    if (quota !== undefined && period !== undefined && quota > 0 && period > 0)
      return Math.max(1, Math.min(machine, Math.floor(quota / period)));
  } catch {
    // No cgroup v2 CPU controller here: the machine's count.
  }
  return machine;
}
const GIB = 1024 ** 3;
@@ -126,6 +190,18 @@
export function childrenForMemory(bytes: number): number {
}
const MIN_CHILDREN = 5;
/**
 * One crew run per 3 GiB of room above the forge's own 2 GiB (32 GiB gives
 * 10, 64 GiB the ceiling of 16), never fewer than two: a big machine runs
 * the crew it can hold without an override the sentinel then flags (#2548).
 */
export function crewForMemory(bytes: number): number {
  const n = Math.floor((bytes - 2 * GIB) / (3 * GIB));
  return Math.min(MAX_CREW, Math.max(MIN_CREW, Number.isFinite(n) ? n : MIN_CREW));
}
const MIN_CREW = 2;
const MAX_CREW = 16;
/** The pod's memory limit (cgroup v2, then v1), else the machine's memory. */
export function memoryLimitBytes(): number {
  for (const f of ["/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory/memory.limit_in_bytes"]) {
@@ -201,10 +277,18 @@
export function egressSetting(db: Database): "enforce" | "report" {
  return capacityValue(db, "cells.egress") === "report" ? "report" : "enforce";
}
/** What a check's commands and a crew shell are told: the shard count, when one is set. */
export function capacityEnv(db: Database): Record<string, string> {
  const shards = capacityValue(db, "checks.shards");
  return shards === "" ? {} : { FORGE_CHECK_SHARDS: shards };
/**
 * What a check's commands and a crew shell are told: the shard count in
 * force. An admin's value or the environment's is passed as it is; the
 * default is sized from `cpus` — a check passes what its cell holds, so a
 * small cell is never told the pod's count.
 */
export function capacityEnv(db: Database, cpus?: number): Record<string, string> {
  const shards =
    storedCapacity(db, "checks.shards") ??
    explicitDefault("checks.shards") ??
    String(shardsForCpus(cpus ?? cpuLimit()));
  return { FORGE_CHECK_SHARDS: shards };
}
export interface CapacityChange {
@@ -120,3 +120,8 @@
export function revokeInvite(
  if (changed === 0) return err("not_found");
  return ok(view(db.query<Row, [string]>("SELECT * FROM invite WHERE id = ?1").get(inviteId)!, now));
}
/** The id of the invite an account spent, if one let it in. */
export function inviteSpentBy(db: Database, userId: string): string | undefined {
  return db.query<{ id: string }, [string]>("SELECT id FROM invite WHERE used_by = ?1").get(userId)?.id;
}
@@ -0,0 +1,119 @@
{
  "0001-foundation": "b6c433603a3fe1c2e79b9d66fe9bc3bc67f403d0f815a9f81dc16a00ef245da5",
  "0002-conversations": "5af40c718d7c91c0340d2e1609ecc2edd69a91373e33d78ea561f4ef4de700e1",
  "0003-pull-requests": "6a53fb594a0bc023a80948c2cde3500a37a95aafe30a896b157794aee1998094",
  "0004-forks": "9c323d30f8afb5b5e0b26dece297d88e41c758f21bf8155c9cc32d9df7d61609",
  "0005-organizations-and-teams": "90ac9619b147b5818762c74e97da40fba00fdfe77071b642ddf4bb45b279f927",
  "0006-ssh-public-keys": "f5bcf371354ab8e32c71e33c220a5d1e9554b4a6bb5b202922a47a07aa645979",
  "0007-per-user-interface-preferences": "97d11a5e8f8d14bd1b4c8027201f6d286bf480d61c7d116ef3abcfa87d4824e8",
  "0008-origins": "ae3db8d70c0890e72362e4c4e246eb4cf34e66354de622338015b5276b22eefb",
  "0009-oauth-for-origins": "5e8796b669bca4271907987f88fab4f37e230247b5f49465a00ce14fedd1666a",
  "0010-a-short-link-for-a": "37d793621278ccb06b0f0e3c9c4f5996fddc10ac4532a24c5ed5fdee5273a931",
  "0011-counts-on-a-list": "a8d5bac07bdbd8eaa77a73e0344dd3b8233fea0791084b247a4d5c85288d5344",
  "0012-the-indexes-a-busy-forge": "6aa333646f3751566cab4892f957e4aa7ffbfd8750fa3e1b76f514eaa46a6e57",
  "0013-review-states": "54c0f87c1f3bbbabf131b1722d811f915c536a752d2ebbdcec3a0fccbe22b6d7",
  "0014-apps": "3508216ce82bf466b5fe9e07bc69dd10c5ddf1d3cbd7e95c1daa7027ab480747",
  "0015-an-app-s-environment": "8949dbd4c0c38431d02cc9818827b68a3f196265624df9d100afc9a0974f84ad",
  "0016-a-paused-run": "af46f93cd74f18e640fd0a91e02dac0c31a5a52c4d354bcdb2856e21b23e234e",
  "0017-checks": "69ca4560ae58d18ee836d835555d436a5ea2a48e79a1467e51b1f937172b7173",
  "0018-a-run-s-lease": "eca89f69fee4c278d27e8764d08ac49a3e5bdeefc54a29e22098a7110daaac2a",
  "0019-how-many-turns-an-agent": "1217cffaad87574c9b786de2cae43e6638a3803b6df75f0e4df5ea16bb895376",
  "0020-issues-that-depend-on-issues": "5f8a0b647e9060606e21eb4a982d1815225ffc365e145eb9471c297c3343af2f",
  "0021-a-repository-can-run-on": "85a05853ae9e636049597fe9036ac536ba3efac6bca66496b6cd75e52a70fc25",
  "0022-the-decision-ledger": "b5e6e4d873686180dd9e76bab6ce0f9ef402a3444d96f2706d62c07cc18e80a3",
  "0023-the-look": "86890cbc353427bd6abe535a5ea2cad2661d437ed285407099b73d484e72071a",
  "0024-a-variable-has-a-scope": "1f9049bb4d0a0843dee7975f6b6201d396ac2e6bc4a155b8c66d65dff47efad5",
  "0025-what-a-running-app-said": "77a8887bfb3309ca024bbd9430240d716454fae860383a6d02e95d8e47223cfa",
  "0026-subscriptions": "85b5a7e228cefb1689c48f6e7862b9539f80d9c4240263a7e428716ccacf123b",
  "0027-the-moment-a-deploy-started": "e43e6b3d4fadba7e03dfb6bb54406ac90326823e9fb8f56b127a551bf4f8ae5f",
  "0028-a-failed-check-s-own": "8b3ff15d07875635f1a5d378f09b0a432fa5650272b621bf1f1ba47c352c2d79",
  "0029-a-third-width": "b4d8527a8a9e5002d2a2b3012fe064c365c180b680a59bb2d19db718f3ec9ad0",
  "0030-take-it-back": "727ee2c2371a095738a3df1079d8f4f98234b695f84d63466d98444a681f0f31",
  "0031-a-look-knows-which-page": "a62ab5826175af737f7b56c2d209ecf61d1bcad1707d0db625ee152bea8cc70f",
  "0032-an-app-s-own-domains": "fa420e16fc6a7723092944018cea038e29bfed19d1cb4c7729fbcb708bf96b69",
  "0033-where-a-repository-s-next": "321bf75d987ac60b6a0dd58b12d461c1758be11a14e84edc8d071ca49ad3ee0d",
  "0034-forge-level-secrets": "2c31a3db69e0e7cb1167ed3efa2d2a496532da896bc90e39979f4793cb70cd49",
  "0035-what-a-viewer-did-with": "fe01b3e75c1f9d4cae068fdb696949ac2691cde2796edb513a175eed3ef2162c",
  "0036-a-share": "dc1c385f26c13547e79eecafaa830a4adc0f4ff0d4856b95e6a50d075f0c94e9",
  "0037-the-usage-ledger": "d5d815aa3bef5e1f4cceb114771cf1a3e57931fdc21235da9208c0122ab93d7b",
  "0038-a-subscription-s-standing": "2445f7d024399cc6f27248ad91ac6693369f6dfbcbd96702e91f1cf7ff022f59",
  "0039-whether-the-agent-s-shell": "cc3a57567f75490d5607e43097ecefc00bf3dc9529304be8874ae697a6231ee7",
  "0040-a-fleet-operation-s-receipt": "ebe243edc22f30a29b6950e0f8cfce4c7ba346d11f7a4a4c61e4a08298b14965",
  "0041-the-meter": "15b995bb9dcfcb16aa9dbee69372ede8e1dcee56857dfa06e5deb2a583520569",
  "0042-every-credential-is-a-subscription": "f56c85102d0ce700a5a3eb958a0134f47a3a0f0cfa1e60c37e0167d2be802012",
  "0043-an-interrupted-run-is-not": "d9949e799843d1221644af17544e4f80fd4e60232c101186432dae0b7d1d2057",
  "0044-why-an-issue-closed": "4f24a266d54eae3910da18f620e396b29b406ac4d993c13158875ce7590687c9",
  "0045-oauth-clients-registered-dynamically": "ce0cf0b16b9094766362a288b9b984a00e8394b5b4821f370acf863b308ace61",
  "0046-which-address-the-forge-s": "96917061cb4b064779a15dba9cbac82c4cba9ff2aefd71acd229e6aca4273190",
  "0047-a-report-from-outside": "be5336004ba3141e71809194b65eb333866a678c5f2bb65c16be8681c65b63e7",
  "0048-an-account-the-owner-suspended": "21c01ec3f30ef0e7f55a293601e95db31c1437f3da33ecf46036d9a425e5a96d",
  "0049-the-asks-ledger": "a29d814f26ba38e871e7271cd955f14022ff923299b7a3af1ba85866f07221e2",
  "0050-outsiders-may-trigger-the-crew": "28be9094e981e547f7f3bd1eaa4b0fd87acf346ccc1188469341c262901de488",
  "0051-the-front-page-s-picks": "e3e9f27841b17e3951e627680cdfcca30e366bc687d8fd6f4c3b56a9661a5b2a",
  "0052-a-whole-life-budget-on": "922ee458bbc52f0d6a9938e36ff09494066a8cb557bd7788495d259c9b335f39",
  "0053-whether-an-account-s-email": "1e64971a56d0c0e16ec513cc506bc3a637efe843092d8a8cc99753a6c77a32e8",
  "0054-the-people-a-private-app": "7670ecd442c0f1a1e23e878fe5e8accb4fd29a58c6bf3b27841a1c2a062352e6",
  "0055-a-hold-on-a-pull": "8a0da478bdb878d0b0bf29d5824735ac3e67a6a504877abaff81bcf36d282597",
  "0056-what-an-account-has-spent": "be08ec2eab632260848ed4c9951291907ace5bd81c514c4e3e53cd6cd0ab85cb",
  "0057-spend-reported-from-outside": "15000df57c5e4f686dc91bb0c937d9484a4466bc424be5b7570c22d4f820c7d4",
  "0058-a-signed-in-browser-has": "32b86a5f3074b1bce792e99aadc1f432b8ccff508703c9fcf6fa3ae537ad4eb4",
  "0059-every-message-the-forge-sent": "4b7829c5fc97384cb4d5d9005168288eeab9419003552d8051ec5561bc9ef242",
  "0060-who-a-picture-was-taken": "7d2319f66c78239cc662477d4ceda2ac50fd670342ebd69db1b7736041c1ed47",
  "0061-an-agent-never-owns-an": "7d4b174818752723caaef32300a0a21783e1db9aca6b72334a44dd7ca092fd1f",
  "0062-only-an-agent-is-ever": "57c2dc3400fc1e2db9acf86f2eb0231561e8e6ed87b54770a9d79493137cfe0e",
  "0063-the-forge-s-capacity": "7721b2146736a958a08eb3e6f77b5ea6e179fd920dfc94e9a9ae98c88ce4eaba",
  "0064-the-front-page-s-settings": "e15834cc1c7e45a58f4849cbad2ee5b627c3ad895c716030431b4961a47f30e9",
  "0065-a-sealed-issue": "46d20691683748274ff0aeab0cd8d2640bb3f0427a4a3235e6a8b828ab0d2396",
  "0066-a-milestone-a-person-watches": "782bfde34aac27ea01782ae22bbf3152e6f32925443e5ce212594f29402af0ff",
  "0067-every-fork-a-person-made": "c2f5bf838522142e42b047e400b6725f711bd0280536a8c143aa5f30a8f24ad9",
  "0068-the-run-a-comment-was": "1a92482c52858a1a326539c0aa1ed9c94d52dd9a2f274c7645a44c4a8840b2a2",
  "0069-an-event-s-place-in": "b266f8f36f9a9a2382e7c5107dc9c5ce70a0c097ad8f5dc0c3f53f893c7a3476",
  "0070-a-sealed-repository-is-never": "be9c7af107676fee4a652dc45580c817309a917211dc90435eb94fa5e2a46e45",
  "0071-an-optional-anthropic-compatible-api": "f5e6a4df5bd7cca6371d164a7e254ec19d5a5db9986c8b120b2f398d2af0fc05",
  "0072-when-a-mirror-lost-its": "34867bc4869e7d8928fa284027179adb6b86680ae540cd31564d4bb941c6eb94",
  "0073-authorization-codes-for-the-mcp": "a1d0617595b908c81a4aba107a00b600c980b8048ffeea98975b6e724dd385e6",
  "0074-a-look-a-person-attached": "4fd216ce408ee877ffbb8b38cab0234c22242da61377e9011f7ccf3189096d74",
  "0075-a-limb-s-place-among": "bc117e5988500afdbc7b4406a232449d1881cd1e78a6f9e9edbabb09cff9c53f",
  "0076-the-sentinel-s-memory": "ff8882d41e1702253a32d09324b1d9f00f29b21a22511779c914dbd7adfd7a01",
  "0077-openid-connect-over-the-same": "1507c0bc5b5168e4b440d9963be59dcb10b94b14d18da1182ca0da6787f22090",
  "0078-variants": "8c983ce83e8cad50f1e015ee5f4eefe77e503471d46bc46a43e5e3b71180d06d",
  "0079-degraded-history": "6c58267cfafaa53117a3945013e9591ae98a8fcc5f1f3998bc9113e4ea62d892",
  "0080-restore-drills": "35147ba9da81cda49cc77346c5c2d5eb71c72afc108676480237bd8d01f98f06",
  "0081-thoughts": "b8fca6511f61669d33d34eeb180a64767b08b744e41496b2968b19943f4a6574",
  "0082-passkeys": "074a5d91370d9d0ec073d9abbeded908a6541a743dcf55456e758422381614e4",
  "0083-releases": "c0e3680642b226841bf039c48b5ae806c97ac0ce18fc8d71fc8eec49aeefe27c",
  "0084-placements": "60036ee7281d17aad4a034409daaaab027ca6c524297da7d29618ef9eaa957df",
  "0085-a-standing-rule-per-repository": "8c47857982bb540589b1b418926218475036050ef9449f3a5f80ef2ed3f5c8f0",
  "0086-what-a-production-deploy-carries": "62379dffd5a0273355fcd1e5e353e046283c65a8ab7c068958efaa866d5db37d",
  "0087-supersession-and-repeats": "bc2647e3fcb0f32b7dda800cb6a38eb0d5e340243e07bd6ad7cf8291d48efbcb",
  "0088-what-the-machine-used": "afc9a4d24ead17763ccd7bdb4aeae80b9bafc3af769c5ded89bef921c7f622ad",
  "0089-a-parked-pull": "7b564005fff642a38dc60cd58c805e97888845f89ad884af9baf79fa615cf77f",
  "0090-a-green-check-proven": "21f33d1dabaf9722611db3a1630bc1c81aca6535a32398b7bbe7ddc47259994f",
  "0091-where-a-check-ran": "8ac84fa10e0079815a48cb7e57dfbf3e5f67bcf820695e57b7841374d570f943",
  "20261015000100-plans-need-a-nod": "754eb716bfe65056915e5f01bcbd6f15cf6b73b413da952c0a9b6ea1990b26ba",
  "20261015090000-a-repository-put-away": "692ca4ed5c82f4c9a281a74ee0716182fd4067a148a2652575c33e9e2a0055c4",
  "20261015090000-feed-chips": "a6057329aa558e6d25916f1edb88fffb30383eac87e24e966994631051029bfb",
  "20261015120000-check-base-tip": "14e4279c9da82e3f7f9a5e163c0f8c75dcaa436e628586af9cab9c25e2db653d",
  "20261015230000-a-domain-drill": "8db343e16d0bfacda42598908cb7ec113485b5aff30c7f663dffe0198cfbeb5c",
  "20261015230532-telemetry-read": "8106ad063b8b13c22c0115dfc23141dab081cb2a23168a1eb891b2fe30b4cc65",
  "20261016090000-sentinel-kept-settings": "03c30ac6d4e9d3164cc0f32af1c0f9d1f69a02e1147dc25bd1fe1838c077d5aa",
  "20261016120000-kept-is-a-person-s": "d12558f4a7395c91e46b6abddf836719509e0531630a046b6114926c5aad4a91",
  "20261016120000-proposal-state": "f24b95f42c1ceee8ed0f69093500365c218be3ec2c00be47a60ab77383a833f7",
  "20261016130000-sign-in-with-the-parent": "cd2a2f60afac864446fee454e746ca3dac0a29848446bd27697a5dc0ba5a7124",
  "20261016130000-usage-count": "1e715bfd5e4e28709903ee3a380fd1e8bf6b2be0a0321186ffa803676e6d5906",
  "20261016140000-a-model-grant": "7ec2dbdfe5e782eb98b5c42f2ae79a686631c3416c9787ab3708adab8fec0777",
  "20261016140000-drill-counts": "2ec120b90753826a5cbb6c7416b84a563566a0995ccd6d0350e00cf63ea16d45",
  "20261016140000-flag-use": "a3375653f892f6eccff8493aebd2b42084d68b4612e80b595cd1f09dcda988da",
  "20261016140000-machine-temperature": "74754feca643996c3b214f799a57868b0ab298a6ac9d9fa85905cd16e8907e9d",
  "20261016150000-memory-hour": "41077b8c7b9b0421255f82b6c14954b4abad249cb51472e064cb130bc0aa0128",
  "20261016160000-cloud-audit": "a64b2356735e172b21a8c462418b84065de9a54f121fa8371e812d97264d00d1",
  "20261016160000-proposal-tokens": "335e0558843d8c103213ef8b9a1202e52cc8bd0674dc0e5db73f3429d38cc7c9",
  "20261016170000-pull-run-grant": "dca1fa79e48ece7ba962b6e2eabeda513b7b364ed68e41a814ca73e123a9b811",
  "20261016170000-upstream-proposals": "ea8c2b809705a5ac38dd94470e3024a560e61f44f01e4f51a84f7d9699df3338",
  "20261016180000-account-ceiling": "0077901b08d41b6af61fe4b7ad7bf3bb16c923e755b5898c2f6d86837e9a42d0",
  "20261016180000-signin-only-code": "c69edd5906a2f3425d6ce867ef9c7b6d59c40fb582ef6cbea0f4c86a67eb0030",
  "20261016190000-member": "9076848fe032dea6905927a1b454c2381a3129acb3ce357c2726e82f1c161bc4",
  "20261016200000-invite": "72bc7e9be6a7223048d1d1cf0226bc93ee4dbea9f00dbb20a6d457305886cbf9",
  "20261017195831-ceiling-reserved": "1e6756752ae06c78a5920ae839e911e837a14cff3ac766cbc38e117c0a40186b",
  "20261018100000-incident-remembered": "61fefa1d0750f3be617fdc23c1e6e17abc1ba610d980d85aff8824f14bf3e253"
}
@@ -4,6 +4,9 @@
// check (`REPO_WRITABLE`'s crew clause) and `onCrew` can never drift.
// A person already there is reported at boot (`peopleOnCrews`), never
// removed by a migration.
// The RAISE messages below are shown to any caller, anonymous ones included
// (#723); they are written for anyone, and are known by name in
// src/http/refusal.ts because the numbered migrations' text is frozen.
export default `
  CREATE TRIGGER repo_crew_agents_only BEFORE INSERT ON repo_crew
  WHEN NOT EXISTS (SELECT 1 FROM user WHERE id = NEW.user_id AND is_agent = 1)
@@ -1,6 +1,9 @@
// 70 — a sealed repository is never public, said by the database (#865's
// review): the rule lived in `updateRepoSettings`; these triggers hold
// it for every writer of `repo.visibility`, now and later.
// The RAISE messages below are shown to any caller, anonymous ones included
// (#723); they are written for anyone, and are known by name in
// src/http/refusal.ts because the numbered migrations' text is frozen.
export default `
  CREATE TRIGGER repo_sealed_never_public BEFORE UPDATE OF visibility, sealed_for ON repo
  WHEN NEW.sealed_for IS NOT NULL AND NEW.visibility = 'public'
@@ -2,6 +2,9 @@
// row per thought, appended and never changed: the triggers refuse an
// UPDATE or DELETE, and there is no foreign key so no cascade can remove
// one when an account goes. The author is an id, not a reference.
// The RAISE messages below are shown to any caller, anonymous ones included
// (#723); they are written for anyone, and are known by name in
// src/http/refusal.ts because the numbered migrations' text is frozen.
export default `
  CREATE TABLE thought (
    id          TEXT PRIMARY KEY,
@@ -0,0 +1,7 @@
// A degraded finding remembers the incident it was filed for (#2553): the
// time the cause first opened, and when its closed issue was last told the
// incident goes on. Both are null for every other finding.
export default `
  ALTER TABLE sentinel_finding ADD COLUMN incident TEXT;
  ALTER TABLE sentinel_finding ADD COLUMN commented_at TEXT;
  `;
@@ -1,5 +1,5 @@
{
  "note": "The migrations each release applied, newest first. The release agent adds the new release here when it cuts one: `migrations` counts the migration ids it applied (every file in src/core/migrations/), and `ids` names the stamped ones among them — every file past 0090, its name without .ts (`ls src/core/migrations | grep -v ^0 | sed s/.ts$//`). A release from before #1837 has a count only: the first that many numbered files. Every migration that is not in both of the two newest releases must be additive (test/schema-additive.test.ts, notAdditive in src/kernel/migrate.ts): the old process keeps serving on the same database while the new one rolls in.",
  "note": "The migrations each release applied, newest first. The release agent adds the new release here when it cuts one: `migrations` counts the migration ids it applied (every file in src/core/migrations/), and `ids` names the stamped ones among them — every file past 0090, its name without .ts (`ls src/core/migrations | grep -v ^0 | sed s/.ts$//`) — and runs `bun scripts/pin-migrations.ts`, which pins each new migration's text in src/core/migration-pins.json. A release from before #1837 has a count only: the first that many numbered files. Every migration that is not in both of the two newest releases must be additive (test/schema-additive.test.ts, notAdditive in src/kernel/migrate.ts): the old process keeps serving on the same database while the new one rolls in.",
  "releases": [
    {
      "version": "0.7.165",
@@ -72,6 +72,7 @@
export const SYSTEM_NAMES: readonly string[] = [
  "flow",
  "front",
  "go",
  "home",
  "icons",
  "live",
  "login",
@@ -9,7 +9,9 @@
 * new one is named by the UTC second it was written
 * (`date -u +%Y%m%d%H%M%S`, e.g. `20261015093000-deploy-notes.ts`), so two
 * pulls that each add one add two files and never touch the same line
 * (#1837). Each file is one `export default` of SQL.
 * (#1837). Each file is one `export default` of SQL. The database records
 * each applied one by name (#783); `migration-pins.json` holds what each
 * said when it was released, and a test fails if one says something else.
 */
import type { Database } from "bun:sqlite";
@@ -13,6 +13,9 @@
import { getUserById, isActive, type User } from "./users.ts";
export type TokenScope = "ro" | "rw" | "admin";
/** What a non-admin is told when they ask for a token that acts as admin. */
export const ADMIN_TOKEN_WORDS = "Only an admin can make a token with admin scope; ro and rw are open to you.";
const rank: Record<TokenScope, number> = { ro: 0, rw: 1, admin: 2 };
export function scopeAllows(held: TokenScope, needed: TokenScope): boolean {
@@ -48,7 +48,7 @@
export const HOLD_MS = 5 * 60_000;
/** The vitals causes this pager owns; the rest are paged elsewhere or are settings. */
export function owned(cause: string): boolean {
  return cause === "errors" || cause === "internal" || cause === "disk" || cause.startsWith("queue:");
  return cause === "errors" || cause === "internal" || cause === "disk" || cause === "release" || cause.startsWith("queue:");
}
/** One owned cause being watched. */
@@ -23,6 +23,7 @@
import type { Database } from "bun:sqlite";
import { getIssueById, standsForPerson } from "../core/issues.ts";
import type { Repo } from "../core/repos.ts";
import {
  alreadyDone,
  BY_CREW,
  DONE_LIKE,
  LATER,
@@ -195,7 +196,7 @@
export interface ParkedTwice {
 * nothing open. So an issue leaves the list when it is closed (split, or
 * replaced by a fresh one), when it is put behind something open (its
 * parts, or the Later gate), or when someone who directs the crew says what
 * to try. One read for every repository, and two more for each that has
 * to try, or when commits on main already name it (`alreadyDone`). One read for every repository, and two more for each that has
 * any.
 */
export function parkedTwice(
@@ -288,9 +289,13 @@
export function parkedTwice(
          : issue;
      });
    if (struck.length === 0) continue;
    // Finished work is the asker's to close, not the planner's to split.
    const done = alreadyDone(db, struck.map((issue) => issue.issueId));
    const open = struck.filter((issue) => !done.has(issue.issueId));
    if (open.length === 0) continue;
    // An open pull that would close it is someone's answer already.
    const proposed = proposedWork(db, repo);
    out.push(...struck.filter((issue) => !proposed.has(issue.number)));
    out.push(...open.filter((issue) => !proposed.has(issue.number)));
  }
  return out;
}
@@ -64,6 +64,8 @@
export interface Finding {
  readonly fix?: string;
  /** A setting finding: the value and `updated_at` a person may have kept (#2001). */
  readonly kept?: { readonly value: string; readonly updatedAt: string };
  /** A degraded finding: when its cause first opened. Same cause and time, same incident (#2553). */
  readonly incident?: string;
}
const clip = (text: string, n = 200): string =>
@@ -386,7 +388,7 @@
export function repeatedErrors(now: number): Finding[] {
 */
export function degradedCauses(db: Database, now: number): Finding[] {
  const from = now - DAY_MS;
  const byCause = new Map<string, { sentence: string; href: string; first: number; ms: number }>();
  const byCause = new Map<string, { sentence: string; href: string; first: number; opened: number; ms: number }>();
  for (const s of spans(db, from, now)) {
    const start = Math.max(s.openedAt, from);
    const held = Math.min(s.closedAt ?? now, now) - start;
@@ -396,6 +398,7 @@
export function degradedCauses(db: Database, now: number): Finding[] {
      sentence: s.sentence,
      href: s.href,
      first: seen?.first ?? start,
      opened: seen?.opened ?? s.openedAt,
      ms: (seen?.ms ?? 0) + Math.max(0, held),
    });
  }
@@ -415,6 +418,7 @@
export function degradedCauses(db: Database, now: number): Finding[] {
      ],
      pillar: "P0 Trust: the forge says when it is not well, and is well again",
      lookFor: `\`${cause}\` stays out of the degraded list for a day, or holds under ${DEGRADED_DAY_MS / 60_000} minutes.`,
      incident: new Date(d.opened).toISOString(),
    });
  }
  return out;
@@ -626,6 +630,42 @@
export function recordKept(db: Database, key: string, value: string, updatedAt:
  ).run(key, value, updatedAt, at);
}
/**
 * A degraded incident whose issue was closed as done and which goes on (same
 * cause, same first-open time) is told to that issue, not filed again: the
 * fix is likely merged but not live. At most one comment a day. True when
 * the incident is that one: the comments made (0 or 1), else undefined.
 */
function continuesDone(app: App, repo: Repo, actor: User, f: Finding, evidence: string, at: string, now: number): 0 | 1 | undefined {
  const prior = app.db
    .query<{ issue_id: string; commented_at: string | null }, [string, string]>(
      `SELECT t.issue_id, t.commented_at FROM sentinel_finding t JOIN issue i ON i.id = t.issue_id
        WHERE t.key = ?1 AND t.incident = ?2 AND i.state = 'closed'
        ORDER BY t.first_seen DESC LIMIT 1`,
    )
    .get(f.key, f.incident ?? "");
  if (prior === null) return undefined;
  const issue = getIssueById(app.db, prior.issue_id);
  if (issue === undefined || issue.closeReason !== "done") return undefined;
  if (prior.commented_at !== null && now - Date.parse(prior.commented_at) < DAY_MS) return 0;
  const minutes = f.evidence.find((e) => e.startsWith("total: "));
  const said = addComment(
    app,
    actor,
    repo,
    issue,
    `Still degraded after the fix; the fix may not be live yet. At ${at}${minutes === undefined ? "" : `, ${minutes.slice(7)}`}; evidence now:\n${f.evidence.map((e) => `- ${e}`).join("\n")}`,
  );
  if (!said.ok) {
    log.warn("sentinel could not comment", { key: f.key, error: String(said.error) });
    return 0;
  }
  app.db
    .query("UPDATE sentinel_finding SET commented_at = ?1, last_seen = ?1, evidence = ?2 WHERE key = ?3 AND issue_id = ?4")
    .run(at, evidence, f.key, prior.issue_id);
  return 1;
}
/**
 * Read, file, record. One open issue per finding: a tracked finding whose
 * issue is open is commented on only when its evidence changed; one whose
@@ -674,6 +714,11 @@
export function hunt(
            .run(at, evidence, f.key, tracked.issue_id);
          continue;
        }
        const told = f.incident === undefined ? undefined : continuesDone(app, repo, actor, f, evidence, at, now);
        if (told !== undefined) {
          commented += told;
          continue;
        }
        const title = `Sentinel: ${clip(f.title, 200)}`;
        // An issue already filed under this title whose record was lost
        // (the insert failed last hunt) is adopted, not filed twice.
@@ -694,8 +739,8 @@
export function hunt(
          if (f.urgent && f.kept === undefined) handOver(app, repo, actor, made.value);
        }
        app.db
          .query("INSERT INTO sentinel_finding (key, issue_id, evidence, first_seen, last_seen) VALUES (?1, ?2, ?3, ?4, ?4)")
          .run(f.key, issueId, evidence, at);
          .query("INSERT INTO sentinel_finding (key, issue_id, evidence, first_seen, last_seen, incident) VALUES (?1, ?2, ?3, ?4, ?4, ?5)")
          .run(f.key, issueId, evidence, at, f.incident ?? null);
      } catch (error) {
        log.error("sentinel filing failed", { key: f.key, error: String(error) });
      }
@@ -160,6 +160,7 @@
import {
  updateProfile,
} from "../actions/users.ts";
import { listEvents } from "../core/events.ts";
import { audienceFor } from "../core/audience.ts";
import { audienceFilter } from "../core/event_audience.ts";
import {
  forkParentFor,
@@ -221,7 +222,7 @@
import {
} from "../core/reviews.ts";
import { createSession } from "../core/sessions.ts";
import { mintProposal } from "../actions/proposal_tokens.ts";
import { listTokens } from "../core/tokens.ts";
import { ADMIN_TOKEN_WORDS, listTokens } from "../core/tokens.ts";
import {
  headInNamespace,
  PROPOSAL_BODY_MAX,
@@ -1854,6 +1855,8 @@
export function apiRoutes(router: Router<Handler>): void {
      str(input.name) || "unnamed",
      scope,
    );
    if (!minted.ok && minted.error === "forbidden")
      return apiError("forbidden", 403, ADMIN_TOKEN_WORDS);
    if (!minted.ok) return unauthorized();
    return json(minted.value, 201);
  });
@@ -2146,7 +2149,12 @@
export function apiRoutes(router: Router<Handler>): void {
  router.add("GET", "/api/v1/repos/:owner/:name", (ctx) => {
    const repo = loadRepo(ctx);
    return repo ? json(repoJson(ctx, repo)) : notFound();
    return repo
      ? json({
          ...repoJson(ctx, repo),
          audience: audienceFor(ctx.app.db, repo, ctx.auth.user),
        })
      : notFound();
  });
  router.add("PATCH", "/api/v1/repos/:owner/:name", async (ctx) => {
@@ -130,7 +130,7 @@
import {
} from "../crew/direct.ts";
import { setArchived, setCrewOutsiders, setDecisionsBy } from "../actions/repos.ts";
import { mapJson, marksFor } from "../web/pages/map.ts";
import { undoFleetOp } from "../actions/fleet.ts";
import { applyFleetIds, previewFleetIds, undoFleetOp } from "../actions/fleet.ts";
import { applyLimb, isLimbAction, previewLimb } from "../actions/limbs.ts";
import {
  milestoneData,
@@ -151,6 +151,7 @@
import {
  isAskedOrder,
  isAskedShow,
} from "../core/asked.ts";
import { audienceFor } from "../core/audience.ts";
import { linksFrom, linksTo } from "../core/thought_links.ts";
import { streamJson, thoughtStream } from "../core/thought_stream.ts";
import {
@@ -264,6 +265,67 @@
function repoArg(ctx: Ctx, user: User, args: Args): Repo | Answer {
    : bad("not_found", "no such app or repository (owner/name)");
}
const MAX_ARCHIVE_REPOS = 50;
/** `archive` over `repos`: the fleet's own operation, each row's outcome and the id to undo. */
async function archiveMany(ctx: Ctx, user: User, args: Args): Promise<Answer> {
  const names = args["repos"];
  if (
    !Array.isArray(names) ||
    names.length === 0 ||
    names.length > MAX_ARCHIVE_REPOS ||
    names.some((n) => typeof n !== "string")
  )
    return bad("invalid", `\`repos\` is one to ${MAX_ARCHIVE_REPOS} names, owner/name each`);
  if (args["repo"] !== undefined)
    return bad("invalid", "give `repo` or `repos`, not both");
  if (args["archived"] !== undefined && typeof args["archived"] !== "boolean")
    return bad("invalid", "`archived` is true or false");
  const archived = args["archived"] ?? true;
  const dryRun = args["dryRun"] !== false;
  if (user.kind === "agent")
    return bad("forbidden", "archiving or waking is a person's call, not an agent's");
  if (!dryRun && !scopeAllows(ctx.auth.scope ?? "ro", "admin"))
    return bad("forbidden", "archiving or waking needs an admin token");
  const ids: string[] = [];
  const missing: string[] = [];
  for (const name of new Set(names as string[])) {
    const repo = repoArg(ctx, user, { repo: name });
    if (isAnswer(repo)) missing.push(name);
    else ids.push(repo.id);
  }
  const query = `mcp: ${ids.length + missing.length} named`;
  const action = { verb: "archive", value: archived };
  const unknown = missing.map((repo) => ({
    repo,
    outcome: "refused",
    reason: "no such app or repository (owner/name)",
  }));
  const rowsOf = (rows: readonly { label: string; outcome: string; reason: string }[]) => [
    ...rows.map((r) => ({ repo: r.label, outcome: r.outcome, reason: r.reason })),
    ...unknown,
  ];
  if (dryRun) {
    const plan = previewFleetIds(ctx.app, user, "repo", ids, query, action);
    if (!plan.ok) return bad(plan.error.code, plan.error.message);
    return good({
      dryRun: true,
      archived,
      rows: rowsOf(plan.value.rows),
      says: "This is the plan; nothing changed. Send dryRun: false to do it.",
    });
  }
  const op = await applyFleetIds(ctx.app, user, "repo", ids, query, action);
  if (!op.ok) return bad(op.error.code, op.error.message);
  return good({
    dryRun: false,
    archived,
    operation: op.value.id,
    rows: rowsOf(op.value.rows),
    says: `Done. The \`limb\` tool with action undo and operation ${op.value.id} reverts it for 24 hours.`,
  });
}
const APP_ARG = {
  type: "string",
  description: "The app, as owner/name.",
@@ -343,6 +405,28 @@
const TOOLS: readonly Tool[] = [
      );
    },
  },
  {
    name: "repo",
    description:
      "One repository you can read: its name, description, visibility and autonomy, and who can see it as `audience` { level, sentence } — the same words as GET /api/v1/repos/:owner/:name.",
    inputSchema: {
      type: "object",
      properties: { repo: REPO_ARG },
      required: ["repo"],
    },
    writes: false,
    run(ctx, user, args) {
      const repo = repoArg(ctx, user, args);
      if (isAnswer(repo)) return repo;
      return good({
        repo: `${repo.ownerName}/${repo.name}`,
        description: repo.description,
        visibility: repo.visibility,
        autonomy: repo.autonomy,
        audience: audienceFor(ctx.app.db, repo, user),
      });
    },
  },
  {
    name: "here",
    description:
@@ -1209,22 +1293,28 @@
const TOOLS: readonly Tool[] = [
  {
    name: "archive",
    description:
      "Put a repository away (archive) or bring it back (wake), by name: nothing is deleted. Without `archived`, says which way it is. With `archived` (true to archive, false to wake), it plans — `dryRun` is true unless you say false — and only an admin of the repository, with an admin token and never as an agent, does it. Each change is recorded in the repository's activity.",
      "Put a repository away (archive) or bring it back (wake), by name: nothing is deleted. Without `archived`, says which way it is. With `archived` (true to archive, false to wake), it plans — `dryRun` is true unless you say false — and only an admin of the repository, with an admin token and never as an agent, does it. Each change is recorded in the repository's activity. To archive several at once, pass `repos` (owner/name each, at most 50) instead of `repo`: it runs the fleet `archive` operation, a dry run unless `dryRun` is false, and answers each repository's outcome and the `operation` id; the `limb` tool with action `undo` and that `operation` (with `app` naming any of them) reverts it for 24 hours.",
    inputSchema: {
      type: "object",
      properties: {
        repo: REPO_ARG,
        repos: {
          type: "array",
          items: { type: "string" },
          maxItems: MAX_ARCHIVE_REPOS,
          description: "Several repositories, as owner/name, to archive in one operation.",
        },
        archived: { type: "boolean" },
        dryRun: {
          type: "boolean",
          description: "Plan only (the default). Send false to do it.",
        },
      },
      required: ["repo"],
    },
    writes: true,
    dryByDefault: true,
    async run(ctx, user, args) {
      if (args["repos"] !== undefined) return archiveMany(ctx, user, args);
      const repo = repoArg(ctx, user, args);
      if (isAnswer(repo)) return repo;
      const archived = args["archived"];
@@ -0,0 +1,51 @@
/**
 * What a database guard's refusal may say to the caller (#723). A trigger's
 * `RAISE(ABORT, '…')` message is answered to anyone, anonymous callers
 * included, so only a message that opts in is shown: it starts with
 * `forge: ` and is written for anyone. Any other message is answered with a
 * generic text and logged in full for the operator.
 */
/** The prefix a guard puts on a message that is meant for the person refused. */
export const REFUSAL_PREFIX = "forge: ";
/** What a caller is told when a guard's message did not opt in. */
export const GENERIC_REFUSAL = "refused";
/**
 * The messages this repository's own triggers (migrations 62, 70, 81) raised
 * before the prefix existed. A migration is never rewritten once it has run,
 * so these stay as written and are known by name.
 */
const BUILT_IN = new Set([
  "only an agent can be on a crew",
  "a sealed repository is never public",
  "a thought is never changed",
  "a thought is never deleted",
]);
/** The words to show for a guard's message: its own, or the generic text. */
export function shownRefusal(message: string): string {
  return message.startsWith(REFUSAL_PREFIX) || BUILT_IN.has(message)
    ? message
    : GENERIC_REFUSAL;
}
/**
 * The original message of a refusal a database trigger raised
 * (`RAISE(ABORT, '…')`), or undefined for any other error.
 */
export function refusedByDatabase(error: unknown): string | undefined {
  if (!(error instanceof Error)) return undefined;
  const code = (error as { code?: unknown }).code;
  return code === "SQLITE_CONSTRAINT_TRIGGER" ? error.message : undefined;
}
/**
 * The words to show for a thrown error if a database trigger refused, else
 * undefined: the one door both the page and the terminal answer through.
 */
export function refusalText(error: unknown): string | undefined {
  const refused = refusedByDatabase(error);
  return refused === undefined ? undefined : shownRefusal(refused);
}
@@ -5,6 +5,7 @@
 * verdict. Auth is resolved once per request and handed to handlers as data.
 */
import { refusedByDatabase, shownRefusal } from "./refusal.ts";
import { NOT_A_MEMBER } from "../core/roles.ts";
import { signupOptions, signupPasskey } from "../web/pages/signup-passkey.ts";
import { toolbarComment, toolbarPage } from "../web/studio/toolbar.ts";
@@ -248,6 +249,7 @@
import {
  asksMute,
  asksPage,
} from "../web/studio/asks_page.ts";
import { homeArchive, homeArchived, homeArchiveUndo } from "../web/studio/home_archive.ts";
import { liveFeedFragment } from "../web/studio/feed.ts";
import { serviceWorker } from "../web/sw.ts";
import { explorePage } from "../web/studio/explore.ts";
@@ -654,6 +656,9 @@
export function buildRouter(): Router<Handler> {
  // Apps and people ranked by real numbers, narrowed in one line (#607).
  router.add("GET", "/explore", explorePage);
  router.add("POST", "/front", frontUpdate);
  router.add("POST", "/home/archive", homeArchive);
  router.add("GET", "/home/archived/:op", homeArchived);
  router.add("POST", "/home/archived/:op/undo", homeArchiveUndo);
  router.add("GET", "/signin", signinPage);
  router.add("POST", "/signin", signinSubmit);
  router.add("GET", "/signin/parent", parentStart);
@@ -1853,25 +1858,16 @@
export interface Servers {
}
/**
 * A refusal a database trigger raised (`RAISE(ABORT, '…')`): an operator's
 * guard saying no, in its own words. It is the answer, not a fault, so the
 * person sees the reason and a 403 rather than "internal".
 */
function refusedByDatabase(error: unknown): string | undefined {
  if (!(error instanceof Error)) return undefined;
  const code = (error as { code?: unknown }).code;
  return code === "SQLITE_CONSTRAINT_TRIGGER" ? error.message : undefined;
}
/**
 * What a thrown handler is answered with: the guard's own words and a 403
 * What a thrown handler is answered with: the guard's words (or a generic
 * "refused") and a 403
 * when a database trigger refused, else a 500 that says nothing of the fault.
 */
function failure(app: App, error: unknown): Response {
  const refused = refusedByDatabase(error);
  if (refused !== undefined) {
    app.log.warn("refused by the database", { reason: refused });
    return securityFloor(json({ error: "refused", message: refused }, 403));
    const shown = shownRefusal(refused);
    app.log.warn("refused by the database", { reason: refused, shown });
    return securityFloor(json({ error: "refused", message: shown }, 403));
  }
  app.log.error("unhandled request error", { error: String(error) });
  return securityFloor(json({ error: "internal" }, 500));
@@ -7,7 +7,10 @@
import { Database } from "bun:sqlite";
import { existsSync, writeFileSync } from "node:fs";
import { migrate } from "./migrate.ts";
import { createLog } from "./log.ts";
import { type Migration, migrate } from "./migrate.ts";
const log = createLog("db");
/**
 * How long a writer waits for another writer before giving up. Long by
@@ -34,7 +37,7 @@
const CACHE_KIB = 64 * 1024;
 */
const MMAP_BYTES = 512 * 1024 * 1024;
export function openDb(path: string, migrations: readonly string[]): Database {
export function openDb(path: string, migrations: readonly Migration[]): Database {
  const memory = path === ":memory:";
  const fresh = memory || !existsSync(path);
  const image = fresh ? migratedEmpty.get(migrations) : undefined;
@@ -47,8 +50,14 @@
export function openDb(path: string, migrations: readonly string[]): Database {
  }
  applyPragmas(db);
  // On a copy every migration is already recorded, so this is a no-op —
  // and a list that grew since the copy was taken still runs what is new.
  migrate(db, migrations);
  // and a list that grew since the copy was taken still runs what is new;
  // one behind the copy boots and says what it does not know (#783).
  const { unknown } = migrate(db, migrations);
  if (unknown.length > 0)
    log.warn("database has migrations this build does not know; left as they are", {
      count: unknown.length,
      unknown: unknown.join(","),
    });
  if (fresh && image === undefined) migratedEmpty.set(migrations, db.serialize());
  return db;
}
@@ -63,7 +72,7 @@
export function openDb(path: string, migrations: readonly string[]): Database {
 * bytes however it was made: the migrations that draw randomness only
 * rewrite rows an empty database does not have.
 */
const migratedEmpty = new WeakMap<readonly string[], Uint8Array>();
const migratedEmpty = new WeakMap<readonly Migration[], Uint8Array>();
/**
 * The image with its header saying "rollback journal" (bytes 18 and 19 of a
@@ -11,7 +11,8 @@
 * migration is a row in `schema_migration`, keyed by a hash of its text, and
 * a boot runs exactly the ones whose text it has not seen. A second boot is
 * a no-op; a migration that moved is still recognised; a sibling's migration
 * this build does not know is left alone.
 * this build does not know is left alone and named, never a refusal to boot
 * (#783) — each row also carries its migration's file name.
 *
 * Each statement is also run so that meeting its own result is a no-op —
 * `CREATE` gains `IF NOT EXISTS`, an `ADD COLUMN` whose column is there is
@@ -77,10 +78,73 @@
export function migrationId(sql: string): string {
  return sha256(sql.replace(/\s+/g, " ").trim());
}
export function migrate(db: Database, migrations: readonly string[]): void {
/** Each pinned migration's id, by name: what it said when it was pinned. */
export type MigrationPins = Readonly<Record<string, string>>;
/**
 * How `files` differ from what was pinned: one line for each pinned
 * migration whose text says something else now, or that is gone. A
 * migration on main is history — an edited one is a new one, and a
 * database that ran the old text runs the new one again on top of it.
 */
export function pinBreaks(
  files: readonly MigrationFile[],
  pins: MigrationPins,
): string[] {
  const now = new Map(files.map((m) => [m.name, migrationId(m.sql)]));
  const breaks: string[] = [];
  for (const [name, id] of Object.entries(pins)) {
    const current = now.get(name);
    if (current === undefined) breaks.push(`${name} was pinned and is gone`);
    else if (current !== id)
      breaks.push(`${name} was edited after it was pinned — add a new migration instead`);
  }
  return breaks;
}
/**
 * `pins` with every migration in `files` it lacks added; a pin already
 * there is never rewritten. Sorted by name, so the file it is written to
 * changes only where a migration was added.
 */
export function pinned(
  files: readonly MigrationFile[],
  pins: MigrationPins,
): MigrationPins {
  const next: Record<string, string> = { ...pins };
  for (const m of files) next[m.name] ??= migrationId(m.sql);
  return Object.fromEntries(
    Object.entries(next).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)),
  );
}
/**
 * A migration as `migrate` takes it: its file, which names it, or bare SQL —
 * a test's own list — which is known by its position alone.
 */
export type Migration = string | MigrationFile;
/** What a boot did to the database, by migration name. */
export interface Migrated {
  /** The migrations this boot applied, in the order it applied them. */
  readonly applied: string[];
  /**
   * What the database has applied that this build does not know: a newer
   * build's or a sibling branch's, on data copied for a preview. Each by
   * the name its build recorded; one recorded with none by its position and
   * the start of its id. Left alone, never undone.
   */
  readonly unknown: string[];
}
export function migrate(db: Database, list: readonly Migration[]): Migrated {
  const migrations = list.map((m) => (typeof m === "string" ? m : m.sql));
  const names = list.map((m, i) =>
    typeof m === "string" ? `migration ${i + 1}` : `migration ${m.name}`,
  );
  const ids = migrations.map((sql, i) => {
    if (sql.trim() === "")
      defect("db.migration_missing", `migration ${i + 1} is empty`);
      defect("db.migration_missing", `${names[i]} is empty`);
    return migrationId(sql);
  });
  const seen = new Set<string>();
@@ -88,22 +152,19 @@
export function migrate(db: Database, migrations: readonly string[]): void {
    if (seen.has(id))
      defect(
        "db.migration_duplicate",
        `migration ${i + 1} repeats an earlier one word for word — it would never run`,
        `${names[i]} repeats an earlier one word for word — it would never run`,
      );
    seen.add(id);
  });
  const applied = appliedIds(db, migrations, ids);
  if (applied.size > migrations.length) {
    defect(
      "db.future_schema",
      `database has applied ${applied.size} migrations, this build only knows ${migrations.length} — refusing to run`,
    );
  }
  recordNames(db, list, ids);
  const ran: string[] = [];
  for (let i = 0; i < migrations.length; i++) {
    const id = ids[i]!;
    if (applied.has(id)) continue;
    const sql = migrations[i]!;
    const entry = list[i]!;
    db.transaction(() => {
      for (const statement of statements(sql)) {
        try {
@@ -111,19 +172,63 @@
export function migrate(db: Database, migrations: readonly string[]): void {
        } catch (error) {
          defect(
            "db.migration_failed",
            `migration ${i + 1} failed at ${quote(statement)}: ${messageOf(error)}`,
            `${names[i]} failed at ${quote(statement)}: ${messageOf(error)}`,
            error,
          );
        }
      }
      db.query(
        "INSERT INTO schema_migration (id, position, applied_at) VALUES (?1, ?2, ?3)",
      ).run(id, i + 1, new Date().toISOString());
        "INSERT INTO schema_migration (id, position, applied_at, name) VALUES (?1, ?2, ?3, ?4)",
      ).run(
        id,
        i + 1,
        new Date().toISOString(),
        typeof entry === "string" ? null : entry.name,
      );
      applied.add(id);
      // PRAGMA can't be parameterized; the value is a count, not input.
      db.exec(`PRAGMA user_version = ${applied.size}`);
    })();
    ran.push(typeof entry === "string" ? String(i + 1) : entry.name);
  }
  const known = new Set(ids);
  const unknown = db
    .query<{ id: string; position: number; name: string | null }, []>(
      "SELECT id, position, name FROM schema_migration ORDER BY position, id",
    )
    .all()
    .filter((row) => !known.has(row.id))
    .map((row) => row.name ?? `#${row.position} (${row.id.slice(0, 12)})`);
  return { applied: ran, unknown };
}
/**
 * Write each known migration's name beside its id, where a build from
 * before names were recorded left none — so the database says which
 * migrations it has by name, and a later build that does not know one can
 * say which. A database written by test lists keeps its blanks.
 */
function recordNames(
  db: Database,
  list: readonly Migration[],
  ids: readonly string[],
): void {
  if (!hasColumn(db, "schema_migration", "name"))
    db.exec("ALTER TABLE schema_migration ADD COLUMN name TEXT");
  const blank = db
    .query<{ n: number }, []>(
      "SELECT COUNT(*) AS n FROM schema_migration WHERE name IS NULL",
    )
    .get()!.n;
  if (blank === 0) return;
  const name = db.query(
    "UPDATE schema_migration SET name = ?2 WHERE id = ?1 AND name IS NULL",
  );
  db.transaction(() => {
    list.forEach((m, i) => {
      if (typeof m !== "string") name.run(ids[i]!, m.name);
    });
  })();
}
/**
@@ -132,7 +237,9 @@
export function migrate(db: Database, migrations: readonly string[]): void {
 * taken as applied — what that number always meant — unless one plainly
 * did not run (a table it creates or a column it adds is missing: a
 * branch's migration sat at that number instead). They are written down
 * once, after which positions never matter again.
 * once, after which positions never matter again. Such a database ahead of
 * this build is still refused: a count names nothing, so there is no
 * telling which of its migrations this build lacks.
 */
function appliedIds(
  db: Database,
@@ -153,7 +260,8 @@
function appliedIds(
      db.exec(`CREATE TABLE schema_migration (
        id         TEXT PRIMARY KEY,
        position   INTEGER NOT NULL,
        applied_at TEXT NOT NULL
        applied_at TEXT NOT NULL,
        name       TEXT
      )`);
      const insert = db.query(
        "INSERT INTO schema_migration (id, position, applied_at) VALUES (?1, ?2, ?3)",
@@ -24,7 +24,7 @@
import { join } from "node:path";
import { parseCidrs } from "./cidr.ts";
import { dataHomeRefusal } from "./data_home.ts";
import { applyPragmas } from "./db.ts";
import { migrate } from "./migrate.ts";
import { type Migration, migrate } from "./migrate.ts";
/** A scratch directory older than this was left by a killed run. */
const STALE_MS = 60 * 60 * 1000;
@@ -81,13 +81,14 @@
export function configProblem(
export function preflight(
  env: Record<string, string | undefined>,
  treeDir: string,
  migrations: readonly string[],
  migrations: readonly Migration[],
): Preflight {
  const bad = configProblem(env, treeDir);
  if (bad !== null) return { ok: false, line: `preflight failed: ${bad}` };
  const dataDir = dataDirOf(env);
  let scratch: string | undefined;
  let unknown: string[] = [];
  try {
    mkdirSync(dataDir, { recursive: true });
    sweepStale(dataDir);
@@ -101,13 +102,17 @@
export function preflight(
    const db = new Database(copy, { create: true, strict: true });
    try {
      applyPragmas(db);
      migrate(db, migrations);
      unknown = migrate(db, migrations).unknown;
    } finally {
      db.close();
    }
    return {
      ok: true,
      line: `preflight ok: ${migrations.length} migrations apply to a copy of the database`,
      line:
        `preflight ok: ${migrations.length} migrations apply to a copy of the database` +
        (unknown.length > 0
          ? `; it also has ${unknown.length} this build does not know (${unknown.join(", ")})`
          : ""),
    };
  } catch (error) {
    const message = (error instanceof Error ? error.message : String(error))
@@ -23,7 +23,9 @@
import {
import { STALE_AFTER_MS } from "../runs/lease.ts";
import { BUDGETS } from "./budgets.ts";
import { lastMirror, mirrorNeed } from "../snapshot/mirror.ts";
import { lastSnapshotOutcome, snapshotNeed } from "../snapshot/outcome.ts";
import { drillNeed, firstSeen as drillFirstSeen, lastVerdict } from "../snapshot/verdict.ts";
import { proofNeed } from "../release/prove.ts";
import { newestDrills } from "../core/domain_drills.ts";
import { DOMAINS } from "./domains.ts";
import { domainDrillNeeds } from "./domain_drill.ts";
@@ -50,7 +52,7 @@
export interface EnvCount {
/**
 * One cause of the forge being off: the `needs` sentence, a `cause` key that
 * holds for as long as its condition does (`disk`, `errors`, `internal`,
 * `queue:<kind>`, `stuck:<runId>`, `red:<repoId>`, `shared:<repoId>:<test>`), when it began by the forge's own evidence
 * `queue:<kind>`, `release`, `stuck:<runId>`, `red:<repoId>`, `shared:<repoId>:<test>`), when it began by the forge's own evidence
 * (the oldest evidence the forge holds, else the first time this process saw
 * it, held until the cause clears; `at` is never the answer to a repeat
 * call), and where to look.
@@ -384,10 +386,18 @@
export function vitals(input: VitalsInput): Vitals {
  ))
    need(owed.cause, owed.sentence, owed.since, "/");
  // A snapshot that failed is a card from that morning until one lands.
  const failing = snapshotNeed(lastSnapshotOutcome(input.dataDir));
  if (failing !== undefined) need("snapshot-failed", failing.sentence, failing.since, "/settings/backups");
  // A snapshot that did not land off the box, or a mirror a day behind, is a card until it does.
  const behind = mirrorNeed(lastMirror(input.dataDir), now);
  if (behind !== undefined) need("mirror", behind.sentence, behind.since, "/settings/backups");
  // A release the forge could not prove against its own public URL is a card until a release does.
  const unproven = proofNeed(input.dataDir);
  if (unproven !== undefined) need("release", unproven.sentence, unproven.since, "/board");
  const apps = { serving: 0, asleep: 0, previews: 0 };
  for (const e of input.envs ?? []) {
    if (e.env !== "production") apps.previews++;
@@ -54,7 +54,9 @@
import { configProblem, dataDirOf, preflight } from "./kernel/preflight.ts";
import { running } from "./kernel/version.ts";
import { record as recordDegraded } from "./kernel/degraded_history.ts";
import { vitals } from "./kernel/vitals.ts";
import { proveRelease } from "./release/prove.ts";
import { snapshotPlace } from "./actions/health.ts";
import { capacityDefault } from "./core/capacity.ts";
// `forge hook <name>` re-enters here from a git hook stub. It relays to the
// running forge over its socket and exits — it must never boot a server.
@@ -72,8 +74,8 @@
if (process.argv[2] === "hook") {
// config parsed, migrations run on a copy of the database. No socket, no
// timers, and the live database is never opened.
if (process.argv.includes("--preflight")) {
  const { migrations } = await import("./core/schema.ts");
  const result = preflight(process.env, import.meta.dir, migrations);
  const { migrationFiles } = await import("./core/schema.ts");
  const result = preflight(process.env, import.meta.dir, migrationFiles);
  // CLI answer on the stream a person or a script reads.
  (result.ok ? console.log : console.error)(result.line);
  process.exit(result.ok ? 0 : 1);
@@ -176,6 +178,8 @@
if (parentProblem !== null) {
const app = createApp({
  dataDir,
  // A new forge starts with signup = invite (FORGE_SIGNUP below still wins).
  seedSignup: true,
  ...(parentUrl === "" ? {} : { parentUrl }),
  port: Number(process.env.PORT ?? 3000),
  ...(process.env.FORGE_HOST === undefined
@@ -332,7 +336,8 @@
const checks =
    ? undefined
    : startChecks(app, {
        forgeUrl,
        concurrency: positive(process.env.FORGE_CHECKS_CONCURRENCY, 1),
        // Sized to the pod's CPUs unless the environment says (#2549).
        concurrency: positive(process.env.FORGE_CHECKS_CONCURRENCY, Number(capacityDefault("checks.concurrency"))),
        redHoldMs: RED_HOLD_MS,
        // A roll lets running checks finish this long before cutting them off.
        drainMs: positive(process.env.FORGE_CHECKS_DRAIN_S, 300) * 1000,
@@ -379,7 +384,7 @@
const crew = appsOn && !crewHeldOff
      forgeUrl,
      localUrl: servers.url,
      binary: process.env.FORGE_CLAUDE ?? "claude",
      concurrency: positive(process.env.FORGE_CREW_CONCURRENCY, hosted ? 1 : 2),
      concurrency: positive(process.env.FORGE_CREW_CONCURRENCY, hosted ? 1 : Number(capacityDefault("crew.concurrency"))),
      budgetUsd: Number(process.env.FORGE_CREW_BUDGET_USD) || 5,
      runTimeoutMs: crewRunMs,
      allowHosts: (process.env.FORGE_CREW_ALLOW_HOSTS ?? "")
@@ -478,6 +483,13 @@
const staleTimer = setInterval(() => void tickAsks(), 120_000);
void tickAsks();
app.log.info("forge is up", { url: servers.url, data: app.config.dataDir });
// A new version proves itself against the public URL once (release/prove.ts).
void proveRelease({
  dataDir: app.config.dataDir,
  version: running().version,
  url: forgeUrl,
  token: process.env["FORGE_TOKEN"],
}).then((p) => p && app.log.info("release proof", { version: p.version, ok: String(p.ok), ...(p.failure ? { failure: p.failure } : {}) }));
// Only an agent is on a crew (the `repo_crew_agents_only` trigger refuses anyone else). A person
// put on one before that is reported here for the owner to take off — the
// forge never removes it on its own.
@@ -0,0 +1,109 @@
/**
 * Is this release so: once per version, after a boot that changed it, the
 * forge waits for its own public URL to answer `/healthz` as that version,
 * then runs the smoke checks (`scripts/smoke.ts`) against it. The verdict is
 * one small file under the data directory (`release/proof.json`), so a
 * restart of the same version never runs it again and vitals reads it
 * without the network. A failure is a `release` card, paged once by the
 * cause pager, until a later version passes. It gives up after ten minutes.
 */
import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { smoke, type Failure } from "../../scripts/smoke.ts";
import { UNSTAMPED } from "../kernel/version.ts";
/** How long the public URL has to answer as the new version. */
export const GIVE_UP_MS = 10 * 60_000;
const POLL_MS = 2_000;
export interface Proof {
  readonly version: string;
  readonly at: string;
  readonly ok: boolean;
  /** The first failing check, or the give-up, when `ok` is false. */
  readonly failure?: string;
}
const proofFile = (dataDir: string): string => join(dataDir, "release", "proof.json");
/** The last verdict kept, or undefined when none is or the file is not one. */
export function lastProof(dataDir: string): Proof | undefined {
  try {
    const p = JSON.parse(readFileSync(proofFile(dataDir), "utf8")) as Proof;
    if (typeof p.version !== "string" || typeof p.at !== "string" || typeof p.ok !== "boolean") return undefined;
    return p;
  } catch {
    return undefined;
  }
}
function keep(dataDir: string, proof: Proof): Proof {
  mkdirSync(join(dataDir, "release"), { recursive: true });
  const tmp = `${proofFile(dataDir)}.tmp`;
  writeFileSync(tmp, JSON.stringify(proof));
  renameSync(tmp, proofFile(dataDir));
  return proof;
}
/** The card an unproven release is, or undefined while the last verdict is a pass. */
export function proofNeed(dataDir: string): { sentence: string; since: string } | undefined {
  const p = lastProof(dataDir);
  if (p === undefined || p.ok) return undefined;
  return { sentence: `Release ${p.version} is unproven: ${p.failure ?? "a smoke check failed"}.`, since: p.at };
}
/** A failing check as one line: where, as whom, and why. */
export function checkLine(f: Failure): string {
  return `${f.url}${f.agent === "-" ? "" : ` as ${f.agent}`} ${f.why} (status ${f.status})`;
}
export interface ProveOptions {
  readonly dataDir: string;
  readonly version: string;
  /** The forge's public URL. */
  readonly url: string;
  /** Signs `/board` in; without it a redirect to sign-in passes. */
  readonly token?: string | undefined;
  readonly fetch?: typeof fetch;
  readonly now?: () => number;
  readonly sleep?: (ms: number) => Promise<void>;
  readonly giveUpMs?: number;
}
/**
 * Proves `version` unless it was proven (or failed) before. Returns what was
 * kept, or undefined when nothing ran. Never throws.
 */
export async function proveRelease(o: ProveOptions): Promise<Proof | undefined> {
  if (o.version === UNSTAMPED || lastProof(o.dataDir)?.version === o.version) return undefined;
  const now = o.now ?? Date.now;
  const sleep = o.sleep ?? Bun.sleep;
  const doFetch = o.fetch ?? fetch;
  const finish = (failure?: string): Proof =>
    keep(o.dataDir, {
      version: o.version,
      at: new Date(now()).toISOString(),
      ok: failure === undefined,
      ...(failure === undefined ? {} : { failure }),
    });
  try {
    const health = new URL("/healthz", o.url).href;
    const deadline = now() + (o.giveUpMs ?? GIVE_UP_MS);
    for (;;) {
      try {
        const res = await doFetch(health, { redirect: "manual", signal: AbortSignal.timeout(5_000) });
        await res.body?.cancel();
        if (res.headers.get("x-forge-version") === o.version) break;
      } catch {
        // not up yet at its public address
      }
      if (now() + POLL_MS > deadline) return finish(`the public URL never answered as ${o.version}`);
      await sleep(POLL_MS);
    }
    const failures = await smoke(o.url, o.token ?? "", { fetch: doFetch, anonymousBoard: true });
    return finish(failures[0] === undefined ? undefined : checkLine(failures[0]));
  } catch (error) {
    return finish(`the smoke checks could not run: ${String(error)}`);
  }
}
@@ -4,8 +4,9 @@
 * each take one line: the release is not kept, or its schema is behind the
 * live database's (a down-migration is not something this forge has).
 * A release whose schema is ahead of the live database is fine: it migrates
 * forward on boot, and the database's own refusal of a future schema is
 * untouched.
 * forward on boot. A release behind it is refused here, not by the
 * database: a build boots on migrations it does not know and only names
 * them (#783).
 */
import { Database } from "bun:sqlite";
@@ -1,7 +1,7 @@
/**
 * Who may make an account: `open`, `invite` or `closed`. Stored in
 * `forge_setting` under `signup`, and every change is kept in
 * `forge_setting_change` (who, when, from what to what). Default `open`.
 * `forge_setting_change` (who, when, from what to what). Default `open` when nothing is stored; a new forge stores `invite` (`seedSignup`).
 * `/signup`, `POST /api/v1/users`, passkey signup and a first edge sign-in ask
 * it (`signupClosed` in `actions/users.ts`); `invite` lets in only a person with
 * an open invite link (`core/invites.ts`), spent with the account.
@@ -36,6 +36,22 @@
export function getSignup(db: Database): SignupMode {
  return parsed.ok ? parsed.value : SIGNUP_DEFAULT;
}
/** The actor kept for the value a new forge starts with. */
export const FRESH_ACTOR = "fresh";
/**
 * What a new forge starts with: `invite`, stored so it is the forge's own
 * value from then on. Only a forge that made its database just now is new
 * (`fresh`); one that stored a value keeps it, and one that never stored any
 * keeps the `open` it has always read as. A spawned child is a fresh data
 * directory, so it starts here too.
 */
export function seedSignup(db: Database, fresh: boolean): void {
  if (!fresh) return;
  const stored = db.query("SELECT 1 FROM forge_setting WHERE key = ?1").get(STORE_KEY);
  if (stored === null) writeSetting(db, STORE_KEY, "invite", SIGNUP_DEFAULT, "invite", FRESH_ACTOR);
}
/** Set the mode and keep the change; the same mode again keeps nothing. */
export function setSignup(db: Database, value: SignupMode, actorId: string): void {
  const from = getSignup(db);
@@ -77,7 +93,9 @@
export function signupChanges(db: Database, limit = 20): SignupChange[] {
    .map((r) => ({
      from: r.from_value,
      to: r.to_value,
      actorName: r.username ?? (r.actor_id === BOOT_ACTOR ? "FORGE_SIGNUP at boot" : ""),
      actorName:
        r.username ??
        (r.actor_id === BOOT_ACTOR ? "FORGE_SIGNUP at boot" : r.actor_id === FRESH_ACTOR ? "new forge" : ""),
      at: r.at,
    }));
}
@@ -56,7 +56,7 @@
class StepTimeout extends Error {}
function within<T>(work: Promise<T>, ms: number): Promise<T> {
  let timer: ReturnType<typeof setTimeout>;
  const late = new Promise<never>((_, reject) => {
    timer = setTimeout(() => reject(new StepTimeout(`no answer in ${ms} ms`)), ms);
    timer = setTimeout(() => reject(new StepTimeout(`no answer in ${Math.round(ms)} ms`)), ms);
  });
  return Promise.race([work, late]).finally(() => clearTimeout(timer));
}
@@ -2,11 +2,21 @@
 * When the weekly drill is owed: every Sunday 09:00 UTC is a slot, and a
 * slot is kept by one drill at or after it, however it was asked for. A
 * forge that was down at the slot runs it on waking; one that restarts
 * after the drill ran does not run it twice. Plain data in, plain data out.
 * after the drill ran does not run it twice. A red drill keeps no slot for
 * long: it is owed again `RED_RETRY_MS` after it ran, and on a boot once it
 * is `BOOT_RED_FLOOR_MS` old, since a restart is most often the deploy of
 * its fix (#2541: a drill red at 09:11 stood as a card for the week after
 * the fix landed at 10:18). The floor keeps a crash loop with a broken
 * restore from running a restore on every boot. Plain data in, plain data out.
 */
const DAY = 86_400_000;
/** A red drill is owed again this long after it ran. */
export const RED_RETRY_MS = 3 * 3_600_000;
/** On a boot, a red drill at least this old is owed again. */
export const BOOT_RED_FLOOR_MS = 30 * 60_000;
/** The latest Sunday 09:00 UTC at or before `now`, in ms. */
export function lastSlot(now: number): number {
  const d = new Date(now);
@@ -17,14 +27,20 @@
export function lastSlot(now: number): number {
}
/**
 * Whether a drill is owed at `now`. `lastAt` is the newest drill's time,
 * `since` when the forge first owed one; a forge younger than the slot
 * owes nothing for it.
 * Whether a drill is owed at `now`. `last` is the newest drill, `since`
 * when the forge first owed one; a forge younger than the slot owes
 * nothing for it. `redRetryMs` is how long a red drill stands before it is
 * owed again; a boot passes `BOOT_RED_FLOOR_MS`.
 */
export function drillDue(lastAt: string | undefined, since: string | undefined, now: number): boolean {
export function drillDue(
  last: { readonly at: string; readonly verdict: "green" | "red" } | undefined,
  since: string | undefined,
  now: number,
  redRetryMs: number = RED_RETRY_MS,
): boolean {
  const slot = lastSlot(now);
  const last = Date.parse(lastAt ?? "");
  if (Number.isFinite(last)) return last < slot;
  const at = Date.parse(last?.at ?? "");
  if (Number.isFinite(at)) return at < slot || (last?.verdict === "red" && now - at >= redRetryMs);
  const began = Date.parse(since ?? "");
  return Number.isFinite(began) && began < slot;
}
@@ -0,0 +1,70 @@
/**
 * How the last snapshot went, kept where the board can read it. A nightly
 * that failed used to be one line in the log and nothing else: two nights
 * of no off-box backup went by unseen (2026-10-17, #2536). Each attempt's
 * outcome is one small file under the data directory
 * (`snapshot/state.json`), written when it ends, so vitals and Backups read
 * it without the network and a restart forgets nothing. A failure is a
 * card from the moment it is written until a snapshot lands.
 */
import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
import { join } from "node:path";
export interface SnapshotOutcome {
  /** When the last attempt ended. */
  readonly at: string;
  /** The newest snapshot that landed, and when; absent until one has. */
  readonly landed?: string;
  readonly landedAt?: string;
  /** Set while attempts are failing: since when, and the last one's reason. */
  readonly failingSince?: string;
  readonly error?: string;
}
const stateFile = (dataDir: string): string => join(dataDir, "snapshot", "state.json");
/** The outcome kept, or undefined when none is or the file is not one. */
export function lastSnapshotOutcome(dataDir: string): SnapshotOutcome | undefined {
  try {
    const s = JSON.parse(readFileSync(stateFile(dataDir), "utf8")) as SnapshotOutcome;
    if (typeof s.at !== "string") return undefined;
    return s;
  } catch {
    return undefined;
  }
}
/** Keep an attempt's end: a name that landed, or the error it failed with. Returns what was kept. */
export function recordSnapshotOutcome(
  dataDir: string,
  result: { readonly landed: string } | { readonly error: string },
  at: string = new Date().toISOString(),
): SnapshotOutcome {
  const prev = lastSnapshotOutcome(dataDir);
  const kept = {
    ...(prev?.landed === undefined ? {} : { landed: prev.landed }),
    ...(prev?.landedAt === undefined ? {} : { landedAt: prev.landedAt }),
  };
  const next: SnapshotOutcome =
    "landed" in result
      ? { at, landed: result.landed, landedAt: at }
      : { at, ...kept, failingSince: prev?.failingSince ?? at, error: result.error };
  mkdirSync(join(dataDir, "snapshot"), { recursive: true });
  const tmp = `${stateFile(dataDir)}.tmp`;
  writeFileSync(tmp, JSON.stringify(next));
  renameSync(tmp, stateFile(dataDir));
  return next;
}
/** The card a failing snapshot is, or undefined while the last one landed. */
export function snapshotNeed(
  s: SnapshotOutcome | undefined,
): { sentence: string; since: string } | undefined {
  if (s?.failingSince === undefined) return undefined;
  const last = s.landedAt ? ` The newest that landed is from ${s.landedAt.slice(0, 10)}.` : "";
  return {
    sentence: `The last snapshot failed: ${(s.error ?? "unknown reason").replace(/\.$/, "")}.${last} The forge tries again within the hour.`,
    since: s.failingSince,
  };
}
@@ -0,0 +1,108 @@
/**
 * A large file packed for a slow uplink: zstd over the whole of it, then
 * cut into parts of `PART_BYTES`, each its own object. Each part is one put
 * with its own deadline and its own retries, so a link that drops halfway
 * loses one part's progress, never the file's; and the mirror, which copies
 * object by object, moves parts the same way.
 *
 * The parts of `forge.db` are `forge.db.zst.0000`, `forge.db.zst.0001`, …;
 * the manifest lists them as files, like any other, and says once in
 * `packed` what they make together and how big that is. A restore reads
 * the parts back in order, joins them, and unpacks to exactly the bytes
 * that went in, or refuses.
 *
 * Plain data in, plain data out: no store, no files.
 */
/** A part's size: one put carries it within its own deadline at the floor rate (27 s at 5 Mbit/s). */
export const PART_BYTES = 16 * 1_048_576;
/** What the manifest says of a file that went up in parts. */
export interface PackedFile {
  /** The file's own path, as a restore writes it back. */
  readonly path: string;
  /** Its size as it was, before compression. */
  readonly bytes: number;
  /** Its size compressed: the parts' sizes added. */
  readonly packedBytes: number;
  readonly parts: number;
}
/** The key of part `n` of `path`, under a snapshot's prefix. */
export const partPath = (path: string, n: number): string =>
  `${path}.zst.${String(n).padStart(4, "0")}`;
/** Every part path of a packed file, in order. */
export const partPaths = (p: PackedFile): string[] =>
  Array.from({ length: p.parts }, (_, n) => partPath(p.path, n));
/** A path a restore may write under its data directory: relative, and never climbing out. */
export const safePath = (path: string): boolean =>
  path !== "" &&
  !path.startsWith("/") &&
  !path.includes("\\") &&
  !path.split("/").some((s) => s === "" || s === "." || s === "..");
/**
 * `body` compressed and cut in parts. The parts are views of one buffer,
 * not copies; an empty body is one empty-content part, so every packed file
 * has at least one.
 */
export async function pack(
  path: string,
  body: Uint8Array,
  partBytes = PART_BYTES,
): Promise<{ packed: PackedFile; parts: Uint8Array[] }> {
  if (!safePath(path)) throw new Error(`refusing to pack ${path}`);
  if (!(partBytes > 0)) throw new Error(`a part of ${partBytes} bytes`);
  const zst = new Uint8Array(await Bun.zstdCompress(body));
  const parts: Uint8Array[] = [];
  for (let at = 0; at < zst.byteLength || parts.length === 0; at += partBytes)
    parts.push(zst.subarray(at, at + partBytes));
  return {
    packed: {
      path,
      bytes: body.byteLength,
      packedBytes: zst.byteLength,
      parts: parts.length,
    },
    parts,
  };
}
/** The file the parts make, checked against what the manifest says it was. */
export async function unpack(
  p: PackedFile,
  parts: readonly Uint8Array[],
): Promise<Uint8Array> {
  if (parts.length !== p.parts)
    throw new Error(`${p.path} came back in ${parts.length} parts, not ${p.parts}`);
  const total = parts.reduce((n, b) => n + b.byteLength, 0);
  if (total !== p.packedBytes)
    throw new Error(`${p.path}'s parts are ${total} bytes, not ${p.packedBytes}`);
  const joined = new Uint8Array(total);
  let at = 0;
  for (const b of parts) {
    joined.set(b, at);
    at += b.byteLength;
  }
  const body = new Uint8Array(await Bun.zstdDecompress(joined));
  if (body.byteLength !== p.bytes)
    throw new Error(`${p.path} unpacks to ${body.byteLength} bytes, not ${p.bytes}`);
  return body;
}
/** Whether a manifest's `packed` entry reads whole: the restore trusts nothing else. */
export function isPackedFile(v: unknown): v is PackedFile {
  if (typeof v !== "object" || v === null) return false;
  const p = v as Record<string, unknown>;
  const count = (n: unknown): boolean => Number.isSafeInteger(n) && (n as number) >= 0;
  return (
    typeof p["path"] === "string" &&
    safePath(p["path"]) &&
    count(p["bytes"]) &&
    count(p["packedBytes"]) &&
    count(p["parts"]) &&
    (p["parts"] as number) >= 1
  );
}
@@ -16,6 +16,10 @@
 * with the same `S3_*` variables the forge runs under, or
 * `FORGE_SNAPSHOT_DIR` for a directory store.
 *
 * A file that went up compressed in parts (the database, `parts.ts`)
 * comes back whole: its parts read in order, joined, unpacked, and checked
 * against the size the manifest gives it.
 *
 * The key comes back only from its wrapped form and the operator's
 * passphrase (`FORGE_SNAPSHOT_PASSPHRASE`). A wrong passphrase refuses the
 * restore; none leaves the key out, and every sealed value that will not
@@ -28,6 +32,8 @@
import { runGit } from "../git/env.ts";
import { installHooks } from "../git/hooks.ts";
import { repoDir } from "../git/repos.ts";
import {
  IN_FLIGHT,
  IN_FLIGHT_BYTES,
  isRunBundle,
  keyOf,
  listSnapshots,
@@ -36,6 +42,7 @@
import {
  type Manifest,
} from "./snapshot.ts";
import { unwrapKey, WRAPPED_KEY } from "./wrap.ts";
import { isPackedFile, partPaths, unpack, type PackedFile } from "./parts.ts";
import { resolveDestination } from "./destination.ts";
import type { Store } from "./store.ts";
@@ -49,6 +56,9 @@
export interface Restored {
  readonly sealedMissing: readonly string[];
}
/** Repositories cloned at once while restoring. */
export const CLONES = 4;
/** Names of every sealed value in a restored database; never the values. */
async function sealedNames(dbFile: string): Promise<string[]> {
  const { Database } = await import("bun:sqlite");
@@ -74,11 +84,67 @@
async function sealedNames(dbFile: string): Promise<string[]> {
  }
}
/**
 * Every file the manifest lists, read with several gets in flight at once
 * and checked against its size, each handed to `each` as it lands. Read
 * one after another, the restore drill of a snapshot with thousands of
 * objects ran out of its ten minutes in the restore step (2026-10-18), as
 * the upload once ran an hour on latency alone; the same limits bound both.
 */
async function fetchAll(
  store: Store,
  prefix: string,
  entries: readonly { readonly path: string; readonly bytes: number }[],
  each: (entry: { readonly path: string; readonly bytes: number }, body: Uint8Array, at: number) => Promise<void>,
  name: string,
  maxBytes: number,
): Promise<void> {
  const pending = new Set<Promise<void>>();
  let failure: unknown;
  let held = 0;
  for (const [at, entry] of entries.entries()) {
    while (
      (pending.size >= IN_FLIGHT ||
        (pending.size > 0 && held + entry.bytes > maxBytes)) &&
      failure === undefined
    )
      await Promise.race(pending);
    if (failure !== undefined) break;
    held += entry.bytes;
    const run: Promise<void> = (async () => {
      const body = await store.get(prefix + entry.path);
      if (body === undefined)
        throw new Error(`snapshot ${name} is missing ${entry.path}`);
      if (body.byteLength !== entry.bytes)
        throw new Error(
          `snapshot ${name}: ${entry.path} is ${body.byteLength} bytes, not ${entry.bytes}`,
        );
      await each(entry, body, at);
    })()
      .catch((error: unknown) => {
        failure ??= error;
      })
      .finally(() => {
        held -= entry.bytes;
        pending.delete(run);
      });
    pending.add(run);
  }
  await Promise.all([...pending]);
  if (failure !== undefined) throw failure;
}
export async function restoreSnapshot(
  store: Store,
  name: string,
  dataDir: string,
  options: { readonly passphrase?: string | undefined } = {},
  options: {
    readonly passphrase?: string | undefined;
    /** Bytes held by reads in flight; `IN_FLIGHT_BYTES` unless a test shrinks it. */
    readonly inFlightBytes?: number | undefined;
    /** Test seam: the git runner the clones go through. */
    readonly git?: typeof runGit | undefined;
  } = {},
): Promise<Restored> {
  if (!/^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}-[0-9]{2}-[0-9]{2}Z$/.test(name))
    throw new Error(`not a snapshot name: ${name}`);
@@ -104,20 +170,29 @@
export async function restoreSnapshot(
        "the passphrase does not open this snapshot's wrapped key; check FORGE_SNAPSHOT_PASSPHRASE, or restore without it to leave sealed values behind",
      );
  }
  const packed: readonly unknown[] = manifest.packed ?? [];
  if (!packed.every(isPackedFile))
    throw new Error(`snapshot ${name}: a packed file in its manifest does not read whole`);
  const partOf = new Map<string, { file: PackedFile; n: number }>();
  for (const file of packed)
    for (const [n, path] of partPaths(file).entries())
      partOf.set(path, { file, n });
  const gathered = new Map<string, Uint8Array[]>();
  mkdirSync(dataDir, { recursive: true });
  let files = 0;
  let repos = 0;
  const bundles: { owner: string; name: string; file: string }[] = [];
  for (const entry of manifest.files) {
    const body = await store.get(prefix + entry.path);
    if (body === undefined)
      throw new Error(`snapshot ${name} is missing ${entry.path}`);
    if (body.byteLength !== entry.bytes)
      throw new Error(
        `snapshot ${name}: ${entry.path} is ${body.byteLength} bytes, not ${entry.bytes}`,
      );
    if (entry.path === WRAPPED_KEY) continue;
  const bundles: { at: number; owner: string; name: string; file: string }[] = [];
  await fetchAll(store, prefix, manifest.files, async (entry, body, at) => {
    if (entry.path === WRAPPED_KEY) return;
    // A part waits for the rest of its file.
    const part = partOf.get(entry.path);
    if (part !== undefined) {
      const got = gathered.get(part.file.path) ?? [];
      got[part.n] = body;
      gathered.set(part.file.path, got);
      return;
    }
    // A day's run logs in one object come back as the logs they were.
    if (isRunBundle(entry.path)) {
      for (const run of unpackRuns(body)) {
@@ -126,7 +201,7 @@
export async function restoreSnapshot(
        await Bun.write(out, run.body);
        files++;
      }
      continue;
      return;
    }
    const bundle = entry.path.match(/^repos\/([^/]+)\/([^/]+)\.bundle$/);
    // A bundle is not a file of the forge's; it becomes a repository below.
@@ -141,23 +216,62 @@
export async function restoreSnapshot(
      chmodSync(target, 0o600);
    files++;
    if (bundle)
      bundles.push({ owner: bundle[1]!, name: bundle[2]!, file: target });
  }
  for (const b of bundles) {
    const dir = repoDir(join(dataDir, "repos"), b.owner, b.name);
    mkdirSync(dirname(dir), { recursive: true });
    const cloned = await runGit(["clone", "--quiet", "--mirror", b.file, dir]);
    if (!cloned.ok)
      throw new Error(
        `could not restore ${b.owner}/${b.name}: ${cloned.error.stderr}`,
      );
    // A mirror clone remembers the bundle as its origin; the forge's
    // repositories have no origin of their own.
    await runGit(["remote", "remove", "origin"], { cwd: dir });
    await runGit(["config", "core.logAllRefUpdates", "true"], { cwd: dir });
    await installHooks(dir);
    repos++;
      bundles.push({ at, owner: bundle[1]!, name: bundle[2]!, file: target });
  }, name, options.inFlightBytes ?? IN_FLIGHT_BYTES);
  // Reads land in any order; repositories come back in the manifest's.
  bundles.sort((a, b) => a.at - b.at);
  for (const file of packed) {
    const got = gathered.get(file.path) ?? [];
    const parts = partPaths(file).map((path, n) => {
      const body = got[n];
      if (body === undefined)
        throw new Error(`snapshot ${name} is missing ${path}`);
      return body;
    });
    gathered.delete(file.path);
    const target = join(dataDir, file.path);
    mkdirSync(dirname(target), { recursive: true });
    await Bun.write(target, await unpack(file, parts));
    files++;
  }
  // Each repository is its own clone, so several at once: a forge of
  // hundreds of them spent the drill's minutes here, one after another.
  const git = options.git ?? runGit;
  let next = 0;
  // Clones finish in any order; a refusal names the earliest bundle in the
  // manifest that failed, so it reads the same however the clones raced.
  const failed: { at: number; error: string }[] = [];
  await Promise.all(
    Array.from({ length: Math.min(CLONES, bundles.length) }, async () => {
      for (let b = bundles[next++]; b !== undefined; b = bundles[next++]) {
        const dir = repoDir(join(dataDir, "repos"), b.owner, b.name);
        mkdirSync(dirname(dir), { recursive: true });
        const cloned = await git([
          "clone",
          "--quiet",
          "--mirror",
          b.file,
          dir,
        ]);
        if (!cloned.ok) {
          next = bundles.length;
          failed.push({
            at: b.at,
            error: `could not restore ${b.owner}/${b.name}: ${cloned.error.stderr}`,
          });
          return;
        }
        // A mirror clone remembers the bundle as its origin; the forge's
        // repositories have no origin of their own.
        await git(["remote", "remove", "origin"], { cwd: dir });
        await git(["config", "core.logAllRefUpdates", "true"], { cwd: dir });
        await installHooks(dir);
        repos++;
      }
    }),
  );
  if (failed.length > 0)
    throw new Error(failed.sort((a, b) => a.at - b.at)[0]!.error);
  const { rmSync } = await import("node:fs");
  rmSync(join(dataDir, "restore-bundles"), { recursive: true, force: true });
  if (key !== undefined) {
@@ -5,7 +5,9 @@
 * snapshots and says so on its settings page, rather than pretending.
 *
 * Runs are serialized: a nightly that meets an on-demand one waits its
 * turn, and a forge stopping waits for the one in flight.
 * turn, and a forge stopping waits for the one in flight. A run that fails
 * is kept (`outcome.ts`) and shows on the board at once, and the next try
 * is within the hour, not the next night.
 */
import type { App } from "../app.ts";
@@ -20,15 +22,29 @@
import {
  type Progress,
} from "./snapshot.ts";
import { runDrill, type DrillDeps, type DrillResult } from "./drill.ts";
import { drillDue } from "./due.ts";
import { BOOT_RED_FLOOR_MS, drillDue, RED_RETRY_MS } from "./due.ts";
import { lastDrill, recordDrill } from "./drills.ts";
import { lastMirror, mirrorPass, MIRROR_EVERY_MS, recordMirror, type MirrorState } from "./mirror.ts";
import { recordSnapshotOutcome } from "./outcome.ts";
import type { Store } from "./store.ts";
const log = createLog("snapshots");
/** When the nightly snapshot is taken, in UTC: before the probe round. */
const NIGHTLY_HOUR_UTC = 8;
/** After a failed snapshot, the next try is this soon (or the nightly, if sooner). */
export const RETRY_MS = 3_600_000;
/**
 * A forge given nowhere to put a snapshot. Not a failure: the Backups page
 * says so calmly, so it is never kept as an outcome, never a card, and
 * never retried sooner than the nightly.
 */
class Nowhere extends Error {
  constructor() {
    super("this forge has nowhere to put a snapshot");
  }
}
export interface SnapshotsHandle {
  /** The store in effect now; undefined when the forge has been given none. */
@@ -45,7 +61,7 @@
export interface SnapshotsHandle {
  readonly last: () => { manifest?: Manifest; error?: string; at?: string };
  /** Drill the newest snapshot now and keep the result; waits for any snapshot in flight. */
  readonly drill: () => Promise<DrillResult>;
  /** Drill if this week's slot has no drill yet; says whether it did. */
  /** Drill if this week's slot has no drill yet, or the last one has stood red long enough; says whether it did. */
  readonly drillIfDue: () => Promise<boolean>;
  /** The off-box mirror's store; undefined when the forge has none. */
  readonly mirror: Store | undefined;
@@ -94,9 +110,15 @@
export function startSnapshots(
    /** Test seam: how often the mirror is looked at. */
    readonly mirrorEveryMs?: number | undefined;
    readonly nightly?: boolean;
    /** Test seam: how soon a failed snapshot is tried again. */
    readonly retryMs?: number | undefined;
    readonly passphrase?: string | undefined;
    /** Test seam: the drill itself. */
    readonly drill?: ((deps: DrillDeps) => Promise<DrillResult>) | undefined;
    /** Test seams: how soon after boot an owed drill runs, and when a red one is owed again. */
    readonly catchUpMs?: number | undefined;
    readonly redRetryMs?: number | undefined;
    readonly bootRedFloorMs?: number | undefined;
  } = {
    version: "0",
  },
@@ -122,12 +144,13 @@
export function startSnapshots(
  let last: { manifest?: Manifest; error?: string; at?: string } = {};
  const take = (): Promise<Manifest> => {
    const run = (inFlight ?? Promise.resolve()).then(async () => {
    // After the one before it, however that ended: chained on a failure as it
    // was, every later run failed at once with the old error until a restart.
    const run = (inFlight ?? Promise.resolve()).catch(() => {}).then(async () => {
      // Resolved per run, so a destination saved on the Backups page is the
      // very next snapshot's.
      const store = current();
      if (store === undefined)
        throw new Error("this forge has nowhere to put a snapshot");
      if (store === undefined) throw new Nowhere();
      running = { objects: 0, bytes: 0, elapsedMs: 0 };
      const manifest = await takeSnapshot(
        {
@@ -153,11 +176,19 @@
export function startSnapshots(
    inFlight = run.then(
      (m) => {
        last = { manifest: m, at: m.finishedAt };
        keep({ landed: m.name });
        arm(untilNightly());
        return m;
      },
      (error) => {
        last = { error: String(error), at: new Date().toISOString() };
        if (error instanceof Nowhere) {
          arm(untilNightly());
          throw error;
        }
        log.error("snapshot failed", { error: String(error) });
        keep({ error: error instanceof Error ? error.message : String(error) });
        arm(Math.min(options.retryMs ?? RETRY_MS, untilNightly()));
        throw error;
      },
    );
@@ -217,20 +248,30 @@
export function startSnapshots(
      return result;
    })().finally(() => {
      drilling = undefined;
      // However it was asked for, a red drill is tried again on its own.
      armRedRetry();
    });
    return drilling;
  };
  let timer: ReturnType<typeof setTimeout> | undefined;
  const schedule = (): void => {
    if (options.nightly === false || stopped) return;
    timer = setTimeout(() => {
      take()
        .catch(() => {})
        .finally(schedule);
    }, untilNightly());
  const keep = (result: { landed: string } | { error: string }): void => {
    try {
      recordSnapshotOutcome(app.config.dataDir, result);
    } catch (error) {
      log.warn("snapshot outcome not kept; the board will not show it", { error: String(error) });
    }
  };
  schedule();
  // One timer for the next snapshot: the nightly, or sooner after a failure.
  // Every run that ends sets it again, whoever asked for the run.
  let timer: ReturnType<typeof setTimeout> | undefined;
  function arm(ms: number): void {
    if (timer !== undefined) clearTimeout(timer);
    timer = undefined;
    if (options.nightly === false || stopped) return;
    timer = setTimeout(() => void take().catch(() => {}), ms);
  }
  arm(untilNightly());
  // The board only reads; the time the forge first owed a drill is kept here.
  try {
@@ -241,22 +282,40 @@
export function startSnapshots(
  let catchUp: ReturnType<typeof setTimeout> | undefined;
  let drillTimer: ReturnType<typeof setTimeout> | undefined;
  let redRetry: ReturnType<typeof setTimeout> | undefined;
  const catchUpMs = options.catchUpMs ?? CATCH_UP_MS;
  const redRetryMs = options.redRetryMs ?? RED_RETRY_MS;
  const scheduleDrill = (): void => {
    if (options.nightly === false || stopped) return;
    drillTimer = setTimeout(() => {
      runIfDue().finally(scheduleDrill);
    }, untilDrill());
  };
  // Owed and not yet kept: run it once. A restart after the drill ran finds it kept.
  const runIfDue = async (): Promise<boolean> => {
    if (!drillDue(lastDrill(app.db)?.at, firstSeen(app.config.dataDir), Date.now())) return false;
  // While the newest drill is red, look again when it is owed again.
  const armRedRetry = (): void => {
    if (redRetry !== undefined) clearTimeout(redRetry);
    redRetry = undefined;
    if (options.nightly === false || stopped) return;
    const newest = lastDrill(app.db);
    if (newest?.verdict !== "red") return;
    const wait = Math.max(catchUpMs, Date.parse(newest.at) + redRetryMs - Date.now());
    redRetry = setTimeout(() => void runIfDue().finally(armRedRetry), wait);
  };
  // Owed and not yet kept: run it once. A restart after a green drill finds it
  // kept; one after a red drill runs it again, once it is past the boot floor.
  const runIfDue = async (redAfterMs: number = redRetryMs): Promise<boolean> => {
    if (!drillDue(lastDrill(app.db), firstSeen(app.config.dataDir), Date.now(), redAfterMs)) return false;
    await drill().catch((error) => log.error("restore drill failed to run", { error: String(error) }));
    return true;
  };
  scheduleDrill();
  // A forge that was down at the slot catches up shortly after boot.
  // A forge that was down at the slot, or whose last drill was red, drills
  // shortly after boot; a red one is looked at again until one passes.
  if (options.nightly !== false) {
    catchUp = setTimeout(() => void runIfDue(), CATCH_UP_MS);
    catchUp = setTimeout(
      () => void runIfDue(options.bootRedFloorMs ?? BOOT_RED_FLOOR_MS).finally(armRedRetry),
      catchUpMs,
    );
  }
  // Within the hour: look for snapshots the mirror lacks, soon after boot and then hourly.
@@ -279,7 +338,7 @@
export function startSnapshots(
    wraps: !!options.passphrase,
    now: take,
    drill,
    drillIfDue: runIfDue,
    drillIfDue: () => runIfDue(),
    list: async () => {
      const store = current();
      return store === undefined ? [] : listSnapshots(store);
@@ -301,6 +360,7 @@
export function startSnapshots(
      if (timer !== undefined) clearTimeout(timer);
      if (drillTimer !== undefined) clearTimeout(drillTimer);
      if (catchUp !== undefined) clearTimeout(catchUp);
      if (redRetry !== undefined) clearTimeout(redRetry);
      if (mirrorTimer !== undefined) clearTimeout(mirrorTimer);
      if (mirroring !== undefined) await mirroring.catch(() => {});
      if (inFlight !== undefined) await inFlight.catch(() => {});
@@ -12,10 +12,13 @@
 * checks, previews, pictures.
 *
 * A snapshot is named by the moment it began, in UTC, so a listing sorts
 * itself. Nothing is compressed or bundled beyond git's own bundles and
 * the crew's older run logs, one compressed bundle per day: a bucket costs
 * a round trip per object, so thousands of tiny logs are one object a day.
 * A restore is reading files back, and a person can read the manifest.
 * itself. Little is compressed or bundled beyond git's own bundles: the
 * crew's older run logs, one compressed bundle per day, because a bucket
 * costs a round trip per object and thousands of tiny logs are one object
 * a day; and the database, compressed and cut in parts (`parts.ts`),
 * because whole it was more than a home uplink carries in one put's
 * deadline. A restore is reading files back, and a person can read the
 * manifest.
 *
 * Objects go up through a bounded queue, eight in flight; the manifest
 * lists them in the order they were handed over, not the order they landed.
@@ -32,6 +35,7 @@
import { runGit } from "../git/env.ts";
import { repoDir } from "../git/repos.ts";
import { createLog } from "../kernel/log.ts";
import { countRows, type DrillCounts } from "./counts.ts";
import { pack, partPath, PART_BYTES, type PackedFile } from "./parts.ts";
import type { Store } from "./store.ts";
import { unwrapKey, wrapKey, WRAPPED_KEY } from "./wrap.ts";
@@ -63,6 +67,8 @@
export interface Manifest {
   * `raw` in snapshots from before the key was wrapped.
   */
  readonly key?: "wrapped" | "none" | "raw";
  /** Files that went up compressed in parts, each listed in `files` by its parts. Absent before the database was packed. */
  readonly packed?: readonly PackedFile[];
}
/** What a manifest says of the key, including manifests older than the field. */
@@ -162,6 +168,8 @@
export interface Snapshotter {
  readonly onLand?: ((p: Progress) => void) | undefined;
  /** Overrides of the progress cadence, for tests. */
  readonly progressEvery?: { objects?: number; ms?: number } | undefined;
  /** Override of the database's part size, for tests. */
  readonly partBytes?: number | undefined;
}
/** `2026-09-22T09-00-00Z`: sortable, and a name a bucket allows. */
@@ -216,6 +224,7 @@
export async function takeSnapshot(
  const prefix = `${SNAPSHOT_PREFIX}${name}/`;
  const scratch = mkdtempSync(join(tmpdir(), "forge-snapshot-"));
  const files: SnapshotFile[] = [];
  const packed: PackedFile[] = [];
  let dbCounts: DrillCounts | undefined;
  const started = Date.now();
  const everyObjects = forge.progressEvery?.objects ?? PROGRESS_OBJECTS;
@@ -305,7 +314,23 @@
export async function takeSnapshot(
    } finally {
      copy.close();
    }
    await putFile("forge.db", dbCopy);
    // Compressed and in parts: each part one put, with its own deadline and
    // retries, so a slow uplink carries it and a dropped one costs a part.
    const database = await pack(
      "forge.db",
      new Uint8Array(await Bun.file(dbCopy).arrayBuffer()),
      forge.partBytes ?? PART_BYTES,
    );
    packed.push(database.packed);
    log.info("database packed", {
      mb: (database.packed.bytes / 1_048_576).toFixed(1),
      packedMb: (database.packed.packedBytes / 1_048_576).toFixed(1),
      ratio: (database.packed.bytes / Math.max(1, database.packed.packedBytes)).toFixed(2),
      parts: database.packed.parts,
    });
    for (const [n, part] of database.parts.entries())
      await put(partPath("forge.db", n), part);
    rmSync(dbCopy, { force: true });
    // The key travels only wrapped. Without a passphrase it stays home.
    let key: Manifest["key"] = "none";
    const keyFile = Bun.file(join(forge.dataDir, "secret.key"));
@@ -434,6 +459,7 @@
export async function takeSnapshot(
      bytes: files.reduce((n, f) => n + f.bytes, 0),
      ...(dbCounts === undefined ? {} : { counts: dbCounts }),
      key,
      packed,
    };
    await store.put(
      `${prefix}manifest.json`,
@@ -51,17 +51,40 @@
export function s3FromEnv(
  };
}
/** The slowest uplink a put may assume: 5 Mbit/s, a third of the home line it runs behind. */
export const FLOOR_BYTES_PER_S = 5_000_000 / 8;
/** What every attempt gets on top of its bytes' time: the handshake, the server's thought, a queue ahead of it. */
export const SLACK_MS = 60_000;
/** How long a read, a listing or a delete may go unanswered. */
export const CALL_DEADLINE_MS = 180_000;
/** How long a put of `bytes` may take before it counts as failed: its bytes at the floor rate, plus slack. */
export const deadlineFor = (
  bytes: number,
  floorBytesPerS = FLOOR_BYTES_PER_S,
  slackMs = SLACK_MS,
): number => slackMs + Math.ceil((bytes / floorBytesPerS) * 1000);
const sizeOf = (body: Uint8Array | string): number =>
  typeof body === "string" ? Buffer.byteLength(body) : body.byteLength;
/**
 * A store that tries again. Object stores answer a burst, a flake or a
 * moment of maintenance with a 5xx — Backblaze says so in its own docs —
 * and one such answer in a thousand puts used to fail the whole snapshot
 * with "S3Error: an unexpected error has occurred" and no key named. Every
 * call is tried `attempts` times with doubling waits, and what finally
 * fails says which key it was. An attempt that has not answered within
 * `deadlineMs` counts as failed too: a connection that died quietly never
 * fails says which key it was. An attempt that has not answered within its
 * deadline counts as failed too: a connection that died quietly never
 * settles on its own, and once held a snapshot on its first object for an
 * hour with the network fine (2026-10-13), so the next attempt opens a
 * fresh request instead.
 *
 * A put's deadline follows its size (`deadlineFor`): a fixed 180 s once
 * failed every attempt at a 334 MB database behind a 15 Mbit/s uplink
 * (2026-10-17). `deadlineMs` fixes every deadline instead, for tests;
 * `floorBytesPerS` and `slackMs` scale the size rule, for tests that run a
 * slow link at a thousand times its speed.
 */
export function withRetries(
  inner: Store,
@@ -69,14 +92,23 @@
export function withRetries(
    attempts?: number;
    baseMs?: number;
    deadlineMs?: number;
    floorBytesPerS?: number;
    slackMs?: number;
    sleep?: (ms: number) => Promise<void>;
  } = {},
): Store {
  const attempts = Math.max(1, options.attempts ?? 5);
  const baseMs = options.baseMs ?? 500;
  const deadlineMs = options.deadlineMs ?? 180_000;
  const fixed = options.deadlineMs;
  const callMs = fixed ?? CALL_DEADLINE_MS;
  const putMs = (body: Uint8Array | string): number =>
    fixed ?? deadlineFor(sizeOf(body), options.floorBytesPerS, options.slackMs);
  const sleep = options.sleep ?? ((ms) => Bun.sleep(ms));
  async function tried<T>(what: string, fn: () => Promise<T>): Promise<T> {
  async function tried<T>(
    what: string,
    deadlineMs: number,
    fn: () => Promise<T>,
  ): Promise<T> {
    let last: unknown;
    for (let n = 0; n < attempts; n++) {
      try {
@@ -93,11 +125,13 @@
export function withRetries(
  return {
    where: inner.where,
    ...(inner.dir === undefined ? {} : { dir: inner.dir }),
    put: (key, body) => tried(`put ${key}`, () => inner.put(key, body)),
    get: (key) => tried(`get ${key}`, () => inner.get(key)),
    list: (prefix) => tried(`list ${prefix}`, () => inner.list(prefix)),
    names: (prefix) => tried(`names ${prefix}`, () => inner.names(prefix)),
    delete: (key) => tried(`delete ${key}`, () => inner.delete(key)),
    put: (key, body) =>
      tried(`put ${key}`, putMs(body), () => inner.put(key, body)),
    get: (key) => tried(`get ${key}`, callMs, () => inner.get(key)),
    list: (prefix) => tried(`list ${prefix}`, callMs, () => inner.list(prefix)),
    names: (prefix) =>
      tried(`names ${prefix}`, callMs, () => inner.names(prefix)),
    delete: (key) => tried(`delete ${key}`, callMs, () => inner.delete(key)),
  };
}
@@ -63,6 +63,7 @@
import {
  type CursorPos,
} from "../ui/widgets.ts";
import { binding, find, keyFor, type Binding } from "./keys.ts";
import { message } from "../screens/common.ts";
import { Palette, type Command, type PaletteSearch } from "./palette.ts";
import { truncate } from "../term/width.ts";
@@ -470,7 +471,7 @@
export class App {
        // Every failure becomes a toast; nothing throws into the loop, and no
        // single bad read may drop the session.
        if (!entry.abort.signal.aborted)
          this.toast(e instanceof Error ? e.message : String(e), "danger");
          this.toast(message(e), "danger");
      })
      .finally(() => {
        entry.loading = false;
@@ -876,7 +877,7 @@
export class App {
  private run(command: Command): void {
    this.overlay = "none";
    void Promise.resolve(command.run()).catch((err: unknown) =>
      this.toast(err instanceof Error ? err.message : String(err), "danger"),
      this.toast(message(err), "danger"),
    );
  }
@@ -15,6 +15,7 @@
 * card is an OSC 8 hyperlink besides.
 */
import { audienceFor } from "../../core/audience.ts";
import { isPinned } from "../../core/pins.ts";
import { parentReleases, releasesSentence } from "../../core/parent_releases.ts";
import { running } from "../../kernel/version.ts";
@@ -128,6 +129,18 @@
export function aboutScreen(
        style(description === "" ? c.faint : c.fg, c.card),
        edge,
      );
      // Who can see it, in the web page's words (#809).
      if (repo !== undefined)
        s.text(
          lead,
          inner.y + 2,
          truncate(
            audienceFor(f.db, repo, f.user).sentence,
            Math.max(0, edge - lead),
          ),
          style(c.mutedFg, c.card),
          edge,
        );
      drawFacts(
        s,
        {
@@ -7,6 +7,7 @@
 * of them. Nothing here knows about repos or issues — that is the point.
 */
import { refusalText } from "../../http/refusal.ts";
import { NOT_A_MEMBER } from "../../core/roles.ts";
import type { Ctx, Screen } from "../app/app.ts";
import { binding, type Binding } from "../app/keys.ts";
@@ -996,6 +997,8 @@
export function errorText(code: string): string {
/** Anything thrown, as a string — the toast text of last resort. */
export function message(error: unknown): string {
  const refused = refusalText(error);
  if (refused !== undefined) return refused;
  return error instanceof Error ? error.message : String(error);
}
@@ -39,7 +39,7 @@
import {
  submitReview,
} from "../../actions/pulls.ts";
import { openEnvUrl } from "../../actions/apps.ts";
import { listRepoRuns } from "../../actions/runs.ts";
import { listRepoRuns, mayReadRunLog } from "../../actions/runs.ts";
import { previewEnv } from "../../apps/data.ts";
import {
  appsRuntime,
@@ -49,7 +49,7 @@
import { proposedByText } from "../../core/proposal_token.ts";
import { getAppByRepo } from "../../core/apps.ts";
import { getIssue, type Issue } from "../../core/issues.ts";
import { isProposed, pullMayRun } from "../../core/pull_trust.ts";
import { runningProgress } from "../../checks/progress.ts";
import { runningProgress, stepsLine } from "../../checks/progress.ts";
import { askCheck } from "../../checks/again.ts";
import { checkStale, checksHold } from "../../checks/stale.ts";
import { checkWords } from "../../checks/summary.ts";
@@ -237,6 +237,8 @@
export function pullScreen(
  /** The pull's base branch, to name what the check was merged with. */
  let checkBase = "";
  let checkProgress = "";
  /** The run page's three step rows as a line; empty when the log is hidden. */
  let checkSteps = "";
  /** The check's green was against a base that has since moved. */
  let checkOld = false;
  /** Agent runs about this pull, newest first. */
@@ -288,6 +290,14 @@
export function pullScreen(
        : checkSummary(f.db, row.id, row.headOid || (facts?.headOid ?? ""));
    checkBase = row?.baseBranch ?? facts?.baseBranch ?? "";
    checkProgress = runningProgress(f.app.config.dataDir, check);
    checkSteps =
      settings === undefined
        ? ""
        : stepsLine(
            f.app.config.dataDir,
            check,
            mayReadRunLog(f.app, f.user, settings),
          );
    checkOld =
      repo !== undefined &&
      row !== undefined &&
@@ -790,7 +800,7 @@
export function pullScreen(
      // The box is exactly as tall as what is in it: the state, then a row
      // for the verdicts when there are any, then a row for the checks when
      // a deploy has run or the gate demands one.
      const checksLine = checksText(check, requireChecks, isApp, checkBase, checkOld, checkProgress);
      const checksLine = checksText(check, requireChecks, isApp, checkBase, checkOld, checkProgress, checkSteps);
      const boxH =
        3 +
        (standingText(standing(), requireReview) === "" ? 0 : 1) +
@@ -1251,6 +1261,20 @@
export function checksText(
  stale = false,
  /** `runningProgress` of the check's log; empty when none is known. */
  progress = "",
  /** `stepsLine` of the check's log; empty when none is shown. */
  steps = "",
): string {
  const line = checksLine(check, required, isApp, baseBranch, stale, progress);
  return line === "" || steps === "" ? line : `${line} · ${steps}`;
}
function checksLine(
  check: CheckSummary,
  required: boolean,
  isApp: boolean,
  baseBranch: string,
  stale: boolean,
  progress: string,
): string {
  if (check.state === "none")
    return required
@@ -8,9 +8,10 @@
 * account while signup allows — not an admin; the operator makes the admin.
 */
import { SIGNUP_CLOSED_MESSAGE, signUp, signupClosed } from "../../actions/users.ts";
import { INVITE_SPENT_MESSAGE, SIGNUP_CLOSED_MESSAGE, signUp, signupClosed } from "../../actions/users.ts";
import { SignupSlot, waitText } from "../../actions/signup-window.ts";
import { hasAnyUser } from "../../core/users.ts";
import { getSignup } from "../../settings/signup.ts";
import type { Ctx, Screen } from "../app/app.ts";
import { binding } from "../app/keys.ts";
import { style } from "../term/style.ts";
@@ -37,36 +38,51 @@
const CREATE: Field[] = [
  { label: "Password", secret: true, hint: "At least 10 characters." },
];
/** While signup is `invite` the create form also asks for the link's token. */
const INVITE_BLANK_MESSAGE = "Paste the token from your invite link.";
const INVITE: Field = { label: "Invite", secret: true, hint: "The token from your invite link." };
export function signInScreen(f: Session, ctx: Ctx, onDone: () => void): Screen {
  // "Is this forge empty?" is a existence question, not a count: `COUNT(*)`
  // walks the whole user table on the one screen every session paints.
  const first = !hasAnyUser(f.db);
  // Closed signup offers no create mode at all, as the page offers no form.
  const closed = signupClosed(f.app);
  let mode: "signin" | "create" = first && !closed ? "create" : "signin";
  let values = ["", "", ""];
  // With signup = invite the form is offered, and asks for the invite.
  // Both are read where they are used, so the screen follows the live setting.
  const invited = (): boolean => getSignup(f.db) === "invite";
  const closed = (): boolean => !invited() && signupClosed(f.app);
  let mode: "signin" | "create" = first && !closed() ? "create" : "signin";
  let values = ["", "", "", ""];
  let focus = 0;
  let error: string | undefined;
  let busy = false;
  const fields = (): Field[] => (mode === "create" ? CREATE : SIGN_IN);
  const fields = (): Field[] => (mode === "create" ? (invited() ? [...CREATE, INVITE] : CREATE) : SIGN_IN);
  const submit = (): void => {
    if (busy) return;
    if (closed && mode === "create") {
    if (closed() && mode === "create") {
      error = SIGNUP_CLOSED_MESSAGE;
      return;
    }
    const needsInvite = mode === "create" && invited();
    if (needsInvite && (values[3] ?? "").trim() === "") {
      error = INVITE_BLANK_MESSAGE;
      focus = 3;
      return;
    }
    busy = true;
    error = undefined;
    runAsync(ctx, async () => {
      try {
        if (mode === "create") {
          const [username, email, password] = values as [
          const [username, email, password, token] = values as [
            string,
            string,
            string,
            string,
          ];
          const invite = needsInvite ? token.trim() : "";
          const slot = SignupSlot.take(f.app, f.identity.remoteAddress);
          if (typeof slot === "number") {
            error = waitText(slot);
@@ -75,10 +91,11 @@
export function signInScreen(f: Session, ctx: Ctx, onDone: () => void): Screen {
          const created = await signUp(
            f.app,
            { username, email, password },
            { slot, address: f.identity.remoteAddress },
            { slot, address: f.identity.remoteAddress, ...(invite === "" ? {} : { invite }) },
          );
          if (!created.ok) {
            error = errorText(created.error);
            // With signup = invite a refusal means the token did not open the door.
            error = needsInvite && created.error === "signup_closed" ? INVITE_SPENT_MESSAGE : errorText(created.error);
            return;
          }
          const adopted = f.adopt(created.value);
@@ -109,21 +126,21 @@
export function signInScreen(f: Session, ctx: Ctx, onDone: () => void): Screen {
    keys: [
      binding(["tab"], "next field"),
      binding(["enter"], mode === "create" ? "create" : "sign in"),
      ...(closed ? [] : [binding(["ctrl+t"], first ? "sign in instead" : "create account")]),
      ...(closed() ? [] : [binding(["ctrl+t"], first ? "sign in instead" : "create account")]),
      binding(["esc"], "back"),
    ],
    render(area, s) {
      const c = ctx.theme.c;
      const [head, body] = split(area, "v", [len(2), fill(1)]);
      segmented(s, head!, c, {
        items: closed ? ["Sign in"] : ["Sign in", "Create account"],
        items: closed() ? ["Sign in"] : ["Sign in", "Create account"],
        active: mode === "create" ? 1 : 0,
      });
      s.text(
        head!.x,
        head!.y + 1,
        truncate(
          closed
          closed()
            ? SIGNUP_CLOSED_MESSAGE
            : first
            ? "This forge has no accounts yet. The operator makes the admin."
@@ -162,9 +179,9 @@
export function signInScreen(f: Session, ctx: Ctx, onDone: () => void): Screen {
      ctx.setCursor(cursor);
    },
    key(e) {
      if (e.ctrl && e.key === "t" && !closed) {
      if (e.ctrl && e.key === "t" && !closed()) {
        mode = mode === "create" ? "signin" : "create";
        values = ["", "", ""];
        values = ["", "", "", ""];
        focus = 0;
        error = undefined;
        return true;
@@ -12,6 +12,7 @@
import { sshCommand } from "../ssh/command.ts";
import { mayDirectCrew } from "../crew/direct.ts";
import { mayReadRuns } from "../actions/runs.ts";
import { getAppByRepo } from "../core/apps.ts";
import { audienceFacts, audienceOf } from "../core/audience.ts";
import { edgeLoginUrl } from "../http/edge.ts";
import {
  canAdmin,
@@ -310,6 +311,7 @@
export function scopeHeader(
    ${facts.visibility === "private"
      ? badge("Private", { icon: "lock", variant: "secondary" })
      : badge("Public", { variant: "secondary" })}
    ${repo === undefined ? null : audienceLine(ctx, repo, admin, base)}
    ${actions.text.trim() !== ""
      ? html`<span class="repo-actions">${actions}</span>`
      : null}`;
@@ -347,6 +349,20 @@
export function scopeHeader(
  </header>`;
}
/**
 * Who can see this repository, in the one sentence `audienceOf` gives, beside
 * the badge. A link to the settings for whoever can open them, plain text for
 * everyone else. One line: CSS truncates it, `title` keeps all of it.
 */
function audienceLine(ctx: Ctx, repo: Repo, admin: boolean, base: string): Html {
  const { sentence } = audienceOf(audienceFacts(ctx.app.db, repo, ctx.auth.user));
  return admin
    ? html`<a class="repo-audience" href="${base}/settings" title="${sentence}"
        >${sentence}</a
      >`
    : html`<span class="repo-audience" title="${sentence}">${sentence}</span>`;
}
export { hostOf };
/** This repo's own rows for the command palette — the studio's nouns for a
@@ -295,6 +295,40 @@
export const studioRunJs = /* js */ `
  ${elapsedTickJs}
  // A check's step rows are server-derived from the log; when a line that
  // can move them lands (a command's echo, the shard runner's banner or a
  // shard header) or the run ends, the rows are fetched again and swapped in.
  var stepsTimer = 0;
  function steps() {
    var cur = document.getElementById('check-steps');
    if (!cur || !window.fetch || !window.DOMParser) return;
    fetch(location.href, { credentials: 'same-origin', headers: { accept: 'text/html' } })
      .then(function (r) { return r.ok ? r.text() : ''; })
      .then(function (body) {
        var doc = new DOMParser().parseFromString(body, 'text/html');
        var next = doc.getElementById('check-steps');
        var now = document.getElementById('check-steps');
        if (next && now && now.parentNode) now.parentNode.replaceChild(next, now);
        // The pin slot is only patched, never replaced: a keyboard user
        // reading a failure keeps focus and scroll.
        var wrap = document.getElementById('check-progress'), pw = doc.getElementById('check-progress');
        var slot = document.getElementById('check-pins'), ps = doc.getElementById('check-pins');
        if (wrap && pw) wrap.setAttribute('data-state', pw.getAttribute('data-state') || '');
        if (slot && ps) {
          if (slot.innerHTML !== ps.innerHTML) slot.innerHTML = ps.innerHTML;
          ['role', 'tabindex', 'aria-label'].forEach(function (a) {
            var v = ps.getAttribute(a);
            if (v === null) slot.removeAttribute(a); else if (slot.getAttribute(a) !== v) slot.setAttribute(a, v);
          });
        }
      })
      .catch(function () {});
  }
  function stepsSoon() {
    if (!document.getElementById('check-steps') || stepsTimer) return;
    stepsTimer = window.setTimeout(function () { stepsTimer = 0; steps(); }, 400);
  }
  if (steer && window.fetch) {
    var text = steer.querySelector('textarea');
    function size() { if (!text) return; text.style.height = 'auto'; text.style.height = Math.min(160, text.scrollHeight) + 'px'; fit(); }
@@ -341,12 +375,15 @@
export const studioRunJs = /* js */ `
    var text = event.data;
    try { var parsed = JSON.parse(text); if (typeof parsed === 'string') text = parsed; } catch (err) {}
    add(draw(text));
    if (/^(\\$ |test-shards: |\\(fail\\) |\u2500\u2500 shard )/.test(text)) stepsSoon();
  });
  source.addEventListener('done', function (event) {
    source.close();
    var run;
    try { run = JSON.parse(event.data); } catch (err) { return; }
    paint(run);
    if (stepsTimer) { window.clearTimeout(stepsTimer); stepsTimer = 0; }
    steps();
    if (steer) {
      steer.setAttribute('data-running', '0');
      var n = document.getElementById('run-steer-note');
@@ -1930,6 +1967,10 @@
export const liveRegionsJs = /* js */ `
      // post is in flight) is never pulled out from under the click; the next
      // matching event tries the swap again.
      if (mine.querySelector('[data-busy]')) return;
      // Likewise a region with a ticked box (apps picked for Archive, or the
      // Select toggle itself): a tap on a phone does not focus a checkbox, so
      // focus cannot be what protects the picks.
      if (mine.querySelector('input[type=checkbox]:checked')) return;
      var active = document.activeElement;
      if (active && mine.contains(active) && (active.tagName === 'INPUT' || active.tagName === 'TEXTAREA')) return;
      if (same(mine.innerHTML, theirs.innerHTML)) return;
@@ -31,7 +31,7 @@
import { ABILITIES, roleOf, rowOf, type Ability } from "../../core/roles.ts";
import { listKeys } from "../../core/keys.ts";
import { endAccountSession } from "../../actions/signins.ts";
import { listSessions, type SessionInfo } from "../../core/sessions.ts";
import { listTokens, type TokenScope } from "../../core/tokens.ts";
import { ADMIN_TOKEN_WORDS, listTokens, type TokenScope } from "../../core/tokens.ts";
import type { CommitEmail, User } from "../../core/users.ts";
import { cookieValue, type Ctx } from "../../http/context.ts";
import { sshCommand } from "../../ssh/command.ts";
@@ -539,7 +539,9 @@
function tokensView(ctx: Ctx): Html {
          options: [
            { value: "rw", label: "read + write — clone, push, open issues" },
            { value: "ro", label: "read only — clone and browse" },
            { value: "admin", label: "admin — everything, including settings" },
            ...(ctx.auth.user?.isAdmin
              ? [{ value: "admin", label: "admin — everything, including settings" }]
              : []),
          ],
          hint: "A token can never mint another token, whatever its scope.",
        })}
@@ -594,6 +596,8 @@
export async function tokenCreate(ctx: Ctx): Promise<Response> {
    field(form, "name").trim() || "token",
    chosen,
  );
  if (!minted.ok && minted.error === "forbidden")
    return flashTo(ctx, "/settings/tokens", { tone: "danger", message: ADMIN_TOKEN_WORDS });
  if (!minted.ok) return signInFirst(ctx);
  // Post-redirect-get: the secret rides a one-shot flash, so a refresh shows
  // the list rather than minting a second token.
@@ -20,7 +20,7 @@
import {
  revokeAccountToken,
} from "../../actions/signins.ts";
import { mailConfigured } from "../../actions/verify.ts";
import type { AccountRow } from "../../core/accounts.ts";
import { originOf, type AccountRow } from "../../core/accounts.ts";
import {
  OWNERS_TEAM,
  ownerlessOrgs,
@@ -73,7 +73,7 @@
function standing(a: AccountRow): Html {
function row(a: AccountRow, sole: readonly string[]): Html {
  const at = `/settings/accounts/${encodeURIComponent(a.username)}`;
  const how = [
    a.via ?? (a.kind === "agent" ? "crew" : ""),
    originOf(a),
    a.address ?? "",
  ].filter((s) => s !== "");
  const holds = [
@@ -20,6 +20,7 @@
import { resolveDestination, sealedSet } from "../../snapshot/destination.ts";
import { keyOf } from "../../snapshot/snapshot.ts";
import { lastDrill } from "../../snapshot/drills.ts";
import { snapshotsOf } from "../../snapshot/service.ts";
import { lastSnapshotOutcome } from "../../snapshot/outcome.ts";
import type { Ctx } from "../../http/context.ts";
import { notFoundPage, settingsLayout, signInFirst } from "../chrome.ts";
import { field, readForm, type Form } from "../form.ts";
@@ -183,9 +184,18 @@
export async function backupsPage(ctx: Ctx): Promise<Response> {
        note: "This forge was given nowhere to put them. On the platform an app's bucket (S3_*) is the place; on a machine, FORGE_SNAPSHOT_DIR names a directory. Seal a bucket below, or restart with one set, and a snapshot is taken every night at 08:00 UTC.",
      })}${form}`,
    );
  // A failure is kept on disk, so it is shown after a restart too; this
  // process's own failure is newer when the file could not be written.
  const kept = lastSnapshotOutcome(ctx.app.config.dataDir);
  const last = svc.last();
  const failing =
    kept?.failingSince !== undefined
      ? { at: kept.at, since: kept.failingSince, error: kept.error ?? "unknown reason" }
      : last.error !== undefined && (kept === undefined || (last.at ?? "") > kept.at)
        ? { at: last.at ?? "", since: last.at ?? "", error: last.error }
        : undefined;
  // The store is somewhere else; a listing that fails is a sentence on the
  // page, never a page that fails.
  const last = svc.last();
  let snapshots: Awaited<ReturnType<typeof svc.list>> = [];
  let listing: string | undefined;
  try {
@@ -227,11 +237,11 @@
bun run src/snapshot/restore.ts ${snapshots[0].name} /path/to/empty/data</pre>`
            </p>`,
        )}
        ${when(
          last.error,
          (error) =>
            html`<p class="backups-error">
              ${icon("alert")} The last snapshot failed ${time(last.at ?? "")}:
              ${error}
          failing,
          (f) =>
            html`<p class="backups-error" role="alert">
              ${icon("alert")} The last snapshot failed ${time(f.at)}, failing
              since ${time(f.since)}: ${f.error}
            </p>`,
        )}
        ${when(
@@ -15,7 +15,7 @@
import { clientAddress } from "../../http/peer.ts";
import { verifyEmail } from "../../actions/verify.ts";
import { STRANGER_MAX } from "../../http/limits.ts";
import { createRepo, repositoryLimitMessage, resolveOwner } from "../../actions/repos.ts";
import { HUMAN_CHECK_MESSAGE, PASSKEY_REQUIRED_MESSAGE, SIGNUP_CLOSED_MESSAGE, signUp, signupClosed } from "../../actions/users.ts";
import { HUMAN_CHECK_MESSAGE, INVITE_SPENT_MESSAGE, PASSKEY_REQUIRED_MESSAGE, SIGNUP_CLOSED_MESSAGE, signUp, signupClosed } from "../../actions/users.ts";
import { accessAllows, listOrgsForUser, orgAccess } from "../../core/orgs.ts";
import { listPins } from "../../core/pins.ts";
import { getReposByIds } from "../../core/repos.ts";
@@ -419,10 +419,6 @@
export function inviteOf(ctx: Ctx): string | undefined {
  return token === "" ? undefined : token;
}
/** What a person with a link that no longer opens the door is told. */
const INVITE_SPENT_MESSAGE =
  "This invite link has been used, revoked or has expired. Ask whoever sent it for a new one.";
/** The whole of `/signup` while the forge makes no new accounts: a sentence, no form. */
function signupClosedPage(ctx: Ctx): Response {
  const said = inviteOf(ctx) === undefined ? SIGNUP_CLOSED_MESSAGE : INVITE_SPENT_MESSAGE;
@@ -27,6 +27,9 @@
export const feedCss = /* css */ `
/* Your apps: a narrow column on a desk, a tab away on a phone. */
.home-tabs { display: none; margin-bottom: 12px; }
.home-app { gap: 8px; }
.home-pick { display: none; }
#apps-select:checked ~ * .home-pick { display: inline-flex; }
#apps-select:focus-visible ~ * .home-apps-verbs label { outline: 2px solid var(--accent, currentColor); }
.home-apps-verbs { flex-wrap: wrap; padding: 10px 14px; }
@media (max-width: 899px) {
  .home-tabs { display: inline-flex; }
@@ -60,7 +60,7 @@
import { passkeyJs } from "../passkey-client.ts";
const ON_YOU_LIVE =
  "issue.assigned,pull.assigned,issue.closed,pull.closed,issue.reopened,pull.reopened,pull.merged,fleet.operated";
/** The events that can change how an app on the rail is. */
const APPS_LIVE = "app.serving,app.stopped,run.finished";
const APPS_LIVE = "app.serving,app.stopped,run.finished,fleet.operated";
/** How many of your apps the rail names before "All apps". */
const RAIL_APPS = 8;
@@ -83,13 +83,23 @@
function yourApps(ctx: Ctx, user: User): Html {
        ? undefined
        : runtime.status(record).find((env) => env.env === "production")?.state;
    return html`<li class="row home-app">
      ${when(
        user.isAdmin,
        () =>
          html`<input
            class="home-pick"
            type="checkbox"
            name="repo"
            value="${repo.id}"
            aria-label="Select ${repo.name}"
          />`,
      )}
      <a class="truncate" href="${repoBase(repo)}">${repo.ownerName}/${repo.name}</a>
      <span class="spacer"></span>
      ${state === undefined ? null : envStateBadge(state)}
    </li>`;
  });
  return card(
    html`${rows.length === 0
  const list = html`${rows.length === 0
        ? html`<p class="muted small">No apps yet.</p>`
        : html`<ul class="list">
            ${rows}
@@ -108,7 +118,24 @@
function yourApps(ctx: Ctx, user: User): Html {
        ${when(user.isAdmin, () =>
          btn("Front page", { href: "/front", size: "sm", icon: "globe" }),
        )}
      </div>`,
        ${when(
          user.isAdmin && rows.length > 0,
          () => html`<label class="btn btn-sm" for="apps-select">Select</label>
            <span class="home-pick">${btn("Archive", { size: "sm" })}</span>`,
        )}
      </div>`;
  return card(
    // An admin's list is a form: ticks cost no space until Select shows them.
    user.isAdmin && rows.length > 0
      ? html`<form method="post" action="/home/archive">
          <input
            class="sr-only"
            type="checkbox"
            id="apps-select"
            aria-label="Select apps"
          />${list}
        </form>`
      : list,
    {
      head: html`${icon("rocket")}
        <h2>Your apps</h2>`,
@@ -0,0 +1,108 @@
/**
 * Archiving apps from the home: an admin ticks apps in the rail and archives
 * them in one step. It is the fleet's own `archive` operation over the ticked
 * repositories (`actions/fleet.ts`) — nothing new is written here — and the
 * page that answers lists what was archived with the operation's undo.
 */
import { applyFleetIds, getFleetOp, undoFleetOp } from "../../actions/fleet.ts";
import type { Ctx } from "../../http/context.ts";
import { signInFirst } from "../chrome.ts";
import { readForm } from "../form.ts";
import { html } from "../html.ts";
import { page, redirect } from "../layout.ts";
import { btn } from "../ui.ts";
import { MAX_ROWS } from "../../fleet/selection.ts";
/** The ticked repository ids, once each, as long as they are plausible ids. */
function picked(form: Awaited<ReturnType<typeof readForm>>): string[] {
  if (form instanceof Response) return [];
  const ids = form
    .getAll("repo")
    .filter((v): v is string => typeof v === "string" && /^[\w-]{1,64}$/.test(v));
  return [...new Set(ids)];
}
// POST /home/archive — archive the ticked apps, a site admin's.
export async function homeArchive(ctx: Ctx): Promise<Response> {
  if (!ctx.auth.user) return signInFirst(ctx);
  const form = await readForm(ctx, "admin");
  if (form instanceof Response) return form;
  if (!ctx.auth.user.isAdmin) return new Response("forbidden", { status: 403 });
  const ids = picked(form);
  if (ids.length === 0 || ids.length > MAX_ROWS)
    return redirect("/", {
      url: ctx.url,
      flash: { tone: "danger", message: "Tick the apps to archive first." },
    });
  const op = await applyFleetIds(
    ctx.app,
    ctx.auth.user,
    "repo",
    ids,
    `home: ${ids.length} selected`,
    { verb: "archive", value: true },
  );
  if (!op.ok)
    return redirect("/", {
      url: ctx.url,
      flash: { tone: "danger", message: op.error.message },
    });
  return redirect(`/home/archived/${op.value.id}`);
}
// GET /home/archived/:op — what was archived, and the undo.
export function homeArchived(ctx: Ctx): Response {
  if (!ctx.auth.user) return signInFirst(ctx);
  const found = getFleetOp(ctx.app, ctx.auth.user, ctx.params.op ?? "");
  if (!found.ok || found.value.verb !== "archive")
    return new Response("not found", { status: 404 });
  const op = found.value;
  const wasUndo = op.undoOf !== null;
  const rows = op.rows.filter((r) => r.outcome === "ok");
  const left = op.rows.filter((r) => r.outcome !== "ok");
  // Only the person who ran it may undo it (`undoFleetOp`), whoever may read it.
  const canUndo =
    op.reversible && op.undoneBy === null && !wasUndo && op.actorId === ctx.auth.user.id;
  const body = html`<h1>${wasUndo ? "Brought back" : "Archived"}</h1>
    <p id="home-archived" role="status">
      ${rows.length} app${rows.length === 1 ? "" : "s"}
      ${wasUndo ? "back on the home" : "archived"}${left.length === 0
        ? ""
        : `, ${left.length} left as they were`}.
    </p>
    <ul class="list">
      ${rows.map((r) => html`<li class="row">${r.label}</li>`)}
    </ul>
    ${left.length === 0
      ? null
      : html`<ul class="list muted">
          ${left.map((r) => html`<li class="row">${r.label}: ${r.reason}</li>`)}
        </ul>`}
    <div class="bar mt">
      ${canUndo
        ? html`<form method="post" action="/home/archived/${op.id}/undo">
            ${btn("Undo", { variant: "primary" })}
          </form>`
        : null}
      ${btn("Home", { href: "/" })}
    </div>
    ${canUndo ? html`<p class="small muted">Undo is kept for a day.</p>` : null}`;
  return page({ ctx, title: wasUndo ? "Brought back" : "Archived", body });
}
// POST /home/archived/:op/undo — the operation's own undo.
export async function homeArchiveUndo(ctx: Ctx): Promise<Response> {
  if (!ctx.auth.user) return signInFirst(ctx);
  const form = await readForm(ctx, "admin");
  if (form instanceof Response) return form;
  const undone = await undoFleetOp(ctx.app, ctx.auth.user, ctx.params.op ?? "");
  if (!undone.ok && undone.error.code === "not_found")
    return new Response("not found", { status: 404 });
  if (!undone.ok)
    return redirect("/", {
      url: ctx.url,
      flash: { tone: "danger", message: undone.error.message },
    });
  return redirect(`/home/archived/${undone.value.id}`);
}
@@ -7,6 +7,7 @@
 * deploy event refreshes (`appStateFragment`) share.
 */
import { audienceFacts, audienceOf } from "../../core/audience.ts";
import { sealedScope } from "../../core/sealed.ts";
import { terminalFor } from "../../actions/apps.ts";
import { envPull, previewEnv } from "../../apps/data.ts";
@@ -298,6 +299,10 @@
export function appPanels(
  // The state in one word, and under it the facts — why it is not up, and
  // the deploy whose log says the rest.
  const live = envPhrase(production);
  // Who the app answers to, in the sentence the forge's own rules produce.
  const audience = audienceOf(
    audienceFacts(db, repo, ctx.auth.user),
  ).sentence;
  const head = html`<div class="page-head studio-head">
    <div class="stack stack-tight">
      <h1>
@@ -320,6 +325,7 @@
export function appPanels(
                >`
            : html`· <span class="mono">${url.replace(/^https?:\/\//, "")}</span>`}
      </p>
      <p class="muted small" id="app-audience">${audience}</p>
      ${when(production, (env) => envFacts(base, env))}
      ${when(writer, () =>
        terminalLine(
@@ -8,6 +8,8 @@
 * at `?all=1`.
 */
import { checkSteps, stepWord } from "../../checks/summary.ts";
import { parseTestsRow, testsRowWords } from "../../checks/tests-row.ts";
import { directionNote } from "../../crew/direct.ts";
import { appsRuntime } from "../../apps/runtime.ts";
import { getAppByRepo } from "../../core/apps.ts";
@@ -24,7 +26,7 @@
import { repoDir } from "../../git/repos.ts";
import type { Ctx } from "../../http/context.ts";
import { crumbsFor, repoBase, repoHeader, repoPalette } from "../chrome.ts";
import { studioRunJs } from "../client.ts";
import { html, when, type Html } from "../html.ts";
import { html, raw, when, type Html } from "../html.ts";
import { runLine } from "../describe.ts";
import { page } from "../layout.ts";
import { btn, sha } from "../ui.ts";
@@ -244,6 +246,58 @@
export async function runView(
          Nothing has been logged yet.
        </p>`,
    )}
    ${when(
      // Progress is read from the log, so a reader the log is hidden from
      // gets no rows either.
      run.kind === "check" && !logHidden,
      () => {
        const steps = checkSteps(run, lines, truncated);
        const row = parseTestsRow(lines);
        const words = testsRowWords(row);
        const known = row.of !== undefined && row.started !== undefined && row.started <= row.of;
        const open = run.state === "queued" || run.state === "running";
        const running = row.running ?? [];
        const runningWords =
          running.length === 0
            ? ""
            : `running ${running.slice(0, 2).join(", ")}${running.length > 2 ? ` +${running.length - 2}` : ""}`;
        // While the run is open every part is rendered at a fixed size (the
        // pin slot by CSS), so the log below never moves as numbers and pins
        // arrive; a run that has ended reserves nothing.
        return html`<div class="check-progress" id="check-progress" data-state="${run.state}">
          <div id="check-steps" role="status" aria-live="polite">
            <ol class="check-steps">
              ${steps.map(
                (s) =>
                  html`<li data-step="${s.name}" data-state="${s.state}">
                    <span class="step-name">${s.name}</span>
                    <span class="step-state">${stepWord(s.state)}</span>
                  </li>`,
              )}
            </ol>
            ${known
              ? html`<progress class="tests-bar" aria-label="Test files started" max="${row.of ?? 0}" value="${row.started ?? 0}"></progress>`
              : open
                ? html`<progress class="tests-bar" aria-label="Test files started"></progress>`
                : ""}
            <p class="tests-words small muted">${[words, runningWords].filter((w) => w !== "").join(" · ")}</p>
          </div>
          <div class="pinned-slot" id="check-pins"${row.failures.length > 0 ? raw(' role="region" tabindex="0" aria-label="Failed tests"') : ""}>
            ${when(
              row.failures.length > 0,
              () => html`<ul class="pinned-failures">
                ${row.failures.map(
                  (f) => html`<li><strong>${f.name}</strong>${when(
                    f.lines.length > 0,
                    () => html`<pre>${f.lines.join("\n")}</pre>`,
                  )}</li>`,
                )}
              </ul>`,
            )}
          </div>
        </div>`;
      },
    )}
    <div class="run-live" id="run-live" data-state="${run.state}">
      ${when(!logHidden, () => logBox({
        id: "run-log",
@@ -25,6 +25,16 @@
export const content = /* css */ `
}
.repo-title .owner { font-weight: 400; color: var(--muted-fg); }
.repo-title .slash { color: var(--faint); font-weight: 400; }
.repo-audience {
  flex: 1 1 16ch; min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
  font-size: 13px; font-weight: 400; letter-spacing: 0; color: var(--muted-fg);
}
a.repo-audience:hover { color: var(--fg); }
/* A phone has no hover for the title: the whole sentence, on its own line. */
@media (max-width: 640px) {
  .repo-audience { flex-basis: 100%; white-space: normal; text-overflow: clip; }
  a.repo-audience { display: block; padding: 12px 0; min-height: 44px; box-sizing: border-box; }
}
.repo-desc { margin-top: 4px; color: var(--muted-fg); font-size: 13.5px; text-wrap: pretty; }
.repo-lineage { margin-top: 3px; font-size: 12.5px; color: var(--faint); }
.repo-head .tabs { margin-top: 12px; }
@@ -865,6 +875,21 @@
form.inline { display: inline; margin-left: auto; }
.checks-line .ok { display: inline-flex; align-items: center; gap: 4px; color: var(--success); }
.checks-line .bad { display: inline-flex; align-items: center; gap: 4px; color: var(--danger); }
.checks-line .now { display: inline-flex; align-items: center; gap: 4px; color: var(--attn); }
.check-steps { list-style: none; margin: 12px 0 0; padding: 0; display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 8px; }
.check-steps li { display: flex; justify-content: space-between; gap: 8px; padding: 8px 12px; min-height: 2.5em; border: 1px solid var(--border); border-radius: var(--radius); }
.tests-bar { display: block; width: 100%; height: 8px; margin-top: 8px; }
.tests-words { margin: 4px 0 0; min-height: 1.4em; }
/* Reserved only while the run is open: pins arriving never move the log. */
.pinned-slot { max-height: 9em; overflow-y: auto; }
.check-progress[data-state="queued"] .pinned-slot, .check-progress[data-state="running"] .pinned-slot { height: 9em; }
.pinned-failures { list-style: none; margin: 8px 0 0; padding: 0; display: grid; gap: 6px; }
.pinned-failures li { padding: 6px 12px; border: 1px solid var(--danger); border-radius: var(--radius); }
.pinned-failures pre { margin: 4px 0 0; white-space: pre-wrap; overflow-wrap: anywhere; font-size: 0.85em; }
@media (max-width: 480px) { .check-steps { grid-template-columns: minmax(0, 1fr); } }
.check-steps .step-state { color: var(--faint); }
.check-steps [data-state="ok"] .step-state { color: var(--success); }
.check-steps [data-state="failed"] .step-state { color: var(--danger); }
.check-steps [data-state="running"] .step-state { color: var(--attn); }
/* A run, live: a transcript with a speaker column, bounded so the page
   head stays reachable, and a composer that sits under it. */
.run-live { position: relative; margin-top: 12px; }
@@ -13,6 +13,7 @@
import {
import { queueAgentRun } from "../src/crew/dispatch.ts";
import { fetchInternal } from "../src/git/hook-address.ts";
import { addComment, openIssue } from "../src/actions/issues.ts";
import { newAccountLines } from "../src/actions/accounts.ts";
import { createRepo } from "../src/actions/repos.ts";
import { createOrg } from "../src/core/orgs.ts";
import { getAgent } from "../src/core/agents.ts";
@@ -175,6 +176,7 @@
describe("accounts: the owner sees, suspends and removes", () => {
      items: {
        username: string;
        via: string | null;
        signsInThrough: string | null;
        repos: number;
        spentUsd: number;
        suspendedAt: string | null;
@@ -186,6 +188,8 @@
describe("accounts: the owner sees, suspends and removes", () => {
    expect(ada.spentUsd).toBe(0);
    expect(ada.suspendedAt).toBeNull();
    expect(items.find((i) => i.username === "edgy")?.via).toBe("edge");
    expect(items.find((i) => i.username === "edgy")?.signsInThrough).toBe("open-platform.sh");
    expect(ada.signsInThrough).toBeNull();
    const page = await fetch(`${base}/settings/accounts`, {
      headers: { authorization: `Bearer ${ownerToken}` },
    });
@@ -193,6 +197,12 @@
describe("accounts: the owner sees, suspends and removes", () => {
    const html = await page.text();
    expect(html).toContain("ada");
    expect(html).toContain("Suspend");
    expect(html).toContain("signs in through open-platform.sh");
    expect(html.match(/signs in through/g)?.length).toBe(1);
    const owner = getUserByName(app.db, "olive")!;
    const lines = newAccountLines(app.db, owner).map((l) => l.line);
    expect(lines).toContain("edgy joined the forge (signs in through open-platform.sh)");
    expect(lines).toContain("ada joined the forge (signup)");
  });
  test("suspending stops every credential at once", async () => {
@@ -598,7 +608,7 @@
describe("accounts: the owner sees, suspends and removes", () => {
  test("a refusal the database raises answers 403 with its reason, not 500", async () => {
    // An operator's guard, the way production holds new accounts back.
    app.db.exec(
      "CREATE TRIGGER test_refuse BEFORE INSERT ON repo BEGIN SELECT RAISE(ABORT, 'new repositories are paused here'); END;",
      "CREATE TRIGGER test_refuse BEFORE INSERT ON repo BEGIN SELECT RAISE(ABORT, 'forge: new repositories are paused here'); END;",
    );
    try {
      const res = await api("/api/v1/repos", {
@@ -609,7 +619,7 @@
describe("accounts: the owner sees, suspends and removes", () => {
      expect(res.status).toBe(403);
      expect(await res.json()).toEqual({
        error: "refused",
        message: "new repositories are paused here",
        message: "forge: new repositories are paused here",
      });
      // The refusal leaves nothing on disk.
      expect(existsSync(repoDir(app.reposRoot, "olive", "guarded"))).toBe(false);
@@ -618,6 +628,40 @@
describe("accounts: the owner sees, suspends and removes", () => {
    }
  });
  test("a refusal without the forge: prefix is answered generically to anyone, and logged in full (#723)", async () => {
    const secret = "constraint on table repo_internal_x failed";
    app.db.exec(
      `CREATE TRIGGER test_refuse_plain BEFORE INSERT ON repo BEGIN SELECT RAISE(ABORT, '${secret}'); END;`,
    );
    const logged: { message: string; fields: Record<string, unknown> | undefined }[] = [];
    const log = app.log;
    (app as { log: typeof log }).log = {
      ...log,
      warn: (message, fields) => {
        logged.push({ message, fields });
        log.warn(message, fields);
      },
    };
    try {
      const res = await api("/api/v1/repos", {
        method: "POST",
        token: ownerToken,
        body: JSON.stringify({ name: "guarded-plain" }),
      });
      expect(res.status).toBe(403);
      expect(await res.json()).toEqual({
        error: "refused",
        message: "refused",
      });
      expect(
        logged.some((l) => l.fields?.reason === secret),
      ).toBe(true);
    } finally {
      (app as { log: typeof log }).log = log;
      app.db.exec("DROP TRIGGER test_refuse_plain");
    }
  });
  test("the owner can verify a newcomer's email, and it can create at once", async () => {
    // Made first: the helper proves every account waiting on a link.
    const { token: other } = await outsider("mallet");
@@ -0,0 +1,89 @@
/**
 * The app page says who the app answers to, in the head beside its state:
 * a public app, a private app, and a private app on a public repository
 * (#620).
 */
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { newApp } from "../src/actions/apps.ts";
import { updateApp } from "../src/core/apps.ts";
import { createApp, type App } from "../src/app.ts";
import { createUser, type User } from "../src/core/users.ts";
import { serve, type Servers } from "../src/http/server.ts";
import { quietLogs } from "./helpers/globals.ts";
quietLogs();
describe("the app page's audience sentence", () => {
  let dataDir: string;
  let app: App;
  let servers: Servers;
  let alice: User;
  let cookie = "";
  const sentence = async (name: string, withCookie = cookie) => {
    const res = await fetch(`${servers.url}/alice/${name}/app`, {
      headers: withCookie ? { cookie: withCookie } : {},
    });
    expect(res.status).toBe(200);
    const body = await res.text();
    return /<p class="muted small" id="app-audience">([^<]*)<\/p>/.exec(body)?.[1];
  };
  beforeAll(async () => {
    dataDir = await mkdtemp(join(tmpdir(), "forge-app-audience-"));
    app = createApp({ dataDir, port: 0, host: "forge.test" });
    servers = await serve(app);
    const made = await createUser(app.db, {
      origin: "seed",
      trusted: true,
      username: "alice",
      email: "[email protected]",
      password: "a-good-passphrase",
      adminIfFirst: true,
    });
    if (!made.ok) throw new Error(made.error);
    alice = made.value;
    for (const [name, visibility, access] of [
      ["open", "public", "public"],
      ["closed", "private", "private"],
      ["split", "public", "private"],
    ] as const) {
      const r = await newApp(app, alice, { name, description: "x", visibility });
      if (!r.ok) throw new Error(String(r.error));
      updateApp(app.db, r.value.app.id, { access });
    }
    const res = await fetch(`${servers.url}/signin`, {
      method: "POST",
      body: new URLSearchParams({ username: "alice", password: "a-good-passphrase" }),
      redirect: "manual",
    });
    cookie = (res.headers.get("set-cookie") ?? "").split(";")[0]!;
  }, 120_000);
  afterAll(async () => {
    servers.stop();
    app.close();
    await rm(dataDir, { recursive: true, force: true });
  });
  test("a public app says anyone can read it", async () => {
    expect(await sentence("open")).toStartWith("Public: anyone can read this;");
  });
  test("a private app says it is private", async () => {
    expect(await sentence("closed")).toStartWith("Private: only you");
  });
  test("a private app on a public repository says the code is public (#620)", async () => {
    expect(await sentence("split")).toStartWith(
      "The code is public; the app answers only to you",
    );
    // Signed out, the page is seen as a stranger: never told "you".
    const out = await sentence("split", "");
    expect(out).toStartWith("The code is public; the app answers only to ");
    expect(out).not.toContain("you");
  });
});
@@ -17,6 +17,7 @@
import { getUserByName } from "../src/core/users.ts";
import { serve, type Servers } from "../src/http/server.ts";
import { startCrew, type CrewHandle } from "../src/crew/dispatch.ts";
import { seedAdmin } from "./helpers/admin.ts";
import { tokenFor } from "./helpers/tokens.ts";
/** Read from an SSE body until `predicate(buffer)` holds or we time out —
 * never to a "done" that #284's still-live stream no longer sends. */
@@ -455,20 +456,7 @@
describe("apps, runs and the crew over the API", () => {
      },
      null,
    );
    const mallory = (
      (await (
        await post(
          "/api/v1/tokens",
          {
            username: "mallory",
            password: "a-good-passphrase",
            name: "admin",
            scope: "admin",
          },
          null,
        )
      ).json()) as { token: string }
    ).token;
    const mallory = tokenFor(app, "mallory", "admin", "admin");
    expect((await api(env, { token: mallory })).status).toBe(404);
    expect((await put("GREETING", "x", mallory)).status).toBe(404);
    expect(
@@ -18,6 +18,7 @@
let dataDir: string;
let app: App;
let servers: Servers;
let repo: Repo;
let second: Repo;
const tok: Record<string, string> = {};
const ok = <T>(r: { ok: true; value: T } | { ok: false; error: unknown }): T => {
@@ -26,7 +27,7 @@
const ok = <T>(r: { ok: true; value: T } | { ok: false; error: unknown }): T =>
};
const archived = (): boolean => getRepoById(app.db, repo.id)!.archivedAt !== null;
async function mcp(token: string, args: unknown): Promise<{ error?: string; value?: any }> {
async function mcp(token: string, args: unknown, tool = "archive"): Promise<{ error?: string; value?: any }> {
  const res = await fetch(`${servers.url}/mcp`, {
    method: "POST",
    headers: {
@@ -38,7 +39,7 @@
async function mcp(token: string, args: unknown): Promise<{ error?: string; valu
      jsonrpc: "2.0",
      id: 1,
      method: "tools/call",
      params: { name: "archive", arguments: args },
      params: { name: tool, arguments: args },
    }),
  });
  const body = (await res.json()) as any;
@@ -65,6 +66,9 @@
beforeAll(async () => {
  repo = ok(
    await createRepo(app.db, app.reposRoot, { ownerId: org.id, name: "svc", visibility: "public" }),
  );
  second = ok(
    await createRepo(app.db, app.reposRoot, { ownerId: org.id, name: "web", visibility: "public" }),
  );
  const bot = ok(await createAgent(app, founder, { username: "bot" }));
  expect(addToCrew(app, founder, repo, "bot").ok).toBe(true);
  const owners = listTeams(app.db, org.id).find((t) => t.name === OWNERS_TEAM)!;
@@ -111,3 +115,50 @@
test("an admin needs an admin token; then archives and wakes", async () => {
  expect(woke.value).toMatchObject({ archived: false, was: true });
  expect(archived()).toBe(false);
});
const asleep = (r: Repo): boolean => getRepoById(app.db, r.id)!.archivedAt !== null;
test("`repos`: a dry run over two changes nothing; applying archives both; the undo wakes both", async () => {
  const repos = ["acme/svc", "acme/web"];
  const planned = await mcp(tok["founder"]!, { repos });
  expect(planned.value.dryRun).toBe(true);
  expect(planned.value.rows.map((r: any) => r.outcome)).toEqual(["change", "change"]);
  expect(asleep(repo) || asleep(second)).toBe(false);
  const done = await mcp(tok["founder"]!, { repos, dryRun: false });
  expect(done.value.rows.map((r: any) => r.repo)).toEqual(["acme/svc", "acme/web"]);
  expect(done.value.rows.every((r: any) => r.outcome === "ok")).toBe(true);
  expect(asleep(repo) && asleep(second)).toBe(true);
  const undone = await mcp(
    tok["founder"]!,
    { app: "acme/svc", action: "undo", operation: done.value.operation, dryRun: false },
    "limb",
  );
  expect(undone.error).toBeUndefined();
  expect(asleep(repo) || asleep(second)).toBe(false);
});
test("`repos`: a non-admin is refused per row, an agent and a small token are refused outright", async () => {
  const repos = ["acme/svc", "acme/web", "acme/nope"];
  const tried = await mcp(tok["mallory"]!, { repos, dryRun: false });
  expect(tried.value.rows.map((r: any) => r.outcome)).toEqual(["refused", "refused", "refused"]);
  expect(asleep(repo) || asleep(second)).toBe(false);
  expect((await mcp(tok["bot"]!, { repos })).error).toContain("forbidden");
  expect((await mcp(tok["founderRw"]!, { repos, dryRun: false })).error).toContain("forbidden");
  expect((await mcp(tok["founder"]!, { repos: [] })).error).toContain("invalid");
  expect((await mcp(tok["founder"]!, { repo: "acme/svc", repos })).error).toContain("invalid");
  expect((await mcp(tok["founder"]!, { repos, archived: "yes" })).error).toContain("invalid");
  expect((await mcp(tok["founder"]!, { repos: Array(51).fill("acme/svc") })).error).toContain("invalid");
  expect(asleep(repo) || asleep(second)).toBe(false);
});
test("`repos` with `archived: false` wakes several in one operation", async () => {
  const repos = ["acme/svc", "acme/web"];
  await mcp(tok["founder"]!, { repos, dryRun: false });
  expect(asleep(repo) && asleep(second)).toBe(true);
  const woke = await mcp(tok["founder"]!, { repos, archived: false, dryRun: false });
  expect(woke.value.rows.map((r: any) => r.outcome)).toEqual(["ok", "ok"]);
  expect(typeof woke.value.operation).toBe("string");
  expect(asleep(repo) || asleep(second)).toBe(false);
});
@@ -0,0 +1,119 @@
/**
 * The page, the API and the MCP say the same sentence about who can see a
 * repository (#809); the wording itself is tested in audience.test.ts.
 */
import { proveAll } from "./helpers/verified.ts";
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createRepo } from "../src/actions/repos.ts";
import { createApp, type App } from "../src/app.ts";
import { audienceFor } from "../src/core/audience.ts";
import { createApp as makeApp } from "../src/core/apps.ts";
import { createToken } from "../src/core/tokens.ts";
import { createUser, type User } from "../src/core/users.ts";
import { serve, type Servers } from "../src/http/server.ts";
const esc = (s: string): string =>
  s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
/** The audience element of a page: its tag, link and text. */
function audienceOn(page: string) {
  const m = /<(a|span) class="repo-audience"(?: href="([^"]*)")? title="([^"]*)"\s*>([^<]*)<\/\1\s*>/.exec(page);
  if (m === null) throw new Error("no audience element on the page");
  return { tag: m[1], href: m[2], title: m[3], text: m[4] };
}
describe("one sentence on every surface", () => {
  let dataDir: string;
  let app: App;
  let servers: Servers;
  let olive: User;
  beforeAll(async () => {
    dataDir = await mkdtemp(join(tmpdir(), "forge-audience-"));
    app = createApp({ dataDir, port: 0 });
    servers = await serve(app);
    const made = await createUser(app.db, {
      origin: "seed",
      username: "olive",
      email: "[email protected]",
      password: "a-good-passphrase",
      isAdmin: true,
    });
    if (!made.ok) throw new Error(made.error);
    olive = made.value;
    proveAll(app);
  });
  afterAll(async () => {
    servers.stop();
    app.close();
    await rm(dataDir, { recursive: true, force: true });
  });
  test("the API and the MCP say the same words, and the page shows them", async () => {
    const made = await createRepo(app, olive, { name: "garden", visibility: "public" });
    if (!made.ok) throw new Error(String(made.error));
    makeApp(app.db, { repoId: made.value.id, slug: "garden", access: "private" });
    const token = createToken(app.db, olive.id, "t", "ro").token;
    const auth = { authorization: `Bearer ${token}` };
    const res = await fetch(`${servers.url}/api/v1/repos/olive/garden`, { headers: auth });
    const body = (await res.json()) as { audience: { level: string; sentence: string } };
    expect(body.audience).toEqual(audienceFor(app.db, made.value, olive));
    expect(body.audience.sentence).toStartWith("The code is public; the app answers only to");
    const mcp = await fetch(`${servers.url}/mcp`, {
      method: "POST",
      headers: {
        ...auth,
        "content-type": "application/json",
        accept: "application/json, text/event-stream",
      },
      body: JSON.stringify({
        jsonrpc: "2.0",
        id: 1,
        method: "tools/call",
        params: { name: "repo", arguments: { repo: "olive/garden" } },
      }),
    });
    expect(await mcp.text()).toContain("The code is public; the app answers only to");
    const page = await (await fetch(`${servers.url}/olive/garden`, { headers: auth })).text();
    const said = esc(body.audience.sentence);
    expect(audienceOn(page)).toEqual({
      tag: "a",
      href: "/olive/garden/settings",
      title: said,
      text: said,
    });
  });
  test("signed out, the page and the API say the same; a non-admin gets no link", async () => {
    const made = await createRepo(app, olive, { name: "meadow", visibility: "public" });
    if (!made.ok) throw new Error(String(made.error));
    const api = (await (await fetch(`${servers.url}/api/v1/repos/olive/meadow`)).json()) as {
      audience: { sentence: string };
    };
    expect(api.audience.sentence).toStartWith("Public: anyone can read this");
    const said = esc(api.audience.sentence);
    const out = await (await fetch(`${servers.url}/olive/meadow`)).text();
    expect(audienceOn(out)).toEqual({ tag: "span", href: undefined, title: said, text: said });
    const wren = await createUser(app.db, {
      origin: "seed",
      username: "wren",
      email: "[email protected]",
      password: "a-good-passphrase",
    });
    if (!wren.ok) throw new Error(wren.error);
    proveAll(app);
    const token = createToken(app.db, wren.value.id, "t", "ro").token;
    const seen = await (
      await fetch(`${servers.url}/olive/meadow`, {
        headers: { authorization: `Bearer ${token}` },
      })
    ).text();
    expect(audienceOn(seen)).toEqual({ tag: "span", href: undefined, title: said, text: said });
  });
});
@@ -0,0 +1,245 @@
/**
 * The audience sentence: each wording from plain facts, and the counts read
 * from a seeded database against what `REPO_VISIBLE` and `REPO_WRITABLE` let
 * through.
 */
import { beforeAll, describe, expect, test } from "bun:test";
import type { Database } from "bun:sqlite";
import {
  audienceFacts,
  audienceOf,
  type AudienceFacts,
} from "../src/core/audience.ts";
import { getRepoById } from "../src/core/repos.ts";
import { getUserById } from "../src/core/users.ts";
import { migrations } from "../src/core/schema.ts";
import { REPO_VISIBLE, REPO_WRITABLE } from "../src/core/visibility.ts";
import { openDb } from "../src/kernel/db.ts";
const base: AudienceFacts = {
  visibility: "private",
  viewer: "person",
  ownerKind: "user",
  ownerName: "ann",
  readers: 0,
  writers: 1,
  viewerWrites: true,
  crew: true,
};
describe("audienceOf", () => {
  test("a private repository of one person", () => {
    expect(audienceOf(base)).toEqual({
      level: "private",
      sentence: "Private: only you and the crew.",
    });
    expect(audienceOf({ ...base, crew: false }).sentence).toBe(
      "Private: only you.",
    );
  });
  test("a private organisation's people", () => {
    const facts = { ...base, ownerKind: "org" as const, ownerName: "acme" };
    expect(audienceOf({ ...facts, readers: 3 }).sentence).toBe(
      "Private: you, 3 people in acme, and the crew.",
    );
    expect(audienceOf({ ...facts, readers: 1, crew: false }).sentence).toBe(
      "Private: you and 1 person in acme.",
    );
  });
  test("a private repository seen by someone who is not its owner", () => {
    expect(audienceOf({ ...base, readers: 1 }).sentence).toBe(
      "Private: you, ann, and the crew.",
    );
  });
  test("a public repository", () => {
    expect(audienceOf({ ...base, visibility: "public", writers: 2 })).toEqual({
      level: "public",
      sentence:
        "Public: anyone can read this; 2 people and the crew can change it.",
    });
    expect(
      audienceOf({ ...base, visibility: "public", writers: 1 }).sentence,
    ).toContain("1 person and the crew can change it");
  });
  test("a signed-out viewer is never called you", () => {
    const out = { ...base, viewer: "signed-out" as const, viewerWrites: false };
    expect(
      audienceOf({
        ...out,
        visibility: "public",
        writers: 2,
        app: { access: "private", writers: 2, named: 1, viewerNamed: false },
      }).sentence,
    ).toBe(
      "The code is public; the app answers only to 2 people who can change it, 1 person it names, and the crew.",
    );
    expect(audienceOf({ ...out, readers: 1 }).sentence).toBe(
      "Private: ann and the crew.",
    );
    expect(audienceOf({ ...out, crew: false }).sentence).toBe("Private: nobody.");
    expect(
      audienceOf({ ...out, visibility: "public", writers: 0, crew: false }).sentence,
    ).toBe("Public: anyone can read this; nobody can change it.");
  });
  test("a private app on a public repository", () => {
    const facts: AudienceFacts = {
      ...base,
      visibility: "public",
      writers: 3,
      app: { access: "private", writers: 2, named: 2, viewerNamed: false },
    };
    expect(audienceOf(facts)).toEqual({
      level: "split",
      sentence:
        "The code is public; the app answers only to you, 2 people who can change it, 2 people it names, and the crew.",
    });
    expect(
      audienceOf({ ...facts, writers: 1, crew: false, app: { access: "private", writers: 0, named: 0, viewerNamed: false } })
        .sentence,
    ).toBe("The code is public; the app answers only to you.");
  });
  test("a public app on a public repository changes nothing", () => {
    const facts = { ...base, visibility: "public" as const };
    expect(audienceOf({ ...facts, app: { access: "public", writers: 0, named: 0, viewerNamed: false } }).level).toBe(
      "public",
    );
  });
});
describe("audienceFacts", () => {
  let db: Database;
  const ins = (sql: string) => db.run(sql);
  const user = (id: string) =>
    ins(
      `INSERT INTO user (id, username, email, password_hash) VALUES ('${id}','${id}','${id}@x','x')`,
    );
  const facts = (repoId: string, viewerId: string | undefined) =>
    audienceFacts(
      db,
      getRepoById(db, repoId)!,
      viewerId ? getUserById(db, viewerId) : undefined,
    );
  beforeAll(() => {
    db = openDb(":memory:", migrations);
    ins("INSERT INTO user (id, username, email, password_hash, kind) VALUES ('acme','acme','a@x','x','org')");
    for (const id of ["ann", "bob", "cy", "dee", "gone", "root"]) user(id);
    ins("UPDATE user SET suspended_at = '2026-01-01T00:00:00Z' WHERE id = 'gone'");
    ins("UPDATE user SET is_admin = 1 WHERE id = 'root'");
    ins("INSERT INTO user (id, username, email, password_hash, is_agent) VALUES ('bot','bot','b@x','x',1)");
    ins("INSERT INTO team (id, org_id, name, permission) VALUES ('t1','acme','read','read'), ('t2','acme','dev','write')");
    ins("INSERT INTO team_member (team_id, user_id) VALUES ('t1','bob'),('t1','cy'),('t2','cy'),('t2','dee'),('t1','gone'),('t2','gone')");
    ins("INSERT INTO repo (id, owner_id, name, visibility) VALUES ('org-priv','acme','p','private'),('org-pub','acme','q','public'),('mine','ann','m','private')");
    ins("INSERT INTO repo_crew (repo_id, user_id) VALUES ('org-priv','bot')");
    ins("INSERT INTO app (id, repo_id, slug, access) VALUES ('a1','org-pub','pub-app','private')");
    ins("INSERT INTO app_member (app_id, user_id) VALUES ('a1','ann'),('a1','gone'),('a1','cy')");
  });
  test("counts match what visibility.ts lets through", () => {
    const every = (rule: string, repoId: string) =>
      db
        .query<{ id: string }, [string]>(
          `SELECT cand.id FROM user AS cand WHERE cand.kind = 'user' AND cand.is_agent = 0
             AND cand.is_admin = 0 AND cand.suspended_at IS NULL
             AND EXISTS (SELECT 1 FROM repo WHERE repo.id = ?1 AND ${rule.replaceAll("?1", "cand.id")})`,
        )
        .all(repoId)
        .map((r) => r.id);
    const readers = every(REPO_VISIBLE, "org-priv");
    expect(readers.sort()).toEqual(["bob", "cy", "dee"]);
    const f = facts("org-priv", "ann");
    expect(f.readers).toBe(readers.length);
    expect(f.writers).toBe(every(REPO_WRITABLE, "org-priv").length);
    expect(f.writers).toBe(2);
    expect(f.crew).toBe(true);
    expect(f.ownerKind).toBe("org");
  });
  test("the viewer is not counted among the readers, and may write", () => {
    const f = facts("org-priv", "cy");
    expect(f.readers).toBe(2);
    expect(f.viewerWrites).toBe(true);
    expect(facts("org-priv", "bob").viewerWrites).toBe(false);
    expect(audienceOf(facts("org-priv", "cy")).sentence).toBe(
      "Private: you, 2 people in acme, and the crew.",
    );
  });
  test("a personal repository has no one but its owner", () => {
    expect(audienceOf(facts("mine", "ann"))).toEqual({
      level: "private",
      sentence: "Private: only you.",
    });
    expect(facts("mine", "ann").writers).toBe(1);
  });
  test("a private app names its active people", () => {
    const f = facts("org-pub", "dee");
    expect(f.app).toEqual({
      access: "private",
      writers: 1,
      named: 1,
      viewerNamed: false,
    });
    expect(audienceOf(f).sentence).toBe(
      "The code is public; the app answers only to you, 1 person who can change it, and 1 person it names.",
    );
  });
  test("a writer who is also named is counted once", () => {
    // cy writes and is named; ann is only named; dee only writes.
    const f = facts("org-pub", undefined);
    expect(f.app).toMatchObject({ writers: 2, named: 1 });
    expect(audienceOf(f).sentence).toBe(
      "The code is public; the app answers only to 2 people who can change it and 1 person it names.",
    );
    const asCy = facts("org-pub", "cy");
    expect(asCy.app).toMatchObject({ writers: 1, named: 1, viewerNamed: true });
  });
  test("a named viewer who cannot write is 'you', and not counted again", () => {
    const f = facts("org-pub", "ann");
    expect(f.viewerWrites).toBe(false);
    expect(f.app).toMatchObject({ writers: 2, named: 0, viewerNamed: true });
    expect(audienceOf(f).sentence).toBe(
      "The code is public; the app answers only to you and 2 people who can change it.",
    );
  });
  test("a viewer who neither writes nor is named is not told 'you'", () => {
    for (const id of ["bob", "root"]) {
      const f = facts("org-pub", id);
      expect(audienceOf(f).sentence).not.toContain("you");
    }
    expect(audienceOf(facts("org-pub", "bob")).sentence).toBe(
      "The code is public; the app answers only to 2 people who can change it and 1 person it names.",
    );
  });
  test("an admin viewer is told 'you' and counted like a person", () => {
    const f = facts("org-priv", "root");
    expect(f.viewer).toBe("admin");
    // Their reach is the forge's: membership of the repository is what counts.
    expect(f.viewerWrites).toBe(false);
    expect(f.readers).toBe(3);
    expect(audienceOf(f).sentence).toBe(
      "Private: you, 3 people in acme, and the crew.",
    );
  });
  test("an anonymous viewer is no one", () => {
    const f = facts("org-pub", undefined);
    expect(f.viewer).toBe("signed-out");
    expect(audienceOf(f).sentence).toBe(
      "The code is public; the app answers only to 2 people who can change it and 1 person it names.",
    );
    expect(f.viewerWrites).toBe(false);
    expect(f.writers).toBe(2);
  });
});
@@ -0,0 +1,143 @@
/**
 * Checks at once and shards per check default to what the pod's CPUs hold
 * (#2549): shards `min(4, cpus)`, checks `floor(cpus / (shards × 3))` held
 * to 1–16, so the vxrail's 80 cores need no override and the sentinel no
 * longer calls 4 shards and 6 checks away from their defaults.
 */
import { afterEach, describe, expect, test } from "bun:test";
import { writeFileSync } from "node:fs";
import { mkdtemp, rm } from "node:fs/promises";
import { availableParallelism, tmpdir } from "node:os";
import { join } from "node:path";
import { createApp } from "../src/app.ts";
import {
  CAPACITY,
  capacityDefault,
  capacityEnv,
  capacityValue,
  checksForCpus,
  cpuLimit,
  provideCpusForTest,
  setCapacity,
  shardsForCpus,
} from "../src/core/capacity.ts";
import { settingsAway } from "../src/crew/sentinel.ts";
import { createUser } from "../src/core/users.ts";
const ENV = ["FORGE_CHECK_SHARDS", "FORGE_CHECKS_CONCURRENCY"] as const;
const saved = ENV.map((k) => process.env[k]);
afterEach(() => {
  ENV.forEach((k, i) => {
    if (saved[i] === undefined) delete process.env[k];
    else process.env[k] = saved[i];
  });
  provideCpusForTest(4);
});
describe("the defaults follow the CPUs", () => {
  // By hand: shards = min(4, cpus); checks = floor(cpus / (shards * 3)), 1..16.
  test.each([
    // cpus, shards, checks
    [4, 4, 1], // 4 / 12 = 0.33 → held to 1
    [16, 4, 1], // 16 / 12 = 1.33 → 1
    [80, 4, 6], // 80 / 12 = 6.67 → 6
    [2, 2, 1], // 2 / 6 = 0.33 → 1
    [1, 1, 1],
    [1000, 4, 16], // 1000 / 12 = 83 → held to 16
  ])("%d CPUs: %d shards, %d checks at once", (cpus, shards, checks) => {
    expect(shardsForCpus(cpus)).toBe(shards);
    expect(checksForCpus(cpus)).toBe(checks);
    delete process.env.FORGE_CHECK_SHARDS;
    delete process.env.FORGE_CHECKS_CONCURRENCY;
    expect(capacityDefault("checks.shards", cpus)).toBe(String(shards));
    expect(capacityDefault("checks.concurrency", cpus)).toBe(String(checks));
    provideCpusForTest(cpus);
    expect(capacityDefault("checks.shards")).toBe(String(shards));
    expect(capacityDefault("checks.concurrency")).toBe(String(checks));
  });
  test("the environment still wins over the CPUs", () => {
    process.env.FORGE_CHECK_SHARDS = "8";
    process.env.FORGE_CHECKS_CONCURRENCY = "3";
    expect(capacityDefault("checks.shards", 80)).toBe("8");
    expect(capacityDefault("checks.concurrency", 80)).toBe("3");
  });
  test("both help lines say how the default is reached", () => {
    expect(CAPACITY["checks.shards"].help).toContain("at most 4");
    expect(CAPACITY["checks.concurrency"].help).toContain("shards × 3");
  });
});
describe("a stored value overrides the default; the sentinel compares with it", () => {
  test("80 CPUs: 4 shards and 6 checks are the defaults, 8 shards is away", async () => {
    delete process.env.FORGE_CHECK_SHARDS;
    delete process.env.FORGE_CHECKS_CONCURRENCY;
    provideCpusForTest(80);
    const dataDir = await mkdtemp(join(tmpdir(), "forge-capacity-cpus-"));
    const app = createApp({ dataDir, port: 0 });
    try {
      const a = await createUser(app.db, {
        origin: "seed",
        trusted: true,
        username: "alice",
        email: "[email protected]",
        password: "a-good-passphrase",
        adminIfFirst: true,
      });
      if (!a.ok) throw new Error(a.error);
      expect(capacityValue(app.db, "checks.shards")).toBe("4");
      expect(capacityEnv(app.db)).toEqual({ FORGE_CHECK_SHARDS: "4" });
      // A check's cell of 2 CPUs is given 2, not the pod's 4.
      expect(capacityEnv(app.db, 2)).toEqual({ FORGE_CHECK_SHARDS: "2" });
      // The owner's choice for the vxrail, set more than a day ago.
      setCapacity(app.db, "checks.shards", "4", a.value.id);
      setCapacity(app.db, "checks.concurrency", "6", a.value.id);
      setCapacity(app.db, "crew.concurrency", "5", a.value.id);
      const old = new Date(Date.now() - 3 * 24 * 3600_000).toISOString();
      app.db.query("UPDATE forge_setting SET updated_at = ?1").run(old);
      expect(settingsAway(app.db, Date.now()).map((f) => f.key)).toEqual(["setting:crew.concurrency"]);
      // A stored value overrides the default, even a cell's smaller one.
      setCapacity(app.db, "checks.shards", "8", a.value.id);
      app.db.query("UPDATE forge_setting SET updated_at = ?1").run(old);
      expect(capacityValue(app.db, "checks.shards")).toBe("8");
      expect(capacityEnv(app.db, 2)).toEqual({ FORGE_CHECK_SHARDS: "8" });
      const away = settingsAway(app.db, Date.now()).find((f) => f.key === "setting:checks.shards");
      expect(away?.evidence[0]).toBe('checks.shards = "8", default "4"');
    } finally {
      app.close();
      await rm(dataDir, { recursive: true, force: true });
    }
  });
});
describe("cpuLimit reads the pod's cgroup", () => {
  const withFile = async (text: string | undefined, run: (file: string) => void) => {
    provideCpusForTest(undefined);
    const dir = await mkdtemp(join(tmpdir(), "forge-cpu-max-"));
    const file = join(dir, "cpu.max");
    if (text !== undefined) writeFileSync(file, text);
    try {
      run(file);
    } finally {
      await rm(dir, { recursive: true, force: true });
    }
  };
  const machine = availableParallelism();
  test("quota over period, rounded down, never more than the machine", async () => {
    await withFile("150000 100000\n", (f) => expect(cpuLimit(f)).toBe(Math.min(machine, 1)));
    await withFile("50000 100000\n", (f) => expect(cpuLimit(f)).toBe(1));
    await withFile(`${machine * 100000} 100000\n`, (f) => expect(cpuLimit(f)).toBe(machine));
    await withFile(`${machine * 200000} 100000\n`, (f) => expect(cpuLimit(f)).toBe(machine));
  });
  test("no quota, garbage or an unreadable file falls back to the machine", async () => {
    await withFile("max 100000\n", (f) => expect(cpuLimit(f)).toBe(machine));
    await withFile("nonsense", (f) => expect(cpuLimit(f)).toBe(machine));
    await withFile(undefined, (f) => expect(cpuLimit(f)).toBe(machine));
  });
});
@@ -0,0 +1,98 @@
/**
 * Crew runs at once defaults to what the pod's memory holds (#2548): one run
 * per 3 GiB above the forge's own 2 GiB, at least 2 and at most 16. The
 * environment still wins, then a stored value; the sentinel's
 * away-from-default check compares with this default, so a big machine
 * running its share is not flagged.
 */
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import {
  CAPACITY,
  capacityDefault,
  capacityValue,
  crewForMemory,
  memoryLimitBytes,
  setCapacity,
} from "../src/core/capacity.ts";
import { settingsAway } from "../src/crew/sentinel.ts";
const GiB = 1024 ** 3;
describe("crewForMemory", () => {
  test("one run per 3 GiB above 2 GiB, between 2 and 16", () => {
    expect(crewForMemory(4 * GiB)).toBe(2); // (4 − 2) / 3 = 0 → floor of 2
    expect(crewForMemory(32 * GiB)).toBe(10); // (32 − 2) / 3 = 10
    expect(crewForMemory(64 * GiB)).toBe(16); // (64 − 2) / 3 = 20 → ceiling of 16
    expect(crewForMemory(11 * GiB)).toBe(3); // (11 − 2) / 3 = 3
    expect(crewForMemory(GiB)).toBe(2);
    expect(crewForMemory(Number.NaN)).toBe(2);
  });
  test("the capacity page says how the default is reached", () => {
    expect(CAPACITY["crew.concurrency"].help).toContain(
      "Empty: one per 3 GiB of memory above the forge's own 2 GiB",
    );
  });
});
describe("the crew's default, and what overrides it", () => {
  let dataDir: string;
  let app: App;
  let saved: string | undefined;
  beforeEach(async () => {
    saved = process.env.FORGE_CREW_CONCURRENCY;
    delete process.env.FORGE_CREW_CONCURRENCY;
    dataDir = await mkdtemp(join(tmpdir(), "forge-capacity-crew-"));
    app = createApp({ dataDir, port: 0 });
  });
  afterEach(async () => {
    if (saved === undefined) delete process.env.FORGE_CREW_CONCURRENCY;
    else process.env.FORGE_CREW_CONCURRENCY = saved;
    app.close();
    await rm(dataDir, { recursive: true, force: true });
  });
  test("unset, it is the pod's memory's share", () => {
    expect(capacityDefault("crew.concurrency")).toBe(String(crewForMemory(memoryLimitBytes())));
  });
  test("a set FORGE_CREW_CONCURRENCY wins over memory", () => {
    process.env.FORGE_CREW_CONCURRENCY = "7";
    expect(capacityDefault("crew.concurrency")).toBe("7");
    process.env.FORGE_CREW_CONCURRENCY = "99"; // out of range: ignored
    expect(capacityDefault("crew.concurrency")).toBe(String(crewForMemory(memoryLimitBytes())));
  });
  test("a stored value overrides the default", () => {
    const other = capacityDefault("crew.concurrency") === "5" ? "6" : "5";
    setCapacity(app.db, "crew.concurrency", other, "system");
    expect(capacityValue(app.db, "crew.concurrency")).toBe(other);
    setCapacity(app.db, "crew.concurrency", null, "system");
    expect(capacityValue(app.db, "crew.concurrency")).toBe(capacityDefault("crew.concurrency"));
  });
  test("the sentinel flags a value away from the memory default, not one at it", () => {
    const now = Date.now();
    const age = () =>
      app.db
        .query("UPDATE forge_setting SET updated_at = ?1 WHERE key = 'crew.concurrency'")
        .run(new Date(now - 3 * 86_400_000).toISOString());
    const crew = () => settingsAway(app.db, now).filter((f) => f.key === "setting:crew.concurrency");
    const fallback = capacityDefault("crew.concurrency");
    setCapacity(app.db, "crew.concurrency", fallback, "system");
    age();
    expect(crew()).toEqual([]);
    const other = fallback === "5" ? "6" : "5";
    setCapacity(app.db, "crew.concurrency", other, "system");
    age();
    const found = crew();
    expect(found).toHaveLength(1);
    expect(found[0]!.evidence[0]).toBe(`crew.concurrency = "${other}", default "${fallback}"`);
  });
});
@@ -23,6 +23,7 @@
import { startChecks, type ChecksHandle } from "../src/checks/service.ts";
import {
  capacityChanges,
  capacityCount,
  capacityDefault,
  noteBooted,
  capacityEnv,
  capacityValue,
@@ -291,14 +292,23 @@
describe("capacity: only a forge admin reads or changes it", () => {
      body: new URLSearchParams({ key: "checks.shards", reset: "1" }),
    });
    expect(reset.status).toBe(303);
    expect(capacityEnv(app.db)).toEqual({});
    // Back at the default, which is sized from the CPUs it is given (#2549).
    const fromEnv = process.env.FORGE_CHECK_SHARDS;
    delete process.env.FORGE_CHECK_SHARDS;
    try {
      expect(capacityEnv(app.db, 80)).toEqual({ FORGE_CHECK_SHARDS: "4" });
      expect(capacityEnv(app.db, 2)).toEqual({ FORGE_CHECK_SHARDS: "2" });
    } finally {
      if (fromEnv !== undefined) process.env.FORGE_CHECK_SHARDS = fromEnv;
    }
    const shardsDefault = capacityDefault("checks.shards");
    const changes = capacityChanges(app.db);
    expect(changes.map((c) => [c.key, c.from, c.to, c.actorName])).toEqual([
      ["checks.shards", "3", "", "alice"],
      ["checks.shards", "3", shardsDefault, "alice"],
      ["cells.egress", "enforce", "report", "alice"],
      ["crew.concurrency", "2", "3", "alice"],
      ["checks.shards", "", "3", "alice"],
      ["crew.concurrency", capacityDefault("crew.concurrency"), "3", "alice"],
      ["checks.shards", shardsDefault, "3", "alice"],
    ]);
    const after = await (
      await fetch(`${servers.url}/settings/capacity`, { headers: { cookie } })
@@ -0,0 +1,170 @@
/**
 * A check's three step rows, read from the run's state and its log
 * (`checkSteps`, `src/checks/summary.ts`): what each row says while the
 * check runs, when it ends red at each step, when it is stopped, for a job
 * placed check, and for a log whose head was cut away.
 */
import { describe, expect, test } from "bun:test";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { stepsLine } from "../src/checks/progress.ts";
import { checkSteps } from "../src/checks/summary.ts";
import type { Run } from "../src/core/runs.ts";
import { appendRunLog } from "../src/runs/log.ts";
const read = (
  state: "queued" | "running" | "ok" | "failed" | "cancelled",
  lines: string[],
  opts: { placement?: string; truncated?: boolean } = {},
) =>
  checkSteps(
    { state, placement: opts.placement ?? "process" },
    lines,
    opts.truncated ?? false,
  ).map((s) => `${s.name}:${s.state}`);
const INSTALL = "$ bun install";
const CHECK = "$ gate && tsc && bun scripts/test-shards.ts";
const BANNER = "test-shards: 9 files";
describe("checkSteps", () => {
  test("a run that has logged nothing is waiting, then running install", () => {
    expect(read("queued", [])).toEqual([
      "install:pending",
      "typecheck:pending",
      "tests:pending",
    ]);
    expect(read("running", [INSTALL])).toEqual([
      "install:running",
      "typecheck:pending",
      "tests:pending",
    ]);
  });
  test("typecheck runs between the check's echo and the shard banner", () => {
    expect(read("running", [INSTALL, CHECK])).toEqual([
      "install:ok",
      "typecheck:running",
      "tests:pending",
    ]);
  });
  test("a red typecheck fails typecheck and leaves the tests unrun", () => {
    expect(read("failed", [INSTALL, CHECK, "src/a.ts(1,1): error TS2322"])).toEqual([
      "install:ok",
      "typecheck:failed",
      "tests:pending",
    ]);
  });
  test("a red install fails install", () => {
    expect(read("failed", [INSTALL, "error: no such package"])).toEqual([
      "install:failed",
      "typecheck:pending",
      "tests:pending",
    ]);
  });
  test("a repository with no install command starts at typecheck", () => {
    expect(read("running", [CHECK])).toEqual([
      "install:ok",
      "typecheck:running",
      "tests:pending",
    ]);
  });
  test("a cancelled run stops at the step it was on", () => {
    expect(read("cancelled", [INSTALL, CHECK, BANNER])).toEqual([
      "install:ok",
      "typecheck:ok",
      "tests:stopped",
    ]);
  });
  test("a passed run is passed at every step", () => {
    expect(read("ok", [CHECK])).toEqual([
      "install:ok",
      "typecheck:ok",
      "tests:ok",
    ]);
  });
  test("a job-placed check echoes every command first: install is done only once output follows", () => {
    expect(read("running", [INSTALL, CHECK], { placement: "job" })).toEqual([
      "install:running",
      "typecheck:pending",
      "tests:pending",
    ]);
    expect(
      read("running", [INSTALL, CHECK, "src/a.ts ok"], { placement: "job" }),
    ).toEqual(["install:ok", "typecheck:running", "tests:pending"]);
  });
  test("a truncated log whose tests cannot be seen reads unknown, not wrong", () => {
    expect(read("running", ["some late output"], { truncated: true })).toEqual([
      "install:unknown",
      "typecheck:unknown",
      "tests:unknown",
    ]);
    expect(
      read("running", ["── shard 2/4: test/a.test.ts"], { truncated: true }),
    ).toEqual(["install:ok", "typecheck:ok", "tests:running"]);
  });
});
describe("the terminal's step line", () => {
  const dir = mkdtempSync(join(tmpdir(), "forge-steps-"));
  const run = { id: "r1", kind: "check", state: "running", placement: "process" } as Run;
  for (const l of [INSTALL, CHECK, BANNER]) appendRunLog(dir, "r1", l);
  test("reads the same rows as the page, through the same function", () => {
    expect(stepsLine(dir, { state: "running", run }, true)).toBe(
      "install passed · typecheck passed · tests running",
    );
    expect(
      stepsLine(dir, { state: "failed", run: { ...run, state: "failed" } }, true),
    ).toBe("install passed · typecheck passed · tests failed");
  });
  test("carries the tests row and the pinned failure names, withheld with the rest", () => {
    const d = mkdtempSync(join(tmpdir(), "forge-steps-"));
    for (const l of [
      INSTALL,
      CHECK,
      "test-shards: 9 files in 3 shards",
      "── shard 1/3: test/a.test.ts test/b.test.ts",
      "error: boom",
      "(fail) b breaks",
      " 5 pass",
      " 1 fail",
      "── shard 1/3 done (exit 1, 2.0s)",
    ])
      appendRunLog(d, "r2", l);
    const r = { ...run, id: "r2" } as Run;
    expect(stepsLine(d, { state: "running", run: r }, true)).toBe(
      "install passed · typecheck passed · tests running · tests 2 of 9 files started · 5 passed, 1 failed · failing: b breaks",
    );
    expect(stepsLine(d, { state: "running", run: r }, false)).toBe("");
    rmSync(d, { recursive: true, force: true });
  });
  test("says nothing to a reader who may not read the log, nor for a deploy", () => {
    expect(stepsLine(dir, { state: "running", run }, false)).toBe("");
    expect(
      stepsLine(dir, { state: "running", run: { ...run, kind: "deploy" } }, true),
    ).toBe("");
    rmSync(dir, { recursive: true, force: true });
  });
});
describe("install detection", () => {
  test("a script with install in its name is not the install", () => {
    expect(read("running", ["$ bun scripts/install-hooks.ts"])).toEqual([
      "install:ok",
      "typecheck:running",
      "tests:pending",
    ]);
  });
});
@@ -47,6 +47,7 @@
import type { AppRecord } from "../src/core/apps.ts";
import type { Repo } from "../src/core/repos.ts";
import { serve, type Servers } from "../src/http/server.ts";
import { readRunLog } from "../src/runs/log.ts";
import { expectContained, probeSource } from "./helpers/cell-probe.ts";
import { missing } from "./helpers/tools.ts";
import { portInsteadOfSocket } from "../src/git/hook-address.ts";
@@ -497,88 +498,6 @@
const noCells =
  process.platform !== "linux" ||
  portInsteadOfSocket();
/** What the probe tries, from inside a deploy or a check. */
function probeSource(targets: {
  dataDir: string;
  otherAppDir: string;
  canary: string;
  listenerPort: number;
  /** The forge's own pid: its environment holds every secret it was started with. */
  forgePid: number;
}): string {
  return `
import { readdirSync, readFileSync } from "node:fs";
const T = ${JSON.stringify(targets)};
const reached = (detail) => "reached: " + String(detail).slice(0, 80);
async function attempt(fn) {
  try {
    const found = await fn();
    return found === false ? "refused" : reached(found);
  } catch {
    return "refused";
  }
}
const answered = async (url, init) => {
  const res = await fetch(url, { ...init, signal: AbortSignal.timeout(5000) });
  await res.body?.cancel();
  return res.ok ? res.status : false;
};
async function probe() {
  return {
    forgeDb: await attempt(() => readFileSync(T.dataDir + "/forge.db").length),
    secretKey: await attempt(() => readFileSync(T.dataDir + "/secret.key").length),
    forgeData: await attempt(() => {
      const names = readdirSync(T.dataDir);
      return names.some((n) => n === "forge.db" || n === "secret.key" || n === "repos") && names.join(",");
    }),
    otherApp: await attempt(() => readdirSync(T.otherAppDir).join(",")),
    environment: await attempt(() => {
      if (Object.values(process.env).some((v) => v?.includes(T.canary))) return "own env";
      // The pid is the forge's only outside a pid namespace of its own; inside
      // one the same number may be a process of the cell's.
      const cmdline = readFileSync("/proc/" + T.forgePid + "/cmdline", "latin1");
      const forge = readFileSync("/proc/" + T.forgePid + "/environ", "latin1");
      if (cmdline.includes("test") && forge.length > 0) return "the forge's environment, " + forge.length + " bytes";
      for (const pid of readdirSync("/proc")) {
        if (!/^\\d+$/.test(pid)) continue;
        try {
          if (readFileSync("/proc/" + pid + "/environ", "latin1").includes(T.canary)) return "pid " + pid;
        } catch {}
      }
      return false;
    }),
    undeclaredHost: await attempt(() => answered("http://example.com/")),
    cluster: await attempt(() => answered("http://127.0.0.1:" + T.listenerPort + "/")),
    declaredButPrivate: await attempt(() =>
      answered("http://localhost:" + T.listenerPort + "/", { proxy: process.env.HTTP_PROXY ?? undefined }),
    ),
    forge: await attempt(() => answered(process.env.FORGE_URL + "/healthz")),
  };
}
if (process.argv[2] === "check") {
  console.log("PROBE " + JSON.stringify(await probe()));
  // A terminal is the app's own client: its app answers it on its port.
  if (process.env.FORGE_APP_TTY === "1")
    console.log("OWNAPP " + (await attempt(() => answered("http://127.0.0.1:" + process.env.PORT + "/"))));
} else {
  Bun.serve({
    port: Number(process.env.PORT),
    hostname: "127.0.0.1",
    fetch: async (req) =>
      new URL(req.url).pathname === "/probe" ? Response.json(await probe()) : new Response("ok"),
  });
}
`;
}
/** Everything refused, and the forge — the one declared way out — answering. */
function expectContained(results: Record<string, string>): void {
  const { forge, ...rest } = results;
  for (const [name, outcome] of Object.entries(rest))
    expect(`${name}: ${outcome}`).toBe(`${name}: refused`);
  expect(forge).toStartWith("reached");
}
describe.skipIf(noCells)(
  "cells: repository code cannot take over the forge",
  () => {
@@ -223,6 +223,43 @@
describe("the tick says why each ready issue cannot start", () => {
    expect(replannable(again).map((i) => i.number)).toEqual([done.number]);
  });
  test("an issue the crew found done on main is held as done_on_main, not replannable", async () => {
    const { repo, mason } = await crewed(app, alice, "finished");
    const oid = "a".repeat(40);
    const at = (id: string, ago: number) =>
      app.db.query("UPDATE comment SET created_at = ?2 WHERE id = ?1").run(id, minutes(ago));
    const say = (i: Issue, body: string, ago: number) => {
      const c = addComment(app, alice, getRepoById(app.db, repo.id)!, getIssueById(app.db, i.id)!, body);
      if (!c.ok) throw new Error(String(c.error));
      at(c.value.id, ago);
    };
    const done = issue(repo, "Already done");
    ran(repo, done, mason, "declined: it is done", 30);
    say(done, `${DONE_NOTE}${oid}`, 20);
    const reparked = issue(repo, "Done, then parked");
    ran(repo, reparked, mason, "declined: it is done", 30);
    say(reparked, `${DONE_NOTE}${oid}`, 20);
    say(reparked, `${PARK_NOTE}not so`, 10);
    ran(repo, reparked, mason, "declined: it is not done", 5);
    const spoken = issue(repo, "Done, then a word");
    ran(repo, spoken, mason, "declined: it is done", 30);
    say(spoken, `${DONE_NOTE}${oid}`, 20);
    say(spoken, "Please try again", 10);
    const held = heldNow(app, getRepoById(app.db, repo.id)!);
    const named = (h: keyof typeof held.by) => held.by[h].map((i) => i.number);
    expect(named("done_on_main")).toContain(done.number);
    expect(named("stalled")).not.toContain(done.number);
    expect(replannable(held).map((i) => i.number)).not.toContain(done.number);
    // A later park is a newer word: it is the planner's to re-plan again.
    expect(named("done_on_main")).not.toContain(reparked.number);
    expect(replannable(held).map((i) => i.number)).toContain(reparked.number);
    // A person's comment after the done note does not make it startable.
    expect(held.startable.map((r) => r.issue.number)).not.toContain(spoken.number);
    expect(named("done_on_main")).toContain(spoken.number);
    expect(heldWords(held)).toContain("done_on_main=");
  });
  test("nothing startable with a slot free asks the planner once, with the breakdown, whatever the plate", async () => {
    const { repo, atlas, mason } = await crewed(app, alice, "stuck");
    const fresh = () => getRepoById(app.db, repo.id)!;
@@ -1615,4 +1615,34 @@
describe("an issue found already done is the asker's to close, not a park", () =
    expect(alreadyDone(e.app.db, [work.id]).has(work.id)).toBe(false);
    expect(alreadyDoneOids("no changes pushed: already done: " + oid)).toEqual([]);
  });
  test("an issue parked twice whose newest crew note is the done note is not handed to the planner", () => {
    const park = (on: Issue) => {
      for (let n = 0; n < 2; n++)
        addComment(e.app, e.mason, e.fresh(), on, "Parked by the crew: " + nothingNew);
    };
    const nothingNew = "the run ended without a pull: nothing to propose";
    const finished = e.issue("Parked, then found done");
    const open = e.issue("Parked, still open");
    park(finished);
    park(open);
    const numbers = () => parkedTwice(e.app.db, [e.fresh()]).map((i) => i.number);
    expect(numbers()).toEqual([finished.number, open.number]);
    addComment(
      e.app,
      e.mason,
      e.fresh(),
      finished,
      "Done on main already, waiting for you to close it: " + oid,
    );
    // Written in the same instant as the park: the clock cannot order them.
    e.app.db.run(
      `UPDATE comment SET created_at = (SELECT MAX(p.created_at) FROM comment p
         WHERE p.issue_id = ?1 AND p.body LIKE 'Parked by the crew%')
       WHERE issue_id = ?1 AND body LIKE 'Done on main already%'`,
      [finished.id],
    );
    expect(alreadyDone(e.app.db, [finished.id]).get(finished.id)).toEqual([oid]);
    expect(numbers()).toEqual([open.number]);
  });
});
@@ -7,6 +7,7 @@
import { serve, type Servers } from "../src/http/server.ts";
import { gitClientEnv } from "./helpers/git-client.ts";
import { proveAll } from "./helpers/verified.ts";
import { promote } from "./helpers/members.ts";
import { tokenFor } from "./helpers/tokens.ts";
const gitEnv = gitClientEnv({
  GIT_AUTHOR_NAME: "X",
@@ -56,18 +57,7 @@
describe("cross-repo pull requests (fork → upstream)", () => {
      redirect: "manual",
    });
    const cookie = (signup.headers.get("set-cookie") ?? "").split(";")[0]!;
    const token = await fetch(`${base}/api/v1/tokens`, {
      method: "POST",
      headers: { "content-type": "application/json" },
      body: JSON.stringify({ adminConfirm: true,
        username,
        password: "cross-repo-pass",
        name: "c",
        scope: "admin",
      }),
    })
      .then((r) => r.json())
      .then((j) => (j as { token: string }).token);
    const token = tokenFor(app, username, "admin", "c");
    return { token, cookie };
  }
@@ -0,0 +1,32 @@
import { expect, test } from "bun:test";
import { forgePoll } from "../scripts/drills/run.ts";
const answers = (health: Response | Error, board: Response | Error): typeof fetch =>
  (async (url: string) => {
    const r = String(url).endsWith("/api/v1/health") ? health : board;
    if (r instanceof Error) throw r;
    return r.clone();
  }) as unknown as typeof fetch;
const html = (status: number, body: string) => new Response(body, { status, headers: { "content-type": "text/html" } });
const ok = () => Response.json({ ok: true });
test("healthy only when health reports ok and the board answers", async () => {
  expect((await forgePoll("http://f", "t", answers(ok(), html(200, "<p>hi</p>")))()).healthy).toBe(true);
  expect((await forgePoll("http://f", "t", answers(ok(), html(503, "<p>stuck</p>")))()).healthy).toBe(false);
  expect((await forgePoll("http://f", "t", answers(Response.json({ ok: false }), html(200, "hi")))()).healthy).toBe(false);
  expect((await forgePoll("http://f", "t", answers(new Response("ok", { status: 200 }), html(200, "hi")))()).healthy).toBe(false);
});
test("what each page said is kept under its own source", async () => {
  const look = await forgePoll("http://f", "t", answers(
    Response.json({ ok: false, note: "Disk is nearly full" }, { status: 503 }),
    html(503, "<p>The runner is stuck</p>"),
  ))();
  expect(look.said).toEqual({ health: ["Disk is nearly full"], board: ["The runner is stuck"] });
});
test("a page that cannot be reached says 'unreachable'", async () => {
  const look = await forgePoll("http://f", "t", answers(new Error("refused"), html(200, "hi")))();
  expect(look).toEqual({ healthy: false, said: { health: ["unreachable"], board: ["hi"] } });
});
@@ -76,10 +76,33 @@
test("goes down and comes back: posts what it saw and prints the verdict", async
  expect(body.seen.health).toContain("Disk is nearly full: 97%");
  expect(body.seen.board).toContain("The runner is stuck");
  expect(body.backSeconds).toBe(3);
  expect(Object.keys(body).sort()).toEqual(["backSeconds", "domain", "seen"]);
  expect(h.lines.filter((l) => l.includes("The runner is stuck"))).toHaveLength(1);
  expect(h.lines).toContain("GREEN: every phrase seen");
});
test("without a token: prints the result, says nothing was posted, posts nothing", async () => {
  const state: Stub = { down: false, posts: [], refuse: false };
  const lines: string[] = [];
  let t = 0;
  let sleeps = 0;
  const code = await runDrill({
    domain: "runner",
    forge: stub(state),
    token: "",
    say: (l) => lines.push(l),
    enter: async () => { state.down = true; },
    now: () => t,
    sleep: async (ms) => { t += ms; if (++sleeps === 3) state.down = false; },
    intervalMs: 1000,
    ceilingMs: 60_000,
  });
  expect(code).toBe(0);
  expect(state.posts).toHaveLength(0);
  expect(lines.some((l) => l.includes("nothing was posted"))).toBe(true);
  expect(JSON.parse(lines.at(-1)!)).toMatchObject({ domain: "runner", backSeconds: 3, posted: false });
});
test("stays down past the ceiling: posts nothing, exits 0, says so", async () => {
  const state: Stub = { down: false, posts: [], refuse: false };
  const h = harness(state, { ceilingMs: 5_000 });
@@ -119,17 +142,16 @@
test("a sign-in redirect from / is up, and a changing counter is one line", asyn
test("checkArgs: each missing piece and the cleartext refusal is an error; the good case is options", () => {
  const env = { FORGE_TOKEN: "t" };
  const bad = (argv: string[], e: Record<string, string | undefined>, tty: boolean) => {
    const r = checkArgs(argv, e, tty);
  const bad = (argv: string[], e: Record<string, string | undefined>) => {
    const r = checkArgs(argv, e);
    expect(r.ok).toBe(false);
    return r.ok ? "" : r.error;
  };
  expect(bad(["--forge", "https://f.example"], env, true)).toContain("domain");
  expect(bad(["runner"], env, true)).toContain("--forge");
  expect(bad(["runner", "--forge", "https://f.example"], { FORGE_TOKEN: "" }, true)).toContain("FORGE_TOKEN");
  expect(bad(["runner", "--forge", "https://f.example"], env, false)).toContain("not a terminal");
  expect(bad(["runner", "--forge", "http://forge.example"], env, true)).toContain("plain http");
  const ok = (argv: string[]) => checkArgs(argv, env, true);
  expect(bad(["--forge", "https://f.example"], env)).toContain("domain");
  expect(bad(["runner"], env)).toContain("--forge");
  expect(checkArgs(["runner", "--forge", "https://f.example"], {}).ok).toBe(true);
  expect(bad(["runner", "--forge", "http://forge.example"], env)).toContain("plain http");
  const ok = (argv: string[]) => checkArgs(argv, env);
  expect(ok(["runner", "--forge", "http://forge.example", "--insecure"]).ok).toBe(true);
  for (const host of ["localhost:3000", "127.0.0.1:1", "[::1]:1"]) expect(ok(["runner", "--forge", `http://${host}/`]).ok).toBe(true);
  expect(ok(["runner", "--forge", "https://f.example/"])).toEqual({
@@ -161,3 +183,46 @@
test("control characters the forge sends are removed before printing or posting;
  expect(said).not.toContain("\x1b");
  expect(JSON.stringify(state.posts[0]!.body)).not.toContain("\\u001b");
});
test("spawned with piped stdin: waits for Enter, watches, prints the JSON; no drill exits non-zero", async () => {
  const state: Stub = { down: false, posts: [], refuse: false };
  const forge = stub(state);
  const spawn = (domain: string) =>
    Bun.spawn(["bun", "scripts/drills/run.ts", domain, "--forge", forge], {
      stdin: "pipe",
      stdout: "pipe",
      stderr: "pipe",
      env: { ...process.env, FORGE_TOKEN: "tok" },
    });
  const none = spawn("state");
  expect(await none.exited).not.toBe(0);
  expect(await new Response(none.stdout).text()).toContain("has no drill");
  const p = spawn("compute");
  const out = new Response(p.stdout).text();
  await Bun.sleep(300);
  expect(state.posts).toEqual([]);
  state.down = true;
  p.stdin.write("\n");
  p.stdin.end();
  await Bun.sleep(1500);
  state.down = false;
  expect(await p.exited).toBe(0);
  const lines = (await out).trim().split("\n");
  expect(lines[0]).toContain("kill-pod");
  const result = JSON.parse(lines[lines.length - 1]!);
  expect(result).toMatchObject({ domain: "compute", posted: true });
  expect(state.posts).toHaveLength(1);
  expect(state.posts[0]!.auth).toBe("Bearer tok");
});
test("stdin closed without Enter exits non-zero and posts nothing", async () => {
  const state: Stub = { down: false, posts: [], refuse: false };
  const forge = stub(state);
  const p = Bun.spawn(["bun", "scripts/drills/run.ts", "compute", "--forge", forge], {
    stdin: "ignore", stdout: "pipe", stderr: "pipe", env: { ...process.env, FORGE_TOKEN: "tok" },
  });
  expect(await p.exited).not.toBe(0);
  expect(state.posts).toEqual([]);
});
@@ -0,0 +1,82 @@
import { expect, test } from "bun:test";
import { watchDrill } from "../scripts/drills/watch.ts";
const reply = (status: number, body: string, type = "text/plain") => ({
  status,
  text: async () => body,
  headers: { get: (n: string) => (n === "content-type" ? type : null) },
});
function script(steps: Array<ReturnType<typeof reply> | Error>) {
  let i = 0;
  const slept: unknown[] = [];
  return {
    slept,
    fetch: async () => {
      const s = steps[Math.min(i++, steps.length - 1)]!;
      if (s instanceof Error) throw s;
      return s;
    },
    sleep: async (ms: number) => { slept.push(ms); },
  };
}
test("returns distinct lines said while unhealthy, with times, without waiting", async () => {
  const s = script([
    reply(200, "ok"),
    reply(503, "database locked\nretry 1"),
    new Error("refused"),
    reply(503, "database locked\nretry 2"),
    reply(200, "ok"),
  ]);
  const r = await watchDrill({ url: "http://x/", fetch: s.fetch, sleep: s.sleep, limit: 10 });
  expect(r).toEqual({ timeout: false, startedAt: 1000, healthyAt: 4000, lines: ["database locked", "retry 1", "unreachable"] });
  expect(s.slept.length).toBe(4);
});
test("reads JSON strings and strips control characters", async () => {
  const s = script([reply(500, JSON.stringify({ error: "bad\u001b[31m" }), "application/json"), reply(200, "ok")]);
  const r = await watchDrill({ url: "http://x/", fetch: s.fetch, sleep: s.sleep, limit: 5 });
  expect(r).toMatchObject({ timeout: false, lines: ["bad[31m"] });
});
test("times out when the forge never comes back, and when it never went down", async () => {
  const down = script([reply(503, "down")]);
  expect(await watchDrill({ url: "u", fetch: down.fetch, sleep: down.sleep, limit: 3 }))
    .toEqual({ timeout: true, startedAt: 0, lines: ["down"] });
  const up = script([reply(200, "ok")]);
  expect(await watchDrill({ url: "u", fetch: up.fetch, sleep: up.sleep, limit: 3 }))
    .toEqual({ timeout: true, startedAt: null, lines: [] });
});
test("a fetch that throws says 'unreachable'; an unreadable 5xx body says nothing but is still down", async () => {
  const bad = { status: 502, text: async () => { throw new Error("reset"); }, headers: { get: () => null } };
  const s = script([new Error("refused"), bad, reply(200, "ok")]);
  const seen: string[] = [];
  const r = await watchDrill({ url: "u", fetch: s.fetch, sleep: s.sleep, limit: 5, onLine: (l) => seen.push(l) });
  expect(r).toEqual({ timeout: false, startedAt: 0, healthyAt: 2000, lines: ["unreachable"] });
  expect(seen).toEqual(["unreachable"]);
});
test("an injected clock measures the times", async () => {
  const s = script([reply(503, "down"), reply(200, "ok")]);
  let t = 500;
  const r = await watchDrill({ url: "u", fetch: s.fetch, sleep: async () => { t += 1700; }, limit: 5, now: () => t });
  expect(r).toEqual({ timeout: false, startedAt: 0, healthyAt: 1700, lines: ["down"] });
});
test("the ceiling is time, not polls: slow polls time out long before the poll limit", async () => {
  let t = 0;
  let polls = 0;
  const r = await watchDrill({
    url: "u",
    fetch: async () => { polls++; t += 4_000; return reply(503, "down"); },
    sleep: async () => { t += 4_000; },
    limit: 1000,
    intervalMs: 1000,
    ceilingMs: 20_000,
    now: () => t,
  });
  expect(r).toEqual({ timeout: true, startedAt: 4_000, lines: ["down"] });
  expect(polls).toBe(3);
});
@@ -12,7 +12,7 @@
import { createApp, type App } from "../src/app.ts";
import { createUser } from "../src/core/users.ts";
import { vitals } from "../src/kernel/vitals.ts";
import { runDrill, type DrillResult } from "../src/snapshot/drill.ts";
import { drillDue, lastSlot } from "../src/snapshot/due.ts";
import { BOOT_RED_FLOOR_MS, drillDue, lastSlot, RED_RETRY_MS } from "../src/snapshot/due.ts";
import { lastDrill, recordDrill } from "../src/snapshot/drills.ts";
import { startSnapshots } from "../src/snapshot/service.ts";
import { takeSnapshot } from "../src/snapshot/snapshot.ts";
@@ -20,6 +20,8 @@
import { dirStore } from "../src/snapshot/store.ts";
import { recordVerdict } from "../src/snapshot/verdict.ts";
const t = (iso: string): number => Date.parse(iso);
const green = (at: string) => ({ at, verdict: "green" as const });
const red = (at: string) => ({ at, verdict: "red" as const });
describe("when a weekly drill is owed", () => {
  // 2026-10-11 is a Sunday.
@@ -30,10 +32,23 @@
describe("when a weekly drill is owed", () => {
  test("owed once: a drill since the slot keeps it, an older one does not", () => {
    const now = t("2026-10-11T09:30:00Z");
    expect(drillDue("2026-10-04T09:05:00.000Z", undefined, now)).toBe(true);
    expect(drillDue("2026-10-11T09:05:00.000Z", undefined, now)).toBe(false);
    expect(drillDue(green("2026-10-04T09:05:00.000Z"), undefined, now)).toBe(true);
    expect(drillDue(green("2026-10-11T09:05:00.000Z"), undefined, now)).toBe(false);
    // a restart later the same day finds it kept
    expect(drillDue("2026-10-11T09:05:00.000Z", undefined, t("2026-10-11T20:00:00Z"))).toBe(false);
    expect(drillDue(green("2026-10-11T09:05:00.000Z"), undefined, t("2026-10-11T20:00:00Z"))).toBe(false);
  });
  test("a red drill is owed again after RED_RETRY_MS, and on a boot past the floor", () => {
    // #2541: red at 09:11, the fix deployed at 10:18; the card stood for the week.
    const at = "2026-10-18T09:11:45.853Z";
    expect(drillDue(red(at), undefined, t(at) + 60_000)).toBe(false);
    expect(drillDue(red(at), undefined, t(at) + RED_RETRY_MS - 1)).toBe(false);
    expect(drillDue(red(at), undefined, t(at) + RED_RETRY_MS)).toBe(true);
    expect(drillDue(red(at), undefined, t("2026-10-18T10:23:00Z"), BOOT_RED_FLOOR_MS)).toBe(true);
    // a boot soon after the red one (a crash loop) does not restore again
    expect(drillDue(red(at), undefined, t(at) + BOOT_RED_FLOOR_MS - 1, BOOT_RED_FLOOR_MS)).toBe(false);
    // a green one keeps its slot on a boot too
    expect(drillDue(green(at), undefined, t("2026-10-18T10:23:00Z"), 0)).toBe(false);
  });
  test("a forge with no drill owes one only if it is older than the slot", () => {
@@ -80,6 +95,152 @@
describe("the service runs a due drill once", () => {
      await rm(root, { recursive: true, force: true });
    }
  });
  test("a red drill that has stood RED_RETRY_MS is run again until one passes", async () => {
    const root = mkdtempSync(join(tmpdir(), "forge-drill-red-"));
    const app = createApp({ dataDir: root, port: 0 });
    const verdicts: DrillResult["verdict"][] = [];
    const svc = startSnapshots(app, dirStore(join(root, "store")), {
      version: "t",
      nightly: false,
      drill: async (): Promise<DrillResult> => {
        const verdict = verdicts.length === 0 ? "red" : "green";
        verdicts.push(verdict);
        return {
          // The red one ran long enough ago to be owed again.
          at: new Date(Date.now() - (verdict === "red" ? RED_RETRY_MS : 0)).toISOString(),
          snapshotId: "s",
          ms: 1,
          counts: { repositories: 0, issues: 0, users: 0, pulls: 0, apps: 0 },
          health: verdict === "red" ? "unreached" : "ok",
          verdict,
          failedStep: verdict === "red" ? "restore" : null,
          reason: verdict === "red" ? "restore timed out: no answer in 600000 ms" : null,
        };
      },
    });
    try {
      mkdirSync(join(root, "drill"), { recursive: true });
      writeFileSync(join(root, "drill", "first-seen"), new Date(Date.now() - 30 * 86_400_000).toISOString());
      expect(await svc.drillIfDue()).toBe(true);
      expect(lastDrill(app.db)?.verdict).toBe("red");
      expect(await svc.drillIfDue()).toBe(true);
      expect(lastDrill(app.db)?.verdict).toBe("green");
      expect(await svc.drillIfDue()).toBe(false);
      expect(verdicts).toEqual(["red", "green"]);
    } finally {
      await svc.stop();
      app.close();
      await rm(root, { recursive: true, force: true });
    }
  });
});
describe("the service's own clock drills a red one again", () => {
  const result = (verdict: DrillResult["verdict"]): DrillResult => ({
    at: new Date().toISOString(),
    snapshotId: "s",
    ms: 1,
    counts: { repositories: 0, issues: 0, users: 0, pulls: 0, apps: 0 },
    health: verdict === "red" ? "unreached" : "ok",
    verdict,
    failedStep: verdict === "red" ? "restore" : null,
    reason: verdict === "red" ? "restore timed out: no answer in 600000 ms" : null,
  });
  /**
   * A forge booted with nightly on and short delays, its newest drill
   * `before` (kept before boot), each drill it runs answering the next of
   * `answers`; and the time it was booted, for the waits.
   */
  async function boot(
    before: DrillResult["verdict"] | undefined,
    answers: DrillResult["verdict"][],
    opts: { bootRedFloorMs?: number } = {},
  ) {
    const root = mkdtempSync(join(tmpdir(), "forge-drill-clock-"));
    mkdirSync(join(root, "drill"), { recursive: true });
    writeFileSync(join(root, "drill", "first-seen"), new Date(Date.now() - 30 * 86_400_000).toISOString());
    const app = createApp({ dataDir: root, port: 0 });
    if (before !== undefined) recordDrill(app.db, result(before));
    const ran: DrillResult["verdict"][] = [];
    const svc = startSnapshots(app, dirStore(join(root, "store")), {
      version: "t",
      mirrorEveryMs: 3_600_000,
      catchUpMs: 20,
      redRetryMs: 60,
      bootRedFloorMs: opts.bootRedFloorMs ?? 0,
      drill: async () => {
        const verdict = answers[ran.length] ?? "green";
        ran.push(verdict);
        return result(verdict);
      },
    });
    const done = async () => {
      await svc.stop();
      app.close();
      await rm(root, { recursive: true, force: true });
    };
    return { app, svc, ran, done };
  }
  test("the boot catch-up drills a red one at once", async () => {
    const f = await boot("red", ["green"]);
    try {
      await Bun.sleep(150);
      expect(f.ran).toEqual(["green"]);
      expect(lastDrill(f.app.db)?.verdict).toBe("green");
    } finally {
      await f.done();
    }
  });
  test("the boot catch-up leaves a green one of this week alone", async () => {
    const f = await boot("green", []);
    try {
      await Bun.sleep(150);
      expect(f.ran).toEqual([]);
    } finally {
      await f.done();
    }
  });
  test("a boot inside the floor does not restore again, and the retry still comes", async () => {
    const f = await boot("red", ["green"], { bootRedFloorMs: 3_600_000 });
    try {
      await Bun.sleep(40);
      expect(f.ran).toEqual([]);
      await Bun.sleep(150);
      expect(f.ran).toEqual(["green"]);
    } finally {
      await f.done();
    }
  });
  test("a red result arms the retry, which drills again and stops after a pass", async () => {
    const f = await boot(undefined, ["red", "red", "green"]);
    try {
      await Bun.sleep(400);
      expect(f.ran).toEqual(["red", "red", "green"]);
      await Bun.sleep(200);
      expect(f.ran).toEqual(["red", "red", "green"]);
    } finally {
      await f.done();
    }
  });
  test("stop clears the retry", async () => {
    const f = await boot(undefined, ["red"]);
    try {
      await Bun.sleep(40);
      expect(f.ran).toEqual(["red"]);
      await f.svc.stop();
      await Bun.sleep(200);
      expect(f.ran).toEqual(["red"]);
    } finally {
      await f.done();
    }
  });
});
describe("a count that differs is red", () => {
@@ -32,6 +32,7 @@
import { createUser, getUserByName } from "../src/core/users.ts";
import { serve, type Servers } from "../src/http/server.ts";
import { quietLogs } from "./helpers/globals.ts";
import { passkeyFields } from "./helpers/passkey.ts";
import { ADMIN_TOKEN_WORDS } from "../src/core/tokens.ts";
quietLogs();
@@ -241,6 +242,26 @@
describe("the everyday account", () => {
    );
  });
  test("the token page keeps a member on the page with the sentence for admin scope, and makes rw", async () => {
    const kim = await signIn("kim");
    const count = () =>
      (
        app.db
          .query("SELECT COUNT(*) AS n FROM token WHERE user_id = ?1")
          .get(getUserByName(app.db, "kim")!.id) as { n: number }
      ).n;
    const before = count();
    expect(await page(kim, "/settings/tokens")).not.toContain('value="admin"');
    const denied = await post(kim, "/settings/tokens", { name: "big", scope: "admin" });
    expect(denied.status).toBe(303);
    expect(new URL(denied.headers.get("location")!, servers.url).pathname).toBe("/settings/tokens");
    const flash = (denied.headers.get("set-cookie") ?? "").split(";")[0]!;
    expect(decodeURIComponent(flash.replace(/\+/g, " "))).toContain(ADMIN_TOKEN_WORDS);
    expect(count()).toBe(before);
    await post(kim, "/settings/tokens", { name: "small", scope: "rw" });
    expect(count()).toBe(before + 1);
  });
  test("from a page's session, the API refuses an admin's token without adminConfirm", async () => {
    const root = await signIn("ada");
    const mint = (body: Record<string, unknown>) =>
@@ -0,0 +1,6 @@
// Fixture for test/shards.test.ts, kept out of the gate: fails alone too.
import { expect, test } from "bun:test";
test("broken", () => {
  expect(1).toBe(2);
});
@@ -0,0 +1,2 @@
// Fixture for test/shards.test.ts, kept out of the gate: errors while loading.
throw new Error("boom at load");
		@@ -0,0 +1,38 @@
	1	name: outside-gate
	2	# Nightly: every file in EXCLUDED (scripts/gate.ts), one at a time with the
	3	# gate's 60 s timeout, so a suite outside the gate that breaks is seen the
	4	# day it breaks. A newly red file opens one `Red: test/….test.ts` issue
	5	# naming its first failing test; a file green again closes it
	6	# (scripts/flake-hunt.ts outside). The list is EXCLUDED itself: a file moved
	7	# into the gate stops being run here.
	8	#
	9	# Needs secret FORGE_TOKEN and variable FORGE_URL, as flake-hunt does.
	10	on:
	11	schedule:
	12	- cron: "47 3 * * *"
	13	workflow_dispatch: {}
	14
	15	jobs:
	16	outside:
	17	runs-on: isolated
	18	timeout-minutes: 30
	19	env:
	20	REGISTRY: forgejo-http.forgejo.svc.cluster.local:3000
	21	steps:
	22	- name: clone main
	23	env:
	24	TOKEN: ${{ github.token }}
	25	run: \|
	26	set -eu
	27	git clone -q "http://x-access-token:$TOKEN@$REGISTRY/$GITHUB_REPOSITORY.git" src
	28	cd src
	29	bun install --frozen-lockfile
	30
	31	- name: run the excluded files
	32	working-directory: src
	33	env:
	34	FORGE_API: ${{ vars.FORGE_URL }}/api/v1
	35	FORGE_REPO: ${{ github.repository }}
	36	FORGE_TOKEN: ${{ secrets.FORGE_TOKEN }}
	37	FORGE_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_number }}
	38	run: bun scripts/flake-hunt.ts outside --file-issues
		@@ -254,9 +254,12 @@ flag exists only in the stdio dev runner and is never reachable over SSH.
254	254	whose default export is the SQL — never the next number: `0001`–`0091`
255	255	are the numbered history, and a shared next number is what parallel pulls
256	256	collided on (#1837). Files run in name order, once each, recognised by a
257		hash of their text in `schema_migration` (`kernel/migrate.ts`), so two
258		pulls' migrations land in either order. Never edit an existing migration
259		— an edited one is a new one and runs again. It must only add
	257	hash of their text in `schema_migration` (`kernel/migrate.ts`, which also
	258	records each name), so two pulls' migrations land in either order, and a
	259	database with one this build does not know boots and names it. Never edit
	260	an existing migration — an edited one is a new one and runs again; once
	261	pinned in `core/migration-pins.json` (`bun scripts/pin-migrations.ts`, run
	262	by the release agent, never by a pull) the schema test fails on an edit. It must only add
260	263	(`test/schema-additive.test.ts`); the release agent lists its id in
261	264	`core/releases.json`.
262	265	- New git read: put it in `git/read.ts`; reuse the long-lived `cat-file`
		@@ -1114,7 +1114,7 @@ a link to the milestone (`docs/PLANNING.md`); its progress never does.
1114	1114	\| `FORGE_APP_DOMAIN` \| the host in `FORGE_URL`/`_HOST` \| what app hostnames attach to (`example` or `--x`) \|
1115	1115	\| `FORGE_APPS_CONCURRENCY` \| `2` \| deploys running their heavy steps at once \|
1116	1116	\| `FORGE_PREVIEW_IDLE_DAYS` \| `3` \| days a pull may sit untouched before its preview sleeps; `0` = never \|
1117		\| `FORGE_CREW_CONCURRENCY` \| `2` \| agent runs at once (live: Settings → Capacity) \|
	1117	\| `FORGE_CREW_CONCURRENCY` \| 1 per 3 GiB above 2 GiB, 2–16 \| agent runs at once (live: Settings → Capacity) \|
1118	1118	\| `FORGE_CHECKS_CONCURRENCY` \| `1` \| checks at once (live: Settings → Capacity) \|
1119	1119	\| `FORGE_CHECKS_PLACEMENT` \| `process` \| `job` runs a check's commands as a cluster Job (see Placement above) \|
1120	1120	\| `FORGE_CHECKS_DRAIN_S` \| `300` \| seconds a roll lets running checks finish first \|
		@@ -176,8 +176,9 @@ NOT NULL column without a default, and `test/schema-additive.test.ts`
176	176	refuses one in any migration not shipped by both of the two newest releases
177	177	recorded in `src/core/releases.json` (by id: a count, and the stamped ids),
178	178	which the release agent bumps when it cuts one.
179		An old process that restarts mid-roll refuses a database ahead of it
180		(`user_version`), as it always has.
	179	An old process that restarts mid-roll boots on a database ahead of it and
	180	logs, by name, the migrations it does not know (`schema_migration` records
	181	each name, #783) — they only add, so it serves as it did.
181	182
182	183	State transitions use compare-and-swap (`UPDATE … WHERE state = ?`), and every
183	184	transition writes its event row in the same transaction. On boot, a
		@@ -35,13 +35,14 @@ two devices backed by the same physical disk, are not.
35	35	## Running a drill
36	36
37	37	`scripts/drills/run.ts` walks you through one. It refuses a domain whose drill
38		is none, prints the fault to cause, and waits for Enter. It then polls
	38	is none, prints the fault to cause, and waits for Enter (a line on stdin, so piping one works; closing stdin aborts). It then polls
39	39	`/api/v1/health` and `/` once a second, keeping each distinct line the forge
40	40	says while unhealthy; when the forge is healthy again it posts what it saw and
41	41	the seconds it took to `POST /api/v1/drills` and prints the verdict. If the
42	42	forge is still down after an hour it posts nothing, says so, and exits 0. A
43	43	refused post exits non-zero with the forge's message. `FORGE_TOKEN` must be a
44		site admin's token.
	44	site admin's token; without it the script still watches, prints the result as
	45	JSON and says that nothing was posted.
45	46
46	47	```
47	48	FORGE_TOKEN=… bun scripts/drills/run.ts runner --forge https://forge.open-platform.sh
		@@ -28,6 +28,7 @@ for a tunnel and a forge that sleeps.
28	28	\| tool \| does \| writes \|
29	29	\| -------------- \| ------------------------------------------------------------------------------------------------- \| ------ \|
30	30	\| `apps` \| the apps you can see: live or not, where served, how each runs itself, what is working \| \|
	31	\| `repo` \| `{ repo }`: one repository's description, visibility, autonomy and `audience { level, sentence }`, the same as `GET /api/v1/repos/:owner/:name` \| \|
31	32	\| `tell` \| a sentence becomes an issue, a builder is picked, a run starts on the crew's next tick \| yes \|
32	33	\| `say` \| speak on an issue or a pull; a working builder hears it as its next step \| yes \|
33	34	\| `pull` \| `{ repo, number }`: a pull's title, state, head and base, and for an open pull whether it merges into the base's current head (`mergeable`) and the paths that stop it (`conflicts`), and `check: { state, baseOid }`: the latest check's state and the base tip it was merged with (empty when it merged nothing); `mergeable` and `conflicts` are the same fields as `GET …/pulls/:number`, `check` is `state` and `run.baseOid` from `GET …/pulls/:number/checks` \| \|
		@@ -39,7 +40,7 @@ for a tunnel and a forge that sleeps.
39	40	\| `audit` \| who can do what, per app, read from its contract.json in git: yes, own or no per door, and whether outsiders may direct its crew (`crewOutsiders`) \| \|
40	41	\| `outsiders` \| `{ repo, on?, dryRun? }`: whether people who cannot write may direct the crew; with `on`, planned first (`dryRun` is true unless `false`); set only by the app's admin with an admin token, never an agent \| yes \|
41	42	\| `decisions_by` \| `{ repo, by?, dryRun? }`: the standing rule for a repository's hand-overs (`me`, `decider`, `decider_tell`); with `by`, planned first (`dryRun` is true unless `false`); set only by the app's admin with an admin token, never an agent \| yes \|
42		\| `archive` \| `{ repo, archived?, dryRun? }`: archive (true) or wake (false) a repository, by name; planned first (`dryRun` is true unless `false`); only the repository's admin with an admin token, never an agent \| yes \|
	43	\| `archive` \| `{ repo \| repos, archived?, dryRun? }`: archive (true) or wake (false) a repository, by name; planned first (`dryRun` is true unless `false`); only the repository's admin with an admin token, never an agent. `repos` (at most 50) runs the fleet `archive` operation over them, answers each one's outcome and the `operation` id; `limb` with `action: "undo"`, that `operation` and any one of the repositories as `app` reverts it for a day \| yes \|
43	44	\| `probe` \| knock on a running app's doors as nobody, a user and an admin; the ones that disobey its contract \| \|
44	45	\| `following` \| where a fork stands against what it follows, and what the crew is doing about it \| \|
45	46	\| `spawn` \| `{ name, apps?, crew?, dryRun? }`: a child forge, planned first (`dryRun` is true unless `false`); an admin's, and an `ro` token may plan but not make \| yes \|
		@@ -16,8 +16,8 @@ Sundays 09:00 UTC the forge drills a restore of the newest snapshot, boots it
16	16	on a side port and reads `/api/v1/health`. One comment lands on the recovery
17	17	issue (`FORGE_RECOVERY_ISSUE` in `FORGE_RECOVERY_REPO`): snapshot, restore
18	18	time, health, counts, pass or fail. A failed drill is an inbox card naming
19		the step (fetch, restore, open, count, unseal, health) until one passes. No
20		comment this week means the backups are unproven.
	19	the step (fetch, restore, open, count, unseal, health), run again at boot and
	20	every three hours until one passes. No comment this week: backups unproven.
21	21
22	22	## What a restore needs
23	23
		@@ -8,7 +8,23 @@ empty data directory. `src/snapshot/`.
8	8	## What a snapshot holds
9	9
10	10	- `forge.db`, copied consistently while the forge runs (`VACUUM INTO`,
11		never a raw copy of a file in WAL mode).
	11	never a raw copy of a file in WAL mode), then zstd-compressed and cut in
	12	16 MB parts: `forge.db.zst.0000`, `forge.db.zst.0001`, … Each part is its
	13	own put with its own deadline and retries, so a slow uplink carries it
	14	and a dropped connection costs one part, not the database. The manifest's
	15	`packed` says what the parts make and how big that is; a restore joins
	16	them, unpacks, and refuses a size that does not match. The log line
	17	`database packed` gives each night's ratio. Snapshots from before this
	18	hold a plain `forge.db` and still restore. The parts go up through the
	19	snapshot's upload queue like every other object: at most eight objects
	20	or 64 MB in flight, so four 16 MB parts at a time.
	21
	22	Packing and restoring both hold the database in memory: the copy, its
	23	compressed form (the parts are views of it, not copies), and on restore
	24	the joined parts and the unpacked file, about three times the database
	25	at the peak (a 334 MB database at about 1 GB). A database that outgrows
	26	the app's memory limit needs a streaming pack; until then that limit,
	27	not the uplink, is the ceiling.
12	28	- The key the database's sealed values need, only wrapped:
13	29	`secret.key.wrapped`, AES-256-GCM under a key scrypt-derived from the
14	30	operator's passphrase. The raw `secret.key` is never written to the store,
		@@ -24,8 +40,8 @@ empty data directory. `src/snapshot/`.
24	40	stay out.
25	41	- The crew's run logs (`runs/`).
26	42	- `manifest.json`: the name, when, the forge's version, counts, every file
27		with its size, and `key`: `wrapped` or `none` (older snapshots: `raw`). A
28		restore checks each size.
	43	with its size, `packed` (the files that went up in parts), and `key`:
	44	`wrapped` or `none` (older snapshots: `raw`). A restore checks each size.
29	45
30	46	What stays out is what a boot or a deploy rebuilds: clones, checks,
31	47	pictures, previews.
		@@ -86,7 +102,24 @@ mirror is copied from the primary store, not written in parallel, so a snapshot
86	102	that never landed on the primary is not seen by it.
87	103
88	104	The store is `Bun.S3Client` or a directory behind one small interface
89		(`src/snapshot/store.ts`): put, get, list, delete. Nothing else.
	105	(`src/snapshot/store.ts`): put, get, list, delete. Nothing else. Every call
	106	is tried five times; an attempt that has not answered by its deadline counts
	107	as failed. A put's deadline follows its size: its bytes at a floor of
	108	5 Mbit/s, plus 60 s. A fixed 180 s once failed every attempt at a 334 MB
	109	database behind a 15 Mbit/s uplink (#2536).
	110
	111	## When a snapshot fails
	112
	113	Its outcome is kept in `snapshot/state.json` under the data directory, so a
	114	restart forgets nothing. A failure shows straight away as a card on the board
	115	(`snapshot-failed`, pointing at Backups), in the terminal's dashboard among
	116	the forge's needs, and as a red line on the Backups page with the reason and
	117	since when. The forge tries again within the hour, not the next night. The
	118	next snapshot that lands, nightly or on demand, clears both.
	119
	120	A forge given nowhere to put a snapshot has not failed: nothing is kept, no
	121	card is raised, and nothing is retried sooner than the nightly. The Backups
	122	page says there is no destination.
90	123
91	124	## The restore drill
92	125
		@@ -96,12 +129,26 @@ scratch directory under the data directory, opens the database read-only,
96	129	counts repositories and issues, opens one sealed value with the restored key,
97	130	and removes the scratch whatever happened. The page waits until it finishes,
98	131	for up to 10 minutes; a step that hangs past that is red and names the step.
	132	The restore reads several objects at a time, under the same limits as the
	133	upload (`IN_FLIGHT` gets, `IN_FLIGHT_BYTES` held): read one after another, a
	134	snapshot of thousands of objects spent the drill's whole 10 minutes in the
	135	restore step (#2539).
	136	Repositories come back four clones at a time, not one after another (#2543).
99	137	One drill runs at a time: a second click, or a retry after a proxy timeout,
100	138	joins the drill already running rather than starting another restore. Every
101	139	run is a row in the `drill` table, and the last one shows as a line on the
102	140	Backups page: when, which snapshot, how long, the counts, and green or the
103	141	step that failed.
104	142
	143	The drill runs on its own every Sunday 09:00 UTC, and a forge that was down
	144	then catches up five minutes after boot. A red drill does not keep the
	145	week's slot: it is owed again `RED_RETRY_MS` (three hours) after it ran,
	146	and at a boot once it is `BOOT_RED_FLOOR_MS` (30 minutes) old, until one
	147	passes (`src/snapshot/due.ts`). The floor keeps a crash loop with a broken
	148	restore from running a restore on every boot. Kept for the
	149	week, a drill red at 09:11 stood as a card long after the fix deployed at
	150	10:18 (#2541).
	151
105	152	## Restore
106	153
107	154	The printable card, seven lines, is [RESTORE.md](RESTORE.md); a test runs it.
		@@ -4,11 +4,13 @@
4	4	* polls `/api/v1/health` and `/` once a second keeping each distinct line the
5	5	* forge says while unhealthy, and when the forge is healthy again posts what
6	6	* it saw and how long it took to `POST /api/v1/drills` (`FORGE_TOKEN` from
7		* the environment). If the forge never comes back within the ceiling it posts
8		* nothing. Zero dependencies; it never shells out.
	7	* the environment). Without a token it prints the result and says nothing was
	8	* posted. If the forge never comes back within the ceiling it posts nothing.
	9	* Zero dependencies; it never shells out.
9	10	*/
10	11
11	12	import { DOMAINS } from "../../src/kernel/domains.ts";
	13	import { UNREACHABLE, clean, linesOf, watchPolls, type Poll } from "./watch.ts";
12	14
13	15	export interface DrillRun {
14	16	domain: string;
		@@ -27,50 +29,17 @@ export interface DrillRun {
27	29	const SEEN_MAX = 8_000;
28	30	const DEFAULT_CEILING_MS = 3_600_000;
29	31
30		/** C0 and C1 control characters out (tab stays): the forge's words never drive the terminal. */
31		export const clean = (s: string): string => s.replace(/[\x00-\x08\x0a-\x1f\x7f-\x9f]/g, "");
	32	export { clean };
32	33
33		/** Every string in a JSON value, or the text's own lines when it is not JSON. */
34		function linesOf(text: string, contentType: string): string[] {
35		return rawLinesOf(text, contentType).map(clean).filter((l) => l.trim() !== "");
36		}
37
38		function rawLinesOf(text: string, contentType: string): string[] {
39		if (contentType.includes("json")) {
40		try {
41		const out: string[] = [];
42		const walk = (v: unknown): void => {
43		if (typeof v === "string") out.push(v);
44		else if (Array.isArray(v)) v.forEach(walk);
45		else if (v && typeof v === "object") Object.values(v).forEach(walk);
46		};
47		walk(JSON.parse(text));
48		return out;
49		} catch {
50		/* fall through to plain lines */
51		}
52		}
53		return text
54		.replace(/<(script\|style)[\s\S]*?<\/\1>/gi, " ")
55		.replace(/<[^>]*>/g, "\n")
56		.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">")
57		.replace(/"/g, '"').replace(/&#0?39;/g, "'")
58		.split("\n")
59		.map((l) => l.trim())
60		.filter((l) => l !== "");
61		}
62
63		interface Look {
64		healthy: boolean;
65		health: string[];
66		board: string[];
67		}
68
69		async function look(forge: string, token: string): Promise<Look> {
	34	/**
	35	* One look at the forge: healthy only when `/api/v1/health` reports ok and `/`
	36	* answers; what each said is kept under `health` and `board`.
	37	*/
	38	export function forgePoll(forge: string, token: string, fetchIt: typeof fetch = fetch): () => Promise<Poll> {
70	39	const get = async (path: string): Promise<{ ok: boolean; lines: string[]; json: unknown }> => {
71	40	try {
72		const res = await fetch(`${forge}${path}`, {
73		headers: { authorization: `Bearer ${token}` },
	41	const res = await fetchIt(`${forge}${path}`, {
	42	headers: token === "" ? {} : { authorization: `Bearer ${token}` },
74	43	redirect: "manual",
75	44	signal: AbortSignal.timeout(5_000),
76	45	});
		@@ -82,17 +51,16 @@ async function look(forge: string, token: string): Promise<Look> {
82	51	}
83	52	return { ok: res.status < 400, lines: linesOf(text, type), json };
84	53	} catch {
85		return { ok: false, lines: [], json: null };
	54	return { ok: false, lines: [UNREACHABLE], json: null };
86	55	}
87	56	};
88		const [h, b] = await Promise.all([get("/api/v1/health"), get("/")]);
89		const reported = (h.json as { ok?: unknown } \| null)?.ok === true;
90		return { healthy: h.ok && b.ok && reported, health: h.lines, board: b.lines };
	57	return async () => {
	58	const [h, b] = await Promise.all([get("/api/v1/health"), get("/")]);
	59	const reported = (h.json as { ok?: unknown } \| null)?.ok === true;
	60	return { healthy: h.ok && b.ok && reported, said: { health: h.lines, board: b.lines } };
	61	};
91	62	}
92	63
93		/** A line with its numbers blanked, so a clock or a counter is not a new line. */
94		const shape = (l: string): string => l.replace(/\d+/g, "#");
95
96	64	const cap = (lines: Iterable<string>): string => [...lines].join("\n").slice(0, SEEN_MAX);
97	65
98	66	/** Runs the drill; resolves to the process exit code. */
		@@ -116,29 +84,29 @@ export async function runDrill(run: DrillRun): Promise<number> {
116	84	run.say(`The forge should be back within ${domain.backMinutes} minutes. Press Enter when you have caused it.`);
117	85	await run.enter();
118	86
119		const started = now();
120		const health = new Map<string, string>();
121		const board = new Map<string, string>();
122		let sawDown = false;
123		for (;;) {
124		const seen = await look(run.forge, run.token);
125		if (seen.healthy && sawDown) break;
126		if (!seen.healthy) {
127		sawDown = true;
128		for (const l of seen.health) if (!health.has(shape(l))) { health.set(shape(l), l); run.say(`health: ${l}`); }
129		for (const l of seen.board) if (!board.has(shape(l))) { board.set(shape(l), l); run.say(`board: ${l}`); }
130		}
131		if (now() - started >= ceilingMs) {
132		const ceilingS = Math.round(ceilingMs / 1000);
133		run.say(`The forge did not come back within the ceiling of ${ceilingS}s; nothing was posted.`);
134		run.say(JSON.stringify({ domain: domain.key, backSeconds: null, posted: false, ceilingSeconds: ceilingS }));
135		return 0;
136		}
137		await sleep(intervalMs);
	87	const watched = await watchPolls({
	88	poll: forgePoll(run.forge, run.token),
	89	sleep,
	90	limit: Infinity,
	91	ceilingMs,
	92	intervalMs,
	93	now,
	94	onLine: (l, source) => run.say(`${source}: ${l}`),
	95	});
	96	if (watched.timeout) {
	97	const ceilingS = Math.round(ceilingMs / 1000);
	98	run.say(`The forge did not come back within the ceiling of ${ceilingS}s; nothing was posted.`);
	99	run.say(JSON.stringify({ domain: domain.key, backSeconds: null, posted: false, ceilingSeconds: ceilingS }));
	100	return 0;
	101	}
	102	const { health = [], board = [] } = watched.said;
	103	const backSeconds = Math.max(0, Math.round(watched.healthyAt / 1000));
	104	const body = { domain: domain.key, seen: { health: cap(health), board: cap(board) }, backSeconds };
	105	if (run.token === "") {
	106	run.say("FORGE_TOKEN is not set; nothing was posted. The result:");
	107	run.say(JSON.stringify({ domain: domain.key, backSeconds, posted: false, body }));
	108	return 0;
138	109	}
139
140		const backSeconds = Math.max(0, Math.round((now() - started) / 1000));
141		const body = { domain: domain.key, seen: { health: cap(health.values()), board: cap(board.values()) }, backSeconds };
142	110	let res: Response;
143	111	try {
144	112	res = await fetch(`${run.forge}/api/v1/drills`, {
		@@ -172,14 +140,13 @@ export interface DrillArgs {
172	140	token: string;
173	141	}
174	142
175		const USAGE = "usage: FORGE_TOKEN=… bun scripts/drills/run.ts <domain> --forge <url> [--insecure]";
	143	const USAGE = "usage: [FORGE_TOKEN=…] bun scripts/drills/run.ts <domain> --forge <url> [--insecure]";
176	144	const LOOPBACK = new Set(["localhost", "127.0.0.1", "[::1]", "::1"]);
177	145
178	146	/** The options, or the one plain line to print before exiting 2. */
179	147	export function checkArgs(
180	148	argv: string[],
181	149	env: Record<string, string \| undefined>,
182		isTTY: boolean,
183	150	): { ok: true; args: DrillArgs } \| { ok: false; error: string } {
184	151	const at = argv.indexOf("--forge");
185	152	const forge = at >= 0 ? argv[at + 1] : undefined;
		@@ -193,16 +160,14 @@ export function checkArgs(
193	160	return { ok: false, error: `--forge must be an http(s) URL; ${USAGE}` };
194	161	}
195	162	const token = env.FORGE_TOKEN ?? "";
196		if (token === "") return { ok: false, error: `FORGE_TOKEN is empty; ${USAGE}` };
197		if (!isTTY) return { ok: false, error: "stdin is not a terminal; run the drill from an interactive shell" };
198		if (url.protocol === "http:" && !LOOPBACK.has(url.hostname) && !insecure) {
	163	if (token !== "" && url.protocol === "http:" && !LOOPBACK.has(url.hostname) && !insecure) {
199	164	return { ok: false, error: `refusing to send the admin token over plain http to ${url.hostname}; use https, or pass --insecure` };
200	165	}
201	166	return { ok: true, args: { domain, forge: forge.replace(/\/+$/, ""), token } };
202	167	}
203	168
204	169	if (import.meta.main) {
205		const checked = checkArgs(process.argv.slice(2), process.env, process.stdin.isTTY === true);
	170	const checked = checkArgs(process.argv.slice(2), process.env);
206	171	if (!checked.ok) {
207	172	console.error(checked.error);
208	173	process.exit(2);
		@@ -211,7 +176,8 @@ if (import.meta.main) {
211	176	...checked.args,
212	177	say: (l) => console.log(clean(l)),
213	178	enter: async () => {
214		for await (const _ of console) break;
	179	for await (const _ of console) return;
	180	throw new Error("stdin closed before Enter; nothing was caused or watched");
215	181	},
216	182	});
217	183	process.exit(code);
		@@ -0,0 +1,140 @@
	1	/**
	2	* `watchDrill`: polls one URL until the forge is healthy again after having
	3	* been unhealthy, and returns each distinct line it said meanwhile. Pure: the
	4	* fetch and the sleep come in as arguments, nothing is read, posted or global.
	5	* Times are milliseconds: the clock you pass, or else polls made times the
	6	* interval. `watchPolls` is the loop for any poll that says healthy or not and
	7	* what each of its sources said; `watchDrill` is it over one URL.
	8	*/
	9
	10	/** C0 and C1 control characters out (tab stays): the forge's words never drive the terminal. */
	11	export const clean = (s: string): string => s.replace(/[\x00-\x08\x0a-\x1f\x7f-\x9f]/g, "");
	12
	13	/** Every string in a JSON value, or the text's own lines when it is not JSON. */
	14	export function linesOf(text: string, contentType: string): string[] {
	15	return rawLinesOf(text, contentType).map(clean).filter((l) => l.trim() !== "");
	16	}
	17
	18	function rawLinesOf(text: string, contentType: string): string[] {
	19	if (contentType.includes("json")) {
	20	try {
	21	const out: string[] = [];
	22	const walk = (v: unknown): void => {
	23	if (typeof v === "string") out.push(v);
	24	else if (Array.isArray(v)) v.forEach(walk);
	25	else if (v && typeof v === "object") Object.values(v).forEach(walk);
	26	};
	27	walk(JSON.parse(text));
	28	return out;
	29	} catch {
	30	/* fall through to plain lines */
	31	}
	32	}
	33	return text
	34	.replace(/<(script\|style)[\s\S]*?<\/\1>/gi, " ")
	35	.replace(/<[^>]*>/g, "\n")
	36	.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">")
	37	.replace(/"/g, '"').replace(/&#0?39;/g, "'")
	38	.split("\n")
	39	.map((l) => l.trim())
	40	.filter((l) => l !== "");
	41	}
	42
	43	/** A line with its numbers blanked, so a clock or a counter is not a new line. */
	44	const shape = (l: string): string => l.replace(/\d+/g, "#");
	45
	46	/** The part of a Response that a watch reads. */
	47	export interface Reply {
	48	status: number;
	49	text: () => Promise<string>;
	50	headers: { get: (name: string) => string \| null };
	51	}
	52
	53	/** What a fetch that threw is recorded as saying. */
	54	export const UNREACHABLE = "unreachable";
	55
	56	/** One look at the forge: is it healthy, and what each source said (named by source). */
	57	export interface Poll {
	58	healthy: boolean;
	59	said: Record<string, string[]>;
	60	}
	61
	62	export interface Watch {
	63	poll: () => Promise<Poll>;
	64	/** Waits one interval. */
	65	sleep: (ms: number) => Promise<unknown>;
	66	/** The most polls to make before giving up. */
	67	limit: number;
	68	intervalMs?: number;
	69	/** Milliseconds on any clock; without it, time is the polls made times `intervalMs`. */
	70	now?: () => number;
	71	/** Gives up once this many milliseconds have passed, however few polls that took. */
	72	ceilingMs?: number;
	73	/** Called once with each new distinct line and its source, as it is first said. */
	74	onLine?: (line: string, source: string) => void;
	75	}
	76
	77	export type Watched =
	78	\| { timeout: false; startedAt: number; healthyAt: number; said: Record<string, string[]> }
	79	\| { timeout: true; startedAt: number \| null; said: Record<string, string[]> };
	80
	81	/** Polls until healthy again after having been unhealthy; keeps each distinct line said while unhealthy. */
	82	export async function watchPolls(w: Watch): Promise<Watched> {
	83	const intervalMs = w.intervalMs ?? 1_000;
	84	const seen = new Map<string, Map<string, string>>();
	85	const said = (): Record<string, string[]> =>
	86	Object.fromEntries([...seen].map(([source, m]) => [source, [...m.values()]]));
	87	const began = w.now?.() ?? 0;
	88	let startedAt: number \| null = null;
	89	for (let n = 0; n < w.limit; n++) {
	90	const look = await w.poll();
	91	const at = w.now ? w.now() - began : n * intervalMs;
	92	if (look.healthy && startedAt !== null) return { timeout: false, startedAt, healthyAt: at, said: said() };
	93	if (!look.healthy) {
	94	startedAt ??= at;
	95	for (const [source, lines] of Object.entries(look.said)) {
	96	const m = seen.get(source) ?? new Map<string, string>();
	97	seen.set(source, m);
	98	for (const l of lines) if (!m.has(shape(l))) { m.set(shape(l), l); w.onLine?.(l, source); }
	99	}
	100	}
	101	if (w.ceilingMs !== undefined && at >= w.ceilingMs) break;
	102	await w.sleep(intervalMs);
	103	}
	104	return { timeout: true, startedAt, said: said() };
	105	}
	106
	107	export interface WatchDrill extends Omit<Watch, "poll" \| "onLine"> {
	108	url: string;
	109	fetch: (url: string) => Promise<Reply>;
	110	onLine?: (line: string) => void;
	111	}
	112
	113	export type WatchedDrill =
	114	\| { timeout: false; startedAt: number; healthyAt: number; lines: string[] }
	115	\| { timeout: true; startedAt: number \| null; lines: string[] };
	116
	117	/** `watchPolls` over one URL: healthy is a status under 400, and the lines are its body's. */
	118	export async function watchDrill(w: WatchDrill): Promise<WatchedDrill> {
	119	const poll = async (): Promise<Poll> => {
	120	try {
	121	const res = await w.fetch(w.url);
	122	if (res.status < 400) return { healthy: true, said: {} };
	123	let lines: string[] = [];
	124	try {
	125	lines = linesOf(await res.text(), res.headers.get("content-type") ?? "");
	126	} catch {
	127	/* an unreadable body is unhealthy and says nothing */
	128	}
	129	return { healthy: false, said: { "": lines } };
	130	} catch {
	131	return { healthy: false, said: { "": [UNREACHABLE] } };
	132	}
	133	};
	134	const { url: _url, fetch: _fetch, onLine, ...rest } = w;
	135	const r = await watchPolls({ ...rest, poll, ...(onLine && { onLine: (l: string) => onLine(l) }) });
	136	const lines = r.said[""] ?? [];
	137	return r.timeout
	138	? { timeout: true, startedAt: r.startedAt, lines }
	139	: { timeout: false, startedAt: r.startedAt, healthyAt: r.healthyAt, lines };
	140	}