13b5480 · b2dc787 · 44e2741 · e55803a · 8ef3372 · 194eebf
@@ -45,7 +45,7 @@
bun src/main.ts --rollback <sha>  # swap back to a kept release: a full 40-hex s
Set `FORGE_REPO=owner/name` to the forge's own code and a pull the crew lands there is written to `$FORGE_DATA/releases/<sha>/` (the newest `FORGE_RELEASES_KEEP`, default 5, are kept), preflighted with that tree's own `--preflight`, and recorded in the `release` table (sha, landed pull, verdict, its line, when). A red preflight is asked again once (the copy of the live database can tear under load); a second red is logged and told to the landed pull. Other repositories are untouched, and nothing is swapped unless `FORGE_SELF_SWAP=1`: then the forge binds its port with SO_REUSEPORT, a green release is started from `releases/<sha>/` beside it on the same port (`FORGE_COMMIT` set to the sha), `/api/v1/health` is read until an answer names that sha and is well (`FORGE_SWAP_BUDGET_S`, default 60), and only then does this process drain as a SIGTERM does — except that it stops accepting at once, so every new connection is the release's, and answers what it already holds (a long push or clone) to the end, for up to five minutes, before it closes. A release not healthy in its budget is killed, the old keeps serving, and the log and the landed pull say so. One swap per process: a landing while another swap is in its budget, or after one succeeded, is released and recorded but not started (the new process runs it from the next restart). The SSH terminal is not handed over yet (the new process cannot bind the old one's port), so a process holding the SSH listener refuses the swap and tells the landed pull; run with `FORGE_SSH_PORT=0` to swap. The swap also needs `net.ipv4.tcp_migrate_req=1` (Linux 5.14+): without it the kernel resets the connections still queued on the old listener when it closes, so a swap is refused (and told to the landed pull) rather than failing requests. The release is a child of the old process, so the swap is for a forge run directly, not under a supervisor that kills its children when the old process exits (PID 1 of a container, a systemd unit with the default `KillMode=control-group`). SO_REUSEPORT lets any process of the forge's user bind the same port and take a share of its connections, so `FORGE_SELF_SWAP=1` assumes nothing else runs as the forge's user on the host network (agents and apps are sandboxed or in cells). `--rollback <sha>` starts the kept release beside the running forge by the same health-read swap (`FORGE_SWAP_BUDGET_S` does not apply: it waits the default 60s), and a release that is not healthy is killed with the old still serving. It refuses, with one line and exit 1, a sha that is not kept (never checked out, or pruned) and a release whose schema has fewer migrations than the live database has applied (`<sha> predates schema N; rolling back would need a down-migration`); a release with a newer schema is allowed, as it migrates forward on boot, and the database's refusal of a future schema is unchanged. It does not drain the forge that was serving: until part 2 of the rollback does, both processes share the port and the operator stops the old one (the command prints that). It does not check `tcp_migrate_req` or SSH either; stopping the old process cleanly needs the same `net.ipv4.tcp_migrate_req=1` setup as the swap, and the SSH port stays with the old process. `/releases` lists the kept releases (short sha, landed pull, when, and since when the live one has been healthy: this process's start); a person who writes the `FORGE_REPO` repository (never an agent: the button is hidden and the POST is a 403) gets a Roll back button on a kept, schema-compatible release, which runs the same refusals and swap and then drains this process (so it needs `FORGE_SELF_SWAP=1`); only readers of that repository and site admins see the page and the home line (anyone else gets a 404), and the forge's SSH terminal does not have the release list or rollback yet (the CLI `--rollback` does); the home carries a `live: <sha> since <time>` line linking there. `FORGE_REPO` also names the parent code that spawned child forges fork.
**Keeping a domain drill**: `POST /api/v1/drills` (a site admin person; an agent is refused with 403, and a session needs a same-origin request) takes `{domain, seen:{health, board}, backSeconds}`: a domain key from `docs/DOMAINS.md`, what the drill read from health and the board (text, at most 8000 characters each), and whole seconds until the forge was well again (0 to 31536000). The body is capped at 32000 bytes (413). It answers 201 `{id, domain, backSeconds, green, why}`: the verdict is judged against the domain's own words and kept whether green or red. An unknown domain, bad `seen` or bad `backSeconds` is 422. Kept drills cannot be read back yet.
**Keeping a domain drill**: `POST /api/v1/drills` (a site admin person; an agent is refused with 403, and a session needs a same-origin request) takes `{domain, seen:{health, board}, backSeconds}`: a domain key from `docs/DOMAINS.md`, what the drill read from health and the board (text, at most 8000 characters each), and whole seconds until the forge was well again (0 to 31536000). The body is capped at 32000 bytes (413). It answers 201 `{id, domain, backSeconds, green, why}`: the verdict is judged against the domain's own words and kept whether green or red. An unknown domain, bad `seen` or bad `backSeconds` is 422. Kept drills are read back on **Settings → Failure domains** (below).
If your shell already has `FORGE_URL` set — a crew workspace sets it to the
forge you work for — this forge takes it as its own canonical address and
@@ -655,6 +655,14 @@
into scratch, opens it, counts what is there and unseals a value; the page
waits for it, up to 10 minutes, and the last drill shows there as one line.
Only one runs at a time. See `docs/SNAPSHOT.md`.
**Failure domains** (`/settings/resilience`, not `/resilience`, which is the THREATS.md page; admins only; a non-admin gets the same
404 as the other admin settings pages): one row per failure domain from
`docs/DOMAINS.md` — what breaks, the words the forge says, and the domain's
last kept drill (when, green or red with its reason, seconds until back) or
the sentence on why it is not drilled. In the terminal it is the **Failure domains**
command (`ctrl+k`; the **Resilience** command is the THREATS.md one), shown to an admin only and read through the same
`listResilience` action. Only the latest drill per domain is shown.
A red drill, or none in eight days, is one line for the forge's admin on the
board's forge strip, in the terminal's dashboard (same `forgeNeeds`), and in
MCP `health` and `asks`; it names the failed step and the next green drill
@@ -0,0 +1,44 @@
/**
 * Each failure domain with its last drill, read by the forge's owner
 * (`kernel/domains.ts`, `core/domain_drills.ts`): a forge admin only, never
 * an agent, and everyone else is told `not_found`. The settings page and the
 * terminal read this and nothing else.
 */
import type { App } from "../app.ts";
import { lastDrill, type LastDrill } from "../core/domain_drills.ts";
import type { User } from "../core/users.ts";
import { DOMAINS } from "../kernel/domains.ts";
import { err, ok, type Result } from "../kernel/result.ts";
export interface ResilienceRow {
  readonly key: string;
  readonly name: string;
  readonly breaks: string;
  readonly says: readonly string[];
  readonly backMinutes: number;
  /** A drill's name and its last result (none yet), or why there is none. */
  readonly drill:
    | { readonly name: string; readonly last: LastDrill | null }
    | { readonly name: null; readonly why: string };
}
export function listResilience(
  app: App,
  actor: User | undefined,
): Result<ResilienceRow[], "not_found"> {
  if (actor?.isAdmin !== true || actor.kind === "agent") return err("not_found");
  return ok(
    DOMAINS.map((d) => ({
      key: d.key,
      name: d.name,
      breaks: d.breaks,
      says: d.says,
      backMinutes: d.backMinutes,
      drill:
        d.drill.name === null
          ? { name: null, why: d.drill.why }
          : { name: d.drill.name, last: lastDrill(app.db, d.key) },
    })),
  );
}
@@ -25,3 +25,21 @@
export function recordDrill(
  ).run(row, domain, by, seen.health, seen.board, backSeconds, verdict.green ? 1 : 0, verdict.why);
  return { id: row, domain, backSeconds, ...verdict };
}
export interface LastDrill {
  readonly green: boolean;
  readonly why: string;
  readonly backSeconds: number;
  readonly at: string;
}
/** The newest drill kept for a domain, or null when none has run. */
export function lastDrill(db: Database, domain: string): LastDrill | null {
  const r = db
    .query<{ green: number; why: string; back_seconds: number; at: string }, [string]>(
      `SELECT green, why, back_seconds, at FROM domain_drill
       WHERE domain = ?1 ORDER BY at DESC, rowid DESC LIMIT 1`,
    )
    .get(domain);
  return r === null ? null : { green: r.green === 1, why: r.why, backSeconds: r.back_seconds, at: r.at };
}
@@ -11,6 +11,7 @@
import { unsealReleased } from "../actions/sealed.ts";
import { accessPage } from "../web/pages/access.ts";
import { capacityPage, capacitySet } from "../web/pages/capacity.ts";
import { flowPage } from "../web/pages/flow.ts";
import { domainsPage } from "../web/pages/domains.ts";
import { resiliencePage } from "../web/pages/resilience.ts";
import { limitsPage, limitsSet } from "../web/pages/admin-limits.ts";
import { boardsPage } from "../web/pages/boards.ts";
@@ -738,6 +739,7 @@
export function buildRouter(): Router<Handler> {
  router.add("POST", "/settings/cloud/audit", cloudAudit);
  router.add("POST", "/settings/cloud/remove", cloudRemove);
  router.add("GET", "/settings/access", accessPage);
  router.add("GET", "/settings/resilience", domainsPage);
  router.add("GET", "/settings/capacity", capacityPage);
  router.add("POST", "/settings/capacity", capacitySet);
  router.add("GET", "/boards", boardsList);
@@ -38,6 +38,7 @@
import { pullScreen } from "./screens/pull.ts";
import { blobScreen } from "./screens/repo.ts";
import { dashboardScreen } from "./screens/dashboard.ts";
import { accessScreen } from "./screens/access.ts";
import { domainsScreen } from "./screens/domains.ts";
import { resilienceLines, resilienceScreen } from "./screens/resilience.ts";
import { flowLines, flowScreen } from "./screens/flow.ts";
import { stalledScreen } from "./screens/stalled.ts";
@@ -257,6 +258,14 @@
export function forgeCommands(session: Session): (ctx: Ctx) => Command[] {
        hint: "what outside agents tried",
        run: () => ctx.push(accessScreen(session, ctx)),
      });
    // Each failure domain and its last drill, as Settings → Failure domains says it (#1945).
    if (session.user?.isAdmin === true && session.user.kind !== "agent")
      commands.push({
        id: "domains",
        title: "Failure domains",
        hint: "failure domains and their last drill",
        run: () => ctx.push(domainsScreen(session, ctx)),
      });
    commands.push({
      id: "family",
      title: "Family",
@@ -0,0 +1,63 @@
/**
 * Failure domains in the terminal, a forge admin's: each failure domain and its
 * last drill — `listResilience`, the action Settings → Failure domains reads.
 * Who may read is the action's call, not this screen's: anyone else gets an
 * empty screen that says nothing exists.
 */
import { listResilience, type ResilienceRow } from "../../actions/resilience.ts";
import type { Ctx, Screen } from "../app/app.ts";
import { binding } from "../app/keys.ts";
import { fill, len, split } from "../ui/layout.ts";
import { style } from "../term/style.ts";
import { truncate } from "../term/width.ts";
import type { Session } from "../session.ts";
import { ListState, drawList, type ListRow } from "./common.ts";
function drillLine(d: ResilienceRow): string {
  if (d.drill.name === null) return `Not drilled: ${d.drill.why}`;
  const last = d.drill.last;
  if (last === null) return `${d.drill.name}: never run`;
  return `${d.drill.name}: ${last.at.slice(0, 16).replace("T", " ")} UTC · ${last.green ? "green" : "red"} — ${last.why} · back in ${last.backSeconds}s (budget ${d.backMinutes} min)`;
}
export function domainsScreen(f: Session, ctx: Ctx): Screen {
  const state = new ListState();
  let domains: ResilienceRow[] = [];
  let loaded = false;
  const rows = (): ListRow[] =>
    domains.map((d) => ({
      title: d.name,
      secondary: d.drill.name === null ? "not drilled" : d.drill.last === null ? "never run" : d.drill.last.green ? "green" : "red",
      meta: `${d.breaks} · ${drillLine(d)}`,
    }));
  const screen: Screen = {
    title: "Failure domains",
    keys: [binding(["j", "k"], "move cursor")],
    load() {
      const listed = listResilience(f.app, f.user);
      domains = listed.ok ? listed.value : [];
      loaded = true;
      state.clamp(domains.length);
      ctx.invalidate();
      return Promise.resolve();
    },
    render(area, s) {
      const c = ctx.theme.c;
      const [head, list] = split(area, "v", [len(1), fill(1)]);
      const note = !loaded
        ? "Loading…"
        : domains.length === 0
          ? "Nothing here."
          : `${domains.length} failure domains, and each one's last drill`;
      s.text(head!.x, head!.y, truncate(note, head!.w), style(c.mutedFg, c.bg), head!.x + head!.w);
      drawList(s, list!, c, state, rows());
    },
    key(e) {
      return state.key(e, domains.length);
    },
  };
  return screen;
}
@@ -0,0 +1,67 @@
/**
 * Failure domains, for a forge admin: each failure domain (`kernel/domains.ts`),
 * what breaks, the words the forge says, and its last drill with the verdict
 * — or the sentence on why it is not drilled (`actions/resilience.ts`).
 */
import { listResilience, type ResilienceRow } from "../../actions/resilience.ts";
import type { Ctx } from "../../http/context.ts";
import { notFoundPage, settingsLayout, signInFirst } from "../chrome.ts";
import { html, type Html } from "../html.ts";
import { page } from "../layout.ts";
import { card, icon, row, time } from "../ui.ts";
import { accountSections } from "./origins.ts";
function drilled(d: ResilienceRow): Html {
  if (d.drill.name === null)
    return html`<span class="muted small">Not drilled: ${d.drill.why}</span>`;
  const last = d.drill.last;
  if (last === null)
    return html`<span class="muted small">${d.drill.name}: never run.</span>`;
  return html`<span class="small"
    >${d.drill.name}: ${time(last.at)} ·
    <strong class="${last.green ? "ok" : "bad"}"
      >${last.green ? "green" : "red"}</strong
    >
    — ${last.why} · back in ${String(last.backSeconds)}s (budget
    ${String(d.backMinutes)} min)</span
  >`;
}
// GET /settings/resilience (named "Failure domains"; /resilience is THREATS.md)
export function domainsPage(ctx: Ctx): Response {
  if (!ctx.auth.user) return signInFirst(ctx);
  const listed = listResilience(ctx.app, ctx.auth.user);
  if (!listed.ok) return notFoundPage(ctx);
  const rows = listed.value.map((d) => {
    const says =
      d.says.length === 0
        ? "Says nothing yet."
        : d.says.map((s) => `“${s}”`).join(" · ");
    return row({
      lead: icon("activity"),
      title: html`${d.name}`,
      meta: html`${d.breaks}<br />${says}<br />${drilled(d)}`,
    });
  });
  const body = card(
    html`<ul class="list" id="resilience-list">
      ${rows}
    </ul>`,
    {
      head: html`${icon("activity")}
        <h2>Failure domains</h2>`,
      pad: false,
    },
  );
  return page({
    ctx,
    title: "Failure domains",
      
        sentinelcrew
        29d ago
        outdated
      
      Same title as main's /resilience page (web/pages/resilience.ts). Title this page and its h1 "Failure domains" so the owner can tell the two apart.
    
    body: html`<div class="page-head"><h1>Failure domains</h1></div>
      ${settingsLayout(
        accountSections("resilience", { admin: true }),
        body,
        "Account settings",
      )}`,
  });
}
@@ -82,6 +82,7 @@
export type AccountSection =
  | "secrets"
  | "capacity"
  | "access"
  | "resilience"
  | "limits"
  | "cloud";
@@ -140,6 +141,12 @@
export function accountSections(
            icon: "activity" as const,
            current: current === "access",
          },
          {
            label: "Failure domains",
      
        sentinelcrew
        29d ago
        outdated
      
      This nav label then reads "Resilience" and opens failure domains, while /resilience is a different page. Label it "Failure domains".
    
            href: "/settings/resilience",
            icon: "activity" as const,
            current: current === "resilience",
          },
          {
            label: "Limits",
            href: "/admin/limits",
@@ -0,0 +1,81 @@
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { recordDrill } from "../src/core/domain_drills.ts";
import { createUser } from "../src/core/users.ts";
import { serve, type Servers } from "../src/http/server.ts";
describe("Settings → Failure domains", () => {
  let dataDir: string;
  let app: App;
  let servers: Servers;
  const login = async (username: string): Promise<string> => {
    const res = await fetch(`${servers.url}/signin`, {
      method: "POST",
      redirect: "manual",
      headers: { "content-type": "application/x-www-form-urlencoded", origin: servers.url },
      body: new URLSearchParams({ username, password: "a-good-passphrase" }),
    });
    return (res.headers.get("set-cookie") ?? "").split(";")[0] ?? "";
  };
  beforeAll(async () => {
    dataDir = await mkdtemp(join(tmpdir(), "forge-resilience-"));
    app = createApp({ dataDir, port: 0 });
    servers = await serve(app);
    for (const [username, adminIfFirst] of [["alice", true], ["bob", false]] as const) {
      const u = await createUser(app.db, {
        trusted: true,
        username,
        email: `${username}@forge.local`,
        password: "a-good-passphrase",
        adminIfFirst,
      });
      if (!u.ok) throw new Error("users");
    }
    recordDrill(app.db, "alice", "snapshots", { health: "", board: "" }, 90, {
      green: true,
      why: "all phrases seen",
    });
    recordDrill(app.db, "alice", "runner", { health: "", board: "" }, 900, {
      green: false,
      why: "<b>late</b> by 5 minutes",
    });
  });
  afterAll(async () => {
    servers.stop();
    app.close();
    await rm(dataDir, { recursive: true, force: true });
  });
  test("an admin sees a green and a red drill, escaped, and why the rest are not drilled", async () => {
    const res = await fetch(`${servers.url}/settings/resilience`, {
      headers: { cookie: await login("alice") },
    });
    expect(res.status).toBe(200);
    const text = await res.text();
    expect(text).toMatch(/class="ok"\s*>green<\/strong\s*>\s*— all phrases seen/);
    expect(text).toMatch(/class="bad"\s*>red<\/strong\s*>\s*— &lt;b&gt;late&lt;\/b&gt; by 5 minutes/);
      
        sentinelcrew
        29d ago
        outdated
      
      toContain("green") and toContain("red") pass on any page that contains those words anywhere, for example in CSS, credential or Ordered. They would stay green if the verdict were never rendered. Assert the rendered verdict itself, for example class="ok" followed by green near all phrases seen, and class="bad" followed by red near the escaped late reason.
    
    expect(text).toContain("&lt;b&gt;late&lt;/b&gt;");
    expect(text).not.toContain("<b>late</b>");
    expect(text).toContain("Filling or detaching the live volume would lose writes.");
    expect(text).toContain("never run");
    expect(text).toContain('href="/settings/resilience"');
    expect(text).toContain("<title>Failure domains");
    expect(text).toContain("<h1>Failure domains</h1>");
  });
  test("a non-admin is refused, an anonymous visitor sent to sign in", async () => {
    const bob = await fetch(`${servers.url}/settings/resilience`, {
      headers: { cookie: await login("bob") },
    });
    expect(bob.status).toBe(404);
    const anon = await fetch(`${servers.url}/settings/resilience`, { redirect: "manual" });
    expect(anon.status).toBe(303);
    expect(anon.headers.get("location")).toBe("/signin?next=%2Fsettings%2Fresilience");
  });
});
@@ -0,0 +1,107 @@
/**
 * Resilience in the terminal (#1853): a forge admin's screen over
      
        sentinelcrew
        29d ago
        
      
      Nit: this comment still cites #1853 and calls it "Resilience". The command is now Failure domains for #1945, and "Resilience" is main's THREATS.md command.
    
 * `listResilience`; anyone else has no such command and, if they reach the
 * action, are told nothing exists.
 */
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App as Forge } from "../src/app.ts";
import { createUser, getUserByName } from "../src/core/users.ts";
import { recordDrill } from "../src/core/domain_drills.ts";
import { listResilience } from "../src/actions/resilience.ts";
import { createForgeApp } from "../src/tui/forge.ts";
import type { TuiIdentity } from "../src/tui/session.ts";
import { headlessTerminal } from "../src/tui/term/terminal.ts";
import { quietLogs } from "./helpers/globals.ts";
import { settleApp } from "./helpers/tui.ts";
quietLogs();
describe("resilience in the terminal", () => {
  let dataDir: string;
  let forge: Forge;
  function boot(username: string) {
    const terminal = headlessTerminal(140, 32);
    const user = getUserByName(forge.db, username)!;
    const identity: TuiIdentity = { sshUsername: "t", remoteAddress: "127.0.0.1", user };
    const { app } = createForgeApp(forge, identity, { terminal, exitHooks: false });
    app.start();
    const key = (k: string) =>
      app.dispatch({ type: "key", key: k, ctrl: k.startsWith("ctrl+"), alt: false, shift: false });
    const press = async (...keys: string[]) => {
      for (const k of keys) key(k.startsWith("ctrl+") ? k.slice(5) : k);
      await settleApp(app);
    };
    const ctrl = async (k: string) => {
      app.dispatch({ type: "key", key: k, ctrl: true, alt: false, shift: false });
      await settleApp(app);
    };
    const type = async (text: string) => {
      for (const ch of text) key(ch);
      await settleApp(app);
    };
    return { text: () => app.screen.toText(), press, ctrl, type, stop: () => app.stop() };
  }
  beforeAll(async () => {
    dataDir = await mkdtemp(join(tmpdir(), "forge-tui-resilience-"));
    forge = createApp({ dataDir, port: 0 });
    for (const [username, adminIfFirst] of [["alice", true], ["bob", false], ["carol", false]] as const) {
      const made = await createUser(forge.db, {
        trusted: true,
        username,
        email: `${username}@forge.local`,
        password: "a-good-passphrase",
        adminIfFirst,
      });
      if (!made.ok) throw new Error(made.error);
    }
    forge.db.query("UPDATE user SET is_admin = 1, is_agent = 1 WHERE username = 'carol'").run();
    recordDrill(forge.db, "alice", "runner", { health: "", board: "" }, 900, {
      green: false,
      why: "late by 5 minutes",
    });
  });
  afterAll(async () => {
    forge.close();
    await rm(dataDir, { recursive: true, force: true });
  });
  const palette = async (h: ReturnType<typeof boot>) => {
    await h.ctrl("k");
    await h.type("Failure domains");
    return h.text();
  };
  test("an admin reads each domain and its last drill", async () => {
    const h = boot("alice");
    expect(await palette(h)).toContain("failure domains and their last drill");
    await h.press("enter");
    expect(h.text()).toContain("Failure domains");
    expect(h.text()).not.toContain("Resilience");
    expect(h.text()).toContain("The runner");
    expect(h.text()).toContain("red");
    expect(h.text()).toContain("not drilled");
    h.stop();
  });
  test("a person who is not an admin has no Failure domains command", async () => {
    const h = boot("bob");
    expect(await palette(h)).not.toContain("failure domains and their last drill");
    await h.press("enter");
    expect(h.text()).not.toContain("The runner");
    h.stop();
  });
  test("the action tells a non-admin and an agent nothing exists", () => {
    for (const name of ["bob", "carol"]) {
      const user = getUserByName(forge.db, name)!;
      const listed = listResilience(forge, user);
      expect(listed.ok).toBe(false);
    }
  });
});
		@@ -45,7 +45,7 @@ bun src/main.ts --rollback <sha> # swap back to a kept release: a full 40-hex s
45	45
46	46	Set `FORGE_REPO=owner/name` to the forge's own code and a pull the crew lands there is written to `$FORGE_DATA/releases/<sha>/` (the newest `FORGE_RELEASES_KEEP`, default 5, are kept), preflighted with that tree's own `--preflight`, and recorded in the `release` table (sha, landed pull, verdict, its line, when). A red preflight is asked again once (the copy of the live database can tear under load); a second red is logged and told to the landed pull. Other repositories are untouched, and nothing is swapped unless `FORGE_SELF_SWAP=1`: then the forge binds its port with SO_REUSEPORT, a green release is started from `releases/<sha>/` beside it on the same port (`FORGE_COMMIT` set to the sha), `/api/v1/health` is read until an answer names that sha and is well (`FORGE_SWAP_BUDGET_S`, default 60), and only then does this process drain as a SIGTERM does — except that it stops accepting at once, so every new connection is the release's, and answers what it already holds (a long push or clone) to the end, for up to five minutes, before it closes. A release not healthy in its budget is killed, the old keeps serving, and the log and the landed pull say so. One swap per process: a landing while another swap is in its budget, or after one succeeded, is released and recorded but not started (the new process runs it from the next restart). The SSH terminal is not handed over yet (the new process cannot bind the old one's port), so a process holding the SSH listener refuses the swap and tells the landed pull; run with `FORGE_SSH_PORT=0` to swap. The swap also needs `net.ipv4.tcp_migrate_req=1` (Linux 5.14+): without it the kernel resets the connections still queued on the old listener when it closes, so a swap is refused (and told to the landed pull) rather than failing requests. The release is a child of the old process, so the swap is for a forge run directly, not under a supervisor that kills its children when the old process exits (PID 1 of a container, a systemd unit with the default `KillMode=control-group`). SO_REUSEPORT lets any process of the forge's user bind the same port and take a share of its connections, so `FORGE_SELF_SWAP=1` assumes nothing else runs as the forge's user on the host network (agents and apps are sandboxed or in cells). `--rollback <sha>` starts the kept release beside the running forge by the same health-read swap (`FORGE_SWAP_BUDGET_S` does not apply: it waits the default 60s), and a release that is not healthy is killed with the old still serving. It refuses, with one line and exit 1, a sha that is not kept (never checked out, or pruned) and a release whose schema has fewer migrations than the live database has applied (`<sha> predates schema N; rolling back would need a down-migration`); a release with a newer schema is allowed, as it migrates forward on boot, and the database's refusal of a future schema is unchanged. It does not drain the forge that was serving: until part 2 of the rollback does, both processes share the port and the operator stops the old one (the command prints that). It does not check `tcp_migrate_req` or SSH either; stopping the old process cleanly needs the same `net.ipv4.tcp_migrate_req=1` setup as the swap, and the SSH port stays with the old process. `/releases` lists the kept releases (short sha, landed pull, when, and since when the live one has been healthy: this process's start); a person who writes the `FORGE_REPO` repository (never an agent: the button is hidden and the POST is a 403) gets a Roll back button on a kept, schema-compatible release, which runs the same refusals and swap and then drains this process (so it needs `FORGE_SELF_SWAP=1`); only readers of that repository and site admins see the page and the home line (anyone else gets a 404), and the forge's SSH terminal does not have the release list or rollback yet (the CLI `--rollback` does); the home carries a `live: <sha> since <time>` line linking there. `FORGE_REPO` also names the parent code that spawned child forges fork.
47	47
48		Keeping a domain drill: `POST /api/v1/drills` (a site admin person; an agent is refused with 403, and a session needs a same-origin request) takes `{domain, seen:{health, board}, backSeconds}`: a domain key from `docs/DOMAINS.md`, what the drill read from health and the board (text, at most 8000 characters each), and whole seconds until the forge was well again (0 to 31536000). The body is capped at 32000 bytes (413). It answers 201 `{id, domain, backSeconds, green, why}`: the verdict is judged against the domain's own words and kept whether green or red. An unknown domain, bad `seen` or bad `backSeconds` is 422. Kept drills cannot be read back yet.
	48	Keeping a domain drill: `POST /api/v1/drills` (a site admin person; an agent is refused with 403, and a session needs a same-origin request) takes `{domain, seen:{health, board}, backSeconds}`: a domain key from `docs/DOMAINS.md`, what the drill read from health and the board (text, at most 8000 characters each), and whole seconds until the forge was well again (0 to 31536000). The body is capped at 32000 bytes (413). It answers 201 `{id, domain, backSeconds, green, why}`: the verdict is judged against the domain's own words and kept whether green or red. An unknown domain, bad `seen` or bad `backSeconds` is 422. Kept drills are read back on Settings → Failure domains (below).
49	49
50	50	If your shell already has `FORGE_URL` set — a crew workspace sets it to the
51	51	forge you work for — this forge takes it as its own canonical address and
		@@ -655,6 +655,14 @@ into scratch, opens it, counts what is there and unseals a value; the page
655	655	waits for it, up to 10 minutes, and the last drill shows there as one line.
656	656	Only one runs at a time. See `docs/SNAPSHOT.md`.
657	657
	658	Failure domains (`/settings/resilience`, not `/resilience`, which is the THREATS.md page; admins only; a non-admin gets the same
	659	404 as the other admin settings pages): one row per failure domain from
	660	`docs/DOMAINS.md` — what breaks, the words the forge says, and the domain's
	661	last kept drill (when, green or red with its reason, seconds until back) or
	662	the sentence on why it is not drilled. In the terminal it is the Failure domains
	663	command (`ctrl+k`; the Resilience command is the THREATS.md one), shown to an admin only and read through the same
	664	`listResilience` action. Only the latest drill per domain is shown.
	665
658	666	A red drill, or none in eight days, is one line for the forge's admin on the
659	667	board's forge strip, in the terminal's dashboard (same `forgeNeeds`), and in
660	668	MCP `health` and `asks`; it names the failed step and the next green drill
		@@ -0,0 +1,44 @@
	1	/**
	2	* Each failure domain with its last drill, read by the forge's owner
	3	* (`kernel/domains.ts`, `core/domain_drills.ts`): a forge admin only, never
	4	* an agent, and everyone else is told `not_found`. The settings page and the
	5	* terminal read this and nothing else.
	6	*/
	7
	8	import type { App } from "../app.ts";
	9	import { lastDrill, type LastDrill } from "../core/domain_drills.ts";
	10	import type { User } from "../core/users.ts";
	11	import { DOMAINS } from "../kernel/domains.ts";
	12	import { err, ok, type Result } from "../kernel/result.ts";
	13
	14	export interface ResilienceRow {
	15	readonly key: string;
	16	readonly name: string;
	17	readonly breaks: string;
	18	readonly says: readonly string[];
	19	readonly backMinutes: number;
	20	/** A drill's name and its last result (none yet), or why there is none. */
	21	readonly drill:
	22	\| { readonly name: string; readonly last: LastDrill \| null }
	23	\| { readonly name: null; readonly why: string };
	24	}
	25
	26	export function listResilience(
	27	app: App,
	28	actor: User \| undefined,
	29	): Result<ResilienceRow[], "not_found"> {
	30	if (actor?.isAdmin !== true \|\| actor.kind === "agent") return err("not_found");
	31	return ok(
	32	DOMAINS.map((d) => ({
	33	key: d.key,
	34	name: d.name,
	35	breaks: d.breaks,
	36	says: d.says,
	37	backMinutes: d.backMinutes,
	38	drill:
	39	d.drill.name === null
	40	? { name: null, why: d.drill.why }
	41	: { name: d.drill.name, last: lastDrill(app.db, d.key) },
	42	})),
	43	);
	44	}
		@@ -25,3 +25,21 @@ export function recordDrill(
25	25	).run(row, domain, by, seen.health, seen.board, backSeconds, verdict.green ? 1 : 0, verdict.why);
26	26	return { id: row, domain, backSeconds, ...verdict };
27	27	}
	28
	29	export interface LastDrill {
	30	readonly green: boolean;
	31	readonly why: string;
	32	readonly backSeconds: number;
	33	readonly at: string;
	34	}
	35
	36	/** The newest drill kept for a domain, or null when none has run. */
	37	export function lastDrill(db: Database, domain: string): LastDrill \| null {
	38	const r = db
	39	.query<{ green: number; why: string; back_seconds: number; at: string }, [string]>(
	40	`SELECT green, why, back_seconds, at FROM domain_drill
	41	WHERE domain = ?1 ORDER BY at DESC, rowid DESC LIMIT 1`,
	42	)
	43	.get(domain);
	44	return r === null ? null : { green: r.green === 1, why: r.why, backSeconds: r.back_seconds, at: r.at };
	45	}
		@@ -11,6 +11,7 @@ import { unsealReleased } from "../actions/sealed.ts";
11	11	import { accessPage } from "../web/pages/access.ts";
12	12	import { capacityPage, capacitySet } from "../web/pages/capacity.ts";
13	13	import { flowPage } from "../web/pages/flow.ts";
	14	import { domainsPage } from "../web/pages/domains.ts";
14	15	import { resiliencePage } from "../web/pages/resilience.ts";
15	16	import { limitsPage, limitsSet } from "../web/pages/admin-limits.ts";
16	17	import { boardsPage } from "../web/pages/boards.ts";
		@@ -738,6 +739,7 @@ export function buildRouter(): Router<Handler> {
738	739	router.add("POST", "/settings/cloud/audit", cloudAudit);
739	740	router.add("POST", "/settings/cloud/remove", cloudRemove);
740	741	router.add("GET", "/settings/access", accessPage);
	742	router.add("GET", "/settings/resilience", domainsPage);
741	743	router.add("GET", "/settings/capacity", capacityPage);
742	744	router.add("POST", "/settings/capacity", capacitySet);
743	745	router.add("GET", "/boards", boardsList);
		@@ -38,6 +38,7 @@ import { pullScreen } from "./screens/pull.ts";
38	38	import { blobScreen } from "./screens/repo.ts";
39	39	import { dashboardScreen } from "./screens/dashboard.ts";
40	40	import { accessScreen } from "./screens/access.ts";
	41	import { domainsScreen } from "./screens/domains.ts";
41	42	import { resilienceLines, resilienceScreen } from "./screens/resilience.ts";
42	43	import { flowLines, flowScreen } from "./screens/flow.ts";
43	44	import { stalledScreen } from "./screens/stalled.ts";
		@@ -257,6 +258,14 @@ export function forgeCommands(session: Session): (ctx: Ctx) => Command[] {
257	258	hint: "what outside agents tried",
258	259	run: () => ctx.push(accessScreen(session, ctx)),
259	260	});
	261	// Each failure domain and its last drill, as Settings → Failure domains says it (#1945).
	262	if (session.user?.isAdmin === true && session.user.kind !== "agent")
	263	commands.push({
	264	id: "domains",
	265	title: "Failure domains",
	266	hint: "failure domains and their last drill",
	267	run: () => ctx.push(domainsScreen(session, ctx)),
	268	});
260	269	commands.push({
261	270	id: "family",
262	271	title: "Family",
		@@ -0,0 +1,63 @@
	1	/**
	2	* Failure domains in the terminal, a forge admin's: each failure domain and its
	3	* last drill — `listResilience`, the action Settings → Failure domains reads.
	4	* Who may read is the action's call, not this screen's: anyone else gets an
	5	* empty screen that says nothing exists.
	6	*/
	7
	8	import { listResilience, type ResilienceRow } from "../../actions/resilience.ts";
	9	import type { Ctx, Screen } from "../app/app.ts";
	10	import { binding } from "../app/keys.ts";
	11	import { fill, len, split } from "../ui/layout.ts";
	12	import { style } from "../term/style.ts";
	13	import { truncate } from "../term/width.ts";
	14	import type { Session } from "../session.ts";
	15	import { ListState, drawList, type ListRow } from "./common.ts";
	16
	17	function drillLine(d: ResilienceRow): string {
	18	if (d.drill.name === null) return `Not drilled: ${d.drill.why}`;
	19	const last = d.drill.last;
	20	if (last === null) return `${d.drill.name}: never run`;
	21	return `${d.drill.name}: ${last.at.slice(0, 16).replace("T", " ")} UTC · ${last.green ? "green" : "red"} — ${last.why} · back in ${last.backSeconds}s (budget ${d.backMinutes} min)`;
	22	}
	23
	24	export function domainsScreen(f: Session, ctx: Ctx): Screen {
	25	const state = new ListState();
	26	let domains: ResilienceRow[] = [];
	27	let loaded = false;
	28
	29	const rows = (): ListRow[] =>
	30	domains.map((d) => ({
	31	title: d.name,
	32	secondary: d.drill.name === null ? "not drilled" : d.drill.last === null ? "never run" : d.drill.last.green ? "green" : "red",
	33	meta: `${d.breaks} · ${drillLine(d)}`,
	34	}));
	35
	36	const screen: Screen = {
	37	title: "Failure domains",
	38	keys: [binding(["j", "k"], "move cursor")],
	39	load() {
	40	const listed = listResilience(f.app, f.user);
	41	domains = listed.ok ? listed.value : [];
	42	loaded = true;
	43	state.clamp(domains.length);
	44	ctx.invalidate();
	45	return Promise.resolve();
	46	},
	47	render(area, s) {
	48	const c = ctx.theme.c;
	49	const [head, list] = split(area, "v", [len(1), fill(1)]);
	50	const note = !loaded
	51	? "Loading…"
	52	: domains.length === 0
	53	? "Nothing here."
	54	: `${domains.length} failure domains, and each one's last drill`;
	55	s.text(head!.x, head!.y, truncate(note, head!.w), style(c.mutedFg, c.bg), head!.x + head!.w);
	56	drawList(s, list!, c, state, rows());
	57	},
	58	key(e) {
	59	return state.key(e, domains.length);
	60	},
	61	};
	62	return screen;
	63	}
		@@ -0,0 +1,67 @@
	1	/**
	2	* Failure domains, for a forge admin: each failure domain (`kernel/domains.ts`),
	3	* what breaks, the words the forge says, and its last drill with the verdict
	4	* — or the sentence on why it is not drilled (`actions/resilience.ts`).
	5	*/
	6
	7	import { listResilience, type ResilienceRow } from "../../actions/resilience.ts";
	8	import type { Ctx } from "../../http/context.ts";
	9	import { notFoundPage, settingsLayout, signInFirst } from "../chrome.ts";
	10	import { html, type Html } from "../html.ts";
	11	import { page } from "../layout.ts";
	12	import { card, icon, row, time } from "../ui.ts";
	13	import { accountSections } from "./origins.ts";
	14
	15	function drilled(d: ResilienceRow): Html {
	16	if (d.drill.name === null)
	17	return html`<span class="muted small">Not drilled: ${d.drill.why}</span>`;
	18	const last = d.drill.last;
	19	if (last === null)
	20	return html`<span class="muted small">${d.drill.name}: never run.</span>`;
	21	return html`<span class="small"
	22	>${d.drill.name}: ${time(last.at)} ·
	23	<strong class="${last.green ? "ok" : "bad"}"
	24	>${last.green ? "green" : "red"}</strong
	25	>
	26	— ${last.why} · back in ${String(last.backSeconds)}s (budget
	27	${String(d.backMinutes)} min)</span
	28	>`;
	29	}
	30
	31	// GET /settings/resilience (named "Failure domains"; /resilience is THREATS.md)
	32	export function domainsPage(ctx: Ctx): Response {
	33	if (!ctx.auth.user) return signInFirst(ctx);
	34	const listed = listResilience(ctx.app, ctx.auth.user);
	35	if (!listed.ok) return notFoundPage(ctx);
	36	const rows = listed.value.map((d) => {
	37	const says =
	38	d.says.length === 0
	39	? "Says nothing yet."
	40	: d.says.map((s) => `“${s}”`).join(" · ");
	41	return row({
	42	lead: icon("activity"),
	43	title: html`${d.name}`,
	44	meta: html`${d.breaks}<br />${says}<br />${drilled(d)}`,
	45	});
	46	});
	47	const body = card(
	48	html`<ul class="list" id="resilience-list">
	49	${rows}
	50	</ul>`,
	51	{
	52	head: html`${icon("activity")}
	53	<h2>Failure domains</h2>`,
	54	pad: false,
	55	},
	56	);
	57	return page({
	58	ctx,
	59	title: "Failure domains",
		sentinelcrew 29d ago outdated Same title as main's `/resilience` page (`web/pages/resilience.ts`). Title this page and its h1 "Failure domains" so the owner can tell the two apart.
	60	body: html`<div class="page-head"><h1>Failure domains</h1></div>
	61	${settingsLayout(
	62	accountSections("resilience", { admin: true }),
	63	body,
	64	"Account settings",
	65	)}`,
	66	});
	67	}
		@@ -82,6 +82,7 @@ export type AccountSection =
82	82	\| "secrets"
83	83	\| "capacity"
84	84	\| "access"
	85	\| "resilience"
85	86	\| "limits"
86	87	\| "cloud";
87	88
		@@ -140,6 +141,12 @@ export function accountSections(
140	141	icon: "activity" as const,
141	142	current: current === "access",
142	143	},
	144	{
	145	label: "Failure domains",
		sentinelcrew 29d ago outdated This nav label then reads "Resilience" and opens failure domains, while `/resilience` is a different page. Label it "Failure domains".
	146	href: "/settings/resilience",
	147	icon: "activity" as const,
	148	current: current === "resilience",
	149	},
143	150	{
144	151	label: "Limits",
145	152	href: "/admin/limits",
		@@ -0,0 +1,81 @@
	1	import { afterAll, beforeAll, describe, expect, test } from "bun:test";
	2	import { mkdtemp, rm } from "node:fs/promises";
	3	import { tmpdir } from "node:os";
	4	import { join } from "node:path";
	5	import { createApp, type App } from "../src/app.ts";
	6	import { recordDrill } from "../src/core/domain_drills.ts";
	7	import { createUser } from "../src/core/users.ts";
	8	import { serve, type Servers } from "../src/http/server.ts";
	9
	10	describe("Settings → Failure domains", () => {
	11	let dataDir: string;
	12	let app: App;
	13	let servers: Servers;
	14
	15	const login = async (username: string): Promise<string> => {
	16	const res = await fetch(`${servers.url}/signin`, {
	17	method: "POST",
	18	redirect: "manual",
	19	headers: { "content-type": "application/x-www-form-urlencoded", origin: servers.url },
	20	body: new URLSearchParams({ username, password: "a-good-passphrase" }),
	21	});
	22	return (res.headers.get("set-cookie") ?? "").split(";")[0] ?? "";
	23	};
	24
	25	beforeAll(async () => {
	26	dataDir = await mkdtemp(join(tmpdir(), "forge-resilience-"));
	27	app = createApp({ dataDir, port: 0 });
	28	servers = await serve(app);
	29	for (const [username, adminIfFirst] of [["alice", true], ["bob", false]] as const) {
	30	const u = await createUser(app.db, {
	31	trusted: true,
	32	username,
	33	email: `${username}@forge.local`,
	34	password: "a-good-passphrase",
	35	adminIfFirst,
	36	});
	37	if (!u.ok) throw new Error("users");
	38	}
	39	recordDrill(app.db, "alice", "snapshots", { health: "", board: "" }, 90, {
	40	green: true,
	41	why: "all phrases seen",
	42	});
	43	recordDrill(app.db, "alice", "runner", { health: "", board: "" }, 900, {
	44	green: false,
	45	why: "<b>late</b> by 5 minutes",
	46	});
	47	});
	48
	49	afterAll(async () => {
	50	servers.stop();
	51	app.close();
	52	await rm(dataDir, { recursive: true, force: true });
	53	});
	54
	55	test("an admin sees a green and a red drill, escaped, and why the rest are not drilled", async () => {
	56	const res = await fetch(`${servers.url}/settings/resilience`, {
	57	headers: { cookie: await login("alice") },
	58	});
	59	expect(res.status).toBe(200);
	60	const text = await res.text();
	61	expect(text).toMatch(/class="ok"\s>green<\/strong\s>\s*— all phrases seen/);
	62	expect(text).toMatch(/class="bad"\s>red<\/strong\s>\s*— <b>late<\/b> by 5 minutes/);
		sentinelcrew 29d ago outdated `toContain("green")` and `toContain("red")` pass on any page that contains those words anywhere, for example in CSS, `credential` or `Ordered`. They would stay green if the verdict were never rendered. Assert the rendered verdict itself, for example `class="ok"` followed by `green` near `all phrases seen`, and `class="bad"` followed by `red` near the escaped `late` reason.
	63	expect(text).toContain("<b>late</b>");
	64	expect(text).not.toContain("<b>late</b>");
	65	expect(text).toContain("Filling or detaching the live volume would lose writes.");
	66	expect(text).toContain("never run");
	67	expect(text).toContain('href="/settings/resilience"');
	68	expect(text).toContain("<title>Failure domains");
	69	expect(text).toContain("<h1>Failure domains</h1>");
	70	});
	71
	72	test("a non-admin is refused, an anonymous visitor sent to sign in", async () => {
	73	const bob = await fetch(`${servers.url}/settings/resilience`, {
	74	headers: { cookie: await login("bob") },
	75	});
	76	expect(bob.status).toBe(404);
	77	const anon = await fetch(`${servers.url}/settings/resilience`, { redirect: "manual" });
	78	expect(anon.status).toBe(303);
	79	expect(anon.headers.get("location")).toBe("/signin?next=%2Fsettings%2Fresilience");
	80	});
	81	});
		@@ -0,0 +1,107 @@
	1	/**
	2	* Resilience in the terminal (#1853): a forge admin's screen over
		sentinelcrew 29d ago Nit: this comment still cites #1853 and calls it "Resilience". The command is now Failure domains for #1945, and "Resilience" is main's THREATS.md command.
	3	* `listResilience`; anyone else has no such command and, if they reach the
	4	* action, are told nothing exists.
	5	*/
	6	import { afterAll, beforeAll, describe, expect, test } from "bun:test";
	7	import { mkdtemp, rm } from "node:fs/promises";
	8	import { tmpdir } from "node:os";
	9	import { join } from "node:path";
	10	import { createApp, type App as Forge } from "../src/app.ts";
	11	import { createUser, getUserByName } from "../src/core/users.ts";
	12	import { recordDrill } from "../src/core/domain_drills.ts";
	13	import { listResilience } from "../src/actions/resilience.ts";
	14	import { createForgeApp } from "../src/tui/forge.ts";
	15	import type { TuiIdentity } from "../src/tui/session.ts";
	16	import { headlessTerminal } from "../src/tui/term/terminal.ts";
	17	import { quietLogs } from "./helpers/globals.ts";
	18	import { settleApp } from "./helpers/tui.ts";
	19
	20	quietLogs();
	21
	22	describe("resilience in the terminal", () => {
	23	let dataDir: string;
	24	let forge: Forge;
	25
	26	function boot(username: string) {
	27	const terminal = headlessTerminal(140, 32);
	28	const user = getUserByName(forge.db, username)!;
	29	const identity: TuiIdentity = { sshUsername: "t", remoteAddress: "127.0.0.1", user };
	30	const { app } = createForgeApp(forge, identity, { terminal, exitHooks: false });
	31	app.start();
	32	const key = (k: string) =>
	33	app.dispatch({ type: "key", key: k, ctrl: k.startsWith("ctrl+"), alt: false, shift: false });
	34	const press = async (...keys: string[]) => {
	35	for (const k of keys) key(k.startsWith("ctrl+") ? k.slice(5) : k);
	36	await settleApp(app);
	37	};
	38	const ctrl = async (k: string) => {
	39	app.dispatch({ type: "key", key: k, ctrl: true, alt: false, shift: false });
	40	await settleApp(app);
	41	};
	42	const type = async (text: string) => {
	43	for (const ch of text) key(ch);
	44	await settleApp(app);
	45	};
	46	return { text: () => app.screen.toText(), press, ctrl, type, stop: () => app.stop() };
	47	}
	48
	49	beforeAll(async () => {
	50	dataDir = await mkdtemp(join(tmpdir(), "forge-tui-resilience-"));
	51	forge = createApp({ dataDir, port: 0 });
	52	for (const [username, adminIfFirst] of [["alice", true], ["bob", false], ["carol", false]] as const) {
	53	const made = await createUser(forge.db, {
	54	trusted: true,
	55	username,
	56	email: `${username}@forge.local`,
	57	password: "a-good-passphrase",
	58	adminIfFirst,
	59	});
	60	if (!made.ok) throw new Error(made.error);
	61	}
	62	forge.db.query("UPDATE user SET is_admin = 1, is_agent = 1 WHERE username = 'carol'").run();
	63	recordDrill(forge.db, "alice", "runner", { health: "", board: "" }, 900, {
	64	green: false,
	65	why: "late by 5 minutes",
	66	});
	67	});
	68
	69	afterAll(async () => {
	70	forge.close();
	71	await rm(dataDir, { recursive: true, force: true });
	72	});
	73
	74	const palette = async (h: ReturnType<typeof boot>) => {
	75	await h.ctrl("k");
	76	await h.type("Failure domains");
	77	return h.text();
	78	};
	79
	80	test("an admin reads each domain and its last drill", async () => {
	81	const h = boot("alice");
	82	expect(await palette(h)).toContain("failure domains and their last drill");
	83	await h.press("enter");
	84	expect(h.text()).toContain("Failure domains");
	85	expect(h.text()).not.toContain("Resilience");
	86	expect(h.text()).toContain("The runner");
	87	expect(h.text()).toContain("red");
	88	expect(h.text()).toContain("not drilled");
	89	h.stop();
	90	});
	91
	92	test("a person who is not an admin has no Failure domains command", async () => {
	93	const h = boot("bob");
	94	expect(await palette(h)).not.toContain("failure domains and their last drill");
	95	await h.press("enter");
	96	expect(h.text()).not.toContain("The runner");
	97	h.stop();
	98	});
	99
	100	test("the action tells a non-admin and an agent nothing exists", () => {
	101	for (const name of ["bob", "carol"]) {
	102	const user = getUserByName(forge.db, name)!;
	103	const listed = listResilience(forge, user);
	104	expect(listed.ok).toBe(false);
	105	}
	106	});
	107	});