6c1c1b1
@@ -0,0 +1,19 @@
# Failure domains
The seven ways the forge stops, and what it says when each does.
`src/kernel/domains.ts` is the data; `test/domains.test.ts` keeps this table
equal to it and proves each quoted phrase is what the forge's own code emits
under that condition. The words are quoted from `kernel/vitals.ts`, never
drafted. "Nothing yet" means the forge has no words for that domain today.
Minutes back are estimates, not measurements. A drill named here is the name
issue #1195 will build; only the restore drill exists today.
| Domain | What breaks | The forge says | Minutes back | The owner | Drill |
| --- | --- | --- | --- | --- | --- |
| Compute (the pod) | Nothing answers until the pod is rescheduled. | nothing yet | 2 | Wait for the platform to restart it; read the pod's events if it does not. | kill-pod |
| State (the volume) | Writes fail once the volume fills; the data is lost with the volume. | "Disk is nearly full:" | 60 | Free or grow the volume; if it is gone, restore the newest snapshot (docs/RECOVERY.md). | none: Filling or detaching the live volume would lose writes. |
| The snapshot store | Backups stop landing or cannot be read back; nothing live is affected. | "The restore drill failed at" | 30 | Fix the bucket or its keys, then run the restore drill again. | restore-drill |
| The edge (tunnel and DNS) | The forge runs but nobody outside can reach it. | nothing yet | 10 | Restart the tunnel; check the DNS record. | none: Cutting the tunnel cuts every person and agent off the forge. |
| Identity | Sign-in fails; sessions and tokens already issued may still work. | nothing yet | 15 | Restore the sign-in provider's settings; sign in with the admin password. | none: Breaking sign-in locks the owner out of the forge that would fix it. |
| The model credential | Agents cannot start work; everything else runs. | nothing yet | 5 | Set a fresh credential in the settings. | expire-token |
| The runner | Checks and runs queue behind a lane that is not moving. | "wait behind one lane; the oldest since"; "is stuck: no heartbeat for" | 10 | Cancel the stuck run; restart the runner if the lane stays empty. | lose-runner |
@@ -165,6 +165,7 @@
export const GATE: readonly string[] = [
  "test/migrate.test.ts",
  "test/schema-additive.test.ts",
  "test/schema-migrations.test.ts",
  "test/domains.test.ts",
  "test/vitals.test.ts",
  "test/vitals-degraded.test.ts",
  "test/red-main.test.ts",
@@ -0,0 +1,95 @@
/**
 * The forge's failure domains as data: what breaks, the words the forge
 * already says when it does (`says`: fragments copied from what
 * `kernel/vitals.ts` emits, each proved by `test/domains.test.ts`), the
 * minutes until it is back (estimates, not measurements), what the owner
 * does, and the drill that exercises it. A domain the forge has no words for
 * yet has an empty `says`; `docs/DOMAINS.md` is the same table for a person.
 */
export interface Domain {
  readonly key: string;
  readonly name: string;
  readonly breaks: string;
  /** Fragments of sentences the forge emits today; empty when it says nothing yet. */
  readonly says: readonly string[];
  readonly backMinutes: number;
  readonly owner: string;
  /** A drill's name, or why it cannot be drilled without harm. */
  readonly drill: { readonly name: string } | { readonly name: null; readonly why: string };
}
export const DOMAINS: readonly Domain[] = [
  {
    key: "compute",
    name: "Compute (the pod)",
    breaks: "Nothing answers until the pod is rescheduled.",
    says: [],
    backMinutes: 2,
    owner: "Wait for the platform to restart it; read the pod's events if it does not.",
    drill: { name: "kill-pod" },
  },
  {
    key: "state",
    name: "State (the volume)",
    breaks: "Writes fail once the volume fills; the data is lost with the volume.",
    says: ["Disk is nearly full:"],
    backMinutes: 60,
    owner: "Free or grow the volume; if it is gone, restore the newest snapshot (docs/RECOVERY.md).",
    drill: { name: null, why: "Filling or detaching the live volume would lose writes." },
  },
  {
    key: "snapshots",
    name: "The snapshot store",
    breaks: "Backups stop landing or cannot be read back; nothing live is affected.",
    says: ["The restore drill failed at"],
    backMinutes: 30,
    owner: "Fix the bucket or its keys, then run the restore drill again.",
    drill: { name: "restore-drill" },
  },
  {
    key: "edge",
    name: "The edge (tunnel and DNS)",
    breaks: "The forge runs but nobody outside can reach it.",
    says: [],
    backMinutes: 10,
    owner: "Restart the tunnel; check the DNS record.",
    drill: { name: null, why: "Cutting the tunnel cuts every person and agent off the forge." },
  },
  {
    key: "identity",
    name: "Identity",
    breaks: "Sign-in fails; sessions and tokens already issued may still work.",
    says: [],
    backMinutes: 15,
    owner: "Restore the sign-in provider's settings; sign in with the admin password.",
    drill: { name: null, why: "Breaking sign-in locks the owner out of the forge that would fix it." },
  },
  {
    key: "credential",
    name: "The model credential",
    breaks: "Agents cannot start work; everything else runs.",
    says: [],
    backMinutes: 5,
    owner: "Set a fresh credential in the settings.",
    drill: { name: "expire-token" },
  },
  {
    key: "runner",
    name: "The runner",
    breaks: "Checks and runs queue behind a lane that is not moving.",
    says: ["wait behind one lane; the oldest since", "is stuck: no heartbeat for"],
    backMinutes: 10,
    owner: "Cancel the stuck run; restart the runner if the lane stays empty.",
    drill: { name: "lose-runner" },
  },
];
const cell = (s: string): string => s.replace(/\|/g, "\\|");
/** One domain as its row in `docs/DOMAINS.md`. */
export function domainRow(d: Domain): string {
  const says = d.says.length === 0 ? "nothing yet" : d.says.map((s) => `"${s}"`).join("; ");
  const drill = d.drill.name === null ? `none: ${d.drill.why}` : d.drill.name;
  return `| ${[d.name, d.breaks, says, String(d.backMinutes), d.owner, drill].map(cell).join(" | ")} |`;
}
@@ -0,0 +1,92 @@
/** The failure-domain map: the doc and the data agree, and the forge really says what the data quotes. */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { readFileSync } from "node:fs";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { createUser } from "../src/core/users.ts";
import { createRun } from "../src/core/runs.ts";
import { DOMAINS, domainRow } from "../src/kernel/domains.ts";
import { vitals } from "../src/kernel/vitals.ts";
import { recordVerdict } from "../src/snapshot/verdict.ts";
let dataDir: string;
let app: App;
let repoId = "";
beforeAll(async () => {
  dataDir = await mkdtemp(join(tmpdir(), "forge-domains-"));
  app = createApp({ dataDir, port: 0 });
  const owner = await createUser(app.db, {
    username: "alice", email: "[email protected]", password: "a-good-passphrase",
  });
  if (!owner.ok) throw new Error("no owner");
  repoId = "repo-domains";
  app.db.query("INSERT INTO repo (id, owner_id, name) VALUES (?1, ?2, 'r')").run(repoId, owner.value.id);
});
afterAll(async () => {
  await rm(dataDir, { recursive: true, force: true });
});
const said = (key: string): string[] => {
  const d = DOMAINS.find((x) => x.key === key)!;
  expect(d.says.length).toBeGreaterThan(0);
  return [...d.says];
};
const read = (now = Date.now()) =>
  vitals({ db: app.db, signals: app.signals, dataDir, now }).needs.join("\n");
test("seven domains, each complete", () => {
  expect(DOMAINS.map((d) => d.key)).toEqual([
    "compute", "state", "snapshots", "edge", "identity", "credential", "runner",
  ]);
  for (const d of DOMAINS) {
    expect(d.breaks && d.owner && d.name).toBeTruthy();
    expect(d.backMinutes).toBeGreaterThan(0);
    if (d.drill.name === null) expect(d.drill.why.trim()).not.toBe("");
  }
});
test("the doc's table is the data", () => {
  const doc = readFileSync(join(import.meta.dir, "..", "docs", "DOMAINS.md"), "utf8");
  const rows = doc.split("\n").filter((l) => l.startsWith("| ") && !l.startsWith("| Domain") && !l.startsWith("| ---"));
  expect(rows).toEqual(DOMAINS.map(domainRow));
});
test("a nearly full disk says the state's words", () => {
  const v = vitals({ db: app.db, signals: app.signals, dataDir, diskFreeMin: 1.1 });
  for (const s of said("state")) expect(v.needs.join("\n")).toContain(s);
});
test("a red restore drill says the snapshot store's words", () => {
  recordVerdict(dataDir, {
    at: new Date().toISOString(), snapshotId: null, ms: 1, counts: { repositories: 0, issues: 0 },
    health: "unreached", verdict: "red", failedStep: "fetch", reason: "no bucket",
  });
  for (const s of said("snapshots")) expect(read()).toContain(s);
});
test("a stalled check queue and a silent run say the runner's words", () => {
  app.db.query("DELETE FROM run").run();
  const now = Date.now();
  const iso = (ms: number) => new Date(ms).toISOString();
  const live = createRun(app.db, { repoId, kind: "check" });
  app.db
    .query("UPDATE run SET state='running', started_at=?2, heartbeat_at=?3 WHERE id=?1")
    .run(live.id, iso(now - 31 * 60_000), iso(now - 4 * 60_000));
  for (let i = 0; i < 2; i++) {
    const waiting = createRun(app.db, { repoId, kind: "check" });
    app.db.query("UPDATE run SET created_at=?2 WHERE id=?1").run(waiting.id, iso(now - 13 * 60_000));
  }
  const text = read(now);
  for (const s of said("runner")) expect(text).toContain(s);
  app.db.query("DELETE FROM run").run();
});
test("a domain with no words says nothing, and the doc says so", () => {
  const silent = DOMAINS.filter((d) => d.says.length === 0).map((d) => d.key);
  expect(silent).toEqual(["compute", "edge", "identity", "credential"]);
  expect(readFileSync(join(import.meta.dir, "..", "docs", "DOMAINS.md"), "utf8")).toContain("nothing yet");
});
		@@ -0,0 +1,19 @@
	1	# Failure domains
	2
	3	The seven ways the forge stops, and what it says when each does.
	4	`src/kernel/domains.ts` is the data; `test/domains.test.ts` keeps this table
	5	equal to it and proves each quoted phrase is what the forge's own code emits
	6	under that condition. The words are quoted from `kernel/vitals.ts`, never
	7	drafted. "Nothing yet" means the forge has no words for that domain today.
	8	Minutes back are estimates, not measurements. A drill named here is the name
	9	issue #1195 will build; only the restore drill exists today.
	10
	11	\| Domain \| What breaks \| The forge says \| Minutes back \| The owner \| Drill \|
	12	\| --- \| --- \| --- \| --- \| --- \| --- \|
	13	\| Compute (the pod) \| Nothing answers until the pod is rescheduled. \| nothing yet \| 2 \| Wait for the platform to restart it; read the pod's events if it does not. \| kill-pod \|
	14	\| State (the volume) \| Writes fail once the volume fills; the data is lost with the volume. \| "Disk is nearly full:" \| 60 \| Free or grow the volume; if it is gone, restore the newest snapshot (docs/RECOVERY.md). \| none: Filling or detaching the live volume would lose writes. \|
	15	\| The snapshot store \| Backups stop landing or cannot be read back; nothing live is affected. \| "The restore drill failed at" \| 30 \| Fix the bucket or its keys, then run the restore drill again. \| restore-drill \|
	16	\| The edge (tunnel and DNS) \| The forge runs but nobody outside can reach it. \| nothing yet \| 10 \| Restart the tunnel; check the DNS record. \| none: Cutting the tunnel cuts every person and agent off the forge. \|
	17	\| Identity \| Sign-in fails; sessions and tokens already issued may still work. \| nothing yet \| 15 \| Restore the sign-in provider's settings; sign in with the admin password. \| none: Breaking sign-in locks the owner out of the forge that would fix it. \|
	18	\| The model credential \| Agents cannot start work; everything else runs. \| nothing yet \| 5 \| Set a fresh credential in the settings. \| expire-token \|
	19	\| The runner \| Checks and runs queue behind a lane that is not moving. \| "wait behind one lane; the oldest since"; "is stuck: no heartbeat for" \| 10 \| Cancel the stuck run; restart the runner if the lane stays empty. \| lose-runner \|
		@@ -165,6 +165,7 @@ export const GATE: readonly string[] = [
165	165	"test/migrate.test.ts",
166	166	"test/schema-additive.test.ts",
167	167	"test/schema-migrations.test.ts",
	168	"test/domains.test.ts",
168	169	"test/vitals.test.ts",
169	170	"test/vitals-degraded.test.ts",
170	171	"test/red-main.test.ts",
		@@ -0,0 +1,95 @@
	1	/**
	2	* The forge's failure domains as data: what breaks, the words the forge
	3	* already says when it does (`says`: fragments copied from what
	4	* `kernel/vitals.ts` emits, each proved by `test/domains.test.ts`), the
	5	* minutes until it is back (estimates, not measurements), what the owner
	6	* does, and the drill that exercises it. A domain the forge has no words for
	7	* yet has an empty `says`; `docs/DOMAINS.md` is the same table for a person.
	8	*/
	9
	10	export interface Domain {
	11	readonly key: string;
	12	readonly name: string;
	13	readonly breaks: string;
	14	/** Fragments of sentences the forge emits today; empty when it says nothing yet. */
	15	readonly says: readonly string[];
	16	readonly backMinutes: number;
	17	readonly owner: string;
	18	/** A drill's name, or why it cannot be drilled without harm. */
	19	readonly drill: { readonly name: string } \| { readonly name: null; readonly why: string };
	20	}
	21
	22	export const DOMAINS: readonly Domain[] = [
	23	{
	24	key: "compute",
	25	name: "Compute (the pod)",
	26	breaks: "Nothing answers until the pod is rescheduled.",
	27	says: [],
	28	backMinutes: 2,
	29	owner: "Wait for the platform to restart it; read the pod's events if it does not.",
	30	drill: { name: "kill-pod" },
	31	},
	32	{
	33	key: "state",
	34	name: "State (the volume)",
	35	breaks: "Writes fail once the volume fills; the data is lost with the volume.",
	36	says: ["Disk is nearly full:"],
	37	backMinutes: 60,
	38	owner: "Free or grow the volume; if it is gone, restore the newest snapshot (docs/RECOVERY.md).",
	39	drill: { name: null, why: "Filling or detaching the live volume would lose writes." },
	40	},
	41	{
	42	key: "snapshots",
	43	name: "The snapshot store",
	44	breaks: "Backups stop landing or cannot be read back; nothing live is affected.",
	45	says: ["The restore drill failed at"],
	46	backMinutes: 30,
	47	owner: "Fix the bucket or its keys, then run the restore drill again.",
	48	drill: { name: "restore-drill" },
	49	},
	50	{
	51	key: "edge",
	52	name: "The edge (tunnel and DNS)",
	53	breaks: "The forge runs but nobody outside can reach it.",
	54	says: [],
	55	backMinutes: 10,
	56	owner: "Restart the tunnel; check the DNS record.",
	57	drill: { name: null, why: "Cutting the tunnel cuts every person and agent off the forge." },
	58	},
	59	{
	60	key: "identity",
	61	name: "Identity",
	62	breaks: "Sign-in fails; sessions and tokens already issued may still work.",
	63	says: [],
	64	backMinutes: 15,
	65	owner: "Restore the sign-in provider's settings; sign in with the admin password.",
	66	drill: { name: null, why: "Breaking sign-in locks the owner out of the forge that would fix it." },
	67	},
	68	{
	69	key: "credential",
	70	name: "The model credential",
	71	breaks: "Agents cannot start work; everything else runs.",
	72	says: [],
	73	backMinutes: 5,
	74	owner: "Set a fresh credential in the settings.",
	75	drill: { name: "expire-token" },
	76	},
	77	{
	78	key: "runner",
	79	name: "The runner",
	80	breaks: "Checks and runs queue behind a lane that is not moving.",
	81	says: ["wait behind one lane; the oldest since", "is stuck: no heartbeat for"],
	82	backMinutes: 10,
	83	owner: "Cancel the stuck run; restart the runner if the lane stays empty.",
	84	drill: { name: "lose-runner" },
	85	},
	86	];
	87
	88	const cell = (s: string): string => s.replace(/\\|/g, "\\\|");
	89
	90	/** One domain as its row in `docs/DOMAINS.md`. */
	91	export function domainRow(d: Domain): string {
	92	const says = d.says.length === 0 ? "nothing yet" : d.says.map((s) => `"${s}"`).join("; ");
	93	const drill = d.drill.name === null ? `none: ${d.drill.why}` : d.drill.name;
	94	return `\| ${[d.name, d.breaks, says, String(d.backMinutes), d.owner, drill].map(cell).join(" \| ")} \|`;
	95	}
		@@ -0,0 +1,92 @@
	1	/** The failure-domain map: the doc and the data agree, and the forge really says what the data quotes. */
	2
	3	import { afterAll, beforeAll, expect, test } from "bun:test";
	4	import { readFileSync } from "node:fs";
	5	import { mkdtemp, rm } from "node:fs/promises";
	6	import { tmpdir } from "node:os";
	7	import { join } from "node:path";
	8	import { createApp, type App } from "../src/app.ts";
	9	import { createUser } from "../src/core/users.ts";
	10	import { createRun } from "../src/core/runs.ts";
	11	import { DOMAINS, domainRow } from "../src/kernel/domains.ts";
	12	import { vitals } from "../src/kernel/vitals.ts";
	13	import { recordVerdict } from "../src/snapshot/verdict.ts";
	14
	15	let dataDir: string;
	16	let app: App;
	17	let repoId = "";
	18
	19	beforeAll(async () => {
	20	dataDir = await mkdtemp(join(tmpdir(), "forge-domains-"));
	21	app = createApp({ dataDir, port: 0 });
	22	const owner = await createUser(app.db, {
	23	username: "alice", email: "[email protected]", password: "a-good-passphrase",
	24	});
	25	if (!owner.ok) throw new Error("no owner");
	26	repoId = "repo-domains";
	27	app.db.query("INSERT INTO repo (id, owner_id, name) VALUES (?1, ?2, 'r')").run(repoId, owner.value.id);
	28	});
	29	afterAll(async () => {
	30	await rm(dataDir, { recursive: true, force: true });
	31	});
	32
	33	const said = (key: string): string[] => {
	34	const d = DOMAINS.find((x) => x.key === key)!;
	35	expect(d.says.length).toBeGreaterThan(0);
	36	return [...d.says];
	37	};
	38	const read = (now = Date.now()) =>
	39	vitals({ db: app.db, signals: app.signals, dataDir, now }).needs.join("\n");
	40
	41	test("seven domains, each complete", () => {
	42	expect(DOMAINS.map((d) => d.key)).toEqual([
	43	"compute", "state", "snapshots", "edge", "identity", "credential", "runner",
	44	]);
	45	for (const d of DOMAINS) {
	46	expect(d.breaks && d.owner && d.name).toBeTruthy();
	47	expect(d.backMinutes).toBeGreaterThan(0);
	48	if (d.drill.name === null) expect(d.drill.why.trim()).not.toBe("");
	49	}
	50	});
	51
	52	test("the doc's table is the data", () => {
	53	const doc = readFileSync(join(import.meta.dir, "..", "docs", "DOMAINS.md"), "utf8");
	54	const rows = doc.split("\n").filter((l) => l.startsWith("\| ") && !l.startsWith("\| Domain") && !l.startsWith("\| ---"));
	55	expect(rows).toEqual(DOMAINS.map(domainRow));
	56	});
	57
	58	test("a nearly full disk says the state's words", () => {
	59	const v = vitals({ db: app.db, signals: app.signals, dataDir, diskFreeMin: 1.1 });
	60	for (const s of said("state")) expect(v.needs.join("\n")).toContain(s);
	61	});
	62
	63	test("a red restore drill says the snapshot store's words", () => {
	64	recordVerdict(dataDir, {
	65	at: new Date().toISOString(), snapshotId: null, ms: 1, counts: { repositories: 0, issues: 0 },
	66	health: "unreached", verdict: "red", failedStep: "fetch", reason: "no bucket",
	67	});
	68	for (const s of said("snapshots")) expect(read()).toContain(s);
	69	});
	70
	71	test("a stalled check queue and a silent run say the runner's words", () => {
	72	app.db.query("DELETE FROM run").run();
	73	const now = Date.now();
	74	const iso = (ms: number) => new Date(ms).toISOString();
	75	const live = createRun(app.db, { repoId, kind: "check" });
	76	app.db
	77	.query("UPDATE run SET state='running', started_at=?2, heartbeat_at=?3 WHERE id=?1")
	78	.run(live.id, iso(now - 31 * 60_000), iso(now - 4 * 60_000));
	79	for (let i = 0; i < 2; i++) {
	80	const waiting = createRun(app.db, { repoId, kind: "check" });
	81	app.db.query("UPDATE run SET created_at=?2 WHERE id=?1").run(waiting.id, iso(now - 13 * 60_000));
	82	}
	83	const text = read(now);
	84	for (const s of said("runner")) expect(text).toContain(s);
	85	app.db.query("DELETE FROM run").run();
	86	});
	87
	88	test("a domain with no words says nothing, and the doc says so", () => {
	89	const silent = DOMAINS.filter((d) => d.says.length === 0).map((d) => d.key);
	90	expect(silent).toEqual(["compute", "edge", "identity", "credential"]);
	91	expect(readFileSync(join(import.meta.dir, "..", "docs", "DOMAINS.md"), "utf8")).toContain("nothing yet");
	92	});