31d4a3b · ae13d91 · e209d31 · 73db761 · 38399ee
@@ -74,6 +74,7 @@
function gatherVitals(app: App): Vitals {
    db: app.db,
    dataDir: app.config.dataDir,
    ...(app.config.disk ? { disk: app.config.disk } : {}),
    ...(app.config.hostFree ? { hostFree: app.config.hostFree } : {}),
    envs,
    signals: app.signals,
    forgeUrl: app.config.url ?? `http://localhost:${app.config.port}`,
@@ -46,6 +46,11 @@
export interface ForgeConfig {
  seedSignup?: boolean;
  /** The disk probe vitals reads; the volume under `dataDir` when unset. Tests give it a reading. */
  disk?: (dir: string) => { freeBytes: number; totalBytes: number };
  /**
   * The host's free memory probe vitals reads against its floor; `freemem()`
   * when unset. Tests give it a reading.
   */
  hostFree?: () => number;
  /**
   * The machine's memory in bytes, which the forges' sizes are summed against
   * (`spawn/memory_fit.ts`); its pod's limit or its own when unset. Tests give it a figure.
@@ -42,6 +42,7 @@
import { rollbackNeed } from "../release/launch.ts";
import { behindStanding } from "../release/behind.ts";
import { newestDrills } from "../core/domain_drills.ts";
import { capacityValue, memoryLimitBytes, podMemoryUsedBytes } from "../core/capacity.ts";
import { limitValue } from "../settings/limits.ts";
import { unwrittenFor, SILENT_AFTER_S } from "../http/access_log.ts";
import { DOMAINS } from "./domains.ts";
import { domainDrillNeeds } from "./domain_drill.ts";
@@ -82,8 +83,9 @@
export interface EnvCount {
 * database over 2 GiB, a drill owed, a queue waiting) the forge serves
 * through. Of those, vitals see `disk-full`, `disk-red`, `crash-loop` and
 * `red:<repoId>`; draining is added by the health answer, and the database
 * not answering is `/healthz`'s 503. Memory below its floor is `broken` by
 * this rule, but nothing raises it yet (#12284).
 * not answering is `/healthz`'s 503. Memory below its floor (`memory`): the
 * host's free memory under `forge.memory_floor_mb`, the share the forge keeps
 * for itself (#12300).
 *
 * `disk` and `disk-floor` are `todo`: a low disk stops new starts (child
 * forges, apps, checks), but the forge keeps serving what is already running
@@ -94,7 +96,7 @@
export type NeedKind = "broken" | "todo";
/**
 * One cause of the forge being off: the `needs` sentence, a `cause` key that
 * holds for as long as its condition does (`disk`, `disk-floor`, `disk-80`, `disk-70`,
 * holds for as long as its condition does (`memory`, `disk`, `disk-floor`, `disk-80`, `disk-70`,
 * `db-size`, `db-growth`, `forges-memory`, `errors`, `internal`,
 * `queue:<kind>`, `release`, `stuck:<runId>`, `red:<repoId>`, `shared:<repoId>:<test>`), when it began by the forge's own evidence
 * (the oldest evidence the forge holds, else the first time this process saw
@@ -188,6 +190,23 @@
export interface VitalsInput {
   * machine's memory (`spawn/memory_fit.ts`); undefined with room.
   */
  readonly forgesMemory?: string | undefined;
  /**
   * The host's free memory, bytes; `freemem()` unless a test gives a reading.
   * Under `bun test` (NODE_ENV=test) with none given, the memory need is not
   * raised: the suite's own parallel load would starve the machine it runs on
   * and turn every test's forge broken.
   */
  readonly hostFree?: () => number;
}
/** The host probe the memory need reads when none is given; none under `bun test`. */
const hostFreeDefault: (() => number) | undefined = process.env["NODE_ENV"] === "test" ? undefined : freemem;
const MIB = 1024 * 1024;
/** The `memory` need's sentence: the reading and the floor it is under. */
export function memorySentence(freeBytes: number, floorMiB: number): string {
  return `Memory ${Math.floor(freeBytes / MIB)} MiB free, floor ${floorMiB} MiB.`;
}
/** The device a path lives on, read from it or its nearest parent that exists. */
@@ -369,9 +388,11 @@
export type PodMemory = { readonly limitBytes: number; readonly usedBytes: numbe
export function memoryReading(
  pod: PodMemory = { limitBytes: memoryLimitBytes(), usedBytes: podMemoryUsedBytes() },
  rssBytes = process.memoryUsage().rss,
  hostFreeBytes = freemem(),
): Vitals["memory"] {
  const used = pod.usedBytes ?? rssBytes;
  return { rssBytes, freeBytes: Math.min(freemem(), Math.max(0, pod.limitBytes - used)), totalBytes: pod.limitBytes };
  const freeBytes = Math.min(hostFreeBytes, Math.max(0, pod.limitBytes - used));
  return { rssBytes, freeBytes, totalBytes: pod.limitBytes };
}
export function vitals(input: VitalsInput): Vitals {
@@ -400,6 +421,13 @@
export function vitals(input: VitalsInput): Vitals {
    degraded.push({ cause, sentence, since, href });
  };
  // Free memory under the floor the forge keeps for itself: what it serves with is gone.
  const hostProbe = input.hostFree ?? hostFreeDefault;
  const hostFreeBytes = hostProbe === undefined ? freemem() : hostProbe();
  const memoryFloorMiB = limitValue(input.db, "forge.memory_floor_mb");
  if (hostProbe !== undefined && memoryFloorMiB > 0 && hostFreeBytes < memoryFloorMiB * MIB)
    need("broken", "memory", memorySentence(hostFreeBytes, memoryFloorMiB), undefined, "/settings/capacity");
  const fullAt = fullSince(input.dataDir);
  if (fullAt !== undefined) need("broken", "disk-full", fullSentence(input.dataDir, true), fullAt, "/system");
  const disk = (input.disk ?? readDisk)(input.dataDir);
@@ -690,7 +718,7 @@
export function vitals(input: VitalsInput): Vitals {
    todo,
    degraded,
    disk,
    memory: memoryReading(input.podMemory),
    memory: memoryReading(input.podMemory, undefined, hostFreeBytes),
    queue,
    lane,
    stuck,
@@ -3,7 +3,7 @@
 * now) or `todo` (would like done), and only `broken` makes ok false. This
 * file is the guard on the broken list: a real failure filed as todo would
 * answer ok:true and fail here. A crash loop is one: three unclean starts in
 * ten minutes (#12285).
 * ten minutes (#12285). Free memory under the forge's floor is another (#12300).
 */
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test";
import type { Database } from "bun:sqlite";
@@ -21,7 +21,10 @@
import { health } from "../src/http/health.ts";
import { openDb } from "../src/kernel/db.ts";
import { ErrorLedger } from "../src/kernel/log.ts";
import { RefusedLedger } from "../src/kernel/internal_refused.ts";
import { vitals, type Vitals } from "../src/kernel/vitals.ts";
import { memorySentence, vitals, type Vitals } from "../src/kernel/vitals.ts";
import { limitValue } from "../src/settings/limits.ts";
import { tool as healthTool } from "../src/http/mcp/tools/health.ts";
import type { Ctx } from "../src/http/context.ts";
import { DB_FILE_CAP, dbFile } from "../src/system/db_size.ts";
import { quietLogs } from "./helpers/globals.ts";
@@ -44,11 +47,14 @@
afterEach(() => {
  rmSync(dataDir, { recursive: true, force: true });
});
const read = (disk = roomy): Vitals =>
const MIB = 1024 * 1024;
const read = (disk = roomy, hostFree = () => 1e15): Vitals =>
  vitals({
    db,
    dataDir,
    disk,
    hostFree,
    podMemory: { limitBytes: 1e15, usedBytes: 0 },
    dbWalk: () => null,
    signals: { errors: new ErrorLedger(), refused: new RefusedLedger() },
  });
@@ -77,6 +83,46 @@
describe("broken makes ok false", () => {
  });
});
describe("memory below its floor", () => {
  test("is broken while free memory is under the floor, and clears above it", () => {
    const floor = limitValue(db, "forge.memory_floor_mb");
    expect(floor).toBeGreaterThan(0);
    const low = read(roomy, () => (floor - 76) * MIB);
    const need = low.degraded.find((d) => d.cause === "memory")!;
    expect(need.sentence).toBe(`Memory ${floor - 76} MiB free, floor ${floor} MiB.`);
    expect(low.ok).toBe(false);
    expect(low.broken).toContain(need.sentence);
    expect(low.todo).not.toContain(need.sentence);
    expect(low.memory.freeBytes).toBe((floor - 76) * MIB);
    const back = read(roomy, () => floor * MIB);
    expect(back.degraded.find((d) => d.cause === "memory")).toBeUndefined();
    expect(back.ok).toBe(true);
  });
  test("a floor of 0 keeps none, so nothing is under it", () => {
    db.run("INSERT INTO forge_setting (key, value, updated_by, updated_at) VALUES ('limit.forge.memory_floor_mb', '0', 'u1', '2026-01-01')");
    expect(read(roomy, () => 0).broken).toEqual([]);
  });
  test("under bun test with no probe given, the host's own memory raises nothing, however high the floor", () => {
    db.run("INSERT INTO forge_setting (key, value, updated_by, updated_at) VALUES ('limit.forge.memory_floor_mb', '1048576', 'u1', '2026-01-01')");
    const v = vitals({
      db,
      dataDir,
      disk: roomy,
      podMemory: { limitBytes: 1e15, usedBytes: 0 },
      dbWalk: () => null,
      signals: { errors: new ErrorLedger(), refused: new RefusedLedger() },
    });
    expect(v.degraded.find((d) => d.cause === "memory")).toBeUndefined();
    expect(read(roomy, () => 0).degraded.find((d) => d.cause === "memory")).toBeDefined();
  });
  test("the sentence names the reading and the floor", () => {
    expect(memorySentence(180 * MIB, 256)).toBe("Memory 180 MiB free, floor 256 MiB.");
  });
});
describe("todo alone keeps ok true", () => {
  test("a database over 2 GiB", () => {
    writeFileSync(dbFile(dataDir), "", { mode: 0o600 });
@@ -114,6 +160,37 @@
describe("the health answer", () => {
    health.draining = false;
    expect((readHealth(app, admin) as Vitals).broken).not.toContain(DRAINING);
  });
  test("memory under its floor answers ok:false on the API and the MCP tool, ok:true once back", () => {
    const floor = limitValue(app.db, "forge.memory_floor_mb");
    let free = (floor - 1) * MIB;
    const hostFree = () => free;
    const mem = createApp({ dataDir: dir, port: 0, hostFree, disk: () => ({ freeBytes: 1e12, totalBytes: 1e12 }) });
    try {
      const sentence = memorySentence(free, floor);
      const api = readHealth(mem, admin) as Vitals;
      expect(api.ok).toBe(false);
      expect(api.broken).toContain(sentence);
      const url = new URL("http://localhost/mcp");
      const ctx: Ctx = {
        app: mem,
        request: new Request(url.href, { method: "POST" }),
        url,
        params: {},
        auth: { user: admin, scope: undefined, via: "session" },
        theme: undefined,
      };
      const mcp = healthTool.run(ctx, admin, {}) as { ok: true; value: Vitals };
      expect(mcp.value.ok).toBe(false);
      expect(mcp.value.broken).toContain(sentence);
      free = (floor + 1) * MIB;
      const again = readHealth(mem, admin) as Vitals;
      expect(again.broken).not.toContain(sentence);
      expect(again.degraded.some((d) => d.cause === "memory")).toBe(false);
    } finally {
      mem.db.close();
    }
  });
});
describe("a crash loop", () => {
@@ -178,8 +178,8 @@
test("the number of queries does not grow with runs", () => {
  // finished checks, the red mains holding pulls' checks, the tests red on many pulls, (the checks
  // red on a full disk only while the disk is full), and the newest drill of every domain, the
  // checks no red main holds, the week's flake list, and the database's daily size readings
  // (#12010, at most 60 rows).
  expect(before).toBe(11);
  // (#12010, at most 60 rows), and the memory floor the forge keeps for itself (#12300).
  expect(before).toBe(12);
});
test("errors are counted, bounded, redacted, and a burst is said", async () => {
		@@ -74,6 +74,7 @@ function gatherVitals(app: App): Vitals {
74	74	db: app.db,
75	75	dataDir: app.config.dataDir,
76	76	...(app.config.disk ? { disk: app.config.disk } : {}),
	77	...(app.config.hostFree ? { hostFree: app.config.hostFree } : {}),
77	78	envs,
78	79	signals: app.signals,
79	80	forgeUrl: app.config.url ?? `http://localhost:${app.config.port}`,
		@@ -46,6 +46,11 @@ export interface ForgeConfig {
46	46	seedSignup?: boolean;
47	47	/** The disk probe vitals reads; the volume under `dataDir` when unset. Tests give it a reading. */
48	48	disk?: (dir: string) => { freeBytes: number; totalBytes: number };
	49	/**
	50	* The host's free memory probe vitals reads against its floor; `freemem()`
	51	* when unset. Tests give it a reading.
	52	*/
	53	hostFree?: () => number;
49	54	/**
50	55	* The machine's memory in bytes, which the forges' sizes are summed against
51	56	* (`spawn/memory_fit.ts`); its pod's limit or its own when unset. Tests give it a figure.
		@@ -42,6 +42,7 @@ import { rollbackNeed } from "../release/launch.ts";
42	42	import { behindStanding } from "../release/behind.ts";
43	43	import { newestDrills } from "../core/domain_drills.ts";
44	44	import { capacityValue, memoryLimitBytes, podMemoryUsedBytes } from "../core/capacity.ts";
	45	import { limitValue } from "../settings/limits.ts";
45	46	import { unwrittenFor, SILENT_AFTER_S } from "../http/access_log.ts";
46	47	import { DOMAINS } from "./domains.ts";
47	48	import { domainDrillNeeds } from "./domain_drill.ts";
		@@ -82,8 +83,9 @@ export interface EnvCount {
82	83	* database over 2 GiB, a drill owed, a queue waiting) the forge serves
83	84	* through. Of those, vitals see `disk-full`, `disk-red`, `crash-loop` and
84	85	* `red:<repoId>`; draining is added by the health answer, and the database
85		* not answering is `/healthz`'s 503. Memory below its floor is `broken` by
86		* this rule, but nothing raises it yet (#12284).
	86	* not answering is `/healthz`'s 503. Memory below its floor (`memory`): the
	87	* host's free memory under `forge.memory_floor_mb`, the share the forge keeps
	88	* for itself (#12300).
87	89	*
88	90	* `disk` and `disk-floor` are `todo`: a low disk stops new starts (child
89	91	* forges, apps, checks), but the forge keeps serving what is already running
		@@ -94,7 +96,7 @@ export type NeedKind = "broken" \| "todo";
94	96
95	97	/**
96	98	* One cause of the forge being off: the `needs` sentence, a `cause` key that
97		* holds for as long as its condition does (`disk`, `disk-floor`, `disk-80`, `disk-70`,
	99	* holds for as long as its condition does (`memory`, `disk`, `disk-floor`, `disk-80`, `disk-70`,
98	100	* `db-size`, `db-growth`, `forges-memory`, `errors`, `internal`,
99	101	* `queue:<kind>`, `release`, `stuck:<runId>`, `red:<repoId>`, `shared:<repoId>:<test>`), when it began by the forge's own evidence
100	102	* (the oldest evidence the forge holds, else the first time this process saw
		@@ -188,6 +190,23 @@ export interface VitalsInput {
188	190	* machine's memory (`spawn/memory_fit.ts`); undefined with room.
189	191	*/
190	192	readonly forgesMemory?: string \| undefined;
	193	/**
	194	* The host's free memory, bytes; `freemem()` unless a test gives a reading.
	195	* Under `bun test` (NODE_ENV=test) with none given, the memory need is not
	196	* raised: the suite's own parallel load would starve the machine it runs on
	197	* and turn every test's forge broken.
	198	*/
	199	readonly hostFree?: () => number;
	200	}
	201
	202	/** The host probe the memory need reads when none is given; none under `bun test`. */
	203	const hostFreeDefault: (() => number) \| undefined = process.env["NODE_ENV"] === "test" ? undefined : freemem;
	204
	205	const MIB = 1024 * 1024;
	206
	207	/** The `memory` need's sentence: the reading and the floor it is under. */
	208	export function memorySentence(freeBytes: number, floorMiB: number): string {
	209	return `Memory ${Math.floor(freeBytes / MIB)} MiB free, floor ${floorMiB} MiB.`;
191	210	}
192	211
193	212	/** The device a path lives on, read from it or its nearest parent that exists. */
		@@ -369,9 +388,11 @@ export type PodMemory = { readonly limitBytes: number; readonly usedBytes: numbe
369	388	export function memoryReading(
370	389	pod: PodMemory = { limitBytes: memoryLimitBytes(), usedBytes: podMemoryUsedBytes() },
371	390	rssBytes = process.memoryUsage().rss,
	391	hostFreeBytes = freemem(),
372	392	): Vitals["memory"] {
373	393	const used = pod.usedBytes ?? rssBytes;
374		return { rssBytes, freeBytes: Math.min(freemem(), Math.max(0, pod.limitBytes - used)), totalBytes: pod.limitBytes };
	394	const freeBytes = Math.min(hostFreeBytes, Math.max(0, pod.limitBytes - used));
	395	return { rssBytes, freeBytes, totalBytes: pod.limitBytes };
375	396	}
376	397
377	398	export function vitals(input: VitalsInput): Vitals {
		@@ -400,6 +421,13 @@ export function vitals(input: VitalsInput): Vitals {
400	421	degraded.push({ cause, sentence, since, href });
401	422	};
402	423
	424	// Free memory under the floor the forge keeps for itself: what it serves with is gone.
	425	const hostProbe = input.hostFree ?? hostFreeDefault;
	426	const hostFreeBytes = hostProbe === undefined ? freemem() : hostProbe();
	427	const memoryFloorMiB = limitValue(input.db, "forge.memory_floor_mb");
	428	if (hostProbe !== undefined && memoryFloorMiB > 0 && hostFreeBytes < memoryFloorMiB * MIB)
	429	need("broken", "memory", memorySentence(hostFreeBytes, memoryFloorMiB), undefined, "/settings/capacity");
	430
403	431	const fullAt = fullSince(input.dataDir);
404	432	if (fullAt !== undefined) need("broken", "disk-full", fullSentence(input.dataDir, true), fullAt, "/system");
405	433	const disk = (input.disk ?? readDisk)(input.dataDir);
		@@ -690,7 +718,7 @@ export function vitals(input: VitalsInput): Vitals {
690	718	todo,
691	719	degraded,
692	720	disk,
693		memory: memoryReading(input.podMemory),
	721	memory: memoryReading(input.podMemory, undefined, hostFreeBytes),
694	722	queue,
695	723	lane,
696	724	stuck,
		@@ -3,7 +3,7 @@
3	3	* now) or `todo` (would like done), and only `broken` makes ok false. This
4	4	* file is the guard on the broken list: a real failure filed as todo would
5	5	* answer ok:true and fail here. A crash loop is one: three unclean starts in
6		* ten minutes (#12285).
	6	* ten minutes (#12285). Free memory under the forge's floor is another (#12300).
7	7	*/
8	8	import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test";
9	9	import type { Database } from "bun:sqlite";
		@@ -21,7 +21,10 @@ import { health } from "../src/http/health.ts";
21	21	import { openDb } from "../src/kernel/db.ts";
22	22	import { ErrorLedger } from "../src/kernel/log.ts";
23	23	import { RefusedLedger } from "../src/kernel/internal_refused.ts";
24		import { vitals, type Vitals } from "../src/kernel/vitals.ts";
	24	import { memorySentence, vitals, type Vitals } from "../src/kernel/vitals.ts";
	25	import { limitValue } from "../src/settings/limits.ts";
	26	import { tool as healthTool } from "../src/http/mcp/tools/health.ts";
	27	import type { Ctx } from "../src/http/context.ts";
25	28	import { DB_FILE_CAP, dbFile } from "../src/system/db_size.ts";
26	29	import { quietLogs } from "./helpers/globals.ts";
27	30
		@@ -44,11 +47,14 @@ afterEach(() => {
44	47	rmSync(dataDir, { recursive: true, force: true });
45	48	});
46	49
47		const read = (disk = roomy): Vitals =>
	50	const MIB = 1024 * 1024;
	51	const read = (disk = roomy, hostFree = () => 1e15): Vitals =>
48	52	vitals({
49	53	db,
50	54	dataDir,
51	55	disk,
	56	hostFree,
	57	podMemory: { limitBytes: 1e15, usedBytes: 0 },
52	58	dbWalk: () => null,
53	59	signals: { errors: new ErrorLedger(), refused: new RefusedLedger() },
54	60	});
		@@ -77,6 +83,46 @@ describe("broken makes ok false", () => {
77	83	});
78	84	});
79	85
	86	describe("memory below its floor", () => {
	87	test("is broken while free memory is under the floor, and clears above it", () => {
	88	const floor = limitValue(db, "forge.memory_floor_mb");
	89	expect(floor).toBeGreaterThan(0);
	90	const low = read(roomy, () => (floor - 76) * MIB);
	91	const need = low.degraded.find((d) => d.cause === "memory")!;
	92	expect(need.sentence).toBe(`Memory ${floor - 76} MiB free, floor ${floor} MiB.`);
	93	expect(low.ok).toBe(false);
	94	expect(low.broken).toContain(need.sentence);
	95	expect(low.todo).not.toContain(need.sentence);
	96	expect(low.memory.freeBytes).toBe((floor - 76) * MIB);
	97	const back = read(roomy, () => floor * MIB);
	98	expect(back.degraded.find((d) => d.cause === "memory")).toBeUndefined();
	99	expect(back.ok).toBe(true);
	100	});
	101
	102	test("a floor of 0 keeps none, so nothing is under it", () => {
	103	db.run("INSERT INTO forge_setting (key, value, updated_by, updated_at) VALUES ('limit.forge.memory_floor_mb', '0', 'u1', '2026-01-01')");
	104	expect(read(roomy, () => 0).broken).toEqual([]);
	105	});
	106
	107	test("under bun test with no probe given, the host's own memory raises nothing, however high the floor", () => {
	108	db.run("INSERT INTO forge_setting (key, value, updated_by, updated_at) VALUES ('limit.forge.memory_floor_mb', '1048576', 'u1', '2026-01-01')");
	109	const v = vitals({
	110	db,
	111	dataDir,
	112	disk: roomy,
	113	podMemory: { limitBytes: 1e15, usedBytes: 0 },
	114	dbWalk: () => null,
	115	signals: { errors: new ErrorLedger(), refused: new RefusedLedger() },
	116	});
	117	expect(v.degraded.find((d) => d.cause === "memory")).toBeUndefined();
	118	expect(read(roomy, () => 0).degraded.find((d) => d.cause === "memory")).toBeDefined();
	119	});
	120
	121	test("the sentence names the reading and the floor", () => {
	122	expect(memorySentence(180 * MIB, 256)).toBe("Memory 180 MiB free, floor 256 MiB.");
	123	});
	124	});
	125
80	126	describe("todo alone keeps ok true", () => {
81	127	test("a database over 2 GiB", () => {
82	128	writeFileSync(dbFile(dataDir), "", { mode: 0o600 });
		@@ -114,6 +160,37 @@ describe("the health answer", () => {
114	160	health.draining = false;
115	161	expect((readHealth(app, admin) as Vitals).broken).not.toContain(DRAINING);
116	162	});
	163
	164	test("memory under its floor answers ok:false on the API and the MCP tool, ok:true once back", () => {
	165	const floor = limitValue(app.db, "forge.memory_floor_mb");
	166	let free = (floor - 1) * MIB;
	167	const hostFree = () => free;
	168	const mem = createApp({ dataDir: dir, port: 0, hostFree, disk: () => ({ freeBytes: 1e12, totalBytes: 1e12 }) });
	169	try {
	170	const sentence = memorySentence(free, floor);
	171	const api = readHealth(mem, admin) as Vitals;
	172	expect(api.ok).toBe(false);
	173	expect(api.broken).toContain(sentence);
	174	const url = new URL("http://localhost/mcp");
	175	const ctx: Ctx = {
	176	app: mem,
	177	request: new Request(url.href, { method: "POST" }),
	178	url,
	179	params: {},
	180	auth: { user: admin, scope: undefined, via: "session" },
	181	theme: undefined,
	182	};
	183	const mcp = healthTool.run(ctx, admin, {}) as { ok: true; value: Vitals };
	184	expect(mcp.value.ok).toBe(false);
	185	expect(mcp.value.broken).toContain(sentence);
	186	free = (floor + 1) * MIB;
	187	const again = readHealth(mem, admin) as Vitals;
	188	expect(again.broken).not.toContain(sentence);
	189	expect(again.degraded.some((d) => d.cause === "memory")).toBe(false);
	190	} finally {
	191	mem.db.close();
	192	}
	193	});
117	194	});
118	195
119	196	describe("a crash loop", () => {
		@@ -178,8 +178,8 @@ test("the number of queries does not grow with runs", () => {
178	178	// finished checks, the red mains holding pulls' checks, the tests red on many pulls, (the checks
179	179	// red on a full disk only while the disk is full), and the newest drill of every domain, the
180	180	// checks no red main holds, the week's flake list, and the database's daily size readings
181		// (#12010, at most 60 rows).
182		expect(before).toBe(11);
	181	// (#12010, at most 60 rows), and the memory floor the forge keeps for itself (#12300).
	182	expect(before).toBe(12);
183	183	});
184	184
185	185	test("errors are counted, bounded, redacted, and a burst is said", async () => {