5ffe00c · 68c0ae6
@@ -28,7 +28,7 @@
import { STALE_AFTER_MS } from "../runs/lease.ts";
import { ownRepoPath } from "./own_repo.ts";
import { BUDGETS } from "./budgets.ts";
import { openDbFiles } from "./file_modes.ts";
import { dbFile, dbGrowth, dbSizeNeed, fileBytes } from "../system/db_size.ts";
import { dbFile, dbGrowth, dbSize, dbSizeNeed, fileBytes, type DbWalk } from "../system/db_size.ts";
import { backupVerdict, isWatched } from "../snapshot/bucket.ts";
import { lastDrill } from "../snapshot/drills.ts";
import { lastMirror, mirrorNeed } from "../snapshot/mirror.ts";
@@ -141,6 +141,11 @@
export interface VitalsInput {
  readonly disk?: (dir: string) => DiskReading;
  /** The free-disk share below which the disk is a need; `DISK_FREE_MIN` unless a test says. */
  readonly diskFreeMin?: number;
  /**
   * The last walk of the database file, asked for only while a db-size need
   * fires; `dbSize.last` unless a test gives one.
   */
  readonly dbWalk?: (db: Database, dataDir: string) => DbWalk | null;
  /** One forge's own signals (`App.signals`). */
  readonly signals: { errors: ErrorLedger; refused: RefusedLedger };
  /** Where snapshots go: the directory, when the store is one, and whether an off-box mirror copies them. */
@@ -370,9 +375,13 @@
export function vitals(input: VitalsInput): Vitals {
  for (const file of openDbFiles(input.dataDir))
    need(`file:${file.name}`, file.sentence, undefined, "/settings/capacity");
  // A database past the 2 GiB Bun maps, or growing faster than 64 MiB a day, is said (#12010).
  // A database past the 2 GiB Bun maps, or growing faster than 64 MiB a day, is said (#12010),
  // with what fills it from the last walk once there is one (#12051): the sentence is the
  // Sentinel issue's evidence, so it names the tables. The walk is only asked for then.
  const dbBytes = fileBytes(dbFile(input.dataDir));
  const dbBig = dbSizeNeed(dbBytes, dbGrowth(input.db, dbBytes, now));
  const dbGrows = dbGrowth(input.db, dbBytes, now);
  let dbBig = dbSizeNeed(dbBytes, dbGrows);
  if (dbBig !== undefined) dbBig = dbSizeNeed(dbBytes, dbGrows, (input.dbWalk ?? dbSize.last)(input.db, input.dataDir));
  if (dbBig !== undefined) need(dbBig.cause, dbBig.sentence, undefined, "/system");
  // Snapshots on the volume they copy, and no mirror, share its one point of failure.
@@ -331,22 +331,52 @@
export function dbGrowth(db: Database, bytesNow: number, now: number = Date.now(
  };
}
/** One sentence for the vitals when the file is past the map's cap or grows too fast. */
/** Bytes said short: GiB with one decimal from 1 GiB, else whole MiB. */
const sizeSaid = (n: number): string => (n >= 1024 ** 3 ? `${(n / 1024 ** 3).toFixed(1)} GiB` : `${Math.round(n / 1024 ** 2)} MiB`);
/** Rows said short: 812, 41k, 3.2M. */
const rowsSaid = (n: number): string =>
  n >= 1e6 ? `${(n / 1e6).toFixed(1)}M` : n >= 1e3 ? `${Math.round(n / 1e3)}k` : String(n);
/** How many tables the alarm names. */
export const NEED_TABLES = 5;
/**
 * What fills the file, from a walk (#12051): its five largest tables, their
 * bytes and rows, and the freelist, as of the walk's own time, since the walk
 * may be up to an hour old.
 */
export function walkSaid(walk: DbWalk): string {
  const tables = walk.tables
    .slice(0, NEED_TABLES)
    .map((t) => `${t.name} ${sizeSaid(t.bytes)} (${rowsSaid(t.rows)} rows)`)
    .join(", ");
  return `as of ${walk.at.slice(11, 16)}Z its largest tables are ${tables}; ${sizeSaid(walk.freeBytes)} free.`;
}
/**
 * One sentence for the vitals when the file is past the map's cap or grows
 * too fast. With a walk it names what fills the file, so the alarm (and the
 * Sentinel issue that quotes it) carries its own evidence; without one it
 * points at /system.
 */
export function dbSizeNeed(
  fileBytes: number,
  growth: Pick<DbGrowth, "perDay" | "days">,
  walk: DbWalk | null = null,
): { cause: "db-size" | "db-growth"; sentence: string } | undefined {
  const gib = (n: number): string => (n / 1024 ** 3).toFixed(1);
  const mib = (n: number): string => String(Math.round(n / 1024 ** 2));
  const tail = walk === null || walk.tables.length === 0 ? "; /system names its largest tables." : `: ${walkSaid(walk)}`;
  if (fileBytes > DB_FILE_CAP)
    return {
      cause: "db-size",
      sentence: `The database is ${gib(fileBytes)} GiB, past the 2 GiB Bun maps; /system names its largest tables.`,
      sentence: `The database is ${gib(fileBytes)} GiB, past the 2 GiB Bun maps${tail}`,
    };
  if (growth.perDay !== null && growth.perDay > DB_GROWTH_MAX)
    return {
      cause: "db-growth",
      sentence: `The database grows ${mib(growth.perDay)} MiB a day over ${growth.days} days, past the ${mib(DB_GROWTH_MAX)} MiB a day worth a look; /system names its largest tables.`,
      sentence: `The database grows ${mib(growth.perDay)} MiB a day over ${growth.days} days, past the ${mib(DB_GROWTH_MAX)} MiB a day worth a look${tail}`,
    };
  return undefined;
}
@@ -392,6 +422,14 @@
export function dbSizeCache(
    return { fileBytes: file, walBytes: fileBytes(`${dbFile(dataDir)}-wal`), growth: dbGrowth(db, file, clock()), walk: w };
  };
  return {
    /**
     * The last walk alone, for the vitals' db-size alarm (#12051); a new walk
     * starts behind it when that is an hour old, never waited on.
     */
    last(db: Database, dataDir: string, log?: Warn): DbWalk | null {
      if (due(dataDir)) void refresh(db, dataDir, log);
      return current(dataDir);
    },
    /** The size now with the last walk; a new walk starts behind it when that is an hour old. */
    peek(db: Database, dataDir: string, log?: Warn): DbSize {
      if (due(dataDir)) void refresh(db, dataDir, log);
@@ -188,6 +188,62 @@
describe("growth and the need", () => {
    });
  });
  const MiB = 1024 ** 2;
  const GiB = 1024 ** 3;
  const fixedWalk = {
    at: "2026-11-13T13:00:41.000Z",
    pageSize: 4096,
    freeBytes: 0.4 * GiB,
    tables: [
      { name: "run_log", bytes: 2.1 * GiB, rows: 3_200_000 },
      { name: "event", bytes: 812 * MiB, rows: 41_300 },
      { name: "check_run", bytes: 300 * MiB, rows: 9_000 },
      { name: "comment", bytes: 120 * MiB, rows: 812 },
      { name: "issue", bytes: 40 * MiB, rows: 12_000 },
      { name: "user", bytes: 1 * MiB, rows: 30 },
    ],
    indexes: [],
  };
  test("with a walk, the need names the five largest tables, their bytes and rows, and the freelist (#12051)", () => {
    expect(dbSizeNeed(4.6 * GiB, { perDay: null, days: 0 }, fixedWalk)).toEqual({
      cause: "db-size",
      sentence:
        "The database is 4.6 GiB, past the 2 GiB Bun maps: as of 13:00Z its largest tables are run_log 2.1 GiB (3.2M rows), event 812 MiB (41k rows), check_run 300 MiB (9k rows), comment 120 MiB (812 rows), issue 40 MiB (12k rows); 410 MiB free.",
    });
    expect(dbSizeNeed(GiB, { perDay: DB_GROWTH_MAX + 1, days: 3.5 }, fixedWalk)?.sentence).toBe(
      "The database grows 64 MiB a day over 3.5 days, past the 64 MiB a day worth a look: as of 13:00Z its largest tables are run_log 2.1 GiB (3.2M rows), event 812 MiB (41k rows), check_run 300 MiB (9k rows), comment 120 MiB (812 rows), issue 40 MiB (12k rows); 410 MiB free.",
    );
    // Quiet stays quiet, walk or no walk.
    expect(dbSizeNeed(GiB, { perDay: null, days: 0 }, fixedWalk)).toBeUndefined();
  });
  test("without a walk, the need keeps pointing at /system", () => {
    for (const w of [null, { ...fixedWalk, tables: [] }])
      expect(dbSizeNeed(4.6 * GiB, { perDay: null, days: 0 }, w)?.sentence).toBe(
        "The database is 4.6 GiB, past the 2 GiB Bun maps; /system names its largest tables.",
      );
  });
  test("the vitals' db need carries the walk, the sentence the Sentinel issue quotes", () => {
    const now = Date.now();
    let asked = 0;
    const dbWalk = () => {
      asked++;
      return fixedWalk;
    };
    const cause = () => vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy, now, dbWalk }).degraded.find((d) => d.cause.startsWith("db-"));
    app.db.run("DELETE FROM db_size_day");
    expect(cause()).toBeUndefined();
    // No need, no walk asked for.
    expect(asked).toBe(0);
    const then = now - 2 * DAY;
    app.db.query("INSERT INTO db_size_day (day, at, bytes) VALUES (?, ?, ?)").run(new Date(then).toISOString().slice(0, 10), new Date(then).toISOString(), -300 * 1024 ** 2);
    expect(cause()?.sentence).toContain(": as of 13:00Z its largest tables are run_log 2.1 GiB (3.2M rows)");
    expect(asked).toBe(1);
    app.db.run("DELETE FROM db_size_day");
  });
  test("the vitals carry the growth need to /system, and drop it when growth is quiet", () => {
    const now = Date.now();
    const cause = () => vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy, now }).degraded.find((d) => d.cause.startsWith("db-"));
@@ -223,6 +279,22 @@
describe("growth and the need", () => {
    expect(cache.peek(app.db, dataDir).walk).toBe(read.walk);
  });
  test("last gives the cached walk without waiting, and starts the first one", async () => {
    let walks = 0;
    let finish = (): void => {};
    const cache = dbSizeCache(async (db, d) => {
      walks++;
      await new Promise<void>((r) => (finish = r));
      return measureDbWalk(db, d);
    });
    expect(cache.last(app.db, dataDir)).toBeNull();
    expect(walks).toBe(1);
    finish();
    const read = await cache.read(app.db, dataDir, 5_000);
    expect(cache.last(app.db, dataDir)).toBe(read.walk);
    expect(walks).toBe(1);
  });
  test("a walk that fails is said in the log, not swallowed, and the page keeps no walk", async () => {
    const said: { message: string; fields: Record<string, unknown> | undefined }[] = [];
    const log = { warn: (message: string, fields?: Record<string, unknown>) => void said.push({ message, fields }) };
		@@ -28,7 +28,7 @@ import { STALE_AFTER_MS } from "../runs/lease.ts";
28	28	import { ownRepoPath } from "./own_repo.ts";
29	29	import { BUDGETS } from "./budgets.ts";
30	30	import { openDbFiles } from "./file_modes.ts";
31		import { dbFile, dbGrowth, dbSizeNeed, fileBytes } from "../system/db_size.ts";
	31	import { dbFile, dbGrowth, dbSize, dbSizeNeed, fileBytes, type DbWalk } from "../system/db_size.ts";
32	32	import { backupVerdict, isWatched } from "../snapshot/bucket.ts";
33	33	import { lastDrill } from "../snapshot/drills.ts";
34	34	import { lastMirror, mirrorNeed } from "../snapshot/mirror.ts";
		@@ -141,6 +141,11 @@ export interface VitalsInput {
141	141	readonly disk?: (dir: string) => DiskReading;
142	142	/** The free-disk share below which the disk is a need; `DISK_FREE_MIN` unless a test says. */
143	143	readonly diskFreeMin?: number;
	144	/**
	145	* The last walk of the database file, asked for only while a db-size need
	146	* fires; `dbSize.last` unless a test gives one.
	147	*/
	148	readonly dbWalk?: (db: Database, dataDir: string) => DbWalk \| null;
144	149	/** One forge's own signals (`App.signals`). */
145	150	readonly signals: { errors: ErrorLedger; refused: RefusedLedger };
146	151	/** Where snapshots go: the directory, when the store is one, and whether an off-box mirror copies them. */
		@@ -370,9 +375,13 @@ export function vitals(input: VitalsInput): Vitals {
370	375	for (const file of openDbFiles(input.dataDir))
371	376	need(`file:${file.name}`, file.sentence, undefined, "/settings/capacity");
372	377
373		// A database past the 2 GiB Bun maps, or growing faster than 64 MiB a day, is said (#12010).
	378	// A database past the 2 GiB Bun maps, or growing faster than 64 MiB a day, is said (#12010),
	379	// with what fills it from the last walk once there is one (#12051): the sentence is the
	380	// Sentinel issue's evidence, so it names the tables. The walk is only asked for then.
374	381	const dbBytes = fileBytes(dbFile(input.dataDir));
375		const dbBig = dbSizeNeed(dbBytes, dbGrowth(input.db, dbBytes, now));
	382	const dbGrows = dbGrowth(input.db, dbBytes, now);
	383	let dbBig = dbSizeNeed(dbBytes, dbGrows);
	384	if (dbBig !== undefined) dbBig = dbSizeNeed(dbBytes, dbGrows, (input.dbWalk ?? dbSize.last)(input.db, input.dataDir));
376	385	if (dbBig !== undefined) need(dbBig.cause, dbBig.sentence, undefined, "/system");
377	386
378	387	// Snapshots on the volume they copy, and no mirror, share its one point of failure.
		@@ -331,22 +331,52 @@ export function dbGrowth(db: Database, bytesNow: number, now: number = Date.now(
331	331	};
332	332	}
333	333
334		/** One sentence for the vitals when the file is past the map's cap or grows too fast. */
	334	/** Bytes said short: GiB with one decimal from 1 GiB, else whole MiB. */
	335	const sizeSaid = (n: number): string => (n >= 1024 3 ? `${(n / 1024 3).toFixed(1)} GiB` : `${Math.round(n / 1024 ** 2)} MiB`);
	336
	337	/** Rows said short: 812, 41k, 3.2M. */
	338	const rowsSaid = (n: number): string =>
	339	n >= 1e6 ? `${(n / 1e6).toFixed(1)}M` : n >= 1e3 ? `${Math.round(n / 1e3)}k` : String(n);
	340
	341	/** How many tables the alarm names. */
	342	export const NEED_TABLES = 5;
	343
	344	/**
	345	* What fills the file, from a walk (#12051): its five largest tables, their
	346	* bytes and rows, and the freelist, as of the walk's own time, since the walk
	347	* may be up to an hour old.
	348	*/
	349	export function walkSaid(walk: DbWalk): string {
	350	const tables = walk.tables
	351	.slice(0, NEED_TABLES)
	352	.map((t) => `${t.name} ${sizeSaid(t.bytes)} (${rowsSaid(t.rows)} rows)`)
	353	.join(", ");
	354	return `as of ${walk.at.slice(11, 16)}Z its largest tables are ${tables}; ${sizeSaid(walk.freeBytes)} free.`;
	355	}
	356
	357	/**
	358	* One sentence for the vitals when the file is past the map's cap or grows
	359	* too fast. With a walk it names what fills the file, so the alarm (and the
	360	* Sentinel issue that quotes it) carries its own evidence; without one it
	361	* points at /system.
	362	*/
335	363	export function dbSizeNeed(
336	364	fileBytes: number,
337	365	growth: Pick<DbGrowth, "perDay" \| "days">,
	366	walk: DbWalk \| null = null,
338	367	): { cause: "db-size" \| "db-growth"; sentence: string } \| undefined {
339	368	const gib = (n: number): string => (n / 1024 ** 3).toFixed(1);
340	369	const mib = (n: number): string => String(Math.round(n / 1024 ** 2));
	370	const tail = walk === null \|\| walk.tables.length === 0 ? "; /system names its largest tables." : `: ${walkSaid(walk)}`;
341	371	if (fileBytes > DB_FILE_CAP)
342	372	return {
343	373	cause: "db-size",
344		sentence: `The database is ${gib(fileBytes)} GiB, past the 2 GiB Bun maps; /system names its largest tables.`,
	374	sentence: `The database is ${gib(fileBytes)} GiB, past the 2 GiB Bun maps${tail}`,
345	375	};
346	376	if (growth.perDay !== null && growth.perDay > DB_GROWTH_MAX)
347	377	return {
348	378	cause: "db-growth",
349		sentence: `The database grows ${mib(growth.perDay)} MiB a day over ${growth.days} days, past the ${mib(DB_GROWTH_MAX)} MiB a day worth a look; /system names its largest tables.`,
	379	sentence: `The database grows ${mib(growth.perDay)} MiB a day over ${growth.days} days, past the ${mib(DB_GROWTH_MAX)} MiB a day worth a look${tail}`,
350	380	};
351	381	return undefined;
352	382	}
		@@ -392,6 +422,14 @@ export function dbSizeCache(
392	422	return { fileBytes: file, walBytes: fileBytes(`${dbFile(dataDir)}-wal`), growth: dbGrowth(db, file, clock()), walk: w };
393	423	};
394	424	return {
	425	/**
	426	* The last walk alone, for the vitals' db-size alarm (#12051); a new walk
	427	* starts behind it when that is an hour old, never waited on.
	428	*/
	429	last(db: Database, dataDir: string, log?: Warn): DbWalk \| null {
	430	if (due(dataDir)) void refresh(db, dataDir, log);
	431	return current(dataDir);
	432	},
395	433	/** The size now with the last walk; a new walk starts behind it when that is an hour old. */
396	434	peek(db: Database, dataDir: string, log?: Warn): DbSize {
397	435	if (due(dataDir)) void refresh(db, dataDir, log);
		@@ -188,6 +188,62 @@ describe("growth and the need", () => {
188	188	});
189	189	});
190	190
	191	const MiB = 1024 ** 2;
	192	const GiB = 1024 ** 3;
	193	const fixedWalk = {
	194	at: "2026-11-13T13:00:41.000Z",
	195	pageSize: 4096,
	196	freeBytes: 0.4 * GiB,
	197	tables: [
	198	{ name: "run_log", bytes: 2.1 * GiB, rows: 3_200_000 },
	199	{ name: "event", bytes: 812 * MiB, rows: 41_300 },
	200	{ name: "check_run", bytes: 300 * MiB, rows: 9_000 },
	201	{ name: "comment", bytes: 120 * MiB, rows: 812 },
	202	{ name: "issue", bytes: 40 * MiB, rows: 12_000 },
	203	{ name: "user", bytes: 1 * MiB, rows: 30 },
	204	],
	205	indexes: [],
	206	};
	207
	208	test("with a walk, the need names the five largest tables, their bytes and rows, and the freelist (#12051)", () => {
	209	expect(dbSizeNeed(4.6 * GiB, { perDay: null, days: 0 }, fixedWalk)).toEqual({
	210	cause: "db-size",
	211	sentence:
	212	"The database is 4.6 GiB, past the 2 GiB Bun maps: as of 13:00Z its largest tables are run_log 2.1 GiB (3.2M rows), event 812 MiB (41k rows), check_run 300 MiB (9k rows), comment 120 MiB (812 rows), issue 40 MiB (12k rows); 410 MiB free.",
	213	});
	214	expect(dbSizeNeed(GiB, { perDay: DB_GROWTH_MAX + 1, days: 3.5 }, fixedWalk)?.sentence).toBe(
	215	"The database grows 64 MiB a day over 3.5 days, past the 64 MiB a day worth a look: as of 13:00Z its largest tables are run_log 2.1 GiB (3.2M rows), event 812 MiB (41k rows), check_run 300 MiB (9k rows), comment 120 MiB (812 rows), issue 40 MiB (12k rows); 410 MiB free.",
	216	);
	217	// Quiet stays quiet, walk or no walk.
	218	expect(dbSizeNeed(GiB, { perDay: null, days: 0 }, fixedWalk)).toBeUndefined();
	219	});
	220
	221	test("without a walk, the need keeps pointing at /system", () => {
	222	for (const w of [null, { ...fixedWalk, tables: [] }])
	223	expect(dbSizeNeed(4.6 * GiB, { perDay: null, days: 0 }, w)?.sentence).toBe(
	224	"The database is 4.6 GiB, past the 2 GiB Bun maps; /system names its largest tables.",
	225	);
	226	});
	227
	228	test("the vitals' db need carries the walk, the sentence the Sentinel issue quotes", () => {
	229	const now = Date.now();
	230	let asked = 0;
	231	const dbWalk = () => {
	232	asked++;
	233	return fixedWalk;
	234	};
	235	const cause = () => vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy, now, dbWalk }).degraded.find((d) => d.cause.startsWith("db-"));
	236	app.db.run("DELETE FROM db_size_day");
	237	expect(cause()).toBeUndefined();
	238	// No need, no walk asked for.
	239	expect(asked).toBe(0);
	240	const then = now - 2 * DAY;
	241	app.db.query("INSERT INTO db_size_day (day, at, bytes) VALUES (?, ?, ?)").run(new Date(then).toISOString().slice(0, 10), new Date(then).toISOString(), -300 * 1024 ** 2);
	242	expect(cause()?.sentence).toContain(": as of 13:00Z its largest tables are run_log 2.1 GiB (3.2M rows)");
	243	expect(asked).toBe(1);
	244	app.db.run("DELETE FROM db_size_day");
	245	});
	246
191	247	test("the vitals carry the growth need to /system, and drop it when growth is quiet", () => {
192	248	const now = Date.now();
193	249	const cause = () => vitals({ db: app.db, signals: app.signals, dataDir, disk: roomy, now }).degraded.find((d) => d.cause.startsWith("db-"));
		@@ -223,6 +279,22 @@ describe("growth and the need", () => {
223	279	expect(cache.peek(app.db, dataDir).walk).toBe(read.walk);
224	280	});
225	281
	282	test("last gives the cached walk without waiting, and starts the first one", async () => {
	283	let walks = 0;
	284	let finish = (): void => {};
	285	const cache = dbSizeCache(async (db, d) => {
	286	walks++;
	287	await new Promise<void>((r) => (finish = r));
	288	return measureDbWalk(db, d);
	289	});
	290	expect(cache.last(app.db, dataDir)).toBeNull();
	291	expect(walks).toBe(1);
	292	finish();
	293	const read = await cache.read(app.db, dataDir, 5_000);
	294	expect(cache.last(app.db, dataDir)).toBe(read.walk);
	295	expect(walks).toBe(1);
	296	});
	297
226	298	test("a walk that fails is said in the log, not swallowed, and the page keeps no walk", async () => {
227	299	const said: { message: string; fields: Record<string, unknown> \| undefined }[] = [];
228	300	const log = { warn: (message: string, fields?: Record<string, unknown>) => void said.push({ message, fields }) };