5d1000f
@@ -15,6 +15,10 @@
keyboard. It does what the steward's `release.sh`, `verify-after.sh` and
   - no merge-queue gate is running;
   - no snapshot is being taken, unless it has run two hours and is stuck
     (a release drains the process the snapshot runs in; #9653);
   - the disk under the forge's data is less than 80% full (a release
     writes a new tree beside the live one; at 80% the tick says
     "the disk is N% full; releases pause at 80% until room is made" and
     cuts on the first tick with room; #11686);
   - 30 minutes have passed since the last cut;
   - no hold stands.
@@ -9,6 +9,7 @@
 *       ∧ no snapshot is being taken, unless it has run `SNAPSHOT_WAIT_MS`
 *       ∧ now − the last cut ≥ the minimum interval
 *       ∧ no hold stands (a failed verify's, or an owner's)
 *       ∧ the forge's disk is less than `RELEASE_DISK_USED_MAX` full
 *
 * The first rule that refuses names itself, so the record says why not.
 * The version is the newest tag's with its patch number plus one.
@@ -25,6 +26,14 @@
export const MIN_INTERVAL_MS = 30 * 60_000;
 */
export const SNAPSHOT_WAIT_MS = 2 * 3_600_000;
/**
 * A release writes a new tree and its install beside the live one, so at
 * this share of the forge's disk used no release is cut (#11686, item 6:
 * a full disk took the forge down once). The cut waits; nothing is held,
 * and the next tick with room cuts.
 */
export const RELEASE_DISK_USED_MAX = 0.8;
export interface CadenceInput {
  /** main's head commit. */
  readonly main: string;
@@ -44,6 +53,8 @@
export interface CadenceInput {
  readonly lastCutAt: number | undefined;
  /** The reason of a standing hold, when one stands. */
  readonly hold: string | undefined;
  /** The share of the forge's disk in use, 0..1; undefined when it could not be read. */
  readonly diskUsed?: number | undefined;
  readonly now: number;
  readonly minIntervalMs?: number;
}
@@ -69,6 +80,8 @@
export function decideCut(input: CadenceInput): Cadence {
  if (input.gating > 0) return no(`the merge queue is landing (${input.gating} gate${input.gating === 1 ? "" : "s"} running)`);
  if (input.snapshotMs !== undefined && input.snapshotMs < SNAPSHOT_WAIT_MS)
    return no(`a snapshot is being taken (${Math.floor(input.snapshotMs / 60_000)} min in); a release would cut it short`);
  if (input.diskUsed !== undefined && input.diskUsed >= RELEASE_DISK_USED_MAX)
    return no(`the disk is ${Math.round(input.diskUsed * 100)}% full; releases pause at ${RELEASE_DISK_USED_MAX * 100}% until room is made`);
  const wait = input.minIntervalMs ?? MIN_INTERVAL_MS;
  if (input.lastCutAt !== undefined && input.now - input.lastCutAt < wait)
    return no(`the last cut was ${Math.floor((input.now - input.lastCutAt) / 60_000)} min ago; the interval is ${wait / 60_000} min`);
@@ -56,6 +56,8 @@
export interface KeeperPorts {
  readonly gating: () => number;
  /** How long the snapshot being taken has run, ms; undefined when none is (#9653). */
  readonly snapshotting?: () => number | undefined;
  /** The share of the forge's disk in use, 0..1; undefined when unread (#11686). */
  readonly diskUsed?: () => number | undefined;
  readonly deployer: Deployer;
  /** Make the annotated tag `v<version>` on `sha` and send it where the release workflow starts; why not. */
  readonly tag: (version: string, sha: string) => Promise<string | undefined>;
@@ -213,6 +215,7 @@
export async function tickStep(p: KeeperPorts): Promise<Step> {
    inFlight: false,
    gating: p.gating(),
    snapshotMs: p.snapshotting?.(),
    diskUsed: p.diskUsed?.(),
    lastCutAt: last === undefined ? undefined : Date.parse(last.cutAt),
    hold: blocking(standingHolds(p.db)),
    now: p.now(),
@@ -35,6 +35,7 @@
import { kubeDeployer } from "./deploy.ts";
import { swapper } from "./swapper.ts";
import type { KeeperPorts } from "./keeper.ts";
import { timedGet, verifyUnderLoad } from "./verify.ts";
import { diskUnder } from "../spawn/disk_floor.ts";
const LABEL = /^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$/;
const OID = /^[0-9a-f]{40}$/;
@@ -123,6 +124,10 @@
export function keeperPorts(app: App, input: { forgeUrl: string; deployment: str
    },
    gating: gatesInFlight,
    snapshotting: () => snapshotsOf(app)?.takingMs(),
    diskUsed: () => {
      const disk = diskUnder(app.config.dataDir);
      return disk === undefined || !(disk.totalBytes > 0) ? undefined : 1 - disk.freeBytes / disk.totalBytes;
    },
    deployer,
    async tag(version, sha) {
      if (!VERSION.test(version) || !OID.test(sha)) return "not a version and a commit";
@@ -1,8 +1,10 @@
// When the release keeper cuts (#7557): each rule refuses on its own, and a cut is a tag, never a commit to main.
import { describe, expect, test } from "bun:test";
import { decideCut, MIN_INTERVAL_MS, nextVersion, SNAPSHOT_WAIT_MS, type CadenceInput } from "../src/keeper/cadence.ts";
import { decideCut, MIN_INTERVAL_MS, nextVersion, RELEASE_DISK_USED_MAX, SNAPSHOT_WAIT_MS, type CadenceInput } from "../src/keeper/cadence.ts";
import { tick } from "../src/keeper/keeper.ts";
import { listKeeperReleases, recordCut } from "../src/keeper/record.ts";
import { keeperPorts } from "../src/keeper/wire.ts";
import { provideDiskForTest } from "../src/kernel/disk_reading.ts";
import { A, B, keeperForge } from "./helpers/keeper.ts";
const NOW = Date.parse("2026-10-27T12:00:00.000Z");
@@ -56,6 +58,35 @@
describe("cadence", () => {
    expect(await tick({ ...f.ports, snapshotting: () => undefined })).toContain("cut 0.7.284");
  });
  test("at 80% of the disk used no release is cut; under it, or unread, the cut goes (#11686)", async () => {
    expect(RELEASE_DISK_USED_MAX).toBe(0.8);
    expect(decideCut({ ...ready, diskUsed: 0.8 })).toEqual({ cut: false, why: "the disk is 80% full; releases pause at 80% until room is made" });
    expect(decideCut({ ...ready, diskUsed: 0.93 }).cut).toBe(false);
    expect(decideCut({ ...ready, diskUsed: 0.79 })).toEqual({ cut: true, version: "0.7.284" });
    expect(decideCut({ ...ready, diskUsed: undefined })).toEqual({ cut: true, version: "0.7.284" });
    const f = await keeperForge();
    expect(await tick({ ...f.ports, diskUsed: () => 0.85 })).toBe("the disk is 85% full; releases pause at 80% until room is made");
    expect(f.world.tags).toEqual([]);
    expect(f.state.set).toEqual([]);
    expect(await tick({ ...f.ports, diskUsed: () => 0.7 })).toContain("cut 0.7.284");
  });
  test("the forge's keeper reads the disk under its data directory for the rule", async () => {
    const f = await keeperForge();
    const ports = keeperPorts(f.app, { forgeUrl: "http://forge.test", deployment: "forge", container: "forge", swap: true });
    if (typeof ports === "string") throw new Error(ports);
    const seen: string[] = [];
    provideDiskForTest((dir) => (seen.push(dir), { freeBytes: 150, totalBytes: 1000 }));
    try {
      expect(ports.diskUsed?.()).toBeCloseTo(0.85);
      expect(seen).toEqual([f.app.config.dataDir]);
      provideDiskForTest(() => ({ freeBytes: 0, totalBytes: 0 }));
      expect(ports.diskUsed?.()).toBeUndefined();
    } finally {
      provideDiskForTest(() => ({ freeBytes: 900 * 1024 ** 3, totalBytes: 1000 * 1024 ** 3 }));
    }
  });
  test("a tag the keeper rolled back is settled: the fix after it may be cut", () => {
    expect(decideCut({ ...ready, newest: { version: "0.7.284", sha: A }, live: "0.7.283", settled: true })).toEqual({ cut: true, version: "0.7.285" });
  });
		@@ -15,6 +15,10 @@ keyboard. It does what the steward's `release.sh`, `verify-after.sh` and
15	15	- no merge-queue gate is running;
16	16	- no snapshot is being taken, unless it has run two hours and is stuck
17	17	(a release drains the process the snapshot runs in; #9653);
	18	- the disk under the forge's data is less than 80% full (a release
	19	writes a new tree beside the live one; at 80% the tick says
	20	"the disk is N% full; releases pause at 80% until room is made" and
	21	cuts on the first tick with room; #11686);
18	22	- 30 minutes have passed since the last cut;
19	23	- no hold stands.
20	24
		@@ -9,6 +9,7 @@
9	9	* ∧ no snapshot is being taken, unless it has run `SNAPSHOT_WAIT_MS`
10	10	* ∧ now − the last cut ≥ the minimum interval
11	11	* ∧ no hold stands (a failed verify's, or an owner's)
	12	* ∧ the forge's disk is less than `RELEASE_DISK_USED_MAX` full
12	13	*
13	14	* The first rule that refuses names itself, so the record says why not.
14	15	* The version is the newest tag's with its patch number plus one.
		@@ -25,6 +26,14 @@ export const MIN_INTERVAL_MS = 30 * 60_000;
25	26	*/
26	27	export const SNAPSHOT_WAIT_MS = 2 * 3_600_000;
27	28
	29	/**
	30	* A release writes a new tree and its install beside the live one, so at
	31	* this share of the forge's disk used no release is cut (#11686, item 6:
	32	* a full disk took the forge down once). The cut waits; nothing is held,
	33	* and the next tick with room cuts.
	34	*/
	35	export const RELEASE_DISK_USED_MAX = 0.8;
	36
28	37	export interface CadenceInput {
29	38	/** main's head commit. */
30	39	readonly main: string;
		@@ -44,6 +53,8 @@ export interface CadenceInput {
44	53	readonly lastCutAt: number \| undefined;
45	54	/** The reason of a standing hold, when one stands. */
46	55	readonly hold: string \| undefined;
	56	/** The share of the forge's disk in use, 0..1; undefined when it could not be read. */
	57	readonly diskUsed?: number \| undefined;
47	58	readonly now: number;
48	59	readonly minIntervalMs?: number;
49	60	}
		@@ -69,6 +80,8 @@ export function decideCut(input: CadenceInput): Cadence {
69	80	if (input.gating > 0) return no(`the merge queue is landing (${input.gating} gate${input.gating === 1 ? "" : "s"} running)`);
70	81	if (input.snapshotMs !== undefined && input.snapshotMs < SNAPSHOT_WAIT_MS)
71	82	return no(`a snapshot is being taken (${Math.floor(input.snapshotMs / 60_000)} min in); a release would cut it short`);
	83	if (input.diskUsed !== undefined && input.diskUsed >= RELEASE_DISK_USED_MAX)
	84	return no(`the disk is ${Math.round(input.diskUsed * 100)}% full; releases pause at ${RELEASE_DISK_USED_MAX * 100}% until room is made`);
72	85	const wait = input.minIntervalMs ?? MIN_INTERVAL_MS;
73	86	if (input.lastCutAt !== undefined && input.now - input.lastCutAt < wait)
74	87	return no(`the last cut was ${Math.floor((input.now - input.lastCutAt) / 60_000)} min ago; the interval is ${wait / 60_000} min`);
		@@ -56,6 +56,8 @@ export interface KeeperPorts {
56	56	readonly gating: () => number;
57	57	/** How long the snapshot being taken has run, ms; undefined when none is (#9653). */
58	58	readonly snapshotting?: () => number \| undefined;
	59	/** The share of the forge's disk in use, 0..1; undefined when unread (#11686). */
	60	readonly diskUsed?: () => number \| undefined;
59	61	readonly deployer: Deployer;
60	62	/** Make the annotated tag `v<version>` on `sha` and send it where the release workflow starts; why not. */
61	63	readonly tag: (version: string, sha: string) => Promise<string \| undefined>;
		@@ -213,6 +215,7 @@ export async function tickStep(p: KeeperPorts): Promise<Step> {
213	215	inFlight: false,
214	216	gating: p.gating(),
215	217	snapshotMs: p.snapshotting?.(),
	218	diskUsed: p.diskUsed?.(),
216	219	lastCutAt: last === undefined ? undefined : Date.parse(last.cutAt),
217	220	hold: blocking(standingHolds(p.db)),
218	221	now: p.now(),
		@@ -35,6 +35,7 @@ import { kubeDeployer } from "./deploy.ts";
35	35	import { swapper } from "./swapper.ts";
36	36	import type { KeeperPorts } from "./keeper.ts";
37	37	import { timedGet, verifyUnderLoad } from "./verify.ts";
	38	import { diskUnder } from "../spawn/disk_floor.ts";
38	39
39	40	const LABEL = /^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$/;
40	41	const OID = /^[0-9a-f]{40}$/;
		@@ -123,6 +124,10 @@ export function keeperPorts(app: App, input: { forgeUrl: string; deployment: str
123	124	},
124	125	gating: gatesInFlight,
125	126	snapshotting: () => snapshotsOf(app)?.takingMs(),
	127	diskUsed: () => {
	128	const disk = diskUnder(app.config.dataDir);
	129	return disk === undefined \|\| !(disk.totalBytes > 0) ? undefined : 1 - disk.freeBytes / disk.totalBytes;
	130	},
126	131	deployer,
127	132	async tag(version, sha) {
128	133	if (!VERSION.test(version) \|\| !OID.test(sha)) return "not a version and a commit";
		@@ -1,8 +1,10 @@
1	1	// When the release keeper cuts (#7557): each rule refuses on its own, and a cut is a tag, never a commit to main.
2	2	import { describe, expect, test } from "bun:test";
3		import { decideCut, MIN_INTERVAL_MS, nextVersion, SNAPSHOT_WAIT_MS, type CadenceInput } from "../src/keeper/cadence.ts";
	3	import { decideCut, MIN_INTERVAL_MS, nextVersion, RELEASE_DISK_USED_MAX, SNAPSHOT_WAIT_MS, type CadenceInput } from "../src/keeper/cadence.ts";
4	4	import { tick } from "../src/keeper/keeper.ts";
5	5	import { listKeeperReleases, recordCut } from "../src/keeper/record.ts";
	6	import { keeperPorts } from "../src/keeper/wire.ts";
	7	import { provideDiskForTest } from "../src/kernel/disk_reading.ts";
6	8	import { A, B, keeperForge } from "./helpers/keeper.ts";
7	9
8	10	const NOW = Date.parse("2026-10-27T12:00:00.000Z");
		@@ -56,6 +58,35 @@ describe("cadence", () => {
56	58	expect(await tick({ ...f.ports, snapshotting: () => undefined })).toContain("cut 0.7.284");
57	59	});
58	60
	61	test("at 80% of the disk used no release is cut; under it, or unread, the cut goes (#11686)", async () => {
	62	expect(RELEASE_DISK_USED_MAX).toBe(0.8);
	63	expect(decideCut({ ...ready, diskUsed: 0.8 })).toEqual({ cut: false, why: "the disk is 80% full; releases pause at 80% until room is made" });
	64	expect(decideCut({ ...ready, diskUsed: 0.93 }).cut).toBe(false);
	65	expect(decideCut({ ...ready, diskUsed: 0.79 })).toEqual({ cut: true, version: "0.7.284" });
	66	expect(decideCut({ ...ready, diskUsed: undefined })).toEqual({ cut: true, version: "0.7.284" });
	67	const f = await keeperForge();
	68	expect(await tick({ ...f.ports, diskUsed: () => 0.85 })).toBe("the disk is 85% full; releases pause at 80% until room is made");
	69	expect(f.world.tags).toEqual([]);
	70	expect(f.state.set).toEqual([]);
	71	expect(await tick({ ...f.ports, diskUsed: () => 0.7 })).toContain("cut 0.7.284");
	72	});
	73
	74	test("the forge's keeper reads the disk under its data directory for the rule", async () => {
	75	const f = await keeperForge();
	76	const ports = keeperPorts(f.app, { forgeUrl: "http://forge.test", deployment: "forge", container: "forge", swap: true });
	77	if (typeof ports === "string") throw new Error(ports);
	78	const seen: string[] = [];
	79	provideDiskForTest((dir) => (seen.push(dir), { freeBytes: 150, totalBytes: 1000 }));
	80	try {
	81	expect(ports.diskUsed?.()).toBeCloseTo(0.85);
	82	expect(seen).toEqual([f.app.config.dataDir]);
	83	provideDiskForTest(() => ({ freeBytes: 0, totalBytes: 0 }));
	84	expect(ports.diskUsed?.()).toBeUndefined();
	85	} finally {
	86	provideDiskForTest(() => ({ freeBytes: 900 * 1024 ** 3, totalBytes: 1000 * 1024 ** 3 }));
	87	}
	88	});
	89
59	90	test("a tag the keeper rolled back is settled: the fix after it may be cut", () => {
60	91	expect(decideCut({ ...ready, newest: { version: "0.7.284", sha: A }, live: "0.7.283", settled: true })).toEqual({ cut: true, version: "0.7.285" });
61	92	});