Public: anyone can read this; the crew can change it.

Release keeper: measure the write pause per release and let it set the cadence #13201

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

torque/forge:torque/7836-release-keeper-measure-the-write-pause-p → main 11 commits fork 5 threads · 5 unresolved

  • Merged. torque/7836-release-keeper-measure-the-write-pause-p was merged into main as 2ef01ec.

Review

Sign in to review this change.
17 files changed +292 −59
docs/features/release-keeper.md +1 −1
@@ -19,7 +19,7 @@ keyboard. It does what the steward's `release.sh`, `verify-after.sh` and
1919 writes a new tree beside the live one; at 80% the tick says
2020 "the disk is N% full; releases pause at 80% until room is made" and
2121 cuts on the first tick with room; #11686);
22 - 30 minutes have passed since the last cut;
22 - 30 minutes have passed since the last cut, or ten times the last release's write pause when that is longer (the new forge's hold on its queue work at boot, kept on the release and shown on /releases, in `release_keeper` and on /maintenance; a 720 s pause waits 120 min; unmeasured leaves 30 min);
2323 - no hold stands.
2424
2525 Before anything moves, it records the known-good version and the exact
scripts/unused-exports-allow.json +1
@@ -125,6 +125,7 @@
125125 "src/http/strangers.ts setStrangerLimits",
126126 "src/http/stripe_hook.ts refusedCount",
127127 "src/http/stripe_hook.ts setStripeHookCaps",
128 "src/keeper/pause.ts forgetWritePause",
128129 "src/kernel/destroy_reach.ts unitsOf",
129130 "src/kernel/db.ts provideEmptyImagesForTest",
130131 "src/kernel/disk_reading.ts provideDiskForTest",
src/core/migration-pins/20261115110000-keeper-write-pause.pin added +1
@@ -0,0 +1,1 @@
1 7f952df7e994536aea30692dd1b2a5d90dae30903a6cf2451deff0869c1d8c41
src/core/migrations/20261115110000-keeper-write-pause.ts added +6
@@ -0,0 +1,6 @@
1 // A roll should not pause queue work (#7836): the new forge times how long it held its queue work
2 // at boot while the old one still wrote, and the keeper keeps it on the release. Null for a release
3 // that did not measure it. Additive: one new nullable column.
4 export default `
5 ALTER TABLE keeper_release ADD COLUMN write_pause_ms INTEGER;
6 `;
src/http/mcp/tools/release_keeper.ts +1 −1
@@ -4,7 +4,7 @@ import { bad, good, type Tool } from "../shared.ts";
44
55 export const tool: Tool = {
66 name: "release_keeper",
7 doc: "`{}`: what the release keeper (#7557) did: its releases newest first (version, commit (short sha), stage — rolling, verifying, verified or rolled_back —, the known-good version it would roll back to, roll probes and non-200s, probes and pages under load, worst ms, seconds dark, reason, landedToLiveSeconds — from the merge of its pull to the new version answering, null until it has (#8042), interruptedRuns — the crew runs the drained process still had running at the switch, null when not counted (#11917)) and its holds (kind failed or owner, version, reason, probe numbers, the pull that said `release hold N`, lifted); `blocked` says what stops the next cut; `heartbeat` is the release tick's last report (at, outcome cut/skipped/failed, reason, silent when over an hour old, #11507); for whoever may read /releases",
7 doc: "`{}`: what the release keeper (#7557) did: its releases newest first (version, commit (short sha), stage — rolling, verifying, verified or rolled_back —, the known-good version it would roll back to, roll probes and non-200s, probes and pages under load, worst ms, seconds dark, reason, landedToLiveSeconds — from the merge of its pull to the new version answering, null until it has (#8042), interruptedRuns — the crew runs the drained process still had running at the switch, null when not counted (#11917), writePauseMs — how long the new forge held its queue work at boot, null when not measured; the next cut waits 10× it, #7836) and its holds (kind failed or owner, version, reason, probe numbers, the pull that said `release hold N`, lifted); `blocked` says what stops the next cut; `heartbeat` is the release tick's last report (at, outcome cut/skipped/failed, reason, silent when over an hour old, #11507); for whoever may read /releases",
88 description: "The release keeper's releases, holds and what blocks the next cut.",
99 inputSchema: { type: "object", properties: {} },
1010 writes: false,
src/keeper/cadence.ts +27 −6
@@ -8,7 +8,8 @@
88 * ∧ no merge-queue gate is running (the queue is not mid-landing)
99 * ∧ no snapshot is being taken, unless it has run `SNAPSHOT_WAIT_MS`
1010 * ∧ no crew run is running, unless the keeper has waited on them `CREW_WAIT_MS`
11 * ∧ now − the last cut ≥ the minimum interval
11 * ∧ now − the last cut ≥ the minimum interval, or `PAUSE_CADENCE_FACTOR` × the last
12 * release's write pause when that is longer (#7836)
1213 * ∧ no hold stands (a failed verify's, or an owner's)
1314 * ∧ the forge's disk is less than `RELEASE_DISK_USED_MAX` full
1415 *
@@ -19,6 +20,14 @@
1920 /** Two releases are at least this far apart: 30 minutes. */
2021 export const MIN_INTERVAL_MS = 30 * 60_000;
2122
23 import { pauseWords } from "./pause.ts";
24
25 /**
26 * The next cut waits this many times the last release's write pause (#7836): 12 minutes held is
27 * 120 minutes before the next roll, so queue work is held at most a tenth of the time.
28 */
29 export const PAUSE_CADENCE_FACTOR = 10;
30
2231 /**
2332 * A release drains the process a snapshot runs in, so a cut waits for the
2433 * snapshot (#9653: swaps every few minutes cut the nightly short all
@@ -67,11 +76,17 @@ export interface CadenceInput {
6776 readonly hold: string | undefined;
6877 /** The share of the forge's disk in use, 0..1; undefined when it could not be read. */
6978 readonly diskUsed?: number | undefined;
79 /**
80 * How long the last release's new forge held its queue work, ms (#7836); undefined when it was
81 * not measured. A roll should not pause queue work: the cut waits ten times the pause.
82 */
83 readonly writePauseMs?: number | undefined;
7084 readonly now: number;
7185 readonly minIntervalMs?: number;
7286 /**
7387 * A person asked for a release now (#11839): the merge queue, a snapshot and the
74 * spacing no longer hold the cut, nor do running crew runs. A release in flight, a hold and a full disk still do.
88 * spacing no longer hold the cut, nor do running crew runs. A release in flight, a hold and a
89 * full disk still do.
7590 */
7691 readonly hotfix?: boolean;
7792 }
@@ -80,7 +95,9 @@ export type Cadence = { readonly cut: true; readonly version: string } | { reado
8095
8196 const SEMVER = /^(\d+)\.(\d+)\.(\d+)$/;
8297
83 /** The next patch version: `0.7.283` → `0.7.284`; undefined when `version` is not major.minor.patch. */
98 /**
99 * The next patch version: `0.7.283` → `0.7.284`; undefined when `version` is not major.minor.patch.
100 */
84101 export function nextVersion(version: string): string | undefined {
85102 const m = SEMVER.exec(version);
86103 if (m === null) return undefined;
@@ -106,9 +123,13 @@ export function decideCut(input: CadenceInput): Cadence {
106123 }
107124 if (input.diskUsed !== undefined && input.diskUsed >= RELEASE_DISK_USED_MAX)
108125 return no(`the disk is ${Math.round(input.diskUsed * 100)}% full; releases pause at ${RELEASE_DISK_USED_MAX * 100}% until room is made`);
109 const wait = input.minIntervalMs ?? MIN_INTERVAL_MS;
110 if (!hotfix && input.lastCutAt !== undefined && input.now - input.lastCutAt < wait)
111 return no(`the last cut was ${Math.floor((input.now - input.lastCutAt) / 60_000)} min ago; the interval is ${wait / 60_000} min`);
126 const floor = input.minIntervalMs ?? MIN_INTERVAL_MS;
127 const pause = input.writePauseMs !== undefined && input.writePauseMs > 0 ? input.writePauseMs * PAUSE_CADENCE_FACTOR : 0;
128 const wait = Math.max(floor, pause);
129 if (!hotfix && input.lastCutAt !== undefined && input.now - input.lastCutAt < wait) {
130 const ago = `the last cut was ${Math.floor((input.now - input.lastCutAt) / 60_000)} min ago; the interval is ${wait / 60_000} min`;
131 return no(pause > floor ? `${ago}, ${PAUSE_CADENCE_FACTOR} × the last release's write pause of ${pauseWords(input.writePauseMs!)}` : ago);
132 }
112133 const version = nextVersion(input.newest.version);
113134 if (version === undefined) return no(`${input.newest.version} is not major.minor.patch`);
114135 return { cut: true, version };
src/keeper/keeper.ts +32 −10
@@ -26,8 +26,8 @@
2626 * `release.verified_live`, `release.rolled_back`, `release.held`.
2727 *
2828 * Guardrail: the keeper's only writes are a tag, the Deployment's image,
29 * its own rows, issues, and the removal of its own old releases. It holds no account that may land a pull or
30 * push main, and it has no code path to either.
29 * its own rows, issues, and the removal of its own old releases. It holds no account that may land
30 * a pull or push main, and it has no code path to either.
3131 */
3232
3333 import type { Database } from "bun:sqlite";
@@ -44,7 +44,10 @@ import { rollBackToNamed } from "./rollback.ts";
4444 import type { Probe, Tally, Verdict } from "./verify.ts";
4545 import type { Step } from "./beat.ts";
4646
47 /** A release not live this long after its cut never rolled: 3 h (the tag's check may take 2 h, the build some more). */
47 /**
48 * A release not live this long after its cut never rolled: 3 h (the tag's check may take 2 h, the
49 * build some more).
50 */
4851 export const ROLL_DEADLINE_MS = 3 * 3_600_000;
4952
5053 export interface KeeperPorts {
@@ -66,15 +69,25 @@ export interface KeeperPorts {
6669 /** The share of the forge's disk in use, 0..1; undefined when unread (#11686). */
6770 readonly diskUsed?: () => number | undefined;
6871 readonly deployer: Deployer;
69 /** Make the annotated tag `v<version>` on `sha` and send it where the release workflow starts; why not. */
72 /**
73 * Make the annotated tag `v<version>` on `sha` and send it where the release workflow starts; why
74 * not.
75 */
7076 readonly tag: (version: string, sha: string) => Promise<string | undefined>;
7177 /** One `/healthz` probe through the public URL. */
7278 readonly probe: () => Promise<Probe>;
7379 /** Ten minutes under page load. */
7480 readonly verify: (onProbe: (t: Tally) => void) => Promise<Verdict>;
7581 readonly minIntervalMs?: number;
76 /** The worst event-loop stall in this process's first five minutes, ms; none when unset (#8848). */
82 /**
83 * The worst event-loop stall in this process's first five minutes, ms; none when unset (#8848).
84 */
7785 readonly bootStallMs?: () => number;
86 /**
87 * How long this process held its queue work at boot for the old forge, ms; none when unset
88 * (#7836).
89 */
90 readonly writePauseMs?: () => number | undefined;
7891 }
7992
8093 /**
@@ -139,7 +152,10 @@ function fileDark(p: KeeperPorts, r: KeeperRelease): number | undefined {
139152 return made.value.number;
140153 }
141154
142 /** The release each keeper is verifying; keyed by its ports, so one keeper (or one test) never sees another's. */
155 /**
156 * The release each keeper is verifying; keyed by its ports, so one keeper (or one test) never sees
157 * another's.
158 */
143159 const verifying = new WeakMap<KeeperPorts, number>();
144160
145161 /** The verify of an in-flight release this process is the new version of, to its end. */
@@ -183,7 +199,8 @@ async function prune(p: KeeperPorts, r: KeeperRelease): Promise<{ pruned: string
183199 const keep: string[] = [];
184200 for (const ref of refs) {
185201 const id = store.idOf(ref);
186 // Fail closed: a live, running or known-good release the store cannot name could be the one pruned.
202 // Fail closed: a live, running or known-good release the store cannot name could be the one
203 // pruned.
187204 if (id === undefined) return { pruned: [], pruneError: `${ref} names no stored release` };
188205 keep.push(id);
189206 }
@@ -215,7 +232,10 @@ function holdStall(p: KeeperPorts, r: KeeperRelease): string | undefined {
215232 return `${reason}; hold ${hold.id} set`;
216233 }
217234
218 /** Hold with the numbers, then set the named known-good image. The hold first: this process may be the one replaced. */
235 /**
236 * Hold with the numbers, then set the named known-good image. The hold first: this process may be
237 * the one replaced.
238 */
219239 async function rollBack(p: KeeperPorts, r: KeeperRelease, reason: string): Promise<string> {
220240 const hold = setHold(p.db, { kind: "failed", version: r.version, reason, probes: r.probes, badInRow: r.badInRow, worstMs: r.worstMs, at: p.now() });
221241 p.bus.append({ kind: "release.held", repoId: p.repo.id, payload: { version: r.version, hold: hold.id, reason, probes: r.probes, badInRow: r.badInRow, worstMs: r.worstMs } });
@@ -244,7 +264,7 @@ export async function tickStep(p: KeeperPorts): Promise<Step> {
244264 const flying = inFlight(p.db);
245265 if (flying !== undefined) {
246266 if (p.live() === flying.version) {
247 markLive(p.db, flying.id, p.now());
267 markLive(p.db, flying.id, p.now(), p.writePauseMs?.());
248268 return verifyAndDecide(p, getKeeperRelease(p.db, flying.id)!);
249269 }
250270 // A cut recorded but never tagged (the process died between the two): tag it now, or drop it.
@@ -276,6 +296,7 @@ export async function tickStep(p: KeeperPorts): Promise<Step> {
276296 crewSince: crewSince(p.db, crewRuns, p.now()),
277297 diskUsed: p.diskUsed?.(),
278298 lastCutAt: last === undefined ? undefined : Date.parse(last.cutAt),
299 writePauseMs: last?.writePauseMs ?? undefined,
279300 hold: blocking(standingHolds(p.db)),
280301 now: p.now(),
281302 ...(asked === undefined ? {} : { hotfix: true }),
@@ -283,7 +304,8 @@ export async function tickStep(p: KeeperPorts): Promise<Step> {
283304 });
284305 if (!decision.cut) return { line: decision.why, outcome: "skipped" };
285306 if (main === undefined) return { line: "main is unreadable", outcome: "failed" };
286 // A tree the deployer cannot release (dependencies changed, preflight red) is not cut: no tag, no row.
307 // A tree the deployer cannot release (dependencies changed, preflight red) is not cut: no tag, no
308 // row.
287309 const refused = await p.deployer.admits?.(main);
288310 if (refused !== undefined) return { line: `not cut: ${refused}`, outcome: "failed" };
289311 const image = await p.deployer.image();
src/keeper/pause.ts added +28
@@ -0,0 +1,28 @@
1 /**
2 * The write pause of this process's boot (#7836): how long the new forge held
3 * its queue work while the forge it replaced still wrote to the same SQLite
4 * file. `main.ts` times `waitForPeer` and notes it here; the keeper, in this
5 * same process, keeps it on the release when the verify begins.
6 */
7
8 let paused: number | undefined;
9
10 /** Note the pause, in ms; a value that is not a finite, non-negative number is ignored. */
11 export function noteWritePause(ms: number): void {
12 if (Number.isFinite(ms) && ms >= 0) paused = Math.round(ms);
13 }
14
15 /** The pause this process measured at boot, ms; undefined when it did not. */
16 export function writePause(): number | undefined {
17 return paused;
18 }
19
20 /** A pause in plain words: whole seconds, `720 s`. */
21 export function pauseWords(ms: number): string {
22 return `${Math.round(ms / 1000)} s`;
23 }
24
25 /** Forget it (tests). */
26 export function forgetWritePause(): void {
27 paused = undefined;
28 }
src/keeper/record.ts +30 −7
@@ -45,6 +45,11 @@ export interface KeeperRelease {
4545 readonly rollbackMs: number | null;
4646 /** Crew runs running in the drained process at the switch (#11917); null when not counted. */
4747 readonly interruptedRuns: number | null;
48 /**
49 * How long the new forge held its queue work at boot while the old one still wrote, ms (#7836);
50 * null when not measured.
51 */
52 readonly writePauseMs: number | null;
4853 /** Why a person cut this release now, outside the spacing; empty for a cut on cadence. */
4954 readonly hotfix: string;
5055 }
@@ -73,6 +78,7 @@ interface Row {
7378 switch_ms: number | null;
7479 rollback_ms: number | null;
7580 interrupted_runs: number | null;
81 write_pause_ms: number | null;
7682 hotfix: string;
7783 }
7884
@@ -99,12 +105,16 @@ const toRelease = (r: Row): KeeperRelease => ({
99105 switchMs: r.switch_ms,
100106 rollbackMs: r.rollback_ms ?? null,
101107 interruptedRuns: r.interrupted_runs,
108 writePauseMs: r.write_pause_ms ?? null,
102109 hotfix: r.hotfix,
103110 });
104111
105112 const iso = (ms: number) => new Date(ms).toISOString();
106113
107 /** The cut, written before the tag is pushed: the known-good image is on record before anything can need it. */
114 /**
115 * The cut, written before the tag is pushed: the known-good image is on record before anything can
116 * need it.
117 */
108118 export function recordCut(
109119 db: Database,
110120 input: { version: string; sha: string; knownGood: { version: string; image: string }; at: number; hotfix?: string },
@@ -119,12 +129,18 @@ export function recordCut(
119129 );
120130 }
121131
122 /** A cut whose tag was refused never happened: its row goes, so the cadence is not left waiting on it. */
132 /**
133 * A cut whose tag was refused never happened: its row goes, so the cadence is not left waiting on
134 * it.
135 */
123136 export function dropCut(db: Database, id: number): void {
124137 db.query(`DELETE FROM keeper_release WHERE id = ?1 AND stage = 'rolling' AND live_at IS NULL`).run(id);
125138 }
126139
127 /** What the deployer timed while promoting: kept on the release still rolling, so a rollback's swap never overwrites them. */
140 /**
141 * What the deployer timed while promoting: kept on the release still rolling, so a rollback's swap
142 * never overwrites them.
143 */
128144 export function keepTimings(db: Database, id: number, t: { buildMs: number; proofMs: number; switchMs: number }): void {
129145 db.query(`UPDATE keeper_release SET build_ms = ?2, proof_ms = ?3, switch_ms = ?4 WHERE id = ?1 AND stage = 'rolling'`).run(
130146 id,
@@ -143,7 +159,10 @@ export function keepRollback(db: Database, id: number, ms: number): void {
143159 db.query(`UPDATE keeper_release SET rollback_ms = ?2 WHERE id = ?1 AND stage IN ('rolling', 'verifying')`).run(id, n);
144160 }
145161
146 /** How many crew runs the old process still had running when the switch drained it: kept on the release still rolling. */
162 /**
163 * How many crew runs the old process still had running when the switch drained it: kept on the
164 * release still rolling.
165 */
147166 export function keepInterrupted(db: Database, id: number, runs: number): void {
148167 const n = Number.isFinite(runs) ? Math.max(0, Math.round(runs)) : 0;
149168 db.query(`UPDATE keeper_release SET interrupted_runs = ?2 WHERE id = ?1 AND stage = 'rolling'`).run(id, n);
@@ -157,9 +176,13 @@ export function countRollProbe(db: Database, id: number, ok: boolean): void {
157176 ).run(id, ok ? 0 : 1);
158177 }
159178
160 /** The new version answers: the verify begins. Only one forge moves it, so a second caller gets false. */
161 export function markLive(db: Database, id: number, at: number): boolean {
162 return db.query(`UPDATE keeper_release SET stage = 'verifying', live_at = ?2 WHERE id = ?1 AND stage = 'rolling'`).run(id, iso(at))
179 /**
180 * The new version answers: the verify begins, with the write pause its process measured at boot
181 * (#7836). Only one forge moves it, so a second caller gets false.
182 */
183 export function markLive(db: Database, id: number, at: number, writePauseMs?: number): boolean {
184 const pause = writePauseMs !== undefined && Number.isFinite(writePauseMs) ? Math.max(0, Math.round(writePauseMs)) : null;
185 return db.query(`UPDATE keeper_release SET stage = 'verifying', live_at = ?2, write_pause_ms = ?3 WHERE id = ?1 AND stage = 'rolling'`).run(id, iso(at), pause)
163186 .changes === 1;
164187 }
165188
src/keeper/wire.ts +8 −4
@@ -27,6 +27,7 @@ import { runGit } from "../git/env.ts";
2727 import { repoDir } from "../git/repos.ts";
2828 import { ownProbeHeaders } from "../kernel/probe.ts";
2929 import { bootStall } from "../perf/live.ts";
30 import { writePause } from "./pause.ts";
3031 import { running } from "../kernel/version.ts";
3132 import { compareVersions, readTags, recordFor } from "../release/shipped.ts";
3233 import { checkoutRelease, preflightRelease } from "../release/checkout.ts";
@@ -83,8 +84,9 @@ export function loadPages(repo: Pick<Repo, "ownerName" | "name">): string[] {
8384 }
8485
8586 /**
86 * What the swap reports just before this process drains (#11839, #11917): the stage timings, and the crew
87 * runs still running here, the ones the switch interrupts, kept on the release still rolling.
87 * What the swap reports just before this process drains (#11839, #11917): the stage timings, and
88 * the crew runs still running here, the ones the switch interrupts, kept on the release still
89 * rolling.
8890 */
8991 export function switched(db: Database, runs: () => number): (t: Timing) => void {
9092 return (t) => {
@@ -137,6 +139,7 @@ export function keeperPorts(app: App, input: { forgeUrl: string; deployment: str
137139 now: () => Date.now(),
138140 live: () => running().version,
139141 bootStallMs: () => bootStall.worst(),
142 writePauseMs: () => writePause(),
140143 async main() {
141144 const head = await runGit(["rev-parse", "--verify", "--quiet", `refs/heads/${repo.defaultBranch}^{commit}`], { cwd: dir });
142145 return head.ok && OID.test(head.value) ? head.value : undefined;
@@ -159,8 +162,9 @@ export function keeperPorts(app: App, input: { forgeUrl: string; deployment: str
159162 deployer,
160163 async tag(version, sha) {
161164 if (!VERSION.test(version) || !OID.test(sha)) return "not a version and a commit";
162 // The record `scripts/cut-release.ts` writes. The migrations are this process's: the keeper cuts only
163 // when main has moved past a live tag, and each forge cuts the code it was built from at the earliest.
165 // The record `scripts/cut-release.ts` writes. The migrations are this process's: the keeper
166 // cuts only when main has moved past a live tag, and each forge cuts the code it was built
167 // from at the earliest.
164168 const record = JSON.stringify(recordFor(version, migrationFiles, NUMBERED));
165169 const made = await runGit(["tag", "--annotate", "--file=-", `v${version}`, sha], {
166170 cwd: dir,
src/main.ts +41 −24
@@ -142,6 +142,7 @@ import { startKeeping } from "./flow/keep.ts";
142142 import { keepForgePages } from "./flow/pages.ts";
143143 import { timeStatements } from "./kernel/slow_sql.ts";
144144 import { waitForPeer } from "./runs/peer.ts";
145 import { noteWritePause } from "./keeper/pause.ts";
145146 import { closeStaleSessions } from "./machines/shell.ts";
146147
147148 // `forge hook <name>` re-enters here from a git hook stub. It relays to the
@@ -383,11 +384,12 @@ loadTrustedHosts(app.db);
383384 // and with nothing asked: one thread, as ever.
384385 const engineOn = engineShape.mode === "worker";
385386 const tail = engineOn ? tailEvents(app.db, app.bus) : undefined;
386 // A write of the engine's is a few milliseconds. This thread waits for one with every request behind it, so
387 // it waits seconds, not the fifteen of a forge whose only other writer is itself: a stuck engine is ended (`engine/host.ts`).
388 // One side checkpoints the log, and it is the engine's (`engine/duties.ts`): a commit of this thread that crossed
389 // the threshold paid for the checkpoint, fsync and all, with every request behind it.
390 // Both are put back if the engine's Worker cannot be had (`engine/run.ts`).
387 // A write of the engine's is a few milliseconds. This thread waits for one with every request
388 // behind it, so it waits seconds, not the fifteen of a forge whose only other writer is itself: a
389 // stuck engine is ended (`engine/host.ts`). One side checkpoints the log, and it is the engine's
390 // (`engine/duties.ts`): a commit of this thread that crossed the threshold paid for the checkpoint,
391 // fsync and all, with every request behind it. Both are put back if the engine's Worker cannot be
392 // had (`engine/run.ts`).
391393 const loneWriter = engineOn ? tuneForSplit(app.db) : () => {};
392394 await refreshCatalogQuietly(app);
393395 if (readOnly) {
@@ -438,9 +440,10 @@ const appDomain = appsOn
438440 })
439441 : undefined;
440442 if (appDomain?.ok === true) holdAppHosts(app, appDomain.domain);
441 // What the first visitors would pay for, read before the listener opens: while this boots the previous
442 // release still answers, so the half second each of these costs on a cold thread is nobody's wait. Each
443 // reads in slices on turns of its own; none of them is needed for the forge to be up.
443 // What the first visitors would pay for, read before the listener opens: while this boots the
444 // previous release still answers, so the half second each of these costs on a cold thread is
445 // nobody's wait. Each reads in slices on turns of its own; none of them is needed for the forge to
446 // be up.
444447 await warmFlow(app);
445448 await warmPlans(app);
446449 await warmDisplays(app).catch(() => undefined);
@@ -449,7 +452,8 @@ markReady();
449452 // From here the flow's forge-wide readings are read once and shared for half a minute
450453 // (`flow/kept.ts`).
451454 keepFlowReadings(KEPT_MS);
452 // The home's picture is read by a worker from here, so the first page finds it (web/pages/home_flow.ts).
455 // The home's picture is read by a worker from here, so the first page finds it
456 // (web/pages/home_flow.ts).
453457 warmHome(app);
454458 warmBoards(app);
455459 // The pictures the home, `/boards`, `/flow` and the front pages of the forge's own repositories
@@ -554,7 +558,11 @@ const leaveEarly = (): void => {
554558 };
555559 const earlySignals = ["SIGINT", "SIGTERM", "SIGHUP"] as const;
556560 for (const signal of earlySignals) process.once(signal, leaveEarly);
561 const peerWaitStarted = performance.now();
557562 await waitForPeer(app.db, app.log);
563 // How long this forge held its queue work for the old one (#7836): the keeper keeps it on the
564 // release.
565 noteWritePause(performance.now() - peerWaitStarted);
558566 for (const signal of earlySignals) process.off(signal, leaveEarly);
559567 // No shell outlives its forge: a row a stop or crash left open is closed. Only
560568 // once the other forge has let go: its shells are live until then, and a write
@@ -623,9 +631,10 @@ const snapshots = startSnapshots(
623631 // What the machine uses (boards/sampler.ts): the pod, each cell and the
624632 // node, every 15 s into a ring the `machine` board source reads.
625633 const sampler = startSampler(app, {});
626 // What has left its window goes a bounded slice at a time on a pace (`system/retention.ts`). The drains no
627 // page reads (machine samples, wake-up events, the old deploy ledger) are this thread's always, kept out of
628 // the write clock; the age rule and the deploy backfill are the engine's when it has a thread of its own.
634 // What has left its window goes a bounded slice at a time on a pace (`system/retention.ts`). The
635 // drains no page reads (machine samples, wake-up events, the old deploy ledger) are this thread's
636 // always, kept out of the write clock; the age rule and the deploy backfill are the engine's when
637 // it has a thread of its own.
629638 const stopRetention = startRetention(app, { local: true, engine: !engineOn });
630639 const memorySampler = startMemorySampler(app);
631640 // The starting boards, once each (actions/boards.ts).
@@ -680,9 +689,9 @@ if (crew !== undefined) {
680689 // (crew/preview.ts).
681690 const previewForge = (process.env.FORGE_APP_ENV ?? "").trim().toLowerCase() === "preview";
682691 void (previewForge ? Promise.resolve() : keepSandboxing(app)).finally(() => bootHostedChildren(app));
683 // The pipeline answer and every repository's display read (#8978, #11007) were filled here, behind the
684 // listener; they are read before it opens now (above).
685 // The release probe's search, grepped once behind the listener so the first visit reads the cache (#11059).
692 // The pipeline answer and every repository's display read (#8978, #11007) were filled here, behind
693 // the listener; they are read before it opens now (above). The release probe's search, grepped once
694 // behind the listener so the first visit reads the cache (#11059).
686695 void warmSearch(app);
687696 // A spawned child's first boot: its direction, opened once for its planner
688697 // (spawn/first-boot.ts). A forge that was not spawned is untouched.
@@ -701,7 +710,8 @@ await seedPreview(app)
701710 const heartbeatTimer = startHeartbeat(app);
702711 // ...and looks upstream once an hour (spawn/upstream.ts).
703712 const upstreamTimer = startUpstreamWatch(app);
704 // ...and this forge asks every child's public addresses what a visitor would get, every few minutes (spawn/reach.ts).
713 // ...and this forge asks every child's public addresses what a visitor would get, every few minutes
714 // (spawn/reach.ts).
705715 const reachTimer = startReachWatch(app, undefined, (name) => !childAwake(app, name));
706716 // Prompts from Instagram (docs/APPS.md): once a day, for each repository
707717 // that connected an account, the top comment on its latest Reel is told to
@@ -817,7 +827,8 @@ const followTimer = setInterval(
817827 );
818828 // An app landing on this forge is told to the children awake here that follow it at once.
819829 const landingsWatch = watchLandings(app);
820 // A child born before its apps were linked is offered the link, and a child offered one takes it: at boot, then every tick.
830 // A child born before its apps were linked is offered the link, and a child offered one takes it:
831 // at boot, then every tick.
821832 void followTick(app, true).catch((error) => app.log.error("could not link the apps", { error: String(error) }));
822833 const staleTimer = setInterval(() => void during("asks", tickAsks, undefined, app), 120_000);
823834 // A desk note whose time is up goes off the home line at once, over the stream.
@@ -864,9 +875,11 @@ const processTick = (): void => {
864875 }
865876 };
866877 // The clog watch (#6524, #6757) reads the flow every minute; its state is the database's.
867 // With an engine thread both are its duties (`engine/duties.ts`), reading what this thread observed from rows.
878 // With an engine thread both are its duties (`engine/duties.ts`), reading what this thread observed
879 // from rows.
868880 const stopClogTick = engineOn ? () => {} : startClogTick(app.db, app.bus, (error) => app.log.warn("clog tick failed", { error: String(error) }), 60_000);
869 // The throughput keeper (#7645) names what binds landings each minute and lifts what is its to lift.
881 // The throughput keeper (#7645) names what binds landings each minute and lifts what is its to
882 // lift.
870883 const stopKeeper = engineOn ? () => {} : startKeeper(app, () => sweepAssigned(app));
871884 // A forge that went down prepared for maintenance comes back by itself. The
872885 // listener is up (`serve` above) and the checks service started; the tunnel
@@ -874,7 +887,8 @@ const stopKeeper = engineOn ? () => {} : startKeeper(app, () => sweepAssigned(ap
874887 void resumeAtBoot(app, undefined, { forgeUrl, runner: checks?.lane }).catch((error) =>
875888 app.log.warn("resume at boot failed", { error: String(error) }),
876889 );
877 // Code scanning's nightly pass (#9593): each hour, the repositories whose last pass is a day old get secrets, advisories and static analysis over their default branch.
890 // Code scanning's nightly pass (#9593): each hour, the repositories whose last pass is a day old
891 // get secrets, advisories and static analysis over their default branch.
878892 const nightlyScanTimer = setInterval(() => {
879893 void during("nightly-scan", async () => {
880894 for (const id of nightlyDue(app.db, new Date(Date.now() - 24 * 3_600_000).toISOString(), 25)) {
@@ -893,7 +907,8 @@ const ledgerGitTimer = readOnly
893907 );
894908 const processTimer = setInterval(() => during("processes", processTick, undefined, app), 60_000);
895909 during("processes", processTick, undefined, app);
896 // A new version proves itself against the public URL; a failed one again, 5 min after and backing off to hourly (release/prove.ts).
910 // A new version proves itself against the public URL; a failed one again, 5 min after and backing
911 // off to hourly (release/prove.ts).
897912 proveRunning({
898913 dataDir: app.config.dataDir,
899914 version: running().version,
@@ -918,7 +933,8 @@ proveRunning({
918933 }, app.log);
919934 // The release tags of the forge's own repository are its shipped releases (release/shipped.ts);
920935 // production behind the newest for over an hour is a card (release/behind.ts).
921 // A FORGE_REPO that is not owner/name is dropped by releaseEnv; say so, or the list stops at the frozen one unexplained.
936 // A FORGE_REPO that is not owner/name is dropped by releaseEnv; say so, or the list stops at the
937 // frozen one unexplained.
922938 if ((process.env.FORGE_REPO ?? "") !== "" && !app.config.forgeRepo)
923939 app.log.warn("FORGE_REPO is not owner/name; release tags are not read", { value: process.env.FORGE_REPO });
924940 if (app.config.forgeRepo) {
@@ -936,8 +952,9 @@ if (app.config.forgeRepo) {
936952 void during("landings", () => readLandings(app), undefined, app);
937953 setInterval(() => void during("landings", () => readLandings(app), undefined, app), 60_000);
938954 }
939 // The release keeper (#7557): cut on cadence, promote, verify under page load, roll back to the named
940 // known-good image and hold. Off unless FORGE_RELEASE_KEEPER=on in a cluster (keeper/wire.ts).
955 // The release keeper (#7557): cut on cadence, promote, verify under page load, roll back to the
956 // named known-good image and hold. Off unless FORGE_RELEASE_KEEPER=on in a cluster
957 // (keeper/wire.ts).
941958 const keeperOn = keeperEnv(process.env);
942959 if (keeperOn.on) {
943960 const ports = keeperPorts(
src/tui/screens/keeper.ts +7 −2
@@ -15,6 +15,7 @@ import { neverTickedLine } from "../../keeper/at_boot.ts";
1515 import { beatLine, lastBeat } from "../../keeper/beat.ts";
1616 import { stallWords } from "../../perf/boot_stall.ts";
1717 import { blocking, holdAs, liftAs, standingHolds } from "../../keeper/hold.ts";
18 import { pauseWords } from "../../keeper/pause.ts";
1819 import { secondsWords, withLandedToLive, type LandedRelease } from "../../keeper/landed.ts";
1920 import { releaseNowAs, pendingNow } from "../../keeper/now.ts";
2021 import { listKeeperReleases, type Stage } from "../../keeper/record.ts";
@@ -38,7 +39,9 @@ const STAGE: Record<Stage, { label: string; tone: Tone }> = {
3839 export interface KeeperRead {
3940 /** What holds the next cut, or the line that says nothing does. */
4041 readonly blocked: string;
41 /** The release tick's last heartbeat as a sentence, said plainly when there is none or it is old. */
42 /**
43 * The release tick's last heartbeat as a sentence, said plainly when there is none or it is old.
44 */
4245 readonly heartbeat: string;
4346 /** Main green and ahead of production, and for how long; absent when it is not. */
4447 readonly ahead?: string;
@@ -79,6 +82,7 @@ export function keeperRows(releases: readonly LandedRelease[]): ListRow[] {
7982 `worst ${r.worstMs} ms`,
8083 r.worstStallMs === 0 ? "" : `boot stall ${stallWords(r.worstStallMs, releases[i + 1]?.worstStallMs)}`,
8184 `${r.secondsDark} s dark`,
outdated

Same unit: print 'write pause 720 s' not '720000 ms', matching the cadence reason; update the test's expectation.

85 r.writePauseMs === null ? "" : `write pause ${pauseWords(r.writePauseMs)}`,
8286 ),
8387 }));
8488 }
@@ -117,7 +121,8 @@ export function keeperScreen(f: Session, ctx: Ctx): Screen {
117121 emptyState(s, area, c, { title: "Not yours to see", body: "The release keeper is for the forge's administrators and its repository's readers." });
118122 return;
119123 }
120 // What holds the cut ends with what to land ("release hold N"): wrapped, up to three lines, never cut short of it.
124 // What holds the cut ends with what to land ("release hold N"): wrapped, up to three lines,
125 // never cut short of it.
121126 const lines = [
122127 ...wrap(read.blocked, area.w).slice(0, 3),
123128 ...wrap(read.heartbeat, area.w).slice(0, 2),
src/web/pages/keeper-panel.ts +4 −2
@@ -12,6 +12,7 @@ import { neverTickedLine } from "../../keeper/at_boot.ts";
1212 import { beatFact, lastBeat } from "../../keeper/beat.ts";
1313 import { blocking, MAX_REASON, standingHolds } from "../../keeper/hold.ts";
1414 import { pendingNow } from "../../keeper/now.ts";
15 import { pauseWords } from "../../keeper/pause.ts";
1516 import { secondsWords, stagesLine, withLandedToLive } from "../../keeper/landed.ts";
1617 import { listKeeperReleases, type Stage } from "../../keeper/record.ts";
1718 import { stallWords } from "../../perf/boot_stall.ts";
@@ -54,7 +55,8 @@ export function keeperPanel(db: Database, mayHold = false, now: Date = new Date(
5455 <td>${r.worstMs} ms</td>
5556 <td>${stallWords(r.worstStallMs, releases[i + 1]?.worstStallMs)}</td>
5657 <td>${r.secondsDark} s</td>
outdated

Principle 3 and 8: a tenth column in a sticky-lead table; and '720000 ms' is hard to read. Show seconds ('720 s'), consistent with the 'Dark' column, or put it in the notes row rather than widening the table.

57 </tr>${when(notes(r) !== "", () => html`<tr><td colspan="9" class="small muted">${notes(r)}</td></tr>`)}`;
58 <td>${r.writePauseMs === null ? "–" : pauseWords(r.writePauseMs)}</td>
59 </tr>${when(notes(r) !== "", () => html`<tr><td colspan="10" class="small muted">${notes(r)}</td></tr>`)}`;
5860 });
5961 return html`<section class="settings-block" id="keeper">
6062 <h2>Release keeper</h2>
@@ -65,7 +67,7 @@ export function keeperPanel(db: Database, mayHold = false, now: Date = new Date(
6567 ${when(ahead !== undefined, () => html`<div class="status-row"><dt>Main vs production</dt><dd id="keeper-ahead">${ahead}</dd></div>`)}
6668 </dl>
6769 ${when(rows.length > 0, () => html`<div class="table-scroll" role="region" tabindex="0" aria-label="Keeper releases"><table class="table" data-sticky-lead>
68 <thead><tr><th>Version</th><th>Cut</th><th>Landed to live</th><th>Known good</th><th>Roll non-200</th><th>Probes / pages</th><th>Worst</th><th>Stall, first 5 min</th><th>Dark</th></tr></thead>
70 <thead><tr><th>Version</th><th>Cut</th><th>Landed to live</th><th>Known good</th><th>Roll non-200</th><th>Probes / pages</th><th>Worst</th><th>Stall, first 5 min</th><th>Dark</th><th>Write pause</th></tr></thead>
6971 <tbody>${rows}</tbody>
7072 </table></div>`)}
7173 ${when(mayHold, () => html`${holds.filter((h) => h.kind === "owner").map(
src/web/pages/maintenance.ts +4 −1
@@ -17,6 +17,7 @@ import { icon } from "../icons.ts";
1717 import { flashTo, page } from "../layout.ts";
1818 import { badge, btn, disclosure, field } from "../ui.ts";
1919 import { STALE_DAYS, STALE_WORDS, closeStale, nudge, stalePulls } from "../../maintain/stale_pulls.ts";
20 import { writePauseFold } from "./maintenance_pause.ts";
2021 import { budgetRows } from "./maintenance_budgets.ts";
2122
2223 function admin(ctx: Ctx): Response | undefined {
@@ -77,6 +78,7 @@ export async function maintenancePage(ctx: Ctx): Promise<Response> {
7778 )}
7879 </dl>`)}
7980 ${fold("maintenance-budgets", "Budgets", String(states.length), false, html`<dl class="status-rows">${budgetRows(ctx.app, states.map((d) => d.name))}</dl>`)}
81 ${writePauseFold(ctx.app.db)}
8082 ${stale > 0
8183 ? fold("stale-pulls", `Pulls untouched for ${String(STALE_DAYS)} days`, String(stale), true, staleRows(ctx.auth.user!.kind !== "agent", stalePending))
8284 : null}
@@ -117,7 +119,8 @@ export async function maintenancePrepare(ctx: Ctx): Promise<Response> {
117119 if (denied) return denied;
118120 const form = await readForm(ctx, "admin");
119121 if (form instanceof Response) return form;
120 // Idle -> prepared stops the crew: asked first. A re-check while prepared has already been agreed.
122 // Idle -> prepared stops the crew: asked first. A re-check while prepared has already been
123 // agreed.
121124 if (status(ctx.app).phase === "idle" && formField(form, "confirm") !== "yes")
122125 return flashTo(ctx, "/maintenance", { tone: "danger", message: "Tick the box to confirm: the crew and releases stop." });
123126 const r = await prepare(ctx.app, ctx.auth.user, formField(form, "reason"));
src/web/pages/maintenance_pause.ts added +29
@@ -0,0 +1,29 @@
1 /**
2 * "Write pause per release" on /maintenance (#7836): how long each release's new forge held its
3 * queue work while the old one still wrote, newest first, in seconds, with the wait the next cut
4 * then keeps. Nothing is shown until a release has measured one.
5 */
6
7 import type { Database } from "bun:sqlite";
8 import { MIN_INTERVAL_MS, PAUSE_CADENCE_FACTOR } from "../../keeper/cadence.ts";
9 import { pauseWords } from "../../keeper/pause.ts";
10 import { listKeeperReleases } from "../../keeper/record.ts";
11 import { disclosure } from "../ui.ts";
12 import { html, type Html } from "../html.ts";
13
14 const SHOWN = 12;
15
16 /** The panel, or an empty fragment when no release has measured a pause. */
17 export function writePauseFold(db: Database): Html {
18 const rows = listKeeperReleases(db, SHOWN).filter((r) => r.writePauseMs !== null);
19 if (rows.length === 0) return html``;
20 const wait = Math.max(MIN_INTERVAL_MS, PAUSE_CADENCE_FACTOR * rows[0]!.writePauseMs!);
21 return disclosure(
22 html`Write pause per release <span class="muted">${pauseWords(rows[0]!.writePauseMs!)}</span>`,
23 html`<p class="small muted" id="write-pause-says">The next cut waits ${String(wait / 60_000)} min.</p>
24 <dl class="status-rows" id="write-pause">
outdated

Title says '(ms)' and the value repeats in the muted span; show seconds, as the cadence reason does.

25 ${rows.map((r) => html`<div class="status-row"><dt>${r.version}</dt><dd>${pauseWords(r.writePauseMs!)}</dd></div>`)}
outdated

Principle 8 and 1: this sentence explains the formula the label already carries. Say it once, in seconds/minutes: 'The next cut waits 120 min.'

26 </dl>`,
27 { id: "maintenance-write-pause", extra: "mb", open: false },
28 );
outdated

Principles 2, 3 and 7: a fact is a dl row; remove the hand-drawn SVG bar (fixed 240 px width overflows at 360 px, fill is hand-rolled). The row with the number is enough.

29 }
test/keeper-language.test.ts +1 −1
@@ -10,7 +10,7 @@ import { keeperPanel } from "../src/web/pages/keeper-panel.ts";
1010 import { keeperForge } from "./helpers/keeper.ts";
1111
1212 const HEAD =
13 "<thead><tr><th>Version</th><th>Cut</th><th>Landed to live</th><th>Known good</th><th>Roll non-200</th><th>Probes / pages</th><th>Worst</th><th>Stall, first 5 min</th><th>Dark</th></tr></thead>";
13 "<thead><tr><th>Version</th><th>Cut</th><th>Landed to live</th><th>Known good</th><th>Roll non-200</th><th>Probes / pages</th><th>Worst</th><th>Stall, first 5 min</th><th>Dark</th><th>Write pause</th></tr></thead>";
1414
1515 test("the keeper is a section with an h2, no card, with its blocked line and its table", async () => {
1616 const f = await keeperForge();
test/release-keeper-write-pause.test.ts added +71
@@ -0,0 +1,71 @@
1 // A roll should not pause queue work (#7836): the new forge's hold is measured, kept on the
2 // release, and the next cut waits ten times it.
3 import { afterEach, expect, test } from "bun:test";
4 import { decideCut } from "../src/keeper/cadence.ts";
5 import { keeperAnswer } from "../src/keeper/landed.ts";
6 import { forgetWritePause, noteWritePause } from "../src/keeper/pause.ts";
7 import { listKeeperReleases, markLive } from "../src/keeper/record.ts";
8 import { tick } from "../src/keeper/keeper.ts";
9 import { keeperPanel } from "../src/web/pages/keeper-panel.ts";
10 import { writePauseFold } from "../src/web/pages/maintenance_pause.ts";
11 import { keeperRows } from "../src/tui/screens/keeper.ts";
12 import { A, B, keeperForge } from "./helpers/keeper.ts";
13
14 afterEach(forgetWritePause);
15
16 const base = { main: B, newest: { version: "0.7.283", sha: A }, live: "0.7.283", inFlight: false, gating: 0, hold: undefined } as const;
17 const MIN = 60_000;
18 const at = 1_000_000_000_000;
19
20 test("a recorded 720 s pause makes the next cut wait 120 min, and says so", () => {
21 const input = { ...base, lastCutAt: at, writePauseMs: 720_000 };
22 const d = decideCut({ ...input, now: at + 119 * MIN });
23 expect(d.cut).toBe(false);
24 if (!d.cut) {
25 expect(d.why).toContain("the interval is 120 min");
26 expect(d.why).toContain("write pause of 720 s");
27 }
28 expect(decideCut({ ...input, now: at + 120 * MIN }).cut).toBe(true);
29 });
30
31 test("an unmeasured pause leaves the 30 min floor, and a short one does not lengthen it", () => {
32 const d = decideCut({ ...base, lastCutAt: at, now: at + 29 * MIN });
33 expect(d.cut).toBe(false);
34 if (!d.cut) expect(d.why).toBe("the last cut was 29 min ago; the interval is 30 min");
35 expect(decideCut({ ...base, lastCutAt: at, now: at + 30 * MIN }).cut).toBe(true);
36 const short = decideCut({ ...base, lastCutAt: at, writePauseMs: 60_000, now: at + 30 * MIN });
37 expect(short.cut).toBe(true);
38 });
39
40 test("the pause this process measured is kept on the release when its verify begins, and shown", async () => {
41 const f = await keeperForge();
42 await tick(f.ports);
43 const r = listKeeperReleases(f.app.db)[0]!;
44 expect(r.writePauseMs).toBeNull();
45 expect(markLive(f.app.db, r.id, f.world.now, 720_000)).toBe(true);
46 const kept = listKeeperReleases(f.app.db)[0]!;
47 expect(kept.writePauseMs).toBe(720_000);
48 expect(keeperAnswer(f.app.db).releases[0]!.writePauseMs).toBe(720_000);
49 expect(String(keeperPanel(f.app.db))).toContain("720 s");
50 expect(String(writePauseFold(f.app.db))).toContain("The next cut waits 120 min");
51 expect(String(writePauseFold(f.app.db))).toContain("Write pause per release");
52 expect(String(writePauseFold(f.app.db))).not.toContain("<svg");
53 const rows = keeperRows(keeperAnswer(f.app.db).releases);
54 expect(JSON.stringify(rows)).toContain("write pause 720 s");
55 });
56
57 test("the keeper hands its process's pause to the release it marks live", async () => {
58 const f = await keeperForge();
59 await tick(f.ports);
60 noteWritePause(5_000);
61 f.world.live = listKeeperReleases(f.app.db)[0]!.version;
62 await tick({ ...f.ports, writePauseMs: () => 5_000 });
63 expect(listKeeperReleases(f.app.db)[0]!.writePauseMs).toBe(5_000);
64 });
65
66 test("an unmeasured release shows no pause", async () => {
67 const f = await keeperForge();
68 await tick(f.ports);
69 expect(String(writePauseFold(f.app.db))).toBe("");
70 expect(String(keeperPanel(f.app.db))).toContain("<th>Write pause</th>");
71 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs