Public: anyone can read this; the crew can change it.
Each release tick leaves a heartbeat row: the keeper writes one per tick with the sentence it already returns #11822
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/11794-each-release-tick-leaves-a-heartbeat-row → main 2 commits fork 1 thread · 1 unresolved
6 files
changed +51
−10
docs/features/release-heartbeat.md +2 −2
| @@ -1,6 +1,6 @@ | ||
| 1 | 1 | # Release heartbeat and the main-ahead alarm (#11507) |
| 2 | 2 | |
| 3 | The release tick records a heartbeat each time it runs (`src/keeper/beat.ts`, table `keeper_beat`): when, whether it cut, skipped or failed, and why. The outcome is the keeper's own (`tickStep` in `src/keeper/keeper.ts` says it where it decides), never read back from the line's words; a tick that throws is recorded as failed (`stepWithBeat`). None for 60 minutes is said plainly ("No heartbeat from the release tick for …"). | |
| 3 | The release tick records a heartbeat each time it runs (`src/keeper/beat.ts`, table `keeper_beat_tick`): when, whether it cut, skipped or failed, and why. The last 50 are kept (`KEPT`); each new one drops the oldest beyond that, and every reader shows the newest (#11794). The outcome is the keeper's own (`tickStep` in `src/keeper/keeper.ts` says it where it decides), never read back from the line's words; a tick that throws is recorded as failed (`stepWithBeat`). None for 60 minutes is said plainly ("No heartbeat from the release tick for …"). | |
| 4 | 4 | |
| 5 | 5 | Where to see it: |
| 6 | 6 | |
| @@ -14,7 +14,7 @@ The flow alarm `main_ahead` (`src/flow/alarm_main_ahead.ts`) fires when main's l | ||
| 14 | 14 | |
| 15 | 15 | ## Recording a tick from outside (#11674) |
| 16 | 16 | |
| 17 | A release path that runs outside the forge (a workflow, a script) leaves the same heartbeat, replacing the last one: | |
| 17 | A release path that runs outside the forge (a workflow, a script) leaves the same heartbeat: it adds a row, and the oldest beyond 50 goes: | |
| 18 | 18 | |
| 19 | 19 | sh |
| 20 | 20 | curl -X POST "/api/v1/release/heartbeat" \ |
src/core/migration-pins/20261112130000-keeper-beat-tick.pin added +1
| @@ -0,0 +1,1 @@ | ||
| 1 | 6d0ccf5b7971877ca4e926340d4bbd7dac7f8991d477523b38f1ef84a52e11a4 |
src/core/migrations/20261112130000-keeper-beat-tick.ts added +14
| @@ -0,0 +1,14 @@ | ||
| 1 | // The release tick's heartbeats (#11794): the last 50 ticks, not only the last, | |
| 2 | // so a quiet six hours can be told from a dead keeper. keeper_beat holds one row | |
| 3 | // by CHECK, and rebuilding it would not be additive across a roll, so the history | |
| 4 | // is its own table, seeded with that row; keeper_beat stays for the release still | |
| 5 | // in play and is read by nothing newer; #11833 retires it. | |
| 6 | export default ` | |
| 7 | CREATE TABLE keeper_beat_tick ( | |
| 8 | id INTEGER PRIMARY KEY AUTOINCREMENT, | |
| 9 | at TEXT NOT NULL, | |
| 10 | outcome TEXT NOT NULL CHECK (outcome IN ('cut', 'skipped', 'failed')), | |
| 11 | reason TEXT NOT NULL | |
| 12 | ); | |
| 13 | INSERT INTO keeper_beat_tick (at, outcome, reason) SELECT at, outcome, reason FROM keeper_beat; | |
| 14 | `; |
src/keeper/beat.ts +10 −6
| @@ -1,7 +1,8 @@ | ||
| 1 | 1 | /** |
| 2 | 2 | * The release tick's heartbeat (#11507): each time the tick runs it reports |
| 3 | 3 | * what it did — cut, skipped and why, or failed and why — and the forge keeps |
| 4 | * the last. The tick runs outside the forge's own judgment, so when it stops | |
| 4 | * the last {@link KEPT} (#11794), so a quiet stretch can be told from a dead | |
| 5 | * keeper. The tick runs outside the forge's own judgment, so when it stops | |
| 5 | 6 | * this is the only trace; `release_keeper`, /releases and the terminal's |
| 6 | 7 | * release keeper screen show it, and a beat older |
| 7 | 8 | * than {@link SILENT_MS} is said plainly. Plain rows in, plain rows out. |
| @@ -16,6 +17,8 @@ export const isOutcome = (v: unknown): v is Outcome => typeof v === "string" && | ||
| 16 | 17 | /** No heartbeat for this long is itself said. */ |
| 17 | 18 | export const SILENT_MS = 60 * 60_000; |
| 18 | 19 | export const REASON_MAX = 300; |
| 20 | /** How many beats are kept; the oldest goes when one more is written. */ | |
| 21 | export const KEPT = 50; | |
| 19 | 22 | |
| 20 | 23 | export interface Beat { |
| 21 | 24 | readonly at: string; |
| @@ -25,15 +28,16 @@ export interface Beat { | ||
| 25 | 28 | |
| 26 | 29 | export function recordBeat(db: Database, outcome: Outcome, reason: string, at: Date = new Date()): Beat { |
| 27 | 30 | const beat: Beat = { at: at.toISOString(), outcome, reason: reason.replace(/\s+/g, " ").trim().slice(0, REASON_MAX) }; |
| 28 | db.query( | |
| 29 | `INSERT INTO keeper_beat (id, at, outcome, reason) VALUES (1, ?1, ?2, ?3) | |
| 30 | ON CONFLICT (id) DO UPDATE SET at = excluded.at, outcome = excluded.outcome, reason = excluded.reason`, | |
| 31 | ).run(beat.at, beat.outcome, beat.reason); | |
| 31 | db.transaction(() => { | |
| 32 | db.query("INSERT INTO keeper_beat_tick (at, outcome, reason) VALUES (?1, ?2, ?3)").run(beat.at, beat.outcome, beat.reason); | |
| 33 | db.query("DELETE FROM keeper_beat_tick WHERE id <= (SELECT MAX(id) FROM keeper_beat_tick) - ?1").run(KEPT); | |
| 34 | })(); | |
| 32 | 35 | return beat; |
| 33 | 36 | } |
| 34 | 37 | |
| 38 | /** The newest beat, or null before the first. */ | |
| 35 | 39 | export function lastBeat(db: Database): Beat | null { |
| 36 | return db.query<Beat, []>("SELECT at, outcome, reason FROM keeper_beat WHERE id = 1").get() ?? null; | |
| 40 | return db.query<Beat, []>("SELECT at, outcome, reason FROM keeper_beat_tick ORDER BY id DESC LIMIT 1").get() ?? null; | |
| 37 | 41 | } |
| 38 | 42 | |
| 39 | 43 | /** What one tick did, as the keeper says it: the line and its outcome, never read back from the words. */ |
test/flow-alarm-main-ahead.test.ts +1 −1
| @@ -146,7 +146,7 @@ describe("main ahead alarm", () => { | ||
| 146 | 146 | |
| 147 | 147 | describe("heartbeat", () => { |
| 148 | 148 | test("release_keeper shows the last heartbeat and its reason, and says when it is quiet", () => { |
| 149 | app.db.query(`DELETE FROM keeper_beat`).run(); | |
| 149 | app.db.query(`DELETE FROM keeper_beat_tick`).run(); | |
| 150 | 150 | expect(keeperAnswer(app.db, NOW).heartbeat).toBeNull(); |
| 151 | 151 | recordBeat(app.db, "skipped", "not cut: a hold stands", new Date(NOW.getTime() - 5 * 60_000)); |
| 152 | 152 | const h = keeperAnswer(app.db, NOW).heartbeat!; |
test/keeper-heartbeat.test.ts +23 −1
| @@ -7,7 +7,7 @@ import { createApp, type App } from "../src/app.ts"; | ||
| 7 | 7 | import { createToken } from "../src/core/tokens.ts"; |
| 8 | 8 | import { createUser } from "../src/core/users.ts"; |
| 9 | 9 | import { serve, type Servers } from "../src/http/server.ts"; |
| 10 | import { lastBeat, recordBeat } from "../src/keeper/beat.ts"; | |
| 10 | import { KEPT, lastBeat, recordBeat, stepWithBeat } from "../src/keeper/beat.ts"; | |
| 11 | 11 | import { keeperAnswer } from "../src/keeper/landed.ts"; |
| 12 | 12 | |
| 13 | 13 | describe("release heartbeat", () => { |
| @@ -91,4 +91,26 @@ describe("release heartbeat", () => { | ||
| 91 | 91 | expect((await post(admin, { outcome: "cut", reason: "x".repeat(301) })).status).toBe(422); |
| 92 | 92 | expect((await post(admin, { outcome: "cut" })).status).toBe(422); |
| 93 | 93 | }); |
| 94 | ||
| 95 | test("each tick leaves a row: two ticks leave two, the 51st drops the oldest (#11794)", async () => { | |
| 96 | app.db.query("DELETE FROM keeper_beat_tick").run(); | |
| 97 | const rows = () => app.db.query<{ at: string; outcome: string; reason: string }, []>( | |
| 98 | "SELECT at, outcome, reason FROM keeper_beat_tick ORDER BY id", | |
| 99 | ).all(); | |
| 100 | const tick = (n: number) => | |
| 101 | stepWithBeat(app.db, async () => ({ line: `tick ${n}`, outcome: "skipped" }), () => new Date(Date.UTC(2026, 10, 12, 0, n))); | |
| 102 | await tick(1); | |
| 103 | await tick(2); | |
| 104 | expect(rows()).toEqual([ | |
| 105 | { at: "2026-11-12T00:01:00.000Z", outcome: "skipped", reason: "tick 1" }, | |
| 106 | { at: "2026-11-12T00:02:00.000Z", outcome: "skipped", reason: "tick 2" }, | |
| 107 | ]); | |
| 108 | for (let n = 3; n <= KEPT + 1; n++) await tick(n); | |
| 109 | const kept = rows(); | |
| 110 | expect(KEPT).toBe(50); | |
| 111 | expect(kept).toHaveLength(50); | |
| 112 | expect(kept[0]!.reason).toBe("tick 2"); | |
| 113 | expect(kept[49]!.reason).toBe("tick 51"); | |
| 114 | expect(lastBeat(app.db)?.reason).toBe("tick 51"); | |
| 115 | }); | |
| 94 | 116 | }); |
new above
offline
The old keeper_beat table is left in place and nothing reads it. That is acceptable for the roll, but open an issue to retire it in a later migration so it does not stay as dead schema.