Public: anyone can read this; the crew can change it.
A release tick not heard from for 60 minutes fires one release_quiet flow alarm #11898
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/11890-a-release-tick-not-heard-from-for-60-min → main 2 commits fork 2 threads · 2 unresolved
4 files
changed +119
−1
docs/features/release-heartbeat.md +2
| @@ -10,6 +10,8 @@ Where to see it: | ||
| 10 | 10 | |
| 11 | 11 | The flow alarm `main_ahead` (`src/flow/alarm_main_ahead.ts`) fires when main's last finished check passed, its head is not the commit production runs, and main has been ahead for more than 90 minutes. Change the limit with the `release.ahead_minutes` row of `forge_setting` (a whole number, 1 to 10080; anything else is 90). It names production's version and commit, the commits and pulls ahead, how long, and when the tick was last heard from. It is an ask for the owner (home) and one Sentinel issue (`src/crew/sentinel_main_ahead.ts`), which closes itself at the next hunt after production catches up, unless a person commented on it. The issue names the tick's last outcome and time, never its reason: the issue may be public, and the reason is shown only to readers of /releases. |
| 12 | 12 | |
| 13 | The flow alarm `release_quiet` (`src/flow/alarm_release_quiet.ts`, #11890) fires when the release tick has not beaten for more than 60 minutes (`SILENT_MS`), so production could fall behind main with nothing deciding to release. It says how long ago the tick was last heard and its outcome, never its reason, for the same cause as the issue above: the alarm reaches home, /boards, the notifier and the MCP `flow` tool, and links to /releases, where the reason is. A forge that has never had a beat but has cut a release fires it too, counted from the newest cut; one that has never released stays quiet. The next beat, of any outcome, clears it. | |
| 14 | ||
| 13 | 15 | `/api/v1/telemetry/home`: `live` is the version and commit this process runs (null when unstamped); `main` is the head of the forge's own default branch and `behind` what main holds that live does not (#11676). |
| 14 | 16 | |
| 15 | 17 | ## Recording a tick from outside (#11674) |
src/flow/alarm_release_quiet.ts added +43
| @@ -0,0 +1,43 @@ | ||
| 1 | /** | |
| 2 | * The "release tick quiet" flow alarm (#11890): the release tick has not | |
| 3 | * reported a heartbeat (`keeper/beat.ts`) for {@link SILENT_MS} (60 min), so | |
| 4 | * production can fall behind main with nothing deciding to release. One alarm, | |
| 5 | * saying when the tick was last heard and its outcome, never its reason: the | |
| 6 | * alarm reaches home, /boards, the notifier and the MCP `flow` tool, and the | |
| 7 | * reason is shown only to readers of /releases (as with `main_ahead`). On a forge whose keeper | |
| 8 | * has cut a release before, never having had a beat fires it too, counted from | |
| 9 | * the newest cut. A fresh beat clears it. Pure of the clock: `now` is passed in. | |
| 10 | */ | |
| 11 | ||
| 12 | import type { Database } from "bun:sqlite"; | |
| 13 | import { agoWords, lastBeat, SILENT_MS } from "../keeper/beat.ts"; | |
| 14 | ||
| 15 | export interface ReleaseQuietAlarm { | |
| 16 | readonly fired: boolean; | |
| 17 | /** When the tick was last heard (or, never heard, the newest cut); only when fired. */ | |
| 18 | readonly since?: string; | |
| 19 | readonly says: string; | |
| 20 | } | |
| 21 | ||
| 22 | export function releaseQuietAlarm(db: Database, now: Date): ReleaseQuietAlarm { | |
| 23 | const beat = lastBeat(db); | |
| 24 | if (beat !== null) { | |
| 25 | const age = now.getTime() - Date.parse(beat.at); | |
| 26 | if (!(age > SILENT_MS)) return { fired: false, says: `The release tick reported ${agoWords(age)}.` }; | |
| 27 | return { | |
| 28 | fired: true, | |
| 29 | since: beat.at, | |
| 30 | says: `The release tick was last heard from ${agoWords(age)} (${beat.outcome}); its reason is on /releases.`, | |
| 31 | }; | |
| 32 | } | |
| 33 | // Never a beat: only a forge that has released before expects one. One bounded row. | |
| 34 | const cut = db.query<{ cut_at: string }, []>("SELECT cut_at FROM keeper_release ORDER BY id DESC LIMIT 1").get(); | |
| 35 | if (cut === null) return { fired: false, says: "The release tick has never reported, and this forge has not released." }; | |
| 36 | const age = now.getTime() - Date.parse(cut.cut_at); | |
| 37 | if (!(age > SILENT_MS)) return { fired: false, says: "The release tick has never reported; the last release was cut within the hour." }; | |
| 38 | return { | |
| 39 | fired: true, | |
| 40 | since: cut.cut_at, | |
| 41 | says: `The release tick has never been heard from: no heartbeat since the last release was cut ${agoWords(age)}.`, | |
| 42 | }; | |
| 43 | } |
src/flow/alarms.ts +4 −1
| @@ -21,11 +21,12 @@ import { mainAheadAlarm } from "./alarm_main_ahead.ts"; | ||
| 21 | 21 | import { noReleaseAlarm } from "./alarm_no_release.ts"; |
| 22 | 22 | import { redMainAlarm } from "./alarm_red_main.ts"; |
outdated
Import order: put this after alarm_red_main/alarm_repeat... as the other imports are alphabetical. |
||
| 23 | 23 | import { alarmRepeat } from "./alarm_repeat.ts"; |
| 24 | import { releaseQuietAlarm } from "./alarm_release_quiet.ts"; | |
| 24 | 25 | import { alarmSilent } from "./alarm_silent.ts"; |
| 25 | 26 | import { alarmStuck } from "./alarm_stuck.ts"; |
| 26 | 27 | |
| 27 | 28 | export interface FiredAlarm { |
| 28 | readonly key: "landing_stopped" | "red_main" | "stuck" | "idle_crew" | "no_release" | "main_ahead" | "lane_stopped" | "bug_clock" | "silent" | "repeat" | "from_app" | "unreachable" | "backup" | "paid_order"; | |
| 29 | readonly key: "landing_stopped" | "red_main" | "stuck" | "idle_crew" | "no_release" | "main_ahead" | "release_quiet" | "lane_stopped" | "bug_clock" | "silent" | "repeat" | "from_app" | "unreachable" | "backup" | "paid_order"; | |
| 29 | 30 | readonly says: string; |
| 30 | 31 | readonly since: string; |
| 31 | 32 | /** A same-origin path to the item the alarm names. */ |
| @@ -74,6 +75,8 @@ export function firedAlarms( | ||
| 74 | 75 | if (release.fired) out.push({ key: "no_release", says: release.says, since: release.since!, href: "/releases" }); |
| 75 | 76 | const ahead = mainAheadAlarm(db, now); |
| 76 | 77 | if (ahead.fired) out.push({ key: "main_ahead", says: ahead.says, since: ahead.since!, href: "/releases" }); |
| 78 | const quiet = releaseQuietAlarm(db, now); | |
| 79 | if (quiet.fired) out.push({ key: "release_quiet", says: quiet.says, since: quiet.since!, href: "/releases" }); | |
| 77 | 80 | const lane = laneAlarm(db, now, { ceilingUsd: opts.ceilingUsd ?? null }); |
| 78 | 81 | if (lane.fired) out.push({ key: "lane_stopped", says: lane.says, since: lane.since!, href: "/flow" }); |
| 79 | 82 | const bug = bugClockAlarm(db, now, opts.publicOnly ? { publicOnly: true } : {}); |
test/alarm-release-quiet.test.ts added +70
| @@ -0,0 +1,70 @@ | ||
| 1 | /** The "release tick quiet" alarm (#11890): no heartbeat from the release tick for 60 min fires one alarm; a fresh beat clears it. */ | |
| 2 | import { describe, expect, test } from "bun:test"; | |
| 3 | import { mkdtempSync } from "node:fs"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { createApp } from "../src/app.ts"; | |
| 7 | import { releaseQuietAlarm } from "../src/flow/alarm_release_quiet.ts"; | |
| 8 | import { firedAlarms } from "../src/flow/alarms.ts"; | |
| 9 | import { recordBeat } from "../src/keeper/beat.ts"; | |
| 10 | import { recordCut } from "../src/keeper/record.ts"; | |
| 11 | ||
| 12 | const NOW = new Date("2026-11-12T12:00:00.000Z"); | |
| 13 | const minAgo = (m: number) => new Date(NOW.getTime() - m * 60_000); | |
| 14 | const fresh = () => createApp({ dataDir: mkdtempSync(join(tmpdir(), "forge-quiet-")), port: 0 }).db; | |
| 15 | const quiet = (db: ReturnType<typeof fresh>) => firedAlarms(db, NOW, { dataDir: null }).filter((a) => a.key === "release_quiet"); | |
| 16 | ||
| 17 | describe("release tick quiet alarm", () => { | |
| 18 | test("59 min since the last beat says nothing", () => { | |
| 19 | const db = fresh(); | |
| 20 | recordBeat(db, "skipped", "main has not moved", minAgo(59)); | |
| 21 | expect(releaseQuietAlarm(db, NOW).fired).toBe(false); | |
| 22 | expect(quiet(db)).toEqual([]); | |
| 23 | }); | |
| 24 | ||
| 25 | test("61 min since the last beat fires one alarm naming when and the outcome", () => { | |
| 26 | const db = fresh(); | |
| 27 | recordBeat(db, "skipped", "main has not moved", minAgo(61)); | |
| 28 | const a = releaseQuietAlarm(db, NOW); | |
| 29 | expect(a.fired).toBe(true); | |
| 30 | expect(a.since).toBe(minAgo(61).toISOString()); | |
| 31 | expect(a.says).toBe("The release tick was last heard from 1 h 1 min ago (skipped); its reason is on /releases."); | |
| 32 | const fired = quiet(db); | |
| 33 | expect(fired).toHaveLength(1); | |
| 34 | expect(fired[0]).toEqual({ key: "release_quiet", says: a.says, since: a.since!, href: "/releases" }); | |
| 35 | }); | |
| 36 | ||
| 37 | test("a public-only reader gets the alarm without the reason, which may be private", () => { | |
| 38 | const db = fresh(); | |
| 39 | recordBeat(db, "failed", "push to acme/secret-infra refused", minAgo(61)); | |
| 40 | const fired = firedAlarms(db, NOW, { dataDir: null, publicOnly: true }).filter((a) => a.key === "release_quiet"); | |
| 41 | expect(fired).toHaveLength(1); | |
| 42 | expect(fired[0]!.says).toContain("(failed)"); | |
| 43 | expect(fired[0]!.says).not.toContain("secret-infra"); | |
| 44 | expect(quiet(db)[0]!.says).not.toContain("secret-infra"); | |
| 45 | }); | |
| 46 | ||
| 47 | test("a fresh beat clears it", () => { | |
| 48 | const db = fresh(); | |
| 49 | recordBeat(db, "failed", "tag push refused", minAgo(120)); | |
| 50 | expect(releaseQuietAlarm(db, NOW).fired).toBe(true); | |
| 51 | recordBeat(db, "cut", "cut v1.2.3", minAgo(1)); | |
| 52 | expect(releaseQuietAlarm(db, NOW).fired).toBe(false); | |
| 53 | expect(quiet(db)).toEqual([]); | |
| 54 | }); | |
| 55 | ||
| 56 | test("never a beat on a forge that has never released says nothing", () => { | |
| 57 | expect(releaseQuietAlarm(fresh(), NOW).fired).toBe(false); | |
| 58 | }); | |
| 59 | ||
| 60 | test("never a beat on a forge that has released fires, counted from the newest cut", () => { | |
| 61 | const db = fresh(); | |
| 62 | recordCut(db, { version: "v1.0.0", sha: "a".repeat(40), knownGood: { version: "v0.9.0", image: "img:0.9" }, at: minAgo(30).getTime() }); | |
| 63 | expect(releaseQuietAlarm(db, NOW).fired).toBe(false); | |
| 64 | const later = new Date(NOW.getTime() + 31 * 60_000); | |
| 65 | const a = releaseQuietAlarm(db, later); | |
| 66 | expect(a.fired).toBe(true); | |
| 67 | expect(a.since).toBe(minAgo(30).toISOString()); | |
| 68 | expect(a.says).toBe("The release tick has never been heard from: no heartbeat since the last release was cut 1 h 1 min ago."); | |
| 69 | }); | |
| 70 | }); |
new above
offline
beat.reason is put in a non-admin alarm. The docs treat the reason as /releases-only. Leave it out here or gate the alarm.