Public: anyone can read this; the crew can change it.
The no-release alarm reaches the owner when production stands still for 90 minutes while main moves: a test replays 2026-11-11 #11680
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/11673-the-no-release-alarm-reaches-the-owner-w → main 2 commits fork 1 thread · 1 unresolved
4 files
changed +135
−19
src/flow/alarm_no_release.ts +15 −13
| @@ -2,11 +2,12 @@ | ||
| 2 | 2 | * The "no release" flow alarm (#5930): main has moved and nothing has gone |
| 3 | 3 | * live for 90 minutes. Reads what `/changelog` reads (`release/landings.ts`): the oldest |
| 4 | 4 | * merge the running commit does not yet contain; pure of the clock — `now` is passed in. |
| 5 | * Its words name what production runs, how many merged pulls wait and for how long (#11673). | |
| 5 | 6 | */ |
| 6 | 7 | |
| 7 | 8 | import type { Database } from "bun:sqlite"; |
| 8 | import { waitingSince } from "../release/landings.ts"; | |
| 9 | import { running } from "../kernel/version.ts"; | |
| 9 | import { waiting } from "../release/landings.ts"; | |
| 10 | import { running, runningLine } from "../kernel/version.ts"; | |
| 10 | 11 | |
| 11 | 12 | export const NO_RELEASE_MS = 90 * 60_000; |
| 12 | 13 | |
| @@ -17,23 +18,24 @@ export interface NoReleaseAlarm { | ||
| 17 | 18 | readonly says: string; |
| 18 | 19 | } |
| 19 | 20 | |
| 20 | function ago(ms: number): string { | |
| 21 | /** A wait in words: `91 min`, `5 h 50 min`. */ | |
| 22 | export function span(ms: number): string { | |
| 21 | 23 | const min = Math.max(0, Math.floor(ms / 60_000)); |
| 22 | if (min < 120) return `${min} min ago`; | |
| 23 | return `${Math.floor(min / 60)} h ago`; | |
| 24 | if (min < 120) return `${min} min`; | |
| 25 | const rest = min % 60; | |
| 26 | return rest === 0 ? `${Math.floor(min / 60)} h` : `${Math.floor(min / 60)} h ${rest} min`; | |
| 24 | 27 | } |
| 25 | 28 | |
| 26 | 29 | export function noReleaseAlarm(db: Database, now: Date): NoReleaseAlarm { |
| 27 | const moved = waitingSince(db); | |
| 28 | const commit = running().commit; | |
| 29 | const live = commit ? `since ${commit.slice(0, 7)}` : "yet"; | |
| 30 | if (moved === undefined || Date.parse(moved) > now.getTime()) | |
| 30 | const moved = waiting(db); | |
| 31 | if (moved === undefined || Date.parse(moved.since) > now.getTime()) | |
| 31 | 32 | return { fired: false, says: "Nothing has moved on main since the last release." }; |
| 32 | const wait = now.getTime() - Date.parse(moved); | |
| 33 | if (wait < NO_RELEASE_MS) return { fired: false, says: `Main moved ${ago(wait)}; a release is not yet due.` }; | |
| 33 | const wait = now.getTime() - Date.parse(moved.since); | |
| 34 | if (wait < NO_RELEASE_MS) return { fired: false, says: `Main moved ${span(wait)} ago; a release is not yet due.` }; | |
| 35 | const pulls = moved.count === 1 ? "1 merged pull has" : `${moved.count} merged pulls have`; | |
| 34 | 36 | return { |
| 35 | 37 | fired: true, |
| 36 | since: moved, | |
| 37 | says: `Main moved ${ago(wait)} and nothing has gone live ${live}.`, | |
| 38 | since: moved.since, | |
| 39 | says: `${pulls} waited ${span(wait)} and nothing has gone live: production runs ${runningLine(running())}.`, | |
| 38 | 40 | }; |
| 39 | 41 | } |
src/release/landings.ts +11 −5
| @@ -114,10 +114,15 @@ function isLive(row: MergedRow, state: LiveState | undefined): boolean { | ||
| 114 | 114 | /** |
| 115 | 115 | * The merged pulls split by whether they are live. A pull of the forge's own |
| 116 | 116 | * repository that is not in the running commit but merged before a live one |
| 117 | * is not on its way, just not in this history: it counts as live. | |
| 117 | * of the same repository is not on its way, just not in this history: it | |
| 118 | * counts as live. Only the own repository's pulls set that mark: another | |
| 119 | * repository's pull is live as it lands, and once one merge elsewhere made | |
| 120 | * every own pull before it "live", so the no-release alarm stayed silent | |
| 121 | * for 5 h 50 min on 2026-11-11 (#11673). | |
| 118 | 122 | */ |
| 119 | 123 | export function split(rows: readonly MergedRow[], state: LiveState | undefined): { live: Landing[]; onItsWay: Landing[] } { |
| 120 | const newestLive = rows.find((r) => isLive(r, state)); | |
| 124 | const own = state?.own.toLowerCase(); | |
| 125 | const newestLive = rows.find((r) => r.repo.toLowerCase() === own && isLive(r, state)); | |
| 121 | 126 | const out: { live: Landing[]; onItsWay: Landing[] } = { live: [], onItsWay: [] }; |
| 122 | 127 | for (const r of rows) { |
| 123 | 128 | const live = isLive(r, state) || (newestLive !== undefined && r.at < newestLive.at); |
| @@ -127,11 +132,12 @@ export function split(rows: readonly MergedRow[], state: LiveState | undefined): | ||
| 127 | 132 | return out; |
| 128 | 133 | } |
| 129 | 134 | |
| 130 | /** The oldest landing still on its way, from what the last read kept; undefined when none or nothing was read. */ | |
| 131 | export function waitingSince(db: Database): string | undefined { | |
| 135 | /** The merged pulls still on their way and when the oldest merged, from what the last read kept; undefined when none or nothing was read. */ | |
| 136 | export function waiting(db: Database): { readonly since: string; readonly count: number } | undefined { | |
| 132 | 137 | if (kept === undefined) return undefined; |
| 133 | 138 | const { onItsWay } = split(mergedRows(db), kept); |
| 134 | return onItsWay.at(-1)?.at; | |
| 139 | const oldest = onItsWay.at(-1); | |
| 140 | return oldest === undefined ? undefined : { since: oldest.at, count: onItsWay.length }; | |
| 135 | 141 | } |
| 136 | 142 | |
| 137 | 143 | /** How long a miss is believed: a repository can turn public, or merge its first pull, after boot. */ |
test/flow-alarm-no-release-1111.test.ts added +108
| @@ -0,0 +1,108 @@ | ||
| 1 | /** | |
| 2 | * The 2026-11-11 replay (#11673): production stood on 0.7.535 for 5 h 50 min | |
| 3 | * while main merged pulls, and the no-release alarm said nothing. A merge to | |
| 4 | * another public repository marked every own pull merged before it as live. | |
| 5 | * Here: a fixed running commit, own pulls merged after it, a merge elsewhere, | |
| 6 | * no keeper rows, no new tag. Silent at 89 min, one alarm and one message at | |
| 7 | * 91, cleared when the running commit contains the merges. | |
| 8 | */ | |
| 9 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 10 | import { mkdtempSync, rmSync } from "node:fs"; | |
| 11 | import { tmpdir } from "node:os"; | |
| 12 | import { join } from "node:path"; | |
| 13 | import { createApp } from "../src/app.ts"; | |
| 14 | import { createRepo } from "../src/core/repos.ts"; | |
| 15 | import { createUser } from "../src/core/users.ts"; | |
| 16 | import { setNotifyUrl, startNotifier } from "../src/crew/notify.ts"; | |
| 17 | import { firedAlarms } from "../src/flow/alarms.ts"; | |
| 18 | import { id } from "../src/kernel/id.ts"; | |
| 19 | import { forgetLive, keepLive } from "../src/release/landings.ts"; | |
| 20 | ||
| 21 | const T0 = Date.parse("2026-11-11T12:00:00.000Z"); | |
| 22 | const at = (min: number) => new Date(T0 + min * 60_000).toISOString(); | |
| 23 | const env: Record<string, string | undefined> = {}; | |
| 24 | const dirs: string[] = []; | |
| 25 | ||
| 26 | beforeAll(() => { | |
| 27 | for (const k of ["FORGE_VERSION", "FORGE_COMMIT", "FORGE_APP_COMMIT"]) env[k] = process.env[k]; | |
| 28 | process.env["FORGE_VERSION"] = "0.7.535"; | |
| 29 | process.env["FORGE_COMMIT"] = "5e1d0c0ffee5e1d0c0ffee5e1d0c0ffee5e1d0c0"; | |
| 30 | delete process.env["FORGE_APP_COMMIT"]; | |
| 31 | }); | |
| 32 | afterAll(() => { | |
| 33 | for (const [k, v] of Object.entries(env)) if (v === undefined) delete process.env[k]; | |
| 34 | else process.env[k] = v; | |
| 35 | for (const d of dirs) rmSync(d, { recursive: true, force: true }); | |
| 36 | forgetLive(); | |
| 37 | }); | |
| 38 | ||
| 39 | describe("no release on 2026-11-11", () => { | |
| 40 | test("silent at 89 min, one alarm and one message at 91, clear once the merges run", async () => { | |
| 41 | const dir = mkdtempSync(join(tmpdir(), "forge-1111-")); | |
| 42 | dirs.push(dir); | |
| 43 | const app = createApp({ dataDir: dir, port: 0 }); | |
| 44 | const u = await createUser(app.db, { username: "alice", email: "[email protected]", password: "a-good-passphrase", origin: "seed", trusted: true, adminIfFirst: true }); | |
| 45 | if (!u.ok) throw new Error(u.error); | |
| 46 | const own = await createRepo(app.db, app.reposRoot, { ownerId: u.value.id, name: "forge", visibility: "public", description: "f" }); | |
| 47 | const other = await createRepo(app.db, app.reposRoot, { ownerId: u.value.id, name: "app", visibility: "public", description: "a" }); | |
| 48 | if (!own.ok || !other.ok) throw new Error("repo"); | |
| 49 | let n = 0; | |
| 50 | const merge = (repo: typeof own.value, min: number, oid: string) => { | |
| 51 | const issueId = id(); | |
| 52 | n++; | |
| 53 | app.db.query(`INSERT INTO issue (id, repo_id, number, kind, title, body, author_id, state, created_at) VALUES (?1, ?2, ?3, 'pull', 'P', '', ?4, 'closed', ?5)`).run(issueId, repo.id, n, u.value.id, at(min)); | |
| 54 | app.db.query(`INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, head_repo_id, merged_at, merged_oid) VALUES (?1, ?2, ?3, '', '', ?4, ?5, ?6)`).run(issueId, repo.defaultBranch, `b/${n}`, repo.id, at(min), oid); | |
| 55 | }; | |
| 56 | // The running commit holds only what merged before it. | |
| 57 | merge(own.value, -30, "a".repeat(40)); | |
| 58 | keepLive("alice/forge", ["a".repeat(40)], T0); | |
| 59 | merge(own.value, 10, "b".repeat(40)); | |
| 60 | merge(own.value, 40, "c".repeat(40)); | |
| 61 | merge(own.value, 70, "d".repeat(40)); | |
| 62 | // The merge elsewhere that once marked every own pull before it live. | |
| 63 | merge(other.value, 95, "e".repeat(40)); | |
| 64 | ||
| 65 | const releaseAlarms = (min: number) => firedAlarms(app.db, new Date(T0 + min * 60_000), { dataDir: null }).filter((a) => a.key === "no_release"); | |
| 66 | expect(releaseAlarms(10 + 89)).toHaveLength(0); | |
| 67 | const fired = releaseAlarms(10 + 91); | |
| 68 | expect(fired).toHaveLength(1); | |
| 69 | expect(fired[0]!.since).toBe(at(10)); | |
| 70 | expect(fired[0]!.says).toBe("3 merged pulls have waited 91 min and nothing has gone live: production runs 0.7.535 · 5e1d0c0."); | |
| 71 | expect(releaseAlarms(10 + 350)[0]!.says).toContain("3 merged pulls have waited 5 h 50 min"); | |
| 72 | ||
| 73 | setNotifyUrl(app.db, u.value.id, "https://ntfy.example.test/topic"); | |
| 74 | const posts: string[] = []; | |
| 75 | let now = T0 + 85 * 60_000; | |
| 76 | const notifier = startNotifier(app, { | |
| 77 | forgeUrl: "https://forge.example.test", | |
| 78 | now: () => now, | |
| 79 | drainingSince: () => null, | |
| 80 | causes: () => [], | |
| 81 | alarms: () => firedAlarms(app.db, new Date(now), { dataDir: null }), | |
| 82 | send: async (_u, body) => { | |
| 83 | posts.push(body); | |
| 84 | return true; | |
| 85 | }, | |
| 86 | }); | |
| 87 | await notifier.settled(); | |
| 88 | const wakeAt = async (min: number) => { | |
| 89 | now = T0 + min * 60_000; | |
| 90 | app.bus.append({ kind: "drill.finished", payload: {} }); | |
| 91 | await notifier.settled(); | |
| 92 | }; | |
| 93 | await wakeAt(10 + 89); | |
| 94 | expect(posts).toHaveLength(0); | |
| 95 | await wakeAt(10 + 91); | |
| 96 | await wakeAt(10 + 120); | |
| 97 | await wakeAt(10 + 350); | |
| 98 | expect(posts).toHaveLength(1); | |
| 99 | expect(posts[0]).toContain("production runs 0.7.535 · 5e1d0c0"); | |
| 100 | ||
| 101 | // A release whose commit contains the merges clears it, and nothing more is said. | |
| 102 | keepLive("alice/forge", ["a", "b", "c", "d"].map((c) => c.repeat(40)), T0 + 360 * 60_000); | |
| 103 | expect(releaseAlarms(370)).toHaveLength(0); | |
| 104 | await wakeAt(370); | |
| 105 | expect(posts).toHaveLength(1); | |
| 106 | await notifier.close(); | |
| 107 | }); | |
| 108 | }); |
test/flow-alarm-no-release.test.ts +1 −1
| @@ -49,7 +49,7 @@ describe("no release alarm", () => { | ||
| 49 | 49 | const a = noReleaseAlarm(s.app.db, NOW); |
| 50 | 50 | expect(a.fired).toBe(true); |
| 51 | 51 | expect(a.since).toBe(minAgo(91)); |
| 52 | expect(a.says).toBe("Main moved 91 min ago and nothing has gone live since abcdef1."); | |
| 52 | expect(a.says).toBe("1 merged pull has waited 91 min and nothing has gone live: production runs dev · abcdef1."); | |
| 53 | 53 | }); |
| 54 | 54 | test("does not fire at 89 min, with no move, or once a release followed the move", async () => { |
| 55 | 55 | const s = await seed(); |
new above
offline
waitingSincehas no remaining caller in the head (alarm_no_release.ts now useswaiting). Remove this wrapper rather than keeping it as dead code.