Public: anyone can read this; the crew can change it.
A main-ahead finding whose release tick was never heard from says the keeper is off and why, and goes to the owner, not the crew #11927
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/11924-a-main-ahead-finding-whose-release-tick → main 8 commits fork 1 thread · 1 unresolved
10 files
changed +176
−7
docs/features/release-keeper.md +10
| @@ -137,6 +137,16 @@ and its container (both default `forge`). The pod's service account needs | ||
| 137 | 137 | `get` and `patch` on that Deployment; the chart does not grant them. The |
| 138 | 138 | boot log says why it is off. |
| 139 | 139 | |
| 140 | While the release tick has never reported, /releases (the "Release tick" | |
| 141 | row) and the terminal's keeper screen say whether the keeper is off and | |
| 142 | why (for example "the in-process release keeper is off: | |
| 143 | FORGE_RELEASE_KEEPER is not on"), or on but not yet ticked. The why names | |
| 144 | only a variable's state, never its value. If main is then ahead of | |
| 145 | production past the limit, the owner gets the `main_ahead` flow alarm with | |
| 146 | the same line, and the sentinel files no crew issue: no code change fixes | |
| 147 | a keeper that is off (#11924). An issue filed before stays open until | |
| 148 | production catches up. | |
| 149 | ||
| 140 | 150 | Guardrail: the keeper writes only a tag, the Deployment's image, its own |
| 141 | 151 | rows and issues. It has no account that may land a pull or push main. |
| 142 | 152 | Rollback is the one action it takes without a canary, because the image it |
src/crew/sentinel_main_ahead.ts +9 −2
| @@ -2,7 +2,8 @@ | ||
| 2 | 2 | * The sentinel's half of the main-ahead alarm (#11507): main green and ahead |
| 3 | 3 | * of production past the setting is one finding, and its issue closes itself |
| 4 | 4 | * at the next hunt after production catches up — unless a person spoke on it, |
| 5 | * then it is theirs to close. | |
| 5 | * then it is theirs to close. When the release tick was never heard from, | |
| 6 | * nothing is filed: the flow alarm asks the owner, naming the keeper's state. | |
| 6 | 7 | */ |
| 7 | 8 | |
| 8 | 9 | import type { Database } from "bun:sqlite"; |
| @@ -22,7 +23,10 @@ export const MAIN_AHEAD_KEY = "main-ahead"; | ||
| 22 | 23 | |
| 23 | 24 | export function mainAhead(db: Database, now: number): Finding[] { |
| 24 | 25 | const a = mainAheadAlarm(db, new Date(now)); |
| 25 | if (!a.fired) return []; | |
| 26 | // A tick never heard from is the deployment's, not the code's (#11924): no builder can | |
| 27 | // fix it, so no issue; the `main_ahead` flow alarm is the owner's ask on home and | |
| 28 | // through the notifier, and says why. | |
| 29 | if (!a.fired || a.neverHeard === true) return []; | |
| 26 | 30 | return [{ |
| 27 | 31 | key: MAIN_AHEAD_KEY, |
| 28 | 32 | title: "Main is green and ahead of production: the release tick has not cut", |
| @@ -45,6 +49,9 @@ export function closeCaughtUp( | ||
| 45 | 49 | cleared: string = "Production caught up with main", |
| 46 | 50 | ): void { |
| 47 | 51 | if (active.has(key)) return; |
| 52 | // Unfiled because the tick was never heard from is not caught up (#11924): an issue | |
| 53 | // opened before stays open while main is still ahead. | |
| 54 | if (key === MAIN_AHEAD_KEY && mainAheadAlarm(app.db, new Date(now)).fired) return; | |
| 48 | 55 | const open = app.db |
| 49 | 56 | .query<{ issue_id: string }, [string]>( |
| 50 | 57 | `SELECT t.issue_id FROM sentinel_finding t JOIN issue i ON i.id = t.issue_id WHERE i.state = 'open' AND t.key = ?1`, |
src/flow/alarm_main_ahead.ts +12 −1
| @@ -10,6 +10,7 @@ | ||
| 10 | 10 | |
| 11 | 11 | import type { Database } from "bun:sqlite"; |
| 12 | 12 | import { getRepo } from "../core/repos.ts"; |
| 13 | import { neverTickedLine } from "../keeper/at_boot.ts"; | |
| 13 | 14 | import { agoWords, lastBeat } from "../keeper/beat.ts"; |
| 14 | 15 | import { ownRepoPath } from "../kernel/own_repo.ts"; |
| 15 | 16 | import { running } from "../kernel/version.ts"; |
| @@ -25,6 +26,11 @@ export interface MainAheadAlarm { | ||
| 25 | 26 | readonly says: string; |
| 26 | 27 | /** Facts for the issue and the ask; only when fired. */ |
| 27 | 28 | readonly facts?: readonly string[]; |
| 29 | /** | |
| 30 | * Fired with no heartbeat ever recorded: no code here can fix it, so it is the owner's | |
| 31 | * ask, not the crew's issue (#11924). | |
| 32 | */ | |
| 33 | readonly neverHeard?: true; | |
| 28 | 34 | } |
| 29 | 35 | |
| 30 | 36 | /** The minutes main may be ahead and green before the alarm; a whole number of 1 or more is the owner's, else 90. */ |
| @@ -76,11 +82,16 @@ export function mainAheadAlarm(db: Database, now: Date): MainAheadAlarm { | ||
| 76 | 82 | // The tick's reason stays out: these facts are a Sentinel issue's, which may be public; /releases shows it to its readers. |
| 77 | 83 | `the release tick was last heard from ${beat === null ? "never" : `${beat.at} (${beat.outcome})`}`, |
| 78 | 84 | ]; |
| 85 | // Never heard from: say what the keeper answered at boot, a variable's state and never | |
| 86 | // its value (#11924). | |
| 87 | const keeper = beat === null ? neverTickedLine() : undefined; | |
| 88 | if (keeper !== undefined) facts.push(keeper); | |
| 79 | 89 | return { |
| 80 | 90 | fired: true, |
| 81 | 91 | since: first.at, |
| 82 | says: `Main is green and ${how} ahead of production (${prod.version} at ${prod.commit.slice(0, 7)}) for ${forHow}; the release tick was last heard from ${tick}.`, | |
| 92 | says: `Main is green and ${how} ahead of production (${prod.version} at ${prod.commit.slice(0, 7)}) for ${forHow}; the release tick was last heard from ${tick}${keeper === undefined ? "" : `; ${keeper}`}.`, | |
| 83 | 93 | facts, |
| 94 | ...(beat === null ? { neverHeard: true as const } : {}), | |
| 84 | 95 | }; |
| 85 | 96 | } |
| 86 | 97 |
src/keeper/at_boot.ts added +30
| @@ -0,0 +1,30 @@ | ||
| 1 | /** | |
| 2 | * What the in-process release keeper answered at boot (#11924): on, or off | |
| 3 | * and why. `main.ts` notes it once; the main-ahead alarm and /releases read it | |
| 4 | * when the release tick was never heard from, so a finding nobody can fix in | |
| 5 | * code says the keeper is off instead of asking a builder to look. Plain data | |
| 6 | * in memory: a restart answers again. The why names only an environment | |
| 7 | * variable's state, never its value: it reaches a possibly public issue. | |
| 8 | */ | |
| 9 | ||
| 10 | export type KeeperAtBoot = { readonly on: true } | { readonly on: false; readonly why: string }; | |
| 11 | ||
| 12 | let atBoot: KeeperAtBoot | undefined; | |
| 13 | ||
| 14 | /** Called once at boot with what `keeperEnv` and `keeperPorts` answered; `undefined` forgets it. */ | |
| 15 | export function noteKeeperAtBoot(k: KeeperAtBoot | undefined): void { | |
| 16 | atBoot = k === undefined ? undefined : k.on ? { on: true } : { on: false, why: k.why }; | |
| 17 | } | |
| 18 | ||
| 19 | export function keeperAtBoot(): KeeperAtBoot | undefined { | |
| 20 | return atBoot; | |
| 21 | } | |
| 22 | ||
| 23 | /** | |
| 24 | * One line for a tick never heard from: the keeper off and why, or on but not ticked; | |
| 25 | * undefined before boot noted it. | |
| 26 | */ | |
| 27 | export function neverTickedLine(): string | undefined { | |
| 28 | if (atBoot === undefined) return undefined; | |
| 29 | return atBoot.on ? "the release keeper is on but has not ticked" : `the in-process release keeper is off: ${atBoot.why}`; | |
| 30 | } |
src/keeper/wire.ts +3 −1
| @@ -105,7 +105,9 @@ export function keeperPorts(app: App, input: { forgeUrl: string; deployment: str | ||
| 105 | 105 | if (named === undefined) return "FORGE_REPO is not set"; |
| 106 | 106 | const [owner = "", name = ""] = named.split("/"); |
| 107 | 107 | const repo = getRepo(app.db, owner, name); |
| 108 | if (repo === undefined) return `FORGE_REPO names ${named}, which is no repository on this forge`; | |
| 108 | // The reason never carries the repository's name: it is noted at boot and may reach a | |
| 109 | // public issue (#11924). | |
| 110 | if (repo === undefined) return "FORGE_REPO names no repository on this forge"; | |
| 109 | 111 | const dir = repoDir(app.reposRoot, repo.ownerName, repo.name); |
| 110 | 112 | const deployer = |
| 111 | 113 | input.swap === true |
src/main.ts +6 −1
| @@ -100,6 +100,7 @@ import { ownTagsReader } from "./release/own_tags.ts"; | ||
| 100 | 100 | import { rollProbe, tickStep as keeperTick } from "./keeper/keeper.ts"; |
| 101 | 101 | import { stepWithBeat } from "./keeper/beat.ts"; |
| 102 | 102 | import { keeperEnv, keeperPorts } from "./keeper/wire.ts"; |
| 103 | import { noteKeeperAtBoot } from "./keeper/at_boot.ts"; | |
| 103 | 104 | import { snapshotPlace } from "./actions/health.ts"; |
| 104 | 105 | import { capacityDefault, egressEnvIgnored } from "./core/capacity.ts"; |
| 105 | 106 | import { bootCounts, recordBoot, recordCleanExit } from "./kernel/boots.ts"; |
| @@ -827,6 +828,7 @@ if (keeperOn.on) { | ||
| 827 | 828 | ? { forgeUrl, deployment: "", container: "", swap: true } |
| 828 | 829 | : { forgeUrl, deployment: keeperOn.deployment, container: keeperOn.container }, |
| 829 | 830 | ); |
| 831 | noteKeeperAtBoot(typeof ports === "string" ? { on: false, why: ports } : { on: true }); | |
| 830 | 832 | if (typeof ports === "string") app.log.warn("the release keeper is off", { why: ports }); |
| 831 | 833 | else { |
| 832 | 834 | let said = ""; |
| @@ -844,7 +846,10 @@ if (keeperOn.on) { | ||
| 844 | 846 | 1000, |
| 845 | 847 | ); |
| 846 | 848 | } |
| 847 | } else app.log.info("the release keeper is off", { why: keeperOn.why }); | |
| 849 | } else { | |
| 850 | noteKeeperAtBoot(keeperOn); | |
| 851 | app.log.info("the release keeper is off", { why: keeperOn.why }); | |
| 852 | } | |
| 848 | 853 | // Only an agent is on a crew (the `repo_crew_agents_only` trigger refuses anyone else). A person |
| 849 | 854 | // put on one before that is reported here for the owner to take off — the |
| 850 | 855 | // forge never removes it on its own. |
src/tui/screens/keeper.ts +5 −1
| @@ -11,6 +11,7 @@ | ||
| 11 | 11 | */ |
| 12 | 12 | |
| 13 | 13 | import { aheadLine } from "../../flow/alarm_main_ahead.ts"; |
| 14 | import { neverTickedLine } from "../../keeper/at_boot.ts"; | |
| 14 | 15 | import { beatLine, lastBeat } from "../../keeper/beat.ts"; |
| 15 | 16 | import { stallWords } from "../../perf/boot_stall.ts"; |
| 16 | 17 | import { blocking, holdAs, liftAs, standingHolds } from "../../keeper/hold.ts"; |
| @@ -51,9 +52,12 @@ export function readKeeper(f: Session, now: Date = new Date()): KeeperRead | und | ||
| 51 | 52 | if (!mayRead(f.app, f.user)) return undefined; |
| 52 | 53 | const ahead = aheadLine(f.db, now); |
| 53 | 54 | const asked = pendingNow(f.db, now.getTime()); |
| 55 | const beat = lastBeat(f.db); | |
| 56 | // Never heard from: the same keeper line /releases and the main-ahead alarm name (#11924). | |
| 57 | const never = beat === null ? neverTickedLine() : undefined; | |
| 54 | 58 | return { |
| 55 | 59 | blocked: blocking(standingHolds(f.db)) ?? "Nothing holds the next cut.", |
| 56 | heartbeat: beatLine(lastBeat(f.db), now), | |
| 60 | heartbeat: never === undefined ? beatLine(beat, now) : `${beatLine(beat, now)} ${never[0]!.toUpperCase()}${never.slice(1)}.`, | |
| 57 | 61 | ...(ahead === undefined ? {} : { ahead }), |
| 58 | 62 | ...(asked === undefined ? {} : { asked: `Release now asked by ${asked.by}: ${asked.reason}` }), |
| 59 | 63 | releases: withLandedToLive(f.db, listKeeperReleases(f.db)), |
src/web/pages/keeper-panel.ts +5 −1
| @@ -8,6 +8,7 @@ | ||
| 8 | 8 | |
| 9 | 9 | import type { Database } from "bun:sqlite"; |
| 10 | 10 | import { aheadLine } from "../../flow/alarm_main_ahead.ts"; |
| 11 | import { neverTickedLine } from "../../keeper/at_boot.ts"; | |
| 11 | 12 | import { beatFact, lastBeat } from "../../keeper/beat.ts"; |
| 12 | 13 | import { blocking, MAX_REASON, standingHolds } from "../../keeper/hold.ts"; |
| 13 | 14 | import { pendingNow } from "../../keeper/now.ts"; |
| @@ -37,6 +38,9 @@ export function keeperPanel(db: Database, mayHold = false, now: Date = new Date( | ||
| 37 | 38 | const beat = lastBeat(db); |
| 38 | 39 | const asked = pendingNow(db, now.getTime()); |
| 39 | 40 | const ahead = aheadLine(db, now); |
| 41 | // Never heard from: the same keeper line the main-ahead alarm names (#11924). | |
| 42 | const never = beat === null ? neverTickedLine() : undefined; | |
| 43 | const tick = never === undefined ? beatFact(beat, now) : `${beatFact(beat, now)}; ${never}`; | |
| 40 | 44 | if (releases.length === 0 && blocked === undefined && beat === null && ahead === undefined && !mayHold) return undefined; |
| 41 | 45 | const rows = releases.map((r, i) => { |
| 42 | 46 | const s = STAGE[r.stage]; |
| @@ -56,7 +60,7 @@ export function keeperPanel(db: Database, mayHold = false, now: Date = new Date( | ||
| 56 | 60 | <h2>Release keeper</h2> |
| 57 | 61 | <p class="small ${blocked === undefined ? "muted" : ""}" id="keeper-blocked">${blocked ?? "Nothing holds the next cut."}</p> |
| 58 | 62 | <dl class="status-rows" id="keeper-facts"> |
| 59 | <div class="status-row"><dt>Release tick</dt><dd id="keeper-heartbeat">${beatFact(beat, now)}</dd></div> | |
| 63 | <div class="status-row"><dt>Release tick</dt><dd id="keeper-heartbeat">${tick}</dd></div> | |
| 60 | 64 | ${when(asked !== undefined, () => html`<div class="status-row"><dt>Release now</dt><dd id="keeper-now">asked by ${asked!.by}: ${asked!.reason}</dd></div>`)} |
| 61 | 65 | ${when(ahead !== undefined, () => html`<div class="status-row"><dt>Main vs production</dt><dd id="keeper-ahead">${ahead}</dd></div>`)} |
| 62 | 66 | </dl> |
test/flow-alarm-main-ahead.test.ts +79
| @@ -17,6 +17,8 @@ import { keeperPanel } from "../src/web/pages/keeper-panel.ts"; | ||
| 17 | 17 | import { keeperAnswer } from "../src/keeper/landed.ts"; |
| 18 | 18 | import { tickStep } from "../src/keeper/keeper.ts"; |
| 19 | 19 | import { keeperForge } from "./helpers/keeper.ts"; |
| 20 | import { noteKeeperAtBoot } from "../src/keeper/at_boot.ts"; | |
| 21 | import { keeperEnv } from "../src/keeper/wire.ts"; | |
| 20 | 22 | import { id } from "../src/kernel/id.ts"; |
| 21 | 23 | import { forgetLive, keepLive } from "../src/release/landings.ts"; |
| 22 | 24 | import { alarmLines } from "../src/crew/notify.ts"; |
| @@ -311,3 +313,80 @@ describe("the keeper's own outcome", () => { | ||
| 311 | 313 | expect(held.outcome).toBe("skipped"); |
| 312 | 314 | }); |
| 313 | 315 | }); |
| 316 | ||
| 317 | describe("a tick never heard from (#11924)", () => { | |
| 318 | const open = () => app.db.query<{ id: string }, []>(`SELECT id FROM issue WHERE state = 'open' AND title LIKE 'Sentinel: Main is green and ahead%'`).all(); | |
| 319 | const closeAll = () => app.db.query(`UPDATE issue SET state = 'closed' WHERE title LIKE 'Sentinel: Main is green and ahead%'`).run(); | |
| 320 | ||
| 321 | test("the alarm and /releases name why the keeper is off, and the sentinel files no crew issue", () => { | |
| 322 | reset(); | |
| 323 | closeAll(); | |
| 324 | app.db.query(`DELETE FROM keeper_beat_tick`).run(); | |
| 325 | check("ok", HEAD); | |
| 326 | merge(120, "6".repeat(40)); | |
| 327 | noteKeeperAtBoot(keeperEnv({ FORGE_RELEASE_KEEPER: "secret-looking-value" })); | |
| 328 | try { | |
| 329 | const a = mainAheadAlarm(app.db, NOW); | |
| 330 | expect(a.fired).toBe(true); | |
| 331 | expect(a.facts).toContain("the in-process release keeper is off: FORGE_RELEASE_KEEPER is not on"); | |
| 332 | expect(a.says).toContain("the in-process release keeper is off: FORGE_RELEASE_KEEPER is not on"); | |
| 333 | expect(JSON.stringify(a)).not.toContain("secret-looking-value"); | |
| 334 | expect(String(keeperPanel(app.db, false, NOW))).toContain( | |
| 335 | '<dd id="keeper-heartbeat">never reported; the in-process release keeper is off: FORGE_RELEASE_KEEPER is not on</dd>', | |
| 336 | ); | |
| 337 | // The owner's ask: the flow alarm reaches the notifier, saying why. | |
| 338 | const lines = alarmLines(app, owner, NOW.getTime()); | |
| 339 | expect(lines.find((l) => l.id.startsWith("alarm:main_ahead@"))?.line).toContain("FORGE_RELEASE_KEEPER is not on"); | |
| 340 | hunt(app, "trevato/forge", NOW.getTime() + 900_000); | |
| 341 | expect(open()).toHaveLength(0); | |
| 342 | noteKeeperAtBoot({ on: true }); | |
| 343 | expect(mainAheadAlarm(app.db, NOW).facts).toContain("the release keeper is on but has not ticked"); | |
| 344 | } finally { | |
| 345 | noteKeeperAtBoot(undefined); | |
| 346 | } | |
| 347 | }); | |
| 348 | ||
| 349 | test("an issue filed before stays open while main is still ahead, and closes once production catches up", () => { | |
| 350 | reset(); | |
| 351 | closeAll(); | |
| 352 | app.db.query(`DELETE FROM keeper_beat_tick`).run(); | |
| 353 | check("ok", HEAD); | |
| 354 | merge(120, "8".repeat(40)); | |
| 355 | // Filed while a beat stood (as every never-heard finding was before #11924), then the | |
| 356 | // beats are gone. | |
| 357 | recordBeat(app.db, "skipped", "a hold stands", new Date(NOW.getTime() - 5 * 60_000)); | |
| 358 | hunt(app, "trevato/forge", NOW.getTime() + 1_100_000); | |
| 359 | expect(open()).toHaveLength(1); | |
| 360 | app.db.query(`DELETE FROM keeper_beat_tick`).run(); | |
| 361 | noteKeeperAtBoot({ on: false, why: "FORGE_RELEASE_KEEPER is not on" }); | |
| 362 | try { | |
| 363 | hunt(app, "trevato/forge", NOW.getTime() + 1_200_000); | |
| 364 | expect(open()).toHaveLength(1); | |
| 365 | keepLive("trevato/forge", ["8".repeat(40)], NOW.getTime()); | |
| 366 | hunt(app, "trevato/forge", NOW.getTime() + 1_300_000); | |
| 367 | expect(open()).toHaveLength(0); | |
| 368 | } finally { | |
| 369 | noteKeeperAtBoot(undefined); | |
| 370 | } | |
| 371 | }); | |
| 372 | ||
| 373 | test("with beats present the finding is filed as before, with no keeper line", () => { | |
| 374 | reset(); | |
| 375 | closeAll(); | |
| 376 | app.db.query(`DELETE FROM keeper_beat_tick`).run(); | |
| 377 | check("ok", HEAD); | |
| 378 | merge(120, "7".repeat(40)); | |
| 379 | recordBeat(app.db, "skipped", "a hold stands", new Date(NOW.getTime() - 5 * 60_000)); | |
| 380 | noteKeeperAtBoot({ on: false, why: "FORGE_RELEASE_KEEPER is not on" }); | |
| 381 | try { | |
| 382 | const a = mainAheadAlarm(app.db, NOW); | |
| 383 | expect(a.facts?.some((f) => f.includes("release keeper is"))).toBe(false); | |
| 384 | expect(a.says).not.toContain("release keeper is"); | |
| 385 | expect(String(keeperPanel(app.db, false, NOW))).not.toContain("release keeper is off"); | |
| 386 | hunt(app, "trevato/forge", NOW.getTime() + 1_000_000); | |
| 387 | expect(open()).toHaveLength(1); | |
| 388 | } finally { | |
| 389 | noteKeeperAtBoot(undefined); | |
| 390 | } | |
| 391 | }); | |
| 392 | }); |
test/tui-release-keeper.test.ts +17
| @@ -8,6 +8,7 @@ import { tick } from "../src/keeper/keeper.ts"; | ||
| 8 | 8 | import type { User } from "../src/core/users.ts"; |
| 9 | 9 | import { createForgeApp, forgeCommands, Session } from "../src/tui/forge.ts"; |
| 10 | 10 | import type { Ctx } from "../src/tui/app/app.ts"; |
| 11 | import { noteKeeperAtBoot } from "../src/keeper/at_boot.ts"; | |
| 11 | 12 | import { holdLine, keeperScreen, readKeeper } from "../src/tui/screens/keeper.ts"; |
| 12 | 13 | import { headlessTerminal } from "../src/tui/term/terminal.ts"; |
| 13 | 14 | import { quietLogs } from "./helpers/globals.ts"; |
| @@ -66,4 +67,20 @@ describe("the release keeper screen", () => { | ||
| 66 | 67 | expect(readKeeper(session)?.asked).toContain("hotfix for the login page"); |
| 67 | 68 | expect(holdLine(session, "now: ").ok).toBe(false); |
| 68 | 69 | }); |
| 70 | ||
| 71 | test("a tick never heard from says the keeper is off and why, as /releases does (#11924)", async () => { | |
| 72 | const f = await keeperForge(); | |
| 73 | noteKeeperAtBoot({ on: false, why: "FORGE_RELEASE_KEEPER is not on" }); | |
| 74 | try { | |
| 75 | expect(readKeeper(new Session(f.app, identity(f.user)))?.heartbeat).toBe( | |
| 76 | "The release tick has never reported a heartbeat. The in-process release keeper is off: FORGE_RELEASE_KEEPER is not on.", | |
| 77 | ); | |
| 78 | noteKeeperAtBoot({ on: true }); | |
| 79 | expect(readKeeper(new Session(f.app, identity(f.user)))?.heartbeat).toBe( | |
| 80 | "The release tick has never reported a heartbeat. The release keeper is on but has not ticked.", | |
| 81 | ); | |
| 82 | } finally { | |
| 83 | noteKeeperAtBoot(undefined); | |
| 84 | } | |
| 85 | }); | |
| 69 | 86 | }); |
new above
offline
Use this line in the TUI keeper screen too (both doors), and add a test for it.