Public: anyone can read this; the crew can change it.
The forge says on the board, and pages once, when production has fallen behind main's newest release for more than an hour #2613
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/2612-the-forge-says-on-the-board-and-pages-on → main 1 commit fork
4 files
changed +264
−0
src/kernel/vitals.ts +5
| @@ -26,6 +26,7 @@ import { lastMirror, mirrorNeed } from "../snapshot/mirror.ts"; | ||
| 26 | 26 | import { lastSnapshotOutcome, snapshotNeed } from "../snapshot/outcome.ts"; |
| 27 | 27 | import { drillNeed, firstSeen as drillFirstSeen, lastVerdict } from "../snapshot/verdict.ts"; |
| 28 | 28 | import { proofNeed } from "../release/prove.ts"; |
| 29 | import { behindStanding } from "../release/behind.ts"; | |
| 29 | 30 | import { newestDrills } from "../core/domain_drills.ts"; |
| 30 | 31 | import { DOMAINS } from "./domains.ts"; |
| 31 | 32 | import { domainDrillNeeds } from "./domain_drill.ts"; |
| @@ -398,6 +399,10 @@ export function vitals(input: VitalsInput): Vitals { | ||
| 398 | 399 | const unproven = proofNeed(input.dataDir); |
| 399 | 400 | if (unproven !== undefined) need("release", unproven.sentence, unproven.since, "/board"); |
| 400 | 401 | |
| 402 | // Production serving a version older than a release main has had over an hour is a card until they match. | |
| 403 | const lagging = behindStanding(now); | |
| 404 | if (lagging !== undefined) need("release:behind", `${lagging.sentence}.`, lagging.since, "/board"); | |
| 405 | ||
| 401 | 406 | const apps = { serving: 0, asleep: 0, previews: 0 }; |
| 402 | 407 | for (const e of input.envs ?? []) { |
| 403 | 408 | if (e.env !== "production") apps.previews++; |
src/main.ts +20
| @@ -55,6 +55,8 @@ import { running } from "./kernel/version.ts"; | ||
| 55 | 55 | import { record as recordDegraded } from "./kernel/degraded_history.ts"; |
| 56 | 56 | import { vitals } from "./kernel/vitals.ts"; |
| 57 | 57 | import { proveRelease } from "./release/prove.ts"; |
| 58 | import { CHECK_EVERY_MS, readNewest, setBehind, watchBehind } from "./release/behind.ts"; | |
| 59 | import { repoDir } from "./git/repos.ts"; | |
| 58 | 60 | import { snapshotPlace } from "./actions/health.ts"; |
| 59 | 61 | import { capacityDefault } from "./core/capacity.ts"; |
| 60 | 62 | |
| @@ -490,6 +492,24 @@ void proveRelease({ | ||
| 490 | 492 | url: forgeUrl, |
| 491 | 493 | token: process.env["FORGE_TOKEN"], |
| 492 | 494 | }).then((p) => p && app.log.info("release proof", { version: p.version, ok: String(p.ok), ...(p.failure ? { failure: p.failure } : {}) })); |
| 495 | // Production behind main's newest release for over an hour is a card (release/behind.ts). | |
| 496 | if (app.config.forgeRepo) { | |
| 497 | const [owner, name] = app.config.forgeRepo.split("/"); | |
| 498 | const watch = watchBehind({ | |
| 499 | live: () => running().version, | |
| 500 | read: async () => { | |
| 501 | const row = app.db | |
| 502 | .query<{ default_branch: string }, [string, string]>( | |
| 503 | "SELECT repo.default_branch FROM repo JOIN user u ON u.id = repo.owner_id WHERE u.username = ?1 AND repo.name = ?2", | |
| 504 | ) | |
| 505 | .get(owner ?? "", name ?? ""); | |
| 506 | return row ? readNewest(repoDir(app.reposRoot, owner!, name!), row.default_branch) : undefined; | |
| 507 | }, | |
| 508 | }); | |
| 509 | setBehind(watch); | |
| 510 | void watch.tick(); | |
| 511 | setInterval(() => void watch.tick(), CHECK_EVERY_MS); | |
| 512 | } | |
| 493 | 513 | // Only an agent is on a crew (the `repo_crew_agents_only` trigger refuses anyone else). A person |
| 494 | 514 | // put on one before that is reported here for the owner to take off — the |
| 495 | 515 | // forge never removes it on its own. |
src/release/behind.ts added +121
| @@ -0,0 +1,121 @@ | ||
| 1 | /** | |
| 2 | * Is production behind main? Every ten minutes the forge compares the version | |
| 3 | * it is running (the one `/healthz` sends as `X-Forge-Version`) with the newest | |
| 4 | * release in `src/core/releases.json` on the default branch of its own | |
| 5 | * repository, read from the git store and never the network. When that | |
| 6 | * release has been on main for over an hour and is not the version serving, | |
| 7 | * vitals carry one `release:behind` cause ("production is 0.7.162; main | |
| 8 | * released 0.7.165 2 h ago"); the incident pager pages it once, as it does | |
| 9 | * any cause, and it clears when the versions match. #2574 proves a release | |
| 10 | * after it boots; this sees the release that never boots. | |
| 11 | * | |
| 12 | * Plain data in, plain data out: the clock and the reader are injected, the | |
| 13 | * answer is kept in memory (it is derived from git and rebuilt on boot). | |
| 14 | */ | |
| 15 | ||
| 16 | import { runGit } from "../git/env.ts"; | |
| 17 | import { UNSTAMPED } from "../kernel/version.ts"; | |
| 18 | ||
| 19 | /** How often the forge looks. */ | |
| 20 | export const CHECK_EVERY_MS = 10 * 60_000; | |
| 21 | /** How long a release may sit on main before production not serving it is a cause. */ | |
| 22 | export const GRACE_MS = 60 * 60_000; | |
| 23 | ||
| 24 | const FILE = "src/core/releases.json"; | |
| 25 | const VERSION = /^[0-9A-Za-z][0-9A-Za-z.+_-]{0,63}$/; | |
| 26 | ||
| 27 | /** The newest release on main and when it landed there (epoch ms). */ | |
| 28 | export interface Newest { | |
| 29 | readonly version: string; | |
| 30 | readonly at: number; | |
| 31 | } | |
| 32 | ||
| 33 | export interface BehindNeed { | |
| 34 | readonly sentence: string; | |
| 35 | readonly since: string; | |
| 36 | } | |
| 37 | ||
| 38 | /** The newest version in a releases document (newest first), or undefined when it names none. */ | |
| 39 | export function newestVersion(text: string): string | undefined { | |
| 40 | try { | |
| 41 | const list = (JSON.parse(text) as { releases?: unknown }).releases; | |
| 42 | const v = Array.isArray(list) ? (list[0] as { version?: unknown } | undefined)?.version : undefined; | |
| 43 | return typeof v === "string" && VERSION.test(v) ? v : undefined; | |
| 44 | } catch { | |
| 45 | return undefined; | |
| 46 | } | |
| 47 | } | |
| 48 | ||
| 49 | /** The newest release of the bare repository at `dir`, on `branch`; undefined when git cannot say. */ | |
| 50 | export async function readNewest(dir: string, branch: string): Promise<Newest | undefined> { | |
| 51 | if (!/^[\w][\w./-]{0,200}$/.test(branch) || branch.includes("..")) return undefined; | |
| 52 | const doc = await runGit(["show", `refs/heads/${branch}:${FILE}`], { cwd: dir }); | |
| 53 | if (!doc.ok) return undefined; | |
| 54 | const version = newestVersion(doc.value); | |
| 55 | if (version === undefined) return undefined; | |
| 56 | // The commit that put this version in the list is when main released it. | |
| 57 | const when = await runGit( | |
| 58 | ["log", "-1", "--format=%ct", `-S"version": "${version}"`, `refs/heads/${branch}`, "--", FILE], | |
| 59 | { cwd: dir }, | |
| 60 | ); | |
| 61 | const secs = when.ok ? Number(when.value) : NaN; | |
| 62 | return Number.isFinite(secs) && secs > 0 ? { version, at: secs * 1000 } : undefined; | |
| 63 | } | |
| 64 | ||
| 65 | /** "40 min ago", "2 h ago", "3 d ago". */ | |
| 66 | function ago(ms: number): string { | |
| 67 | const minutes = Math.floor(ms / 60_000); | |
| 68 | if (minutes < 60) return `${minutes} min ago`; | |
| 69 | if (minutes < 24 * 60) return `${Math.floor(minutes / 60)} h ago`; | |
| 70 | return `${Math.floor(minutes / (24 * 60))} d ago`; | |
| 71 | } | |
| 72 | ||
| 73 | /** The cause, or undefined while the forge serves the newest release or it is still new. */ | |
| 74 | export function behindNeed(live: string, newest: Newest | undefined, now: number): BehindNeed | undefined { | |
| 75 | if (newest === undefined || live === UNSTAMPED || live === newest.version) return undefined; | |
| 76 | const age = now - newest.at; | |
| 77 | if (age <= GRACE_MS) return undefined; | |
| 78 | const at = new Date(newest.at).toISOString(); | |
| 79 | return { | |
| 80 | sentence: `production is ${live}; main released ${newest.version} ${ago(age)}`, | |
| 81 | since: at, | |
| 82 | }; | |
| 83 | } | |
| 84 | ||
| 85 | export interface Watch { | |
| 86 | /** Read main once and keep the answer. Never throws. */ | |
| 87 | tick(): Promise<void>; | |
| 88 | /** The cause as of `now`, from what the last tick read. */ | |
| 89 | need(now?: number): BehindNeed | undefined; | |
| 90 | } | |
| 91 | ||
| 92 | export function watchBehind(o: { | |
| 93 | readonly live: () => string; | |
| 94 | readonly read: () => Promise<Newest | undefined>; | |
| 95 | readonly now?: () => number; | |
| 96 | }): Watch { | |
| 97 | const clock = o.now ?? Date.now; | |
| 98 | let newest: Newest | undefined; | |
| 99 | return { | |
| 100 | async tick() { | |
| 101 | try { | |
| 102 | newest = await o.read(); | |
| 103 | } catch { | |
| 104 | // A read that fails keeps the last answer; it must not clear a cause. | |
| 105 | } | |
| 106 | }, | |
| 107 | need: (now = clock()) => behindNeed(o.live(), newest, now), | |
| 108 | }; | |
| 109 | } | |
| 110 | ||
| 111 | let current: Watch | undefined; | |
| 112 | ||
| 113 | /** The watch the vitals read; main sets it once. */ | |
| 114 | export function setBehind(w: Watch | undefined): void { | |
| 115 | current = w; | |
| 116 | } | |
| 117 | ||
| 118 | /** The standing cause, if a watch is set and it holds one. */ | |
| 119 | export function behindStanding(now: number): BehindNeed | undefined { | |
| 120 | return current?.need(now); | |
| 121 | } |
test/release-behind.test.ts added +118
| @@ -0,0 +1,118 @@ | ||
| 1 | /** | |
| 2 | * Production behind main's newest release for over an hour is one | |
| 3 | * `release:behind` cause, and catching up clears it (#2612). | |
| 4 | */ | |
| 5 | ||
| 6 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 7 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 8 | import { tmpdir } from "node:os"; | |
| 9 | import { join } from "node:path"; | |
| 10 | import { createApp, type App } from "../src/app.ts"; | |
| 11 | import { vitals } from "../src/kernel/vitals.ts"; | |
| 12 | import { newestVersion, readNewest, setBehind, watchBehind, type Newest } from "../src/release/behind.ts"; | |
| 13 | ||
| 14 | let dataDir: string; | |
| 15 | let app: App; | |
| 16 | beforeAll(async () => { | |
| 17 | dataDir = await mkdtemp(join(tmpdir(), "forge-behind-")); | |
| 18 | app = createApp({ dataDir, port: 0 }); | |
| 19 | }); | |
| 20 | afterAll(async () => { | |
| 21 | setBehind(undefined); | |
| 22 | app.db.close(); | |
| 23 | await rm(dataDir, { recursive: true, force: true }); | |
| 24 | }); | |
| 25 | ||
| 26 | const T0 = Date.parse("2026-10-16T17:00:00Z"); | |
| 27 | const MIN = 60_000; | |
| 28 | ||
| 29 | function harness(live: string, newest: Newest | undefined) { | |
| 30 | let clock = T0; | |
| 31 | const state = { live, newest }; | |
| 32 | const watch = watchBehind({ live: () => state.live, read: async () => state.newest, now: () => clock }); | |
| 33 | setBehind(watch); | |
| 34 | const card = () => | |
| 35 | vitals({ db: app.db, signals: app.signals, dataDir, now: clock }).degraded.find((d) => d.cause === "release:behind"); | |
| 36 | return { state, watch, card, advance: (ms: number) => (clock += ms) }; | |
| 37 | } | |
| 38 | ||
| 39 | test("matching versions say nothing", async () => { | |
| 40 | const h = harness("0.7.165", { version: "0.7.165", at: T0 - 5 * 3_600_000 }); | |
| 41 | await h.watch.tick(); | |
| 42 | expect(h.card()).toBeUndefined(); | |
| 43 | }); | |
| 44 | ||
| 45 | test("behind by 30 minutes says nothing", async () => { | |
| 46 | const h = harness("0.7.162", { version: "0.7.165", at: T0 - 30 * MIN }); | |
| 47 | await h.watch.tick(); | |
| 48 | expect(h.card()).toBeUndefined(); | |
| 49 | }); | |
| 50 | ||
| 51 | test("behind by 2 hours raises the cause once, and catching up clears it", async () => { | |
| 52 | const h = harness("0.7.162", { version: "0.7.165", at: T0 - 120 * MIN }); | |
| 53 | await h.watch.tick(); | |
| 54 | const first = h.card(); | |
| 55 | expect(first).toMatchObject({ | |
| 56 | sentence: "production is 0.7.162; main released 0.7.165 2 h ago.", | |
| 57 | href: "/board", | |
| 58 | }); | |
| 59 | h.advance(10 * MIN); | |
| 60 | await h.watch.tick(); | |
| 61 | const again = vitals({ db: app.db, signals: app.signals, dataDir, now: T0 + 10 * MIN }).degraded.filter( | |
| 62 | (d) => d.cause === "release:behind", | |
| 63 | ); | |
| 64 | expect(again).toHaveLength(1); | |
| 65 | expect(again[0]!.since).toBe(first!.since); | |
| 66 | ||
| 67 | h.state.live = "0.7.165"; | |
| 68 | expect(h.card()).toBeUndefined(); | |
| 69 | }); | |
| 70 | ||
| 71 | test("a read that finds nothing clears the cause; an unstamped forge says nothing", async () => { | |
| 72 | const h = harness("0.7.162", { version: "0.7.165", at: T0 - 120 * MIN }); | |
| 73 | await h.watch.tick(); | |
| 74 | h.state.newest = undefined; | |
| 75 | await h.watch.tick(); | |
| 76 | expect(h.card()).toBeUndefined(); // read gave nothing: no cause, not a stale one | |
| 77 | h.state.newest = { version: "0.7.165", at: T0 - 120 * MIN }; | |
| 78 | h.state.live = "dev"; | |
| 79 | await h.watch.tick(); | |
| 80 | expect(h.card()).toBeUndefined(); | |
| 81 | }); | |
| 82 | ||
| 83 | test("the newest version is the first of the document, or nothing", () => { | |
| 84 | expect(newestVersion(JSON.stringify({ releases: [{ version: "1.2.3" }, { version: "1.2.2" }] }))).toBe("1.2.3"); | |
| 85 | expect(newestVersion("{")).toBeUndefined(); | |
| 86 | expect(newestVersion(JSON.stringify({ releases: [{ version: "a b; rm" }] }))).toBeUndefined(); | |
| 87 | }); | |
| 88 | ||
| 89 | test("readNewest reads the list and its landing time from the git store", async () => { | |
| 90 | const dir = join(dataDir, "r.git"); | |
| 91 | const run = async (...a: string[]) => { | |
| 92 | const p = Bun.spawn(["git", ...a], { | |
| 93 | cwd: dir, | |
| 94 | stdout: "ignore", | |
| 95 | stderr: "ignore", | |
| 96 | env: { ...process.env, GIT_AUTHOR_NAME: "t", GIT_AUTHOR_EMAIL: "t@t", GIT_COMMITTER_NAME: "t", GIT_COMMITTER_EMAIL: "t@t" }, | |
| 97 | }); | |
| 98 | await p.exited; | |
| 99 | }; | |
| 100 | await Bun.spawn(["git", "init", "-q", "-b", "main", dir]).exited; | |
| 101 | const commit = async (v: string[], date: string) => { | |
| 102 | await Bun.$`mkdir -p ${dir}/src/core`.quiet(); | |
| 103 | await Bun.write( | |
| 104 | join(dir, "src/core/releases.json"), | |
| 105 | JSON.stringify({ releases: v.map((version) => ({ version })) }, null, 2), | |
| 106 | ); | |
| 107 | await run("add", "."); | |
| 108 | await Bun.spawn(["git", "commit", "-q", "-m", v[0]!, "--date", date], { | |
| 109 | cwd: dir, | |
| 110 | env: { ...process.env, GIT_AUTHOR_NAME: "t", GIT_AUTHOR_EMAIL: "t@t", GIT_COMMITTER_NAME: "t", GIT_COMMITTER_EMAIL: "t@t", GIT_COMMITTER_DATE: date }, | |
| 111 | }).exited; | |
| 112 | }; | |
| 113 | await commit(["0.1.0"], "2026-10-16T10:00:00Z"); | |
| 114 | await commit(["0.1.1", "0.1.0"], "2026-10-16T12:00:00Z"); | |
| 115 | expect(await readNewest(dir, "main")).toEqual({ version: "0.1.1", at: Date.parse("2026-10-16T12:00:00Z") }); | |
| 116 | expect(await readNewest(dir, "nope")).toBeUndefined(); | |
| 117 | expect(await readNewest(dir, "--bad")).toBeUndefined(); | |
| 118 | }); |
new above
offline