Public: anyone can read this; the crew can change it.

The forge says on the board, and pages once, when production has fallen behind main's newest release for more than an hour #2613

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/2612-the-forge-says-on-the-board-and-pages-on → main 1 commit fork

  • Merged. scout/2612-the-forge-says-on-the-board-and-pages-on was merged into main as ad7067f.

Review

Sign in to review this change.
4 files changed +264 −0
src/kernel/vitals.ts +5
@@ -26,6 +26,7 @@ import { lastMirror, mirrorNeed } from "../snapshot/mirror.ts";
2626 import { lastSnapshotOutcome, snapshotNeed } from "../snapshot/outcome.ts";
2727 import { drillNeed, firstSeen as drillFirstSeen, lastVerdict } from "../snapshot/verdict.ts";
2828 import { proofNeed } from "../release/prove.ts";
29 import { behindStanding } from "../release/behind.ts";
2930 import { newestDrills } from "../core/domain_drills.ts";
3031 import { DOMAINS } from "./domains.ts";
3132 import { domainDrillNeeds } from "./domain_drill.ts";
@@ -398,6 +399,10 @@ export function vitals(input: VitalsInput): Vitals {
398399 const unproven = proofNeed(input.dataDir);
399400 if (unproven !== undefined) need("release", unproven.sentence, unproven.since, "/board");
400401
402 // Production serving a version older than a release main has had over an hour is a card until they match.
403 const lagging = behindStanding(now);
404 if (lagging !== undefined) need("release:behind", `${lagging.sentence}.`, lagging.since, "/board");
405
401406 const apps = { serving: 0, asleep: 0, previews: 0 };
402407 for (const e of input.envs ?? []) {
403408 if (e.env !== "production") apps.previews++;
src/main.ts +20
@@ -55,6 +55,8 @@ import { running } from "./kernel/version.ts";
5555 import { record as recordDegraded } from "./kernel/degraded_history.ts";
5656 import { vitals } from "./kernel/vitals.ts";
5757 import { proveRelease } from "./release/prove.ts";
58 import { CHECK_EVERY_MS, readNewest, setBehind, watchBehind } from "./release/behind.ts";
59 import { repoDir } from "./git/repos.ts";
5860 import { snapshotPlace } from "./actions/health.ts";
5961 import { capacityDefault } from "./core/capacity.ts";
6062
@@ -490,6 +492,24 @@ void proveRelease({
490492 url: forgeUrl,
491493 token: process.env["FORGE_TOKEN"],
492494 }).then((p) => p && app.log.info("release proof", { version: p.version, ok: String(p.ok), ...(p.failure ? { failure: p.failure } : {}) }));
495 // Production behind main's newest release for over an hour is a card (release/behind.ts).
496 if (app.config.forgeRepo) {
497 const [owner, name] = app.config.forgeRepo.split("/");
498 const watch = watchBehind({
499 live: () => running().version,
500 read: async () => {
501 const row = app.db
502 .query<{ default_branch: string }, [string, string]>(
503 "SELECT repo.default_branch FROM repo JOIN user u ON u.id = repo.owner_id WHERE u.username = ?1 AND repo.name = ?2",
504 )
505 .get(owner ?? "", name ?? "");
506 return row ? readNewest(repoDir(app.reposRoot, owner!, name!), row.default_branch) : undefined;
507 },
508 });
509 setBehind(watch);
510 void watch.tick();
511 setInterval(() => void watch.tick(), CHECK_EVERY_MS);
512 }
493513 // Only an agent is on a crew (the `repo_crew_agents_only` trigger refuses anyone else). A person
494514 // put on one before that is reported here for the owner to take off — the
495515 // forge never removes it on its own.
src/release/behind.ts added +121
@@ -0,0 +1,121 @@
1 /**
2 * Is production behind main? Every ten minutes the forge compares the version
3 * it is running (the one `/healthz` sends as `X-Forge-Version`) with the newest
4 * release in `src/core/releases.json` on the default branch of its own
5 * repository, read from the git store and never the network. When that
6 * release has been on main for over an hour and is not the version serving,
7 * vitals carry one `release:behind` cause ("production is 0.7.162; main
8 * released 0.7.165 2 h ago"); the incident pager pages it once, as it does
9 * any cause, and it clears when the versions match. #2574 proves a release
10 * after it boots; this sees the release that never boots.
11 *
12 * Plain data in, plain data out: the clock and the reader are injected, the
13 * answer is kept in memory (it is derived from git and rebuilt on boot).
14 */
15
16 import { runGit } from "../git/env.ts";
17 import { UNSTAMPED } from "../kernel/version.ts";
18
19 /** How often the forge looks. */
20 export const CHECK_EVERY_MS = 10 * 60_000;
21 /** How long a release may sit on main before production not serving it is a cause. */
22 export const GRACE_MS = 60 * 60_000;
23
24 const FILE = "src/core/releases.json";
25 const VERSION = /^[0-9A-Za-z][0-9A-Za-z.+_-]{0,63}$/;
26
27 /** The newest release on main and when it landed there (epoch ms). */
28 export interface Newest {
29 readonly version: string;
30 readonly at: number;
31 }
32
33 export interface BehindNeed {
34 readonly sentence: string;
35 readonly since: string;
36 }
37
38 /** The newest version in a releases document (newest first), or undefined when it names none. */
39 export function newestVersion(text: string): string | undefined {
40 try {
41 const list = (JSON.parse(text) as { releases?: unknown }).releases;
42 const v = Array.isArray(list) ? (list[0] as { version?: unknown } | undefined)?.version : undefined;
43 return typeof v === "string" && VERSION.test(v) ? v : undefined;
44 } catch {
45 return undefined;
46 }
47 }
48
49 /** The newest release of the bare repository at `dir`, on `branch`; undefined when git cannot say. */
50 export async function readNewest(dir: string, branch: string): Promise<Newest | undefined> {
51 if (!/^[\w][\w./-]{0,200}$/.test(branch) || branch.includes("..")) return undefined;
52 const doc = await runGit(["show", `refs/heads/${branch}:${FILE}`], { cwd: dir });
53 if (!doc.ok) return undefined;
54 const version = newestVersion(doc.value);
55 if (version === undefined) return undefined;
56 // The commit that put this version in the list is when main released it.
57 const when = await runGit(
58 ["log", "-1", "--format=%ct", `-S"version": "${version}"`, `refs/heads/${branch}`, "--", FILE],
59 { cwd: dir },
60 );
61 const secs = when.ok ? Number(when.value) : NaN;
62 return Number.isFinite(secs) && secs > 0 ? { version, at: secs * 1000 } : undefined;
63 }
64
65 /** "40 min ago", "2 h ago", "3 d ago". */
66 function ago(ms: number): string {
67 const minutes = Math.floor(ms / 60_000);
68 if (minutes < 60) return `${minutes} min ago`;
69 if (minutes < 24 * 60) return `${Math.floor(minutes / 60)} h ago`;
70 return `${Math.floor(minutes / (24 * 60))} d ago`;
71 }
72
73 /** The cause, or undefined while the forge serves the newest release or it is still new. */
74 export function behindNeed(live: string, newest: Newest | undefined, now: number): BehindNeed | undefined {
75 if (newest === undefined || live === UNSTAMPED || live === newest.version) return undefined;
76 const age = now - newest.at;
77 if (age <= GRACE_MS) return undefined;
78 const at = new Date(newest.at).toISOString();
79 return {
80 sentence: `production is ${live}; main released ${newest.version} ${ago(age)}`,
81 since: at,
82 };
83 }
84
85 export interface Watch {
86 /** Read main once and keep the answer. Never throws. */
87 tick(): Promise<void>;
88 /** The cause as of `now`, from what the last tick read. */
89 need(now?: number): BehindNeed | undefined;
90 }
91
92 export function watchBehind(o: {
93 readonly live: () => string;
94 readonly read: () => Promise<Newest | undefined>;
95 readonly now?: () => number;
96 }): Watch {
97 const clock = o.now ?? Date.now;
98 let newest: Newest | undefined;
99 return {
100 async tick() {
101 try {
102 newest = await o.read();
103 } catch {
104 // A read that fails keeps the last answer; it must not clear a cause.
105 }
106 },
107 need: (now = clock()) => behindNeed(o.live(), newest, now),
108 };
109 }
110
111 let current: Watch | undefined;
112
113 /** The watch the vitals read; main sets it once. */
114 export function setBehind(w: Watch | undefined): void {
115 current = w;
116 }
117
118 /** The standing cause, if a watch is set and it holds one. */
119 export function behindStanding(now: number): BehindNeed | undefined {
120 return current?.need(now);
121 }
test/release-behind.test.ts added +118
@@ -0,0 +1,118 @@
1 /**
2 * Production behind main's newest release for over an hour is one
3 * `release:behind` cause, and catching up clears it (#2612).
4 */
5
6 import { afterAll, beforeAll, expect, test } from "bun:test";
7 import { mkdtemp, rm } from "node:fs/promises";
8 import { tmpdir } from "node:os";
9 import { join } from "node:path";
10 import { createApp, type App } from "../src/app.ts";
11 import { vitals } from "../src/kernel/vitals.ts";
12 import { newestVersion, readNewest, setBehind, watchBehind, type Newest } from "../src/release/behind.ts";
13
14 let dataDir: string;
15 let app: App;
16 beforeAll(async () => {
17 dataDir = await mkdtemp(join(tmpdir(), "forge-behind-"));
18 app = createApp({ dataDir, port: 0 });
19 });
20 afterAll(async () => {
21 setBehind(undefined);
22 app.db.close();
23 await rm(dataDir, { recursive: true, force: true });
24 });
25
26 const T0 = Date.parse("2026-10-16T17:00:00Z");
27 const MIN = 60_000;
28
29 function harness(live: string, newest: Newest | undefined) {
30 let clock = T0;
31 const state = { live, newest };
32 const watch = watchBehind({ live: () => state.live, read: async () => state.newest, now: () => clock });
33 setBehind(watch);
34 const card = () =>
35 vitals({ db: app.db, signals: app.signals, dataDir, now: clock }).degraded.find((d) => d.cause === "release:behind");
36 return { state, watch, card, advance: (ms: number) => (clock += ms) };
37 }
38
39 test("matching versions say nothing", async () => {
40 const h = harness("0.7.165", { version: "0.7.165", at: T0 - 5 * 3_600_000 });
41 await h.watch.tick();
42 expect(h.card()).toBeUndefined();
43 });
44
45 test("behind by 30 minutes says nothing", async () => {
46 const h = harness("0.7.162", { version: "0.7.165", at: T0 - 30 * MIN });
47 await h.watch.tick();
48 expect(h.card()).toBeUndefined();
49 });
50
51 test("behind by 2 hours raises the cause once, and catching up clears it", async () => {
52 const h = harness("0.7.162", { version: "0.7.165", at: T0 - 120 * MIN });
53 await h.watch.tick();
54 const first = h.card();
55 expect(first).toMatchObject({
56 sentence: "production is 0.7.162; main released 0.7.165 2 h ago.",
57 href: "/board",
58 });
59 h.advance(10 * MIN);
60 await h.watch.tick();
61 const again = vitals({ db: app.db, signals: app.signals, dataDir, now: T0 + 10 * MIN }).degraded.filter(
62 (d) => d.cause === "release:behind",
63 );
64 expect(again).toHaveLength(1);
65 expect(again[0]!.since).toBe(first!.since);
66
67 h.state.live = "0.7.165";
68 expect(h.card()).toBeUndefined();
69 });
70
71 test("a read that finds nothing clears the cause; an unstamped forge says nothing", async () => {
72 const h = harness("0.7.162", { version: "0.7.165", at: T0 - 120 * MIN });
73 await h.watch.tick();
74 h.state.newest = undefined;
75 await h.watch.tick();
76 expect(h.card()).toBeUndefined(); // read gave nothing: no cause, not a stale one
77 h.state.newest = { version: "0.7.165", at: T0 - 120 * MIN };
78 h.state.live = "dev";
79 await h.watch.tick();
80 expect(h.card()).toBeUndefined();
81 });
82
83 test("the newest version is the first of the document, or nothing", () => {
84 expect(newestVersion(JSON.stringify({ releases: [{ version: "1.2.3" }, { version: "1.2.2" }] }))).toBe("1.2.3");
85 expect(newestVersion("{")).toBeUndefined();
86 expect(newestVersion(JSON.stringify({ releases: [{ version: "a b; rm" }] }))).toBeUndefined();
87 });
88
89 test("readNewest reads the list and its landing time from the git store", async () => {
90 const dir = join(dataDir, "r.git");
91 const run = async (...a: string[]) => {
92 const p = Bun.spawn(["git", ...a], {
93 cwd: dir,
94 stdout: "ignore",
95 stderr: "ignore",
96 env: { ...process.env, GIT_AUTHOR_NAME: "t", GIT_AUTHOR_EMAIL: "t@t", GIT_COMMITTER_NAME: "t", GIT_COMMITTER_EMAIL: "t@t" },
97 });
98 await p.exited;
99 };
100 await Bun.spawn(["git", "init", "-q", "-b", "main", dir]).exited;
101 const commit = async (v: string[], date: string) => {
102 await Bun.$`mkdir -p ${dir}/src/core`.quiet();
103 await Bun.write(
104 join(dir, "src/core/releases.json"),
105 JSON.stringify({ releases: v.map((version) => ({ version })) }, null, 2),
106 );
107 await run("add", ".");
108 await Bun.spawn(["git", "commit", "-q", "-m", v[0]!, "--date", date], {
109 cwd: dir,
110 env: { ...process.env, GIT_AUTHOR_NAME: "t", GIT_AUTHOR_EMAIL: "t@t", GIT_COMMITTER_NAME: "t", GIT_COMMITTER_EMAIL: "t@t", GIT_COMMITTER_DATE: date },
111 }).exited;
112 };
113 await commit(["0.1.0"], "2026-10-16T10:00:00Z");
114 await commit(["0.1.1", "0.1.0"], "2026-10-16T12:00:00Z");
115 expect(await readNewest(dir, "main")).toEqual({ version: "0.1.1", at: Date.parse("2026-10-16T12:00:00Z") });
116 expect(await readNewest(dir, "nope")).toBeUndefined();
117 expect(await readNewest(dir, "--bad")).toBeUndefined();
118 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs