Public: anyone can read this; the crew can change it.
The forge computes the "red main" flow alarm (#5930) #6361
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/6343-the-forge-computes-the-red-main-flow-ala → main 1 commit fork 1 thread · 1 unresolved
2 files
changed +143
−0
src/flow/alarm_red_main.ts added +55
| @@ -0,0 +1,55 @@ | ||
| 1 | /** | |
| 2 | * The "red main" flow alarm (#5930): the forge repository's default branch | |
| 3 | * has had a red latest check for {@link RED_MAIN_MIN} minutes or more. | |
| 4 | * Plain data in (the db, `now`), plain data out; nothing draws it here. | |
| 5 | */ | |
| 6 | ||
| 7 | import type { Database } from "bun:sqlite"; | |
| 8 | import { redMain } from "../checks/red_main.ts"; | |
| 9 | import { getRepo } from "../core/repos.ts"; | |
| 10 | import { ownRepoPath } from "../kernel/own_repo.ts"; | |
| 11 | ||
| 12 | /** How long main may stay red before the alarm fires. */ | |
| 13 | export const RED_MAIN_MIN = 10; | |
| 14 | ||
| 15 | export interface RedMainAlarm { | |
| 16 | readonly fired: boolean; | |
| 17 | /** When the current red streak began, while main is red. */ | |
| 18 | readonly since?: string; | |
| 19 | readonly says: string; | |
| 20 | } | |
| 21 | ||
| 22 | /** The first failure after main's last green check: when the streak began. */ | |
| 23 | function streakStart(db: Database, repoId: string, fallback: string): string { | |
| 24 | const row = db | |
| 25 | .query<{ at: string | null }, [string]>( | |
| 26 | `SELECT MIN(COALESCE(r.finished_at, r.created_at)) AS at | |
| 27 | FROM run r JOIN repo ON repo.id = r.repo_id | |
| 28 | WHERE r.repo_id = ?1 AND r.kind = 'check' AND r.issue_id IS NULL | |
| 29 | AND r.env = repo.default_branch AND r.state = 'failed' | |
| 30 | AND r.id > COALESCE(( | |
| 31 | SELECT MAX(g.id) FROM run g | |
| 32 | WHERE g.repo_id = r.repo_id AND g.kind = 'check' AND g.issue_id IS NULL | |
| 33 | AND g.env = repo.default_branch AND g.state = 'ok'), '')`, | |
| 34 | ) | |
| 35 | .get(repoId); | |
| 36 | return row?.at ?? fallback; | |
| 37 | } | |
| 38 | ||
| 39 | export function redMainAlarm(db: Database, now: Date): RedMainAlarm { | |
| 40 | const [owner, name] = ownRepoPath().split("/"); | |
| 41 | const repo = owner && name ? getRepo(db, owner, name) : undefined; | |
| 42 | const red = repo ? redMain(db, repo.id) : undefined; | |
| 43 | if (!repo || !red) return { fired: false, says: "Main is green." }; | |
| 44 | const since = streakStart(db, repo.id, red.since); | |
| 45 | const minutes = Math.floor((now.getTime() - Date.parse(since)) / 60_000); | |
| 46 | if (!(minutes >= RED_MAIN_MIN)) { | |
| 47 | return { fired: false, since, says: `Main went red ${Math.max(minutes, 0)} min ago.` }; | |
| 48 | } | |
| 49 | const what = red.failing[0] ?? red.summary.replace(/\s+/g, " ").trim().slice(0, 120); | |
| 50 | return { | |
| 51 | fired: true, | |
| 52 | since, | |
| 53 | says: `Main has been red for ${minutes} min${what ? `: ${what}` : ""}.`, | |
| 54 | }; | |
| 55 | } |
test/flow-alarm-red-main.test.ts added +88
| @@ -0,0 +1,88 @@ | ||
| 1 | /** The red-main flow alarm (#6343): fires at 10 min of red, names the failing test; green or fresh red does not fire. */ | |
| 2 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 3 | import { mkdtempSync, rmSync } from "node:fs"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { createApp, type App } from "../src/app.ts"; | |
| 7 | import { createRepo } from "../src/core/repos.ts"; | |
| 8 | import { createUser } from "../src/core/users.ts"; | |
| 9 | import { id } from "../src/kernel/id.ts"; | |
| 10 | import { redMainAlarm } from "../src/flow/alarm_red_main.ts"; | |
| 11 | ||
| 12 | const NOW = new Date("2026-10-26T12:00:00.000Z"); | |
| 13 | const ago = (min: number): string => new Date(NOW.getTime() - min * 60_000).toISOString(); | |
| 14 | ||
| 15 | let root: string; | |
| 16 | let app: App; | |
| 17 | let repoId: string; | |
| 18 | let branch: string; | |
| 19 | let n = 0; | |
| 20 | ||
| 21 | beforeAll(async () => { | |
| 22 | root = mkdtempSync(join(tmpdir(), "forge-alarm-red-")); | |
| 23 | app = createApp({ dataDir: root, port: 0 }); | |
| 24 | const made = await createUser(app.db, { username: "trevato", email: "[email protected]", password: "a-good-passphrase", origin: "seed", trusted: true, adminIfFirst: true }); | |
| 25 | if (!made.ok) throw new Error(made.error); | |
| 26 | const repo = await createRepo(app.db, app.reposRoot, { ownerId: made.value.id, name: "forge", visibility: "public", description: "f" }); | |
| 27 | if (!repo.ok) throw new Error(repo.error); | |
| 28 | repoId = repo.value.id; | |
| 29 | branch = repo.value.defaultBranch; | |
| 30 | }); | |
| 31 | afterAll(() => { | |
| 32 | app.close(); | |
| 33 | rmSync(root, { recursive: true, force: true }); | |
| 34 | }); | |
| 35 | ||
| 36 | function check(state: "ok" | "failed", finishedMinAgo: number, names: string[] = []): void { | |
| 37 | // ids sort in insertion order | |
| 38 | app.db | |
| 39 | .query(`INSERT INTO run (id, repo_id, kind, env, state, summary, check_detail, created_at, finished_at) VALUES (?1, ?2, 'check', ?3, ?4, ?5, ?6, ?7, ?7)`) | |
| 40 | .run(`${String(++n).padStart(4, "0")}${id()}`, repoId, branch, state, state === "ok" ? "ok" : "check failed", JSON.stringify({ names }), ago(finishedMinAgo)); | |
| 41 | } | |
| 42 | const clear = (): void => void app.db.query(`DELETE FROM run`).run(); | |
| 43 | ||
| 44 | describe("red main alarm", () => { | |
| 45 | test("no check yet does not fire", () => { | |
| 46 | expect(redMainAlarm(app.db, NOW).fired).toBe(false); | |
| 47 | }); | |
| 48 | ||
| 49 | test("red for 11 min fires and names the test", () => { | |
| 50 | clear(); | |
| 51 | check("failed", 11, ["test/standards.test.ts"]); | |
| 52 | const a = redMainAlarm(app.db, NOW); | |
| 53 | expect(a.fired).toBe(true); | |
| 54 | expect(a.since).toBe(ago(11)); | |
| 55 | expect(a.says).toBe("Main has been red for 11 min: test/standards.test.ts."); | |
| 56 | }); | |
| 57 | ||
| 58 | test("red for 9 min does not fire", () => { | |
| 59 | clear(); | |
| 60 | check("failed", 9, ["x.test.ts"]); | |
| 61 | expect(redMainAlarm(app.db, NOW).fired).toBe(false); | |
| 62 | }); | |
| 63 | ||
| 64 | test("green does not fire, even after an old red", () => { | |
| 65 | clear(); | |
| 66 | check("failed", 60); | |
| 67 | check("ok", 5); | |
| 68 | expect(redMainAlarm(app.db, NOW)).toEqual({ fired: false, says: "Main is green." }); | |
| 69 | }); | |
| 70 | ||
| 71 | test("the streak counts from the first red after the last green", () => { | |
| 72 | clear(); | |
| 73 | check("ok", 90); | |
| 74 | check("failed", 25, ["a.test.ts"]); | |
| 75 | check("failed", 3, ["a.test.ts"]); | |
| 76 | const a = redMainAlarm(app.db, NOW); | |
| 77 | expect(a.fired).toBe(true); | |
| 78 | expect(a.says).toBe("Main has been red for 25 min: a.test.ts."); | |
| 79 | }); | |
| 80 | ||
| 81 | test("a pull's red check is not main's", () => { | |
| 82 | clear(); | |
| 83 | const issue = id(); | |
| 84 | app.db.query(`INSERT INTO issue (id, repo_id, number, kind, title, body, author_id, state, created_at) VALUES (?1, ?2, 1, 'pull', 'p', '', (SELECT id FROM user LIMIT 1), 'open', ?3)`).run(issue, repoId, ago(100)); | |
| 85 | app.db.query(`INSERT INTO run (id, repo_id, kind, issue_id, env, state, created_at, finished_at) VALUES ('zzz1', ?1, 'check', ?2, ?3, 'failed', ?4, ?4)`).run(repoId, issue, branch, ago(30)); | |
| 86 | expect(redMainAlarm(app.db, NOW).fired).toBe(false); | |
| 87 | }); | |
| 88 | }); |
new above
offline
The fallback to
red.summaryhas no test. Add a case withnames: []and assert thesaystext.