Public: anyone can read this; the crew can change it.
Production's /status reads Down only when a stopping cause is real: find which cause still says down while every page serves #14230
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/14228-production-s-status-reads-down-only-when → main 1 commit fork 1 thread · 1 unresolved
4 files
changed +54
−4
src/kernel/boots.ts +20 −3
| @@ -28,6 +28,8 @@ interface Start { | ||
| 28 | 28 | /** When it started, ms since the epoch. */ |
| 29 | 29 | readonly at: number; |
| 30 | 30 | clean: boolean; |
| 31 | /** The commit this start ran, when known (#14228); absent in older ledgers. */ | |
| 32 | readonly commit?: string; | |
| 31 | 33 | } |
| 32 | 34 | |
| 33 | 35 | interface Ledger { |
| @@ -62,7 +64,8 @@ function read(dataDir: string): Ledger { | ||
| 62 | 64 | s !== null && |
| 63 | 65 | typeof (s as Start).id === "string" && |
| 64 | 66 | typeof (s as Start).at === "number" && |
| 65 | typeof (s as Start).clean === "boolean", | |
| 67 | typeof (s as Start).clean === "boolean" && | |
| 68 | ((s as Start).commit === undefined || typeof (s as Start).commit === "string"), | |
| 66 | 69 | ) |
| 67 | 70 | : []; |
| 68 | 71 | const total = |
| @@ -99,13 +102,14 @@ export function recordBoot( | ||
| 99 | 102 | dataDir: string, |
| 100 | 103 | id: string = randomUUID(), |
| 101 | 104 | now = Date.now(), |
| 105 | commit: string | null = null, | |
| 102 | 106 | ): BootCounts { |
| 103 | 107 | thisStart = id; |
| 104 | 108 | const ledger = read(dataDir); |
| 105 | 109 | // An older ledger that never trimmed still knows its first start; one that did, only from here. |
| 106 | 110 | ledger.firstAt ??= ledger.starts[0]?.at ?? now; |
| 107 | 111 | ledger.total += 1; |
| 108 | ledger.starts = [...ledger.starts, { id, at: now, clean: false }].slice(-KEPT); | |
| 112 | ledger.starts = [...ledger.starts, commit === null ? { id, at: now, clean: false } : { id, at: now, clean: false, commit }].slice(-KEPT); | |
| 109 | 113 | try { |
| 110 | 114 | write(dataDir, ledger); |
| 111 | 115 | } catch { |
| @@ -168,8 +172,21 @@ export const LOOP_WINDOW_MS = 10 * 60_000; | ||
| 168 | 172 | * Starts that never exited cleanly within the last {@link LOOP_WINDOW_MS} |
| 169 | 173 | * when there are {@link LOOP_STARTS} or more, else null. The newest start is |
| 170 | 174 | * left out: it is the one running. |
| 175 | * | |
| 176 | * A start the next one replaced with a different commit is a release, not a | |
| 177 | * crash (#14228): a busy main releases every minute or two, and an old | |
| 178 | * process cut before it marks itself clean is no crash. Ten minutes of merges | |
| 179 | * read as a loop and put `/status` at Down while every page served. A crash | |
| 180 | * loop restarts the same commit, so it still counts; a start whose commit, or | |
| 181 | * whose successor's, is unknown counts as before. | |
| 171 | 182 | */ |
| 172 | 183 | export function crashLoop(dataDir: string, now = Date.now()): { unclean: number; since: number } | null { |
| 173 | const bad = read(dataDir).starts.slice(0, -1).filter((s) => !s.clean && now - s.at <= LOOP_WINDOW_MS); | |
| 184 | const starts = read(dataDir).starts; | |
| 185 | const released = (i: number) => { | |
| 186 | const a = starts[i]?.commit; | |
| 187 | const b = starts[i + 1]?.commit; | |
| 188 | return a !== undefined && b !== undefined && a !== b; | |
| 189 | }; | |
| 190 | const bad = starts.slice(0, -1).filter((s, i) => !s.clean && now - s.at <= LOOP_WINDOW_MS && !released(i)); | |
| 174 | 191 | return bad.length >= LOOP_STARTS ? { unclean: bad.length, since: bad[0]!.at } : null; |
| 175 | 192 | } |
src/main.ts +2 −1
| @@ -906,7 +906,8 @@ void during("asks", tickAsks, undefined, app); | ||
| 906 | 906 | |
| 907 | 907 | // Every start is counted, and one that follows a process that never said |
| 908 | 908 | // goodbye is a crash the log could not record (kernel/boots.ts, #3708). |
| 909 | const boot = recordBoot(app.config.dataDir); | |
| 909 | // The commit too: a start replaced by another commit is a release, not a crash (#14228). | |
| 910 | const boot = recordBoot(app.config.dataDir, undefined, undefined, running().commit); | |
| 910 | 911 | app.log.info("forge is up", { |
| 911 | 912 | url: servers.url, |
| 912 | 913 | data: app.config.dataDir, |
test/health-broken.test.ts +13
| @@ -214,6 +214,19 @@ describe("a crash loop", () => { | ||
| 214 | 214 | expect(crashLoop(dir, 3 * min)).toEqual({ unclean: 3, since: 0 }); |
| 215 | 215 | }); |
| 216 | 216 | |
| 217 | test("a start replaced by another commit is a release, not a crash; unknown commits count (#14228)", () => { | |
| 218 | const dir = ledger("releases"); | |
| 219 | for (const [i, c] of ["a", "b", "c", "d"].entries()) recordBoot(dir, `r${i}`, i * min, OID(c)); | |
| 220 | expect(crashLoop(dir, 3 * min)).toBeNull(); | |
| 221 | // A release that crashes on boot restarts its own commit: that is a loop. | |
| 222 | for (const i of [4, 5, 6, 7]) recordBoot(dir, `r${i}`, i * min, OID("e")); | |
| 223 | expect(crashLoop(dir, 7 * min)).toEqual({ unclean: 3, since: 4 * min }); | |
| 224 | // Older ledgers carry no commit: counted as before. | |
| 225 | const old = ledger("unknown"); | |
| 226 | for (const [i, id] of ["a", "b", "c", "d"].entries()) recordBoot(old, id, i * min); | |
| 227 | expect(crashLoop(old, 3 * min)).toEqual({ unclean: 3, since: 0 }); | |
| 228 | }); | |
| 229 | ||
| 217 | 230 | test("clean exits and old crashes are not a loop", () => { |
| 218 | 231 | const dir = ledger("calm"); |
| 219 | 232 | for (const [i, id] of ["a", "b", "c", "d"].entries()) { |
test/status-page.test.ts +19
| @@ -9,6 +9,7 @@ import { statusBuilds } from "../src/web/pages/status.ts"; | ||
| 9 | 9 | import { createForgeApp, forgeCommands, Session } from "../src/tui/forge.ts"; |
| 10 | 10 | import { statusLines } from "../src/tui/screens/status.ts"; |
| 11 | 11 | import { headlessTerminal } from "../src/tui/term/terminal.ts"; |
| 12 | import { recordBoot } from "../src/kernel/boots.ts"; | |
| 12 | 13 | |
| 13 | 14 | let dir: string; |
| 14 | 15 | let app: App; |
| @@ -120,6 +121,24 @@ describe("statusWords", () => { | ||
| 120 | 121 | expect(statusLines(new Session(failing, user))[0]).toBe(w.sentence); |
| 121 | 122 | }); |
| 122 | 123 | |
| 124 | test("releases cut before a clean exit are not a crash loop; the same commit restarting is (#14228)", () => { | |
| 125 | const now = Date.now(); | |
| 126 | const min = 60_000; | |
| 127 | // A busy main: four releases in four minutes, each old process cut before it marked itself | |
| 128 | // clean. | |
| 129 | const busy = createApp({ dataDir: join(dir, "releases"), port: 0 }); | |
| 130 | for (const [i, c] of ["a", "b", "c", "d"].entries()) recordBoot(busy.config.dataDir, `r${i}`, now - (4 - i) * min, c.repeat(40)); | |
| 131 | const w = statusWords(busy); | |
| 132 | expect(w.state).toBe("working"); | |
| 133 | expect(w.rows[0]).toEqual({ name: "pages and sign-in", state: "ok" }); | |
| 134 | // The same commit dying and starting again is a crash loop, and /status still says so. | |
| 135 | const looping = createApp({ dataDir: join(dir, "looping"), port: 0 }); | |
| 136 | for (const i of [0, 1, 2, 3]) recordBoot(looping.config.dataDir, `l${i}`, now - (4 - i) * min, "e".repeat(40)); | |
| 137 | const down = statusWords(looping); | |
| 138 | expect(down.state).toBe("down"); | |
| 139 | expect(down.sentence).toBe(`Down since ${new Date(now - 4 * min).toISOString().slice(11, 16)} UTC.`); | |
| 140 | }); | |
| 141 | ||
| 123 | 142 | test("inside the reuse window a second call returns the same words, after it a fresh read", () => { |
| 124 | 143 | const was = statusReuse.ms; |
| 125 | 144 | statusReuse.ms = 10_000; |
new above
offline
This line is very long; build the Start object in a const first for readability.