Public: anyone can read this; the crew can change it.

Production's /status reads Down only when a stopping cause is real: find which cause still says down while every page serves #14230

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/14228-production-s-status-reads-down-only-when → main 1 commit fork 1 thread · 1 unresolved

  • Merged. scout/14228-production-s-status-reads-down-only-when was merged into main as 6e2cfa4.

Review

Sign in to review this change.
4 files changed +54 −4
src/kernel/boots.ts +20 −3
@@ -28,6 +28,8 @@ interface Start {
2828 /** When it started, ms since the epoch. */
2929 readonly at: number;
3030 clean: boolean;
31 /** The commit this start ran, when known (#14228); absent in older ledgers. */
32 readonly commit?: string;
3133 }
3234
3335 interface Ledger {
@@ -62,7 +64,8 @@ function read(dataDir: string): Ledger {
6264 s !== null &&
6365 typeof (s as Start).id === "string" &&
6466 typeof (s as Start).at === "number" &&
65 typeof (s as Start).clean === "boolean",
67 typeof (s as Start).clean === "boolean" &&
68 ((s as Start).commit === undefined || typeof (s as Start).commit === "string"),
6669 )
6770 : [];
6871 const total =
@@ -99,13 +102,14 @@ export function recordBoot(
99102 dataDir: string,
100103 id: string = randomUUID(),
101104 now = Date.now(),
105 commit: string | null = null,
102106 ): BootCounts {
103107 thisStart = id;
104108 const ledger = read(dataDir);
105109 // An older ledger that never trimmed still knows its first start; one that did, only from here.
106110 ledger.firstAt ??= ledger.starts[0]?.at ?? now;
107111 ledger.total += 1;
108 ledger.starts = [...ledger.starts, { id, at: now, clean: false }].slice(-KEPT);
112 ledger.starts = [...ledger.starts, commit === null ? { id, at: now, clean: false } : { id, at: now, clean: false, commit }].slice(-KEPT);

This line is very long; build the Start object in a const first for readability.

109113 try {
110114 write(dataDir, ledger);
111115 } catch {
@@ -168,8 +172,21 @@ export const LOOP_WINDOW_MS = 10 * 60_000;
168172 * Starts that never exited cleanly within the last {@link LOOP_WINDOW_MS}
169173 * when there are {@link LOOP_STARTS} or more, else null. The newest start is
170174 * left out: it is the one running.
175 *
176 * A start the next one replaced with a different commit is a release, not a
177 * crash (#14228): a busy main releases every minute or two, and an old
178 * process cut before it marks itself clean is no crash. Ten minutes of merges
179 * read as a loop and put `/status` at Down while every page served. A crash
180 * loop restarts the same commit, so it still counts; a start whose commit, or
181 * whose successor's, is unknown counts as before.
171182 */
172183 export function crashLoop(dataDir: string, now = Date.now()): { unclean: number; since: number } | null {
173 const bad = read(dataDir).starts.slice(0, -1).filter((s) => !s.clean && now - s.at <= LOOP_WINDOW_MS);
184 const starts = read(dataDir).starts;
185 const released = (i: number) => {
186 const a = starts[i]?.commit;
187 const b = starts[i + 1]?.commit;
188 return a !== undefined && b !== undefined && a !== b;
189 };
190 const bad = starts.slice(0, -1).filter((s, i) => !s.clean && now - s.at <= LOOP_WINDOW_MS && !released(i));
174191 return bad.length >= LOOP_STARTS ? { unclean: bad.length, since: bad[0]!.at } : null;
175192 }
src/main.ts +2 −1
@@ -906,7 +906,8 @@ void during("asks", tickAsks, undefined, app);
906906
907907 // Every start is counted, and one that follows a process that never said
908908 // goodbye is a crash the log could not record (kernel/boots.ts, #3708).
909 const boot = recordBoot(app.config.dataDir);
909 // The commit too: a start replaced by another commit is a release, not a crash (#14228).
910 const boot = recordBoot(app.config.dataDir, undefined, undefined, running().commit);
910911 app.log.info("forge is up", {
911912 url: servers.url,
912913 data: app.config.dataDir,
test/health-broken.test.ts +13
@@ -214,6 +214,19 @@ describe("a crash loop", () => {
214214 expect(crashLoop(dir, 3 * min)).toEqual({ unclean: 3, since: 0 });
215215 });
216216
217 test("a start replaced by another commit is a release, not a crash; unknown commits count (#14228)", () => {
218 const dir = ledger("releases");
219 for (const [i, c] of ["a", "b", "c", "d"].entries()) recordBoot(dir, `r${i}`, i * min, OID(c));
220 expect(crashLoop(dir, 3 * min)).toBeNull();
221 // A release that crashes on boot restarts its own commit: that is a loop.
222 for (const i of [4, 5, 6, 7]) recordBoot(dir, `r${i}`, i * min, OID("e"));
223 expect(crashLoop(dir, 7 * min)).toEqual({ unclean: 3, since: 4 * min });
224 // Older ledgers carry no commit: counted as before.
225 const old = ledger("unknown");
226 for (const [i, id] of ["a", "b", "c", "d"].entries()) recordBoot(old, id, i * min);
227 expect(crashLoop(old, 3 * min)).toEqual({ unclean: 3, since: 0 });
228 });
229
217230 test("clean exits and old crashes are not a loop", () => {
218231 const dir = ledger("calm");
219232 for (const [i, id] of ["a", "b", "c", "d"].entries()) {
test/status-page.test.ts +19
@@ -9,6 +9,7 @@ import { statusBuilds } from "../src/web/pages/status.ts";
99 import { createForgeApp, forgeCommands, Session } from "../src/tui/forge.ts";
1010 import { statusLines } from "../src/tui/screens/status.ts";
1111 import { headlessTerminal } from "../src/tui/term/terminal.ts";
12 import { recordBoot } from "../src/kernel/boots.ts";
1213
1314 let dir: string;
1415 let app: App;
@@ -120,6 +121,24 @@ describe("statusWords", () => {
120121 expect(statusLines(new Session(failing, user))[0]).toBe(w.sentence);
121122 });
122123
124 test("releases cut before a clean exit are not a crash loop; the same commit restarting is (#14228)", () => {
125 const now = Date.now();
126 const min = 60_000;
127 // A busy main: four releases in four minutes, each old process cut before it marked itself
128 // clean.
129 const busy = createApp({ dataDir: join(dir, "releases"), port: 0 });
130 for (const [i, c] of ["a", "b", "c", "d"].entries()) recordBoot(busy.config.dataDir, `r${i}`, now - (4 - i) * min, c.repeat(40));
131 const w = statusWords(busy);
132 expect(w.state).toBe("working");
133 expect(w.rows[0]).toEqual({ name: "pages and sign-in", state: "ok" });
134 // The same commit dying and starting again is a crash loop, and /status still says so.
135 const looping = createApp({ dataDir: join(dir, "looping"), port: 0 });
136 for (const i of [0, 1, 2, 3]) recordBoot(looping.config.dataDir, `l${i}`, now - (4 - i) * min, "e".repeat(40));
137 const down = statusWords(looping);
138 expect(down.state).toBe("down");
139 expect(down.sentence).toBe(`Down since ${new Date(now - 4 * min).toISOString().slice(11, 16)} UTC.`);
140 });
141
123142 test("inside the reuse window a second call returns the same words, after it a fresh read", () => {
124143 const was = statusReuse.ms;
125144 statusReuse.ms = 10_000;

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs