Public: anyone can read this; the crew can change it.

/status says down only when pages or sign-in actually fail, not for any broken cause #14222

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/14219-status-says-down-only-when-pages-or-sign → main 2 commits fork 2 threads · 2 unresolved

  • Merged. scout/14219-status-says-down-only-when-pages-or-sign was merged into main as 08f443c.

Review

Sign in to review this change.
2 files changed +38 −13
src/core/status_words.ts +18 −6
@@ -6,8 +6,8 @@
66 *
77 * A row is shown only where the forge has a real signal behind it:
88 * - "pages and sign-in": the forge's own health (it answers, the database
9 * answers, it is not restarting) and the vitals that stop or slow serving
10 * (memory, disk, crash loop, error burst);
9 * answers, it is not restarting) and the vitals that stop serving (crash
10 * loop, full disk: down) or slow it (memory, disk, error burst: slow);
1111 * - "building": the check lane (queue, stuck runs, a red main);
1212 * - "billing records": the cost ledger's chain.
1313 * There is no row for previews or apps: nothing here measures them, and an
@@ -67,12 +67,21 @@ const WISHES = starts(
6767 );
6868 /** What touches the check lane. */
6969 const BUILDING = starts("queue", "stuck", "red", "shared");
70 /**
71 * The broken causes vitals raise that stop serving (#14219): a crash loop, a full disk refusing
72 * writes. Any other broken cause (memory under its floor, a red disk, a red main) is one the forge
73 * serves through: its row is slow, never down. A database that does not answer is no cause here:
74 * `/healthz` answers 503 for it.
75 */
outdated

database/db are never emitted by vitals.ts; remove them or show the source of the cause.

76 const STOPS = starts("crash-loop", "disk-full");
7077
7178 const PLAIN: Readonly<Record<Row, string>> = {
7279 "pages and sign-in": "pages are slower than usual",
7380 building: "building is slower than usual",
7481 "billing records": "billing records need a look",
7582 };
83 /** The causes said with their row's plain words; any other is said generically. */
84 const KNOWN = starts("queue", "stuck", "red", "shared", "memory", "memory-growth", "disk", "disk-floor", "disk-red", "disk-full", "errors", "internal", "crash-loop");
7685 /** A cause this table has never heard of is said, not hidden. */
7786 const GENERIC = "something is slower than usual";
7887
@@ -94,10 +103,11 @@ export function statusOf(input: StatusInput): StatusWords {
94103 for (const c of input.causes) {
95104 if (!c.broken && WISHES(c.cause)) continue;
96105 const row: Row = BUILDING(c.cause) ? "building" : "pages and sign-in";
97 raise(row, c.broken ? "down" : "slow");
106 const stops = c.broken && STOPS(c.cause);
107 raise(row, stops ? "down" : "slow");
98108 if (at(c.since) !== null && (since.get(row) === undefined || c.since < since.get(row)!)) since.set(row, c.since);
99 if (c.broken) brokenSince.push(c.since);
100 else slow.push(starts("queue", "stuck", "red", "shared", "memory", "memory-growth", "disk", "disk-floor", "errors", "internal", "crash-loop")(c.cause) ? PLAIN[row] : GENERIC);
109 if (stops) brokenSince.push(c.since);
110 else slow.push(KNOWN(c.cause) ? PLAIN[row] : GENERIC);
101111 }
102112 if (input.ledgerBroken) {
103113 raise("billing records", "slow");
@@ -150,7 +160,9 @@ export function statusWords(app: App, now: number = Date.now()): StatusWords {
150160 const v = forgeVitals(app);
151161 const words = statusOf({
152162 asOf: new Date(now).toISOString(),
153 ok: health.ok && v.ok,
163 // The forge's own health only: `v.ok` is false for any broken need, and which of those stop
164 // serving is `statusOf`'s to say from the causes (#14219).
165 ok: health.ok,
154166 draining: health.draining,
155167 causes: v.degraded.map((d) => ({ cause: d.cause, broken: d.broken, since: d.since })),
156168 ledgerBroken: ledgerNeed() !== null,
test/status-page.test.ts +20 −7
@@ -58,7 +58,7 @@ describe("status words", () => {
5858 expect(ok.asOf).toBe(AS_OF);
5959 expect(asOfWords(ok)).toBe("as of 14:05 UTC");
6060 expect(ok.rows.some((r) => r.since !== undefined)).toBe(false);
61 const d = statusOf({ ...healthy, causes: [cause("disk-full", true, "2026-11-15T14:02:00Z"), cause("disk-full2", true, "2026-11-15T14:03:00Z")] });
61 const d = statusOf({ ...healthy, causes: [cause("disk-full", true, "2026-11-15T14:02:00Z"), cause("crash-loop", true, "2026-11-15T14:03:00Z")] });
6262 const row = d.rows[0]!;
6363 expect(row.since).toBe("2026-11-15T14:02:00Z");
6464 expect(rowWords(row)).toBe("down since 14:02 UTC");
@@ -73,8 +73,17 @@ describe("status words", () => {
7373 expect(statusOf({ ...healthy, ok: false }).sentence).toBe("Down.");
7474 });
7575
76 test("broken is a key, not a sentence: a broken cause with any key is down", () => {
77 expect(statusOf({ ...healthy, causes: [cause("something-new", true)] }).state).toBe("down");
76 test("only a broken cause that stops serving is down; any other broken cause is slow (#14219)", () => {
77 for (const c of ["crash-loop", "disk-full"]) expect(statusOf({ ...healthy, causes: [cause(c, true)] }).state).toBe("down");
outdated

Asserting database is down pins a cause that does not exist. Use only real causes (crash-loop, disk-full).

78 const m = statusOf({ ...healthy, causes: [cause("memory", true, "2026-11-16T13:40:06Z")] });
79 expect(m.state).toBe("slowed");
80 expect(m.sentence).toBe("Slowed: pages are slower than usual.");
81 expect(m.rows[0]).toEqual({ name: "pages and sign-in", state: "slow", since: "2026-11-16T13:40:06Z" });
82 expect(statusOf({ ...healthy, causes: [cause("disk-red", true)] }).sentence).toBe("Slowed: pages are slower than usual.");
83 const r = statusOf({ ...healthy, causes: [cause("red:7", true)] });
84 expect(r.sentence).toBe("Slowed: building is slower than usual.");
85 expect(r.rows.map((x) => x.state)).toEqual(["ok", "slow", "ok"]);
86 expect(statusOf({ ...healthy, causes: [cause("something-new", true)] }).sentence).toBe("Slowed: something is slower than usual.");
7887 });
7988
8089 test("a cause the table does not know is said generically, a standing wish is not", () => {
@@ -85,7 +94,10 @@ describe("status words", () => {
8594 expect(statusOf({ ...healthy, causes: [cause("drill", false), cause("snapshots", false)] }).sentence).toBe("All working.");
8695 const d = statusOf({ ...healthy, causes: [cause(secret, true, secret)] });
8796 expect(JSON.stringify(d)).not.toContain("10.0.0.7");
88 expect(d.sentence).toBe("Down.");
97 expect(d.sentence).toBe("Slowed: something is slower than usual.");
98 const down = statusOf({ ...healthy, ok: false, causes: [cause(secret, true, secret)] });
99 expect(JSON.stringify(down)).not.toContain("10.0.0.7");
100 expect(down.sentence).toBe("Down.");
89101 });
90102
91103 test("a broken ledger slows the billing row and the sentence", () => {
@@ -97,10 +109,11 @@ describe("status words", () => {
97109
98110 describe("statusWords", () => {
99111 test("a real failing vital turns the words, the page, the JSON and the terminal", async () => {
112 // Memory under its floor is broken but the forge serves through it: slow, not down (#14219).
100113 const w = statusWords(failing);
101 expect(w.state).toBe("down");
102 expect(w.sentence).toMatch(/^Down since \d\d:\d\d UTC\.$/);
103 expect(w.rows[0]).toMatchObject({ name: "pages and sign-in", state: "down" });
114 expect(w.state).toBe("slowed");
115 expect(w.sentence).toBe("Slowed: pages are slower than usual.");
116 expect(w.rows[0]).toMatchObject({ name: "pages and sign-in", state: "slow" });
104117 expect(w.rows[0]!.since).toBeString();
105118 expect(statusWords(app).state).toBe("working");
106119 const user = { sshUsername: "git", remoteAddress: "127.0.0.1", fingerprint: "SHA256:test", user: undefined };

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs