Public: anyone can read this; the crew can change it.
/status says down only when pages or sign-in actually fail, not for any broken cause #14222
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/14219-status-says-down-only-when-pages-or-sign → main 2 commits fork 2 threads · 2 unresolved
2 files
changed +38
−13
src/core/status_words.ts +18 −6
| @@ -6,8 +6,8 @@ | ||
| 6 | 6 | * |
| 7 | 7 | * A row is shown only where the forge has a real signal behind it: |
| 8 | 8 | * - "pages and sign-in": the forge's own health (it answers, the database |
| 9 | * answers, it is not restarting) and the vitals that stop or slow serving | |
| 10 | * (memory, disk, crash loop, error burst); | |
| 9 | * answers, it is not restarting) and the vitals that stop serving (crash | |
| 10 | * loop, full disk: down) or slow it (memory, disk, error burst: slow); | |
| 11 | 11 | * - "building": the check lane (queue, stuck runs, a red main); |
| 12 | 12 | * - "billing records": the cost ledger's chain. |
| 13 | 13 | * There is no row for previews or apps: nothing here measures them, and an |
| @@ -67,12 +67,21 @@ const WISHES = starts( | ||
| 67 | 67 | ); |
| 68 | 68 | /** What touches the check lane. */ |
| 69 | 69 | const BUILDING = starts("queue", "stuck", "red", "shared"); |
| 70 | /** | |
| 71 | * The broken causes vitals raise that stop serving (#14219): a crash loop, a full disk refusing | |
| 72 | * writes. Any other broken cause (memory under its floor, a red disk, a red main) is one the forge | |
| 73 | * serves through: its row is slow, never down. A database that does not answer is no cause here: | |
| 74 | * `/healthz` answers 503 for it. | |
| 75 | */ | |
| 76 | const STOPS = starts("crash-loop", "disk-full"); | |
| 70 | 77 | |
| 71 | 78 | const PLAIN: Readonly<Record<Row, string>> = { |
| 72 | 79 | "pages and sign-in": "pages are slower than usual", |
| 73 | 80 | building: "building is slower than usual", |
| 74 | 81 | "billing records": "billing records need a look", |
| 75 | 82 | }; |
| 83 | /** The causes said with their row's plain words; any other is said generically. */ | |
| 84 | const KNOWN = starts("queue", "stuck", "red", "shared", "memory", "memory-growth", "disk", "disk-floor", "disk-red", "disk-full", "errors", "internal", "crash-loop"); | |
| 76 | 85 | /** A cause this table has never heard of is said, not hidden. */ |
| 77 | 86 | const GENERIC = "something is slower than usual"; |
| 78 | 87 | |
| @@ -94,10 +103,11 @@ export function statusOf(input: StatusInput): StatusWords { | ||
| 94 | 103 | for (const c of input.causes) { |
| 95 | 104 | if (!c.broken && WISHES(c.cause)) continue; |
| 96 | 105 | const row: Row = BUILDING(c.cause) ? "building" : "pages and sign-in"; |
| 97 | raise(row, c.broken ? "down" : "slow"); | |
| 106 | const stops = c.broken && STOPS(c.cause); | |
| 107 | raise(row, stops ? "down" : "slow"); | |
| 98 | 108 | if (at(c.since) !== null && (since.get(row) === undefined || c.since < since.get(row)!)) since.set(row, c.since); |
| 99 | if (c.broken) brokenSince.push(c.since); | |
| 100 | else slow.push(starts("queue", "stuck", "red", "shared", "memory", "memory-growth", "disk", "disk-floor", "errors", "internal", "crash-loop")(c.cause) ? PLAIN[row] : GENERIC); | |
| 109 | if (stops) brokenSince.push(c.since); | |
| 110 | else slow.push(KNOWN(c.cause) ? PLAIN[row] : GENERIC); | |
| 101 | 111 | } |
| 102 | 112 | if (input.ledgerBroken) { |
| 103 | 113 | raise("billing records", "slow"); |
| @@ -150,7 +160,9 @@ export function statusWords(app: App, now: number = Date.now()): StatusWords { | ||
| 150 | 160 | const v = forgeVitals(app); |
| 151 | 161 | const words = statusOf({ |
| 152 | 162 | asOf: new Date(now).toISOString(), |
| 153 | ok: health.ok && v.ok, | |
| 163 | // The forge's own health only: `v.ok` is false for any broken need, and which of those stop | |
| 164 | // serving is `statusOf`'s to say from the causes (#14219). | |
| 165 | ok: health.ok, | |
| 154 | 166 | draining: health.draining, |
| 155 | 167 | causes: v.degraded.map((d) => ({ cause: d.cause, broken: d.broken, since: d.since })), |
| 156 | 168 | ledgerBroken: ledgerNeed() !== null, |
test/status-page.test.ts +20 −7
| @@ -58,7 +58,7 @@ describe("status words", () => { | ||
| 58 | 58 | expect(ok.asOf).toBe(AS_OF); |
| 59 | 59 | expect(asOfWords(ok)).toBe("as of 14:05 UTC"); |
| 60 | 60 | expect(ok.rows.some((r) => r.since !== undefined)).toBe(false); |
| 61 | const d = statusOf({ ...healthy, causes: [cause("disk-full", true, "2026-11-15T14:02:00Z"), cause("disk-full2", true, "2026-11-15T14:03:00Z")] }); | |
| 61 | const d = statusOf({ ...healthy, causes: [cause("disk-full", true, "2026-11-15T14:02:00Z"), cause("crash-loop", true, "2026-11-15T14:03:00Z")] }); | |
| 62 | 62 | const row = d.rows[0]!; |
| 63 | 63 | expect(row.since).toBe("2026-11-15T14:02:00Z"); |
| 64 | 64 | expect(rowWords(row)).toBe("down since 14:02 UTC"); |
| @@ -73,8 +73,17 @@ describe("status words", () => { | ||
| 73 | 73 | expect(statusOf({ ...healthy, ok: false }).sentence).toBe("Down."); |
| 74 | 74 | }); |
| 75 | 75 | |
| 76 | test("broken is a key, not a sentence: a broken cause with any key is down", () => { | |
| 77 | expect(statusOf({ ...healthy, causes: [cause("something-new", true)] }).state).toBe("down"); | |
| 76 | test("only a broken cause that stops serving is down; any other broken cause is slow (#14219)", () => { | |
| 77 | for (const c of ["crash-loop", "disk-full"]) expect(statusOf({ ...healthy, causes: [cause(c, true)] }).state).toBe("down"); | |
outdated
Asserting |
||
| 78 | const m = statusOf({ ...healthy, causes: [cause("memory", true, "2026-11-16T13:40:06Z")] }); | |
| 79 | expect(m.state).toBe("slowed"); | |
| 80 | expect(m.sentence).toBe("Slowed: pages are slower than usual."); | |
| 81 | expect(m.rows[0]).toEqual({ name: "pages and sign-in", state: "slow", since: "2026-11-16T13:40:06Z" }); | |
| 82 | expect(statusOf({ ...healthy, causes: [cause("disk-red", true)] }).sentence).toBe("Slowed: pages are slower than usual."); | |
| 83 | const r = statusOf({ ...healthy, causes: [cause("red:7", true)] }); | |
| 84 | expect(r.sentence).toBe("Slowed: building is slower than usual."); | |
| 85 | expect(r.rows.map((x) => x.state)).toEqual(["ok", "slow", "ok"]); | |
| 86 | expect(statusOf({ ...healthy, causes: [cause("something-new", true)] }).sentence).toBe("Slowed: something is slower than usual."); | |
| 78 | 87 | }); |
| 79 | 88 | |
| 80 | 89 | test("a cause the table does not know is said generically, a standing wish is not", () => { |
| @@ -85,7 +94,10 @@ describe("status words", () => { | ||
| 85 | 94 | expect(statusOf({ ...healthy, causes: [cause("drill", false), cause("snapshots", false)] }).sentence).toBe("All working."); |
| 86 | 95 | const d = statusOf({ ...healthy, causes: [cause(secret, true, secret)] }); |
| 87 | 96 | expect(JSON.stringify(d)).not.toContain("10.0.0.7"); |
| 88 | expect(d.sentence).toBe("Down."); | |
| 97 | expect(d.sentence).toBe("Slowed: something is slower than usual."); | |
| 98 | const down = statusOf({ ...healthy, ok: false, causes: [cause(secret, true, secret)] }); | |
| 99 | expect(JSON.stringify(down)).not.toContain("10.0.0.7"); | |
| 100 | expect(down.sentence).toBe("Down."); | |
| 89 | 101 | }); |
| 90 | 102 | |
| 91 | 103 | test("a broken ledger slows the billing row and the sentence", () => { |
| @@ -97,10 +109,11 @@ describe("status words", () => { | ||
| 97 | 109 | |
| 98 | 110 | describe("statusWords", () => { |
| 99 | 111 | test("a real failing vital turns the words, the page, the JSON and the terminal", async () => { |
| 112 | // Memory under its floor is broken but the forge serves through it: slow, not down (#14219). | |
| 100 | 113 | const w = statusWords(failing); |
| 101 | expect(w.state).toBe("down"); | |
| 102 | expect(w.sentence).toMatch(/^Down since \d\d:\d\d UTC\.$/); | |
| 103 | expect(w.rows[0]).toMatchObject({ name: "pages and sign-in", state: "down" }); | |
| 114 | expect(w.state).toBe("slowed"); | |
| 115 | expect(w.sentence).toBe("Slowed: pages are slower than usual."); | |
| 116 | expect(w.rows[0]).toMatchObject({ name: "pages and sign-in", state: "slow" }); | |
| 104 | 117 | expect(w.rows[0]!.since).toBeString(); |
| 105 | 118 | expect(statusWords(app).state).toBe("working"); |
| 106 | 119 | const user = { sshUsername: "git", remoteAddress: "127.0.0.1", fingerprint: "SHA256:test", user: undefined }; |
new above
offline
database/dbare never emitted by vitals.ts; remove them or show the source of the cause.