Public: anyone can read this; the crew can change it.
/healthz answers 503 naming the failed check when the database is missing a column the first paint reads, so a broken upgrade never looks healthy #186
Check · Review · Land · Live done — Landed and live in production.
mason/deal-clock:mason/184-healthz-answers-503-naming-the-failed-ch → main 4 commits fork 2 threads · 2 unresolved
4 files
changed +100
−8
docs/TECHNICAL.md +7 −1
| @@ -291,7 +291,13 @@ decide nothing themselves. `AGENTS.md` says how to add to it. | ||
| 291 | 291 | It needs a signed-in user: no door of the table is open to a stranger, and |
| 292 | 292 | the description is shut to them when none is. |
| 293 | 293 | - **Up, or not.** `GET /healthz` answers `{ ok, version, uptime }`; the |
| 294 | forge waits on it before it sends anyone to a deploy. | |
| 294 | forge waits on it before it sends anyone to a deploy. It runs the bounded | |
| 295 | reads the first paint depends on (`src/health.ts`: the profile row, one | |
| 296 | deal row with its newer columns, one date, and the waiting-reminder | |
| 297 | count's sources, the sent, declined and held reminders, each counted to a | |
| 298 | bound of 1000; `LIMIT`ed, no writes). If one throws it answers 503 with `ok:false` and `failed` naming | |
| 299 | the check (`profile`, `deals`, `dates`, `waiting`), never the SQL or the | |
| 300 | error, so a database a broken upgrade left behind never looks healthy. | |
| 295 | 301 | - **The look.** `design/tokens.json` holds every colour, space and size, |
| 296 | 302 | light and dark; nothing else spells out a colour (`bun run lint:design`), |
| 297 | 303 | and every pair a person reads meets 4.5:1 in both themes. |
src/health.test.ts added +51
| @@ -0,0 +1,51 @@ | ||
| 1 | import { expect, test } from "bun:test"; | |
| 2 | import { Database } from "bun:sqlite"; | |
| 3 | import { migrate, migrations } from "./db.ts"; | |
| 4 | import { routes } from "./routes.ts"; | |
| 5 | ||
| 6 | const handler = routes.find((r) => r.path === "/healthz")!.handle as (c: unknown) => Response; | |
| 7 | const ask = (db: Database) => handler({ db }); | |
| 8 | ||
| 9 | test("a migrated database is healthy, quickly", async () => { | |
| 10 | const db = new Database(":memory:"); | |
| 11 | migrate(db); | |
| 12 | const t = performance.now(); | |
| 13 | const response = ask(db); | |
| 14 | expect(performance.now() - t).toBeLessThan(50); | |
| 15 | expect(response.status).toBe(200); | |
| 16 | expect(response.headers.get("cache-control")).toBe("no-store"); | |
| 17 | expect(response.headers.get("x-content-type-options")).toBe("nosniff"); | |
| 18 | const body = (await response.json()) as Record<string, unknown>; | |
| 19 | expect(body.ok).toBe(true); | |
| 20 | expect("failed" in body).toBe(false); | |
| 21 | }); | |
| 22 | ||
| 23 | test("a database missing profile.line answers 503 naming profile, and nothing else", async () => { | |
| 24 | const db = new Database(":memory:"); | |
| 25 | db.exec("CREATE TABLE schema_version (version INTEGER PRIMARY KEY, applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')))"); | |
| 26 | migrations | |
| 27 | .filter((m) => !/profile ADD COLUMN line/.test(m)) | |
| 28 | .forEach((sql, i) => { | |
| 29 | db.exec(sql); | |
| 30 | db.query("INSERT INTO schema_version (version) VALUES (?1)").run(i + 1); | |
| 31 | }); | |
| 32 | const response = ask(db); | |
| 33 | expect(response.status).toBe(503); | |
| 34 | expect(response.headers.get("cache-control")).toBe("no-store"); | |
| 35 | expect(response.headers.get("x-content-type-options")).toBe("nosniff"); | |
| 36 | const text = await response.text(); | |
| 37 | const body = JSON.parse(text) as Record<string, unknown>; | |
| 38 | expect(body.ok).toBe(false); | |
| 39 | expect(body.failed).toBe("profile"); | |
| 40 | expect(Object.keys(body).sort()).toEqual(["commit", "env", "failed", "ok", "uptime", "version"]); | |
| 41 | expect(text).not.toMatch(/SELECT|no such column|line/); | |
| 42 | }); | |
| 43 | ||
| 44 | test("a database missing a table the waiting count reads answers 503 naming waiting", async () => { | |
| 45 | const db = new Database(":memory:"); | |
| 46 | migrate(db); | |
| 47 | db.exec("DROP TABLE reminder_declined"); | |
| 48 | const response = ask(db); | |
| 49 | expect(response.status).toBe(503); | |
| 50 | expect(((await response.json()) as { failed: string }).failed).toBe("waiting"); | |
| 51 | }); |
src/health.ts added +38
| @@ -0,0 +1,38 @@ | ||
| 1 | import type { Database } from "bun:sqlite"; | |
| 2 | ||
| 3 | /** | |
| 4 | * What the first paint reads, as the cheapest bounded query each: a database | |
| 5 | * that lacks a column one of them names is broken however well `SELECT 1` | |
| 6 | * answers. Reads only: `LIMIT 1`, never a scan, never a write. | |
| 7 | */ | |
| 8 | const CHECKS: readonly (readonly [name: string, sql: string])[] = [ | |
| 9 | ["profile", "SELECT name, brokerage, phone, email, line, zone FROM profile WHERE id = 1"], | |
| 10 | [ | |
| 11 | "deals", | |
| 12 | "SELECT side, cash, closing_time, closing_place, sale_price, earnest_amount, commission_rate FROM deal LIMIT 1", | |
| 13 | ], | |
| 14 | ["dates", "SELECT time, place, lead_days FROM deadline LIMIT 1"], | |
| 15 | // The waiting-reminder count the first paint shows is the drafts due less the rows below: | |
| 16 | // sent, declined and held. Each count is bounded, so a large book costs no more than a small one. | |
| 17 | [ | |
| 18 | "waiting", | |
| 19 | `SELECT | |
| 20 | (SELECT count(*) FROM (SELECT 1 FROM reminder WHERE subject IS NOT NULL AND body IS NOT NULL AND moment IS NOT NULL AND due IS NOT NULL AND move_id IS NOT NULL LIMIT 1000)) | |
| 21 | + (SELECT count(*) FROM (SELECT 1 FROM reminder_declined WHERE reason IS NOT NULL AND moment IS NOT NULL AND due IS NOT NULL AND move_id IS NOT NULL LIMIT 1000)) | |
| 22 | + (SELECT count(*) FROM (SELECT 1 FROM reminder_hold WHERE until IS NOT NULL AND held_on IS NOT NULL AND moment IS NOT NULL AND due IS NOT NULL AND move_id IS NOT NULL LIMIT 1000)) | |
| 23 | AS n`, | |
| 24 | ], | |
| 25 | ]; | |
| 26 | ||
| 27 | /** The name of the first check that throws, else null. Never the error: it can carry SQL. */ | |
| 28 | export function failedCheck(db: Database): string | null { | |
| 29 | for (const [name, sql] of CHECKS) { | |
| 30 | try { | |
| 31 | db.query(sql).get(); | |
| 32 | } catch { | |
| 33 | // Neither the error nor the SQL leaves: the answer names the check and nothing else. | |
| 34 | return name; | |
| 35 | } | |
| 36 | } | |
| 37 | return null; | |
| 38 | } |
src/routes.ts +4 −7
| @@ -109,6 +109,7 @@ import { weekPage } from "./pages/week.ts"; | ||
| 109 | 109 | import { filePage } from "./pages/file.ts"; |
| 110 | 110 | import { timelinePage } from "./pages/timeline.ts"; |
| 111 | 111 | import { welcomePage } from "./pages/welcome.ts"; |
| 112 | import { failedCheck } from "./health.ts"; | |
| 112 | 113 | import { dueOn, remindsLine, todaysDrafts, unsent, waitingCount, type Agent, type Draft } from "./reminders.ts"; |
| 113 | 114 | import { token, tokensCss } from "./tokens.ts"; |
| 114 | 115 | import type { Who } from "./who.ts"; |
| @@ -1973,15 +1974,11 @@ export const routes: readonly Route[] = [ | ||
| 1973 | 1974 | method: "GET", |
| 1974 | 1975 | path: "/healthz", |
| 1975 | 1976 | handle: (c) => { |
| 1976 | let ok = true; | |
| 1977 | try { | |
| 1978 | c.db.query("SELECT 1").get(); | |
| 1979 | } catch { | |
| 1980 | ok = false; | |
| 1981 | } | |
| 1977 | const failed = failedCheck(c.db); | |
| 1978 | const ok = failed === null; | |
| 1982 | 1979 | const uptime = Math.floor((Date.now() - startedAt) / 1000); |
| 1983 | 1980 | return new Response( |
| 1984 | JSON.stringify({ ok, version, commit, env, uptime }), | |
| 1981 | JSON.stringify(ok ? { ok, version, commit, env, uptime } : { ok, failed, version, commit, env, uptime }), | |
| 1985 | 1982 | { |
| 1986 | 1983 | status: ok ? 200 : 503, |
| 1987 | 1984 | headers: { |
new above
offline
Raw error to stderr on every probe: floods the log and can carry SQL/data. Log once per check name or remove.