Public: anyone can read this; the crew can change it.

/healthz answers 503 naming the failed check when the database is missing a column the first paint reads, so a broken upgrade never looks healthy #186

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/deal-clock:mason/184-healthz-answers-503-naming-the-failed-ch → main 4 commits fork 2 threads · 2 unresolved

  • Merged. mason/184-healthz-answers-503-naming-the-failed-ch was merged into main as 8d3ee4f.

Review

Sign in to review this change.
4 files changed +100 −8
docs/TECHNICAL.md +7 −1
@@ -291,7 +291,13 @@ decide nothing themselves. `AGENTS.md` says how to add to it.
291291 It needs a signed-in user: no door of the table is open to a stranger, and
292292 the description is shut to them when none is.
293293 - **Up, or not.** `GET /healthz` answers `{ ok, version, uptime }`; the
294 forge waits on it before it sends anyone to a deploy.
294 forge waits on it before it sends anyone to a deploy. It runs the bounded
295 reads the first paint depends on (`src/health.ts`: the profile row, one
296 deal row with its newer columns, one date, and the waiting-reminder
297 count's sources, the sent, declined and held reminders, each counted to a
298 bound of 1000; `LIMIT`ed, no writes). If one throws it answers 503 with `ok:false` and `failed` naming
299 the check (`profile`, `deals`, `dates`, `waiting`), never the SQL or the
300 error, so a database a broken upgrade left behind never looks healthy.
295301 - **The look.** `design/tokens.json` holds every colour, space and size,
296302 light and dark; nothing else spells out a colour (`bun run lint:design`),
297303 and every pair a person reads meets 4.5:1 in both themes.
src/health.test.ts added +51
@@ -0,0 +1,51 @@
1 import { expect, test } from "bun:test";
2 import { Database } from "bun:sqlite";
3 import { migrate, migrations } from "./db.ts";
4 import { routes } from "./routes.ts";
5
6 const handler = routes.find((r) => r.path === "/healthz")!.handle as (c: unknown) => Response;
7 const ask = (db: Database) => handler({ db });
8
9 test("a migrated database is healthy, quickly", async () => {
10 const db = new Database(":memory:");
11 migrate(db);
12 const t = performance.now();
13 const response = ask(db);
14 expect(performance.now() - t).toBeLessThan(50);
15 expect(response.status).toBe(200);
16 expect(response.headers.get("cache-control")).toBe("no-store");
17 expect(response.headers.get("x-content-type-options")).toBe("nosniff");
18 const body = (await response.json()) as Record<string, unknown>;
19 expect(body.ok).toBe(true);
20 expect("failed" in body).toBe(false);
21 });
22
23 test("a database missing profile.line answers 503 naming profile, and nothing else", async () => {
24 const db = new Database(":memory:");
25 db.exec("CREATE TABLE schema_version (version INTEGER PRIMARY KEY, applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')))");
26 migrations
27 .filter((m) => !/profile ADD COLUMN line/.test(m))
28 .forEach((sql, i) => {
29 db.exec(sql);
30 db.query("INSERT INTO schema_version (version) VALUES (?1)").run(i + 1);
31 });
32 const response = ask(db);
33 expect(response.status).toBe(503);
34 expect(response.headers.get("cache-control")).toBe("no-store");
35 expect(response.headers.get("x-content-type-options")).toBe("nosniff");
36 const text = await response.text();
37 const body = JSON.parse(text) as Record<string, unknown>;
38 expect(body.ok).toBe(false);
39 expect(body.failed).toBe("profile");
40 expect(Object.keys(body).sort()).toEqual(["commit", "env", "failed", "ok", "uptime", "version"]);
41 expect(text).not.toMatch(/SELECT|no such column|line/);
42 });
43
44 test("a database missing a table the waiting count reads answers 503 naming waiting", async () => {
45 const db = new Database(":memory:");
46 migrate(db);
47 db.exec("DROP TABLE reminder_declined");
48 const response = ask(db);
49 expect(response.status).toBe(503);
50 expect(((await response.json()) as { failed: string }).failed).toBe("waiting");
51 });
src/health.ts added +38
@@ -0,0 +1,38 @@
1 import type { Database } from "bun:sqlite";
2
3 /**
4 * What the first paint reads, as the cheapest bounded query each: a database
5 * that lacks a column one of them names is broken however well `SELECT 1`
6 * answers. Reads only: `LIMIT 1`, never a scan, never a write.
7 */
8 const CHECKS: readonly (readonly [name: string, sql: string])[] = [
9 ["profile", "SELECT name, brokerage, phone, email, line, zone FROM profile WHERE id = 1"],
10 [
11 "deals",
12 "SELECT side, cash, closing_time, closing_place, sale_price, earnest_amount, commission_rate FROM deal LIMIT 1",
13 ],
14 ["dates", "SELECT time, place, lead_days FROM deadline LIMIT 1"],
15 // The waiting-reminder count the first paint shows is the drafts due less the rows below:
16 // sent, declined and held. Each count is bounded, so a large book costs no more than a small one.
17 [
18 "waiting",
19 `SELECT
20 (SELECT count(*) FROM (SELECT 1 FROM reminder WHERE subject IS NOT NULL AND body IS NOT NULL AND moment IS NOT NULL AND due IS NOT NULL AND move_id IS NOT NULL LIMIT 1000))
21 + (SELECT count(*) FROM (SELECT 1 FROM reminder_declined WHERE reason IS NOT NULL AND moment IS NOT NULL AND due IS NOT NULL AND move_id IS NOT NULL LIMIT 1000))
22 + (SELECT count(*) FROM (SELECT 1 FROM reminder_hold WHERE until IS NOT NULL AND held_on IS NOT NULL AND moment IS NOT NULL AND due IS NOT NULL AND move_id IS NOT NULL LIMIT 1000))
23 AS n`,
24 ],
outdated

Raw error to stderr on every probe: floods the log and can carry SQL/data. Log once per check name or remove.

25 ];
26
27 /** The name of the first check that throws, else null. Never the error: it can carry SQL. */
28 export function failedCheck(db: Database): string | null {
29 for (const [name, sql] of CHECKS) {
30 try {
31 db.query(sql).get();
32 } catch {
33 // Neither the error nor the SQL leaves: the answer names the check and nothing else.
34 return name;
35 }
36 }
37 return null;
38 }
src/routes.ts +4 −7
@@ -109,6 +109,7 @@ import { weekPage } from "./pages/week.ts";
109109 import { filePage } from "./pages/file.ts";
110110 import { timelinePage } from "./pages/timeline.ts";
111111 import { welcomePage } from "./pages/welcome.ts";
112 import { failedCheck } from "./health.ts";
112113 import { dueOn, remindsLine, todaysDrafts, unsent, waitingCount, type Agent, type Draft } from "./reminders.ts";
113114 import { token, tokensCss } from "./tokens.ts";
114115 import type { Who } from "./who.ts";
@@ -1973,15 +1974,11 @@ export const routes: readonly Route[] = [
19731974 method: "GET",
19741975 path: "/healthz",
19751976 handle: (c) => {
1976 let ok = true;
1977 try {
1978 c.db.query("SELECT 1").get();
1979 } catch {
1980 ok = false;
1981 }
1977 const failed = failedCheck(c.db);
1978 const ok = failed === null;
19821979 const uptime = Math.floor((Date.now() - startedAt) / 1000);
19831980 return new Response(
1984 JSON.stringify({ ok, version, commit, env, uptime }),
1981 JSON.stringify(ok ? { ok, version, commit, env, uptime } : { ok, failed, version, commit, env, uptime }),
19851982 {
19861983 status: ok ? 200 : 503,
19871984 headers: {

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs