Public: anyone can read this; the crew can change it.
A site admin's health names a broken cost-ledger chain, from a verdict kept at boot and once a day #12281
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/12273-a-site-admin-s-health-names-a-broken-cos → main 7 commits fork 1 thread · 1 unresolved
6 files
changed +177
−2
docs/features/cost-ledger.md +14
| @@ -20,6 +20,20 @@ Prints `ok N entries`, or `broken at <file>:<line>: <why>` and exits 1. | ||
| 20 | 20 | |
| 21 | 21 | The tests are `test/cost-ledger.test.ts`, not `test/ledger.test.ts`: that name is the decision ledger's. |
| 22 | 22 | |
| 23 | ## Health: a broken chain is a need | |
| 24 | ||
| 25 | The chain is walked once at boot and then once a day, on the serving thread (`checkLedger` in `src/core/ledger.ts`, run under `during("ledger", …)` from `src/main.ts`; not in read-only mode). The walk logs `ledger: verified in N ms: ok, N entries` or `broken at <file>:<line>: <why>`, and keeps the verdict in memory. | |
| 26 | ||
| 27 | When the verdict is broken, a site admin (and no one else) reads one more line in `needs` of `GET /api/v1/health` (and the MCP `health` tool), with `ok: false`, and on the terminal's and the board's forge strip: | |
| 28 | ||
| 29 | The cost ledger's chain is broken at <file>:<line>: <why>. Run bun scripts/ledger-verify.ts to see it; the ledger's records after that line cannot be trusted. | |
| 30 | ||
| 31 | Health only reads the kept verdict; it never touches the files, so a request stays fast however big the ledger is. | |
| 32 | ||
| 33 | **Clearing it.** Run `bun scripts/ledger-verify.ts` (see Verify) to see the line, then repair or move aside the damaged file (keep a copy: the chain is evidence). The need clears at the next walk: restart the forge, or wait for the daily one. Nothing clears it sooner, so a repaired ledger reads broken until then. | |
| 34 | ||
| 35 | **Cost.** The walk is synchronous: a big ledger holds the event loop for the length of the walk, once at boot and once a day. The boot log line says how long it took; if that grows to something a person would feel, the walk should yield between month files. | |
| 36 | ||
| 23 | 37 | ## Where the chain lives: git |
| 24 | 38 | |
| 25 | 39 | The jsonl is the spool; git is truth. `src/core/ledger_git.ts` commits the chain to the `ledger` branch of a repository the forge owns: `FORGE_LEDGER_REPO` (`owner/name`), else `FORGE_REPO`; with neither, nothing is committed. A timer (`ledger-git`) fires hourly and, only when a month file has lines the branch lacks, appends them (one file per month, `<yyyy-mm>.jsonl` at the branch root) in **one commit**, signed with the forge's SSH host key (`<data>/ssh/host_ed25519`, an `SSHSIG` in the commit's `gpgsig` header, namespace `git`). No new lines, no commit. If the branch holds lines the spool does not, nothing is committed over it and the failure is logged. Entry text reaches git only as blob content, never as an argument. |
src/actions/health.ts +7 −1
| @@ -20,6 +20,7 @@ import { flakeLine, flakeList, flakeNeeds } from "../checks/flake.ts"; | ||
| 20 | 20 | import type { App } from "../app.ts"; |
| 21 | 21 | import { appsRuntime } from "../apps/runtime.ts"; |
| 22 | 22 | import { listApps } from "../core/apps.ts"; |
| 23 | import { ledgerNeed } from "../core/ledger.ts"; | |
| 23 | 24 | import type { User } from "../core/users.ts"; |
| 24 | 25 | import { forgeHealth, rolloutLine, type ForgeHealth } from "../kernel/guard.ts"; |
| 25 | 26 | import { |
| @@ -100,7 +101,10 @@ export function readHealth( | ||
| 100 | 101 | } |
| 101 | 102 | if (!actor?.isAdmin) return forge; |
| 102 | 103 | const v = forgeVitals(app); |
| 103 | return { ...v, ...forge, ok: forge.ok && v.ok }; | |
| 104 | // A broken cost-ledger chain is one more need, from the verdict kept in memory (#12273). | |
| 105 | const ledger = ledgerNeed(); | |
| 106 | if (ledger === null) return { ...v, ...forge, ok: forge.ok && v.ok }; | |
| 107 | return { ...v, ...forge, ok: false, needs: [...v.needs, ledger] }; | |
| 104 | 108 | } |
| 105 | 109 | |
| 106 | 110 | /** |
| @@ -113,11 +117,13 @@ export function forgeNeeds(app: App, actor: User | undefined): string[] | undefi | ||
| 113 | 117 | if (!actor?.isAdmin) return undefined; |
| 114 | 118 | const h = readHealth(app, actor) as Vitals & ForgeHealth; |
| 115 | 119 | const rollout = rolloutLine(h); |
| 120 | const ledger = ledgerNeed(); | |
| 116 | 121 | return [ |
| 117 | 122 | ...(rollout === null ? [] : [rollout]), |
| 118 | 123 | ...h.incidents.map((i) => `${i.key}: ${i.message}`), |
| 119 | 124 | ...h.degraded.map(needLine), |
| 120 | 125 | ...flakeNeeds(h.flakes), |
| 126 | ...(ledger === null ? [] : [ledger]), | |
| 121 | 127 | ]; |
| 122 | 128 | } |
| 123 | 129 |
src/core/ledger.ts +32
| @@ -250,5 +250,37 @@ export function verifyFiles(files: readonly { name: string; text: string }[]): V | ||
| 250 | 250 | return { ok: true, entries: n }; |
| 251 | 251 | } |
| 252 | 252 | |
| 253 | let kept: Verdict | undefined; | |
| 254 | ||
| 255 | /** | |
| 256 | * Walk the chain and keep the verdict in memory, logging how long the walk | |
| 257 | * took (a big ledger makes it slow, so it is never run on a request). Boot | |
| 258 | * and a daily timer call this; health only reads `keptVerdict`. | |
| 259 | */ | |
| 260 | export function checkLedger(d: string, log: (line: string) => void = console.log): Verdict { | |
| 261 | const t0 = performance.now(); | |
| 262 | let verdict: Verdict; | |
| 263 | try { | |
| 264 | verdict = verifyDir(d); | |
| 265 | } catch (error) { | |
| 266 | verdict = { ok: false, at: d, why: `could not be read: ${(error as Error).message}` }; | |
| 267 | } | |
| 268 | kept = verdict; | |
| 269 | const ms = Math.round(performance.now() - t0); | |
| 270 | log(`ledger: verified in ${ms} ms: ${verdict.ok ? `ok, ${verdict.entries} entries` : `broken at ${verdict.at}: ${verdict.why}`}`); | |
| 271 | return verdict; | |
| 272 | } | |
| 273 | ||
| 274 | /** The last verdict `checkLedger` kept; undefined until the first walk. No file is read. */ | |
| 275 | export function keptVerdict(): Verdict | undefined { | |
| 276 | return kept; | |
| 277 | } | |
| 278 | ||
| 279 | /** The kept verdict as one plain sentence when the chain is broken; null otherwise. */ | |
| 280 | export function ledgerNeed(): string | null { | |
| 281 | if (kept === undefined || kept.ok) return null; | |
| 282 | return `The cost ledger's chain is broken at ${kept.at}: ${kept.why}. Run bun scripts/ledger-verify.ts to see it; the ledger's records after that line cannot be trusted.`; | |
| 283 | } | |
| 284 | ||
| 253 | 285 | /** The month files of a ledger directory, oldest first. */ |
| 254 | 286 | export const monthFiles = months; |
src/main.ts +10 −1
| @@ -8,7 +8,7 @@ import { join } from "node:path"; | ||
| 8 | 8 | import { sweepEntitlements } from "./billing/sync.ts"; |
| 9 | 9 | import { scanNightly } from "./checks/scan.ts"; |
| 10 | 10 | import { nightlyDue } from "./checks/scan_store.ts"; |
| 11 | import { setLedgerDir } from "./core/ledger.ts"; | |
| 11 | import { checkLedger, setLedgerDir } from "./core/ledger.ts"; | |
| 12 | 12 | import { BATCH_EVERY_MS, ledgerGitTick } from "./core/ledger_git.ts"; |
| 13 | 13 | import { getRepoById } from "./core/repos.ts"; |
| 14 | 14 | import { BOOT_ACTOR, setSignup, signupFromEnv } from "./settings/signup.ts"; |
| @@ -770,6 +770,14 @@ const staleTimer = setInterval(() => void during("asks", tickAsks), 120_000); | ||
| 770 | 770 | // Every paid entitlement is re-read from Stripe once a day, so a missed webhook heals (#11247). |
| 771 | 771 | const entitlementTimer = setInterval(() => void during("entitlements", () => sweepEntitlements(app)), 86_400_000); |
| 772 | 772 | void during("entitlements", () => sweepEntitlements(app)); |
| 773 | // The cost ledger's chain is walked at boot and once a day (#12273), on this thread: | |
| 774 | // health reads the verdict from this thread's memory. | |
| 775 | const walkLedger = (): void => | |
| 776 | void checkLedger(join(dataDir, "ledger"), (line) => app.log.info(line)); | |
| 777 | const ledgerTimer = readOnly | |
| 778 | ? undefined | |
| 779 | : setInterval(() => during("ledger", walkLedger), 86_400_000); | |
| 780 | if (!readOnly) during("ledger", walkLedger); | |
| 773 | 781 | void during("asks", tickAsks); |
| 774 | 782 | |
| 775 | 783 | // Every start is counted, and one that follows a process that never said |
| @@ -991,6 +999,7 @@ function drain(signal: NodeJS.Signals): Promise<unknown> { | ||
| 991 | 999 | clearInterval(syncTimer); |
| 992 | 1000 | clearInterval(staleTimer); |
| 993 | 1001 | clearInterval(entitlementTimer); |
| 1002 | if (ledgerTimer !== undefined) clearInterval(ledgerTimer); | |
| 994 | 1003 | clearInterval(followTimer); |
| 995 | 1004 | landingsWatch(); |
| 996 | 1005 | stopClogTick(); |
test/job-slices.test.ts +1
| @@ -241,6 +241,7 @@ function jobs(): Record<string, () => unknown> { | ||
| 241 | 241 | * The jobs not timed here, each with why. The release keeper's are off unless a cluster turns them on. |
| 242 | 242 | */ |
| 243 | 243 | const UNTIMED: Record<string, string> = { |
| 244 | ledger: "walks the cost ledger's hash chain at boot and once a day (#12273): file reads proportional to the ledger, off the request path; the fixture has no ledger", | |
| 244 | 245 | "full-disk-probe": "one 64 KiB write every 2 s, only while the disk is full; the full-disk test drives probeFull directly", |
| 245 | 246 | "reach-asks": "a one-shot timer in actions/ask_host.ts, armed only while a hosted forge has an ask under an hour old: one signed call to its host over the network and a few rows; the fixture has no host", |
| 246 | 247 | lane: "a closure in startChecks; starting the checks service starts cell checks — follow-up", |
test/ledger-health.test.ts added +113
| @@ -0,0 +1,113 @@ | ||
| 1 | import { afterEach, beforeEach, expect, test } from "bun:test"; | |
| 2 | import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; | |
| 3 | import { tmpdir } from "node:os"; | |
| 4 | import { join } from "node:path"; | |
| 5 | import { createApp } from "../src/app.ts"; | |
| 6 | import { forgeNeeds } from "../src/actions/health.ts"; | |
| 7 | import { checkLedger, entryOf, verifyDir } from "../src/core/ledger.ts"; | |
| 8 | import { serve } from "../src/http/server.ts"; | |
| 9 | import { createForgeApp } from "../src/tui/forge.ts"; | |
| 10 | import { headlessTerminal } from "../src/tui/term/terminal.ts"; | |
| 11 | import { createUser, type User } from "../src/core/users.ts"; | |
| 12 | import { settleApp } from "./helpers/tui.ts"; | |
| 13 | import { tokenFor } from "./helpers/tokens.ts"; | |
| 14 | import type { Run } from "../src/core/runs.ts"; | |
| 15 | ||
| 16 | let dir: string; | |
| 17 | beforeEach(() => { | |
| 18 | dir = join(mkdtempSync(join(process.env.TMPDIR ?? tmpdir(), "ledger-health-")), "ledger"); | |
| 19 | mkdirSync(dir); | |
| 20 | }); | |
| 21 | afterEach(() => void checkLedger(dir, () => {})); // an empty chain is ok: the next test starts with no broken verdict | |
| 22 | ||
| 23 | function fixture(): string { | |
| 24 | const run = (id: string): Run => ({ id, kind: "check", repoId: "r", env: "pull-1", issueId: null, startedAt: null, finishedAt: "2026-11-01T00:00:00.000Z", costUsd: 1 }) as unknown as Run; | |
| 25 | const a = entryOf(run("a"), "0".repeat(64)); | |
| 26 | const b = entryOf(run("b"), a.hash); | |
| 27 | const file = join(dir, "2026-11.jsonl"); | |
| 28 | writeFileSync(file, `${JSON.stringify(a)}\n${JSON.stringify(b)}\n`); | |
| 29 | return file; | |
| 30 | } | |
| 31 | ||
| 32 | async function admin() { | |
| 33 | const app = createApp({ dataDir: mkdtempSync(join(process.env.TMPDIR ?? tmpdir(), "ledger-health-app-")), port: 0 }); | |
| 34 | const a = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true }); | |
| 35 | if (!a.ok) throw new Error(a.error); | |
| 36 | return { app, admin: a.value }; | |
| 37 | } | |
| 38 | const mentions = (needs: string[] | undefined): string[] => (needs ?? []).filter((n) => n.includes("cost ledger")); | |
| 39 | ||
| 40 | test("a clean chain adds no need", async () => { | |
| 41 | fixture(); | |
| 42 | const { app, admin: a } = await admin(); | |
| 43 | expect(checkLedger(dir, () => {})).toEqual({ ok: true, entries: 2 }); | |
| 44 | expect(mentions(forgeNeeds(app, a))).toEqual([]); | |
| 45 | }); | |
| 46 | ||
| 47 | test("a broken field at boot names the line and why, to the admin only, with no file read", async () => { | |
| 48 | const file = fixture(); | |
| 49 | writeFileSync(file, readFileSync(file, "utf8").replace('"kind":"check"', '"kind":"merge"')); | |
| 50 | expect(verifyDir(dir).ok).toBe(false); | |
| 51 | const logged: string[] = []; | |
| 52 | checkLedger(dir, (l) => logged.push(l)); | |
| 53 | expect(logged[0]).toMatch(/verified in \d+ ms/); | |
| 54 | const { app, admin: a } = await admin(); | |
| 55 | // The files are gone: health answers from the kept verdict, not the disk. | |
| 56 | rmSync(dir, { recursive: true }); | |
| 57 | const need = mentions(forgeNeeds(app, a)); | |
| 58 | expect(need.length).toBe(1); | |
| 59 | expect(need[0]).toContain("2026-11.jsonl:1"); | |
| 60 | expect(need[0]).toContain("hash does not match the entry"); | |
| 61 | expect(forgeNeeds(app, undefined)).toBeUndefined(); | |
| 62 | }); | |
| 63 | ||
| 64 | function broken(): void { | |
| 65 | const file = fixture(); | |
| 66 | writeFileSync(file, readFileSync(file, "utf8").replace('"kind":"check"', '"kind":"merge"')); | |
| 67 | checkLedger(dir, () => {}); | |
| 68 | rmSync(dir, { recursive: true }); // the answers below come from memory | |
| 69 | } | |
| 70 | ||
| 71 | test("GET /api/v1/health carries the need for an admin, and not for a guest or a stranger", async () => { | |
| 72 | broken(); | |
| 73 | const { app } = await admin(); | |
| 74 | const bob = await createUser(app.db, { origin: "seed", trusted: true, username: "bob", email: "[email protected]", password: "a-good-passphrase" }); | |
| 75 | if (!bob.ok) throw new Error(bob.error); | |
| 76 | const servers = await serve(app); | |
| 77 | try { | |
| 78 | const read = async (token?: string) => | |
| 79 | (await (await fetch(`${servers.url}/api/v1/health`, { headers: token === undefined ? {} : { authorization: `Bearer ${token}` } })).json()) as { ok: boolean; needs?: string[] }; | |
| 80 | const a = await read(tokenFor(app, "alice", "ro")); | |
| 81 | expect(a.ok).toBe(false); | |
| 82 | expect(mentions(a.needs).length).toBe(1); | |
| 83 | expect(mentions(a.needs)[0]).toContain("2026-11.jsonl:1"); | |
| 84 | expect(JSON.stringify(await read(tokenFor(app, "bob", "ro")))).not.toContain("cost ledger"); | |
| 85 | expect(JSON.stringify(await read())).not.toContain("cost ledger"); | |
| 86 | } finally { | |
| 87 | servers.stop(); | |
| 88 | app.close(); | |
| 89 | } | |
| 90 | }); | |
| 91 | ||
| 92 | async function screenOf(app: Awaited<ReturnType<typeof admin>>["app"], user: User): Promise<string> { | |
| 93 | const terminal = headlessTerminal(200, 32); | |
| 94 | const { app: tui } = createForgeApp(app, { sshUsername: user.username, remoteAddress: "127.0.0.1", fingerprint: "SHA256:t", user }, { terminal, exitHooks: false }); | |
| 95 | tui.start(); | |
| 96 | await settleApp(tui); | |
| 97 | const text = tui.screen.toText(); | |
| 98 | tui.stop(); | |
| 99 | return text; | |
| 100 | } | |
| 101 | ||
| 102 | test("the terminal's forge strip shows the need to an admin and not to a guest", async () => { | |
| 103 | broken(); | |
| 104 | const { app, admin: a } = await admin(); | |
| 105 | const bob = await createUser(app.db, { origin: "seed", trusted: true, username: "bob", email: "[email protected]", password: "a-good-passphrase" }); | |
| 106 | if (!bob.ok) throw new Error(bob.error); | |
| 107 | try { | |
| 108 | expect(await screenOf(app, a)).toContain("cost ledger"); | |
| 109 | expect(await screenOf(app, bob.value)).not.toContain("cost ledger"); | |
| 110 | } finally { | |
| 111 | app.close(); | |
| 112 | } | |
| 113 | }); |
new above
offline