Public: anyone can read this; the crew can change it.

A site admin's health names a broken cost-ledger chain, from a verdict kept at boot and once a day #12281

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

torque/forge:torque/12273-a-site-admin-s-health-names-a-broken-cos → main 7 commits fork 1 thread · 1 unresolved

  • Merged. torque/12273-a-site-admin-s-health-names-a-broken-cos was merged into main as da65c36.

Review

Sign in to review this change.
6 files changed +177 −2
docs/features/cost-ledger.md +14
@@ -20,6 +20,20 @@ Prints `ok N entries`, or `broken at <file>:<line>: <why>` and exits 1.
2020
2121 The tests are `test/cost-ledger.test.ts`, not `test/ledger.test.ts`: that name is the decision ledger's.
2222
23 ## Health: a broken chain is a need
24
25 The chain is walked once at boot and then once a day, on the serving thread (`checkLedger` in `src/core/ledger.ts`, run under `during("ledger", …)` from `src/main.ts`; not in read-only mode). The walk logs `ledger: verified in N ms: ok, N entries` or `broken at <file>:<line>: <why>`, and keeps the verdict in memory.
26
27 When the verdict is broken, a site admin (and no one else) reads one more line in `needs` of `GET /api/v1/health` (and the MCP `health` tool), with `ok: false`, and on the terminal's and the board's forge strip:
28
29 The cost ledger's chain is broken at <file>:<line>: <why>. Run bun scripts/ledger-verify.ts to see it; the ledger's records after that line cannot be trusted.
30
31 Health only reads the kept verdict; it never touches the files, so a request stays fast however big the ledger is.
32
33 **Clearing it.** Run `bun scripts/ledger-verify.ts` (see Verify) to see the line, then repair or move aside the damaged file (keep a copy: the chain is evidence). The need clears at the next walk: restart the forge, or wait for the daily one. Nothing clears it sooner, so a repaired ledger reads broken until then.
34
35 **Cost.** The walk is synchronous: a big ledger holds the event loop for the length of the walk, once at boot and once a day. The boot log line says how long it took; if that grows to something a person would feel, the walk should yield between month files.
36
2337 ## Where the chain lives: git
2438
2539 The jsonl is the spool; git is truth. `src/core/ledger_git.ts` commits the chain to the `ledger` branch of a repository the forge owns: `FORGE_LEDGER_REPO` (`owner/name`), else `FORGE_REPO`; with neither, nothing is committed. A timer (`ledger-git`) fires hourly and, only when a month file has lines the branch lacks, appends them (one file per month, `<yyyy-mm>.jsonl` at the branch root) in **one commit**, signed with the forge's SSH host key (`<data>/ssh/host_ed25519`, an `SSHSIG` in the commit's `gpgsig` header, namespace `git`). No new lines, no commit. If the branch holds lines the spool does not, nothing is committed over it and the failure is logged. Entry text reaches git only as blob content, never as an argument.
src/actions/health.ts +7 −1
@@ -20,6 +20,7 @@ import { flakeLine, flakeList, flakeNeeds } from "../checks/flake.ts";
2020 import type { App } from "../app.ts";
2121 import { appsRuntime } from "../apps/runtime.ts";
2222 import { listApps } from "../core/apps.ts";
23 import { ledgerNeed } from "../core/ledger.ts";
2324 import type { User } from "../core/users.ts";
2425 import { forgeHealth, rolloutLine, type ForgeHealth } from "../kernel/guard.ts";
2526 import {
@@ -100,7 +101,10 @@ export function readHealth(
100101 }
101102 if (!actor?.isAdmin) return forge;
102103 const v = forgeVitals(app);
103 return { ...v, ...forge, ok: forge.ok && v.ok };
104 // A broken cost-ledger chain is one more need, from the verdict kept in memory (#12273).
105 const ledger = ledgerNeed();
106 if (ledger === null) return { ...v, ...forge, ok: forge.ok && v.ok };
107 return { ...v, ...forge, ok: false, needs: [...v.needs, ledger] };
104108 }
105109
106110 /**
@@ -113,11 +117,13 @@ export function forgeNeeds(app: App, actor: User | undefined): string[] | undefi
113117 if (!actor?.isAdmin) return undefined;
114118 const h = readHealth(app, actor) as Vitals & ForgeHealth;
115119 const rollout = rolloutLine(h);
120 const ledger = ledgerNeed();
116121 return [
117122 ...(rollout === null ? [] : [rollout]),
118123 ...h.incidents.map((i) => `${i.key}: ${i.message}`),
119124 ...h.degraded.map(needLine),
120125 ...flakeNeeds(h.flakes),
126 ...(ledger === null ? [] : [ledger]),
121127 ];
122128 }
123129
src/core/ledger.ts +32
@@ -250,5 +250,37 @@ export function verifyFiles(files: readonly { name: string; text: string }[]): V
250250 return { ok: true, entries: n };
251251 }
252252
253 let kept: Verdict | undefined;
254
255 /**
256 * Walk the chain and keep the verdict in memory, logging how long the walk
257 * took (a big ledger makes it slow, so it is never run on a request). Boot
258 * and a daily timer call this; health only reads `keptVerdict`.
259 */
260 export function checkLedger(d: string, log: (line: string) => void = console.log): Verdict {
261 const t0 = performance.now();
262 let verdict: Verdict;
263 try {
264 verdict = verifyDir(d);
265 } catch (error) {
266 verdict = { ok: false, at: d, why: `could not be read: ${(error as Error).message}` };
267 }
268 kept = verdict;
269 const ms = Math.round(performance.now() - t0);
270 log(`ledger: verified in ${ms} ms: ${verdict.ok ? `ok, ${verdict.entries} entries` : `broken at ${verdict.at}: ${verdict.why}`}`);
271 return verdict;
272 }
273
274 /** The last verdict `checkLedger` kept; undefined until the first walk. No file is read. */
275 export function keptVerdict(): Verdict | undefined {
276 return kept;
277 }
278
279 /** The kept verdict as one plain sentence when the chain is broken; null otherwise. */
280 export function ledgerNeed(): string | null {
281 if (kept === undefined || kept.ok) return null;
282 return `The cost ledger's chain is broken at ${kept.at}: ${kept.why}. Run bun scripts/ledger-verify.ts to see it; the ledger's records after that line cannot be trusted.`;
283 }
284
253285 /** The month files of a ledger directory, oldest first. */
254286 export const monthFiles = months;
src/main.ts +10 −1
@@ -8,7 +8,7 @@ import { join } from "node:path";
88 import { sweepEntitlements } from "./billing/sync.ts";
99 import { scanNightly } from "./checks/scan.ts";
1010 import { nightlyDue } from "./checks/scan_store.ts";
11 import { setLedgerDir } from "./core/ledger.ts";
11 import { checkLedger, setLedgerDir } from "./core/ledger.ts";
1212 import { BATCH_EVERY_MS, ledgerGitTick } from "./core/ledger_git.ts";
1313 import { getRepoById } from "./core/repos.ts";
1414 import { BOOT_ACTOR, setSignup, signupFromEnv } from "./settings/signup.ts";
@@ -770,6 +770,14 @@ const staleTimer = setInterval(() => void during("asks", tickAsks), 120_000);
770770 // Every paid entitlement is re-read from Stripe once a day, so a missed webhook heals (#11247).
771771 const entitlementTimer = setInterval(() => void during("entitlements", () => sweepEntitlements(app)), 86_400_000);
772772 void during("entitlements", () => sweepEntitlements(app));
773 // The cost ledger's chain is walked at boot and once a day (#12273), on this thread:
774 // health reads the verdict from this thread's memory.
775 const walkLedger = (): void =>
776 void checkLedger(join(dataDir, "ledger"), (line) => app.log.info(line));
777 const ledgerTimer = readOnly
778 ? undefined
779 : setInterval(() => during("ledger", walkLedger), 86_400_000);
780 if (!readOnly) during("ledger", walkLedger);
773781 void during("asks", tickAsks);
774782
775783 // Every start is counted, and one that follows a process that never said
@@ -991,6 +999,7 @@ function drain(signal: NodeJS.Signals): Promise<unknown> {
991999 clearInterval(syncTimer);
9921000 clearInterval(staleTimer);
9931001 clearInterval(entitlementTimer);
1002 if (ledgerTimer !== undefined) clearInterval(ledgerTimer);
9941003 clearInterval(followTimer);
9951004 landingsWatch();
9961005 stopClogTick();
test/job-slices.test.ts +1
@@ -241,6 +241,7 @@ function jobs(): Record<string, () => unknown> {
241241 * The jobs not timed here, each with why. The release keeper's are off unless a cluster turns them on.
242242 */
243243 const UNTIMED: Record<string, string> = {
244 ledger: "walks the cost ledger's hash chain at boot and once a day (#12273): file reads proportional to the ledger, off the request path; the fixture has no ledger",
244245 "full-disk-probe": "one 64 KiB write every 2 s, only while the disk is full; the full-disk test drives probeFull directly",
245246 "reach-asks": "a one-shot timer in actions/ask_host.ts, armed only while a hosted forge has an ask under an hour old: one signed call to its host over the network and a few rows; the fixture has no host",
246247 lane: "a closure in startChecks; starting the checks service starts cell checks — follow-up",
test/ledger-health.test.ts added +113
@@ -0,0 +1,113 @@
1 import { afterEach, beforeEach, expect, test } from "bun:test";
2 import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
3 import { tmpdir } from "node:os";
4 import { join } from "node:path";
5 import { createApp } from "../src/app.ts";
6 import { forgeNeeds } from "../src/actions/health.ts";
7 import { checkLedger, entryOf, verifyDir } from "../src/core/ledger.ts";
8 import { serve } from "../src/http/server.ts";
9 import { createForgeApp } from "../src/tui/forge.ts";
10 import { headlessTerminal } from "../src/tui/term/terminal.ts";
11 import { createUser, type User } from "../src/core/users.ts";
12 import { settleApp } from "./helpers/tui.ts";
13 import { tokenFor } from "./helpers/tokens.ts";
14 import type { Run } from "../src/core/runs.ts";
15
16 let dir: string;
17 beforeEach(() => {
18 dir = join(mkdtempSync(join(process.env.TMPDIR ?? tmpdir(), "ledger-health-")), "ledger");
19 mkdirSync(dir);
20 });
21 afterEach(() => void checkLedger(dir, () => {})); // an empty chain is ok: the next test starts with no broken verdict
22
23 function fixture(): string {
24 const run = (id: string): Run => ({ id, kind: "check", repoId: "r", env: "pull-1", issueId: null, startedAt: null, finishedAt: "2026-11-01T00:00:00.000Z", costUsd: 1 }) as unknown as Run;
25 const a = entryOf(run("a"), "0".repeat(64));
26 const b = entryOf(run("b"), a.hash);
27 const file = join(dir, "2026-11.jsonl");
28 writeFileSync(file, `${JSON.stringify(a)}\n${JSON.stringify(b)}\n`);
29 return file;
30 }
31
32 async function admin() {
33 const app = createApp({ dataDir: mkdtempSync(join(process.env.TMPDIR ?? tmpdir(), "ledger-health-app-")), port: 0 });
34 const a = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
35 if (!a.ok) throw new Error(a.error);
36 return { app, admin: a.value };
37 }
38 const mentions = (needs: string[] | undefined): string[] => (needs ?? []).filter((n) => n.includes("cost ledger"));
39
40 test("a clean chain adds no need", async () => {
41 fixture();
42 const { app, admin: a } = await admin();
43 expect(checkLedger(dir, () => {})).toEqual({ ok: true, entries: 2 });
44 expect(mentions(forgeNeeds(app, a))).toEqual([]);
45 });
46
47 test("a broken field at boot names the line and why, to the admin only, with no file read", async () => {
48 const file = fixture();
49 writeFileSync(file, readFileSync(file, "utf8").replace('"kind":"check"', '"kind":"merge"'));
50 expect(verifyDir(dir).ok).toBe(false);
51 const logged: string[] = [];
52 checkLedger(dir, (l) => logged.push(l));
53 expect(logged[0]).toMatch(/verified in \d+ ms/);
54 const { app, admin: a } = await admin();
55 // The files are gone: health answers from the kept verdict, not the disk.
56 rmSync(dir, { recursive: true });
57 const need = mentions(forgeNeeds(app, a));
58 expect(need.length).toBe(1);
59 expect(need[0]).toContain("2026-11.jsonl:1");
60 expect(need[0]).toContain("hash does not match the entry");
61 expect(forgeNeeds(app, undefined)).toBeUndefined();
62 });
63
64 function broken(): void {
65 const file = fixture();
66 writeFileSync(file, readFileSync(file, "utf8").replace('"kind":"check"', '"kind":"merge"'));
67 checkLedger(dir, () => {});
68 rmSync(dir, { recursive: true }); // the answers below come from memory
69 }
70
71 test("GET /api/v1/health carries the need for an admin, and not for a guest or a stranger", async () => {
72 broken();
73 const { app } = await admin();
74 const bob = await createUser(app.db, { origin: "seed", trusted: true, username: "bob", email: "[email protected]", password: "a-good-passphrase" });
75 if (!bob.ok) throw new Error(bob.error);
76 const servers = await serve(app);
77 try {
78 const read = async (token?: string) =>
79 (await (await fetch(`${servers.url}/api/v1/health`, { headers: token === undefined ? {} : { authorization: `Bearer ${token}` } })).json()) as { ok: boolean; needs?: string[] };
80 const a = await read(tokenFor(app, "alice", "ro"));
81 expect(a.ok).toBe(false);
82 expect(mentions(a.needs).length).toBe(1);
83 expect(mentions(a.needs)[0]).toContain("2026-11.jsonl:1");
84 expect(JSON.stringify(await read(tokenFor(app, "bob", "ro")))).not.toContain("cost ledger");
85 expect(JSON.stringify(await read())).not.toContain("cost ledger");
86 } finally {
87 servers.stop();
88 app.close();
89 }
90 });
91
92 async function screenOf(app: Awaited<ReturnType<typeof admin>>["app"], user: User): Promise<string> {
93 const terminal = headlessTerminal(200, 32);
94 const { app: tui } = createForgeApp(app, { sshUsername: user.username, remoteAddress: "127.0.0.1", fingerprint: "SHA256:t", user }, { terminal, exitHooks: false });
95 tui.start();
96 await settleApp(tui);
97 const text = tui.screen.toText();
98 tui.stop();
99 return text;
100 }
101
102 test("the terminal's forge strip shows the need to an admin and not to a guest", async () => {
103 broken();
104 const { app, admin: a } = await admin();
105 const bob = await createUser(app.db, { origin: "seed", trusted: true, username: "bob", email: "[email protected]", password: "a-good-passphrase" });
106 if (!bob.ok) throw new Error(bob.error);
107 try {
108 expect(await screenOf(app, a)).toContain("cost ledger");
109 expect(await screenOf(app, bob.value)).not.toContain("cost ledger");
110 } finally {
111 app.close();
112 }
113 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs