Public: anyone can read this; the crew can change it.
Cost ledger: snapshots append an entry too #12291
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/12256-cost-ledger-snapshots-append-an-entry-to → main 2 commits fork
4 files
changed +110
−6
docs/features/cost-ledger.md +1 −1
| @@ -1,6 +1,6 @@ | ||
| 1 | 1 | # Cost ledger |
| 2 | 2 | |
| 3 | Every run that ends (`finishRun`, and `cancelRun`: agent, check, deploy) appends one line to `$FORGE_DATA/ledger/<yyyy-mm>.jsonl` (by the month it ended). Nothing is written on a day with no runs. Snapshots are not runs (they have no `run` row) and are not recorded yet; that is a follow-up. | |
| 3 | Every run that ends (`finishRun`, and `cancelRun`: agent, check, deploy) appends one line to `$FORGE_DATA/ledger/<yyyy-mm>.jsonl` (by the month it ended). Nothing is written on a day with no runs. A snapshot has no `run` row, so the snapshot service appends its own entry when one ends, taken or failed: `kind: "snapshot"`, `run: "snapshot:<startedAt>"`, `startedAt`, `endedAt`, `wall_s`, and `failed: true` only when it failed. It has no `repo`: a snapshot is the whole forge's, so `repo` is optional and absent, never a made-up id. A forge with nowhere to put a snapshot writes nothing. | |
| 4 | 4 | |
| 5 | 5 | ## An entry |
| 6 | 6 |
src/core/ledger.ts +42 −5
| @@ -18,7 +18,10 @@ export const GENESIS = "0".repeat(64); | ||
| 18 | 18 | export interface LedgerEntry { |
| 19 | 19 | readonly run: string; |
| 20 | 20 | readonly kind: string; |
| 21 | readonly repo: string; | |
| 21 | /** Absent for an instance-wide entry (a snapshot): it belongs to no repository. */ | |
| 22 | readonly repo?: string; | |
| 23 | /** Present, and true, only on a snapshot that failed. */ | |
| 24 | readonly failed?: boolean; | |
| 22 | 25 | readonly issue?: string; |
| 23 | 26 | readonly pull?: number; |
| 24 | 27 | readonly startedAt?: string; |
| @@ -121,21 +124,55 @@ function lastHash(d: string): string { | ||
| 121 | 124 | return GENESIS; |
| 122 | 125 | } |
| 123 | 126 | |
| 124 | /** Append the entry for a finished run. Never throws. */ | |
| 125 | export function appendRun(run: Run): void { | |
| 127 | function appendBody(make: (prev: string) => LedgerEntry, what: string): void { | |
| 126 | 128 | if (dir === undefined) return; |
| 127 | 129 | try { |
| 128 | 130 | mkdirSync(dir, { recursive: true }); |
| 129 | const entry = entryOf(run, lastHash(dir)); | |
| 131 | const entry = make(lastHash(dir)); | |
| 130 | 132 | const file = join(dir, `${entry.endedAt.slice(0, 7)}.jsonl`); |
| 131 | 133 | // A torn last line (no newline) must not swallow this entry. |
| 132 | 134 | const torn = existsSync(file) && !readFileSync(file, "utf8").endsWith("\n") && statSync(file).size > 0; |
| 133 | 135 | appendFileSync(file, `${torn ? "\n" : ""}${JSON.stringify(entry)}\n`); |
| 134 | 136 | } catch (error) { |
| 135 | console.error(`ledger: could not append run ${run.id}: ${(error as Error).message}`); | |
| 137 | console.error(`ledger: could not append ${what}: ${(error as Error).message}`); | |
| 136 | 138 | } |
| 137 | 139 | } |
| 138 | 140 | |
| 141 | /** Append the entry for a finished run. Never throws. */ | |
| 142 | export function appendRun(run: Run): void { | |
| 143 | appendBody((prev) => entryOf(run, prev), `run ${run.id}`); | |
| 144 | } | |
| 145 | ||
| 146 | /** A snapshot that ended: when it began and ended, and whether it failed. */ | |
| 147 | export interface SnapshotEnd { | |
| 148 | readonly startedAt: string; | |
| 149 | readonly endedAt: string; | |
| 150 | readonly failed: boolean; | |
| 151 | } | |
| 152 | ||
| 153 | /** | |
| 154 | * The entry a snapshot makes: kind `snapshot`, no repo (it is the whole | |
| 155 | * forge's), keyed by its start. | |
| 156 | */ | |
| 157 | export function snapshotEntryOf(s: SnapshotEnd, prev: string): LedgerEntry { | |
| 158 | const wall = (Date.parse(s.endedAt) - Date.parse(s.startedAt)) / 1000; | |
| 159 | const body: EntryBody = { | |
| 160 | run: `snapshot:${s.startedAt}`, | |
| 161 | kind: "snapshot", | |
| 162 | startedAt: s.startedAt, | |
| 163 | endedAt: s.endedAt, | |
| 164 | ...(Number.isFinite(wall) ? { wall_s: Math.max(0, Math.round(wall * 1000) / 1000) } : {}), | |
| 165 | ...(s.failed ? { failed: true } : {}), | |
| 166 | prev, | |
| 167 | }; | |
| 168 | return { ...body, hash: hashOf(body) }; | |
| 169 | } | |
| 170 | ||
| 171 | /** Append the entry for a snapshot that ended, taken or failed. Never throws. */ | |
| 172 | export function appendSnapshot(s: SnapshotEnd): void { | |
| 173 | appendBody((prev) => snapshotEntryOf(s, prev), "snapshot"); | |
| 174 | } | |
| 175 | ||
| 139 | 176 | export type Verdict = { ok: true; entries: number } | { ok: false; at: string; why: string }; |
| 140 | 177 | |
| 141 | 178 | /** Walk every month in order; the first broken line is named with the reason. */ |
src/snapshot/service.ts +5
| @@ -12,6 +12,7 @@ | ||
| 12 | 12 | |
| 13 | 13 | import type { App } from "../app.ts"; |
| 14 | 14 | import { activeCrewRuns, crewHoldsCore, forgeCpus, holdClock } from "../core/one_core.ts"; |
| 15 | import { appendSnapshot } from "../core/ledger.ts"; | |
| 15 | 16 | import { createLog } from "../kernel/log.ts"; |
| 16 | 17 | import { firstSeen, reportVerdict, seedFirstSeen } from "./verdict.ts"; |
| 17 | 18 | import { |
| @@ -203,6 +204,7 @@ export function startSnapshots( | ||
| 203 | 204 | const take = (resume?: Date): Promise<Manifest> => { |
| 204 | 205 | // Whether this run's store is off the machine, for home's backup age (#3701). |
| 205 | 206 | let offBox: boolean | undefined; |
| 207 | let began = new Date().toISOString(); | |
| 206 | 208 | // After the one before it, however that ended: chained on a failure as it |
| 207 | 209 | // was, every later run failed at once with the old error until a restart. |
| 208 | 210 | wait.release("snapshot"); |
| @@ -214,6 +216,7 @@ export function startSnapshots( | ||
| 214 | 216 | offBox = isOffBox(store); |
| 215 | 217 | running = { objects: 0, bytes: 0, elapsedMs: 0 }; |
| 216 | 218 | startedAt = Date.now(); |
| 219 | began = new Date(startedAt).toISOString(); | |
| 217 | 220 | // Admin-only, no repository, nothing /backups does not already show. |
| 218 | 221 | app.bus.append({ kind: "backup.started" }); |
| 219 | 222 | let toldAt = Date.now(); |
| @@ -246,6 +249,7 @@ export function startSnapshots( | ||
| 246 | 249 | run.then(() => mirrorNow()).catch(() => {}); |
| 247 | 250 | inFlight = run.then( |
| 248 | 251 | (m) => { |
| 252 | appendSnapshot({ startedAt: m.startedAt, endedAt: m.finishedAt, failed: false }); | |
| 249 | 253 | last = { manifest: m, at: m.finishedAt }; |
| 250 | 254 | keep({ landed: m.name, ...(offBox === undefined ? {} : { offBox }) }); |
| 251 | 255 | app.bus.append({ kind: "backup.finished", payload: { ok: true } }); |
| @@ -258,6 +262,7 @@ export function startSnapshots( | ||
| 258 | 262 | arm(untilNightly()); |
| 259 | 263 | throw error; |
| 260 | 264 | } |
| 265 | appendSnapshot({ startedAt: began, endedAt: new Date().toISOString(), failed: true }); | |
| 261 | 266 | log.error("snapshot failed", { error: String(error) }); |
| 262 | 267 | keep({ error: error instanceof Error ? error.message : String(error) }); |
| 263 | 268 | app.bus.append({ kind: "backup.finished", payload: { ok: false } }); |
test/cost-ledger-snapshot.test.ts added +62
| @@ -0,0 +1,62 @@ | ||
| 1 | import { afterEach, beforeEach, expect, test } from "bun:test"; | |
| 2 | import { existsSync, mkdtempSync, readFileSync, readdirSync } from "node:fs"; | |
| 3 | import { tmpdir } from "node:os"; | |
| 4 | import { join } from "node:path"; | |
| 5 | import { createApp } from "../src/app.ts"; | |
| 6 | import { setLedgerDir, verifyDir } from "../src/core/ledger.ts"; | |
| 7 | import { startSnapshots } from "../src/snapshot/service.ts"; | |
| 8 | import { dirStore, type Store } from "../src/snapshot/store.ts"; | |
| 9 | import { quietLogs } from "./helpers/globals.ts"; | |
| 10 | ||
| 11 | quietLogs(); | |
| 12 | let dir: string; | |
| 13 | let ledger: string; | |
| 14 | beforeEach(() => { | |
| 15 | dir = mkdtempSync(join(process.env.TMPDIR ?? tmpdir(), "ledger-snap-")); | |
| 16 | ledger = join(dir, "ledger"); | |
| 17 | setLedgerDir(ledger); | |
| 18 | }); | |
| 19 | afterEach(() => setLedgerDir(undefined)); | |
| 20 | ||
| 21 | const entries = (): Record<string, unknown>[] => | |
| 22 | (existsSync(ledger) ? readdirSync(ledger) : []).flatMap((f) => | |
| 23 | readFileSync(join(ledger, f), "utf8").split("\n").filter(Boolean).map((l) => JSON.parse(l)), | |
| 24 | ); | |
| 25 | ||
| 26 | test("a snapshot that lands appends one entry of kind snapshot, with no repo", async () => { | |
| 27 | const app = createApp({ dataDir: dir, port: 0 }); | |
| 28 | const svc = startSnapshots(app, dirStore(join(dir, "store")), { version: "t", nightly: false }); | |
| 29 | const m = await svc.now(); | |
| 30 | await svc.stop(); | |
| 31 | app.close(); | |
| 32 | const [e, ...rest] = entries(); | |
| 33 | expect(rest.length).toBe(0); | |
| 34 | expect(e?.kind).toBe("snapshot"); | |
| 35 | expect(e?.run).toBe(`snapshot:${m.startedAt}`); | |
| 36 | expect(e?.endedAt).toBe(m.finishedAt); | |
| 37 | expect("repo" in (e ?? {})).toBe(false); | |
| 38 | expect("failed" in (e ?? {})).toBe(false); | |
| 39 | expect(verifyDir(ledger)).toEqual({ ok: true, entries: 1 }); | |
| 40 | }); | |
| 41 | ||
| 42 | test("a snapshot that fails is recorded as failed", async () => { | |
| 43 | const app = createApp({ dataDir: dir, port: 0 }); | |
| 44 | const bad = dirStore(join(dir, "store")); | |
| 45 | const broken: Store = { ...bad, put: async () => { throw new Error("disk full"); } } as Store; | |
| 46 | const svc = startSnapshots(app, broken, { version: "t", nightly: false, retryMs: 3_600_000 }); | |
| 47 | await svc.now().catch(() => {}); | |
| 48 | await svc.stop(); | |
| 49 | app.close(); | |
| 50 | const [e] = entries(); | |
| 51 | expect(e?.kind).toBe("snapshot"); | |
| 52 | expect(e?.failed).toBe(true); | |
| 53 | }); | |
| 54 | ||
| 55 | test("a forge with nowhere to put a snapshot writes nothing", async () => { | |
| 56 | const app = createApp({ dataDir: dir, port: 0 }); | |
| 57 | const svc = startSnapshots(app, () => undefined, { version: "t", nightly: false }); | |
| 58 | await svc.now().catch(() => {}); | |
| 59 | await svc.stop(); | |
| 60 | app.close(); | |
| 61 | expect(entries().length).toBe(0); | |
| 62 | }); |
new above
offline