trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRuns13g rEventsFlows

The forge says why a rollout is blocked: /api/v1/health surfaces it and the board shows it #540

Merged scoutcrew opened this in answer to trevatoowner

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

scout/forge:scout/476-the-forge-says-why-a-rollout-is-blocked → main 5 commits fork

  • Merged. scout/476-the-forge-says-why-a-rollout-is-blocked was merged into main as a4b42b5.

Review

Sign in to review this change.
9 files changed +304 −17
charts/app/templates/deployment.yaml +8
@@ -125,6 +125,9 @@ spec:
125125 - name: plat-ca
126126 mountPath: /etc/plat-ca
127127 readOnly: true
128 - name: forge-guard
129 mountPath: /etc/forge-guard
130 readOnly: true
128131 # `/` answers 200 for anyone; it is the landing page. Boot reconciles
129132 # the index from git first, which grows with the number of repos.
130133 startupProbe:
@@ -148,4 +151,9 @@ spec:
148151 - name: plat-ca
149152 secret:
150153 secretName: plat-local-ca-tls
154 # The watchdog's view of the platform; absent is fine.
155 - name: forge-guard
156 configMap:
157 name: forge-guard
158 optional: true
151159 optional: true
docs/ARCHITECTURE.md +8 −3
@@ -514,9 +514,14 @@ spike #339) implements the same interface and must provide:
514514
515515 ## The forge's standing
516516
517 `GET /api/v1/health` (`kernel/vitals.ts`) answers a site admin, by session or
518 any token scope, with `{ ok, needs, disk, memory, queue, stuck, apps, at }`;
519 everyone else gets a 404. `needs` is one sentence per thing off: disk under
517 `GET /api/v1/health` answers anyone with `{ ok, version, draining, rollout:
518 { state: ok|blocked|held, reason }, incidents, stale, checkedAt }` — the
519 platform's word, read from the read-only `forge-guard` ConfigMap mounted at
520 `FORGE_GUARD_DIR` (default `/etc/forge-guard`, `kernel/guard.ts`): a `stuck`
521 incident blocks, `held` holds, a view older than ten minutes is stale. A
522 site admin, by session or any token scope, also gets `{ needs, disk, memory,
523 queue, stuck, apps, at }` (`kernel/vitals.ts`), and the board shows
524 `rollout blocked: <reason>` in its status line. `needs` is one sentence per thing off: disk under
520525 10% free on the `FORGE_DATA` volume, a run running with a lapsed lease (or
521526 past six hours), a queue whose oldest run has waited over thirty minutes.
522527 It is a fixed handful of queries however many runs and apps exist, measured
forge.json +1 −1
@@ -6,7 +6,7 @@
66 "ready": 90
77 },
88 "check": {
9 "run": "bun x tsc --noEmit && bun scripts/test-shards.ts --timeout 60000 -- crew crew-wrapup crew-limits crew-sandbox steer landing decide ledger checks dependencies autonomy studio studio-words forks fork-flow cross-repo-pull pulls apps apps-adversarial apps-busy apps-token web git-trust helpers shards migrate vitals asks feed feed-page offline share cloud run-secrets forge-cloudflare usage standing board-windows spawn-plan spawn web-apps fleet-ops meter",
9 "run": "bun x tsc --noEmit && bun scripts/test-shards.ts --timeout 60000 -- crew crew-wrapup crew-limits crew-sandbox steer landing decide ledger checks dependencies autonomy studio studio-words forks fork-flow cross-repo-pull pulls apps apps-adversarial apps-busy apps-token web git-trust helpers shards migrate vitals guard asks feed feed-page offline share cloud run-secrets forge-cloudflare usage standing board-windows spawn-plan spawn web-apps fleet-ops meter",
1010 "timeout": 1800
1111 }
1212 }
src/http/api.ts +8 −6
@@ -8,6 +8,7 @@
88 * back to continue); errors are `{ error, message }`.
99 */
1010
11 import { forgeHealth } from "../kernel/guard.ts";
1112 import { listSecrets, putSecret, removeSecret } from "../actions/secrets.ts";
1213 import { proposeUpstream } from "../actions/upstream.ts";
1314 import { relationOf, syncFork, takeUpdates } from "../actions/forks.ts";
@@ -2752,17 +2753,18 @@ export function apiRoutes(router: Router<Handler>): void {
27522753
27532754 // ---- apps, runs, and the crew (docs/APPS.md) ----
27542755
2755 // How the forge itself is: a site admin's to read, and nobody else's to
2756 // know exists — a 404, never the numbers.
2756 // How the forge is, and the platform's word on its rollout: anyone may
2757 // read it (a monitor has no account). A site admin also gets the forge's
2758 // own vitals alongside, `ok` being both.
27572759 router.add("GET", "/api/v1/health", (ctx) => {
2758 if (!ctx.auth.user?.isAdmin) return notFound();
2760 const forge = forgeHealth();
2761 if (!ctx.auth.user?.isAdmin) return json(forge);
27592762 const runtime = appsRuntime(ctx.app);
27602763 const envs = runtime
27612764 ? listApps(ctx.app.db).flatMap((record) => runtime.status(record))
27622765 : [];
2763 return json(
2764 vitals({ db: ctx.app.db, dataDir: ctx.app.config.dataDir, envs }),
2765 );
2766 const v = vitals({ db: ctx.app.db, dataDir: ctx.app.config.dataDir, envs });
2767 return json({ ...v, ...forge, ok: forge.ok && v.ok });
27662768 });
27672769
27682770 router.add("GET", "/api/v1/apps", (ctx) => {
src/kernel/guard.ts added +131
@@ -0,0 +1,131 @@
1 /**
2 * The platform's view of the forge, as the droplet's watchdog leaves it: a
3 * ConfigMap (`forge-guard`) mounted read-only, no Kubernetes API needed.
4 * `status.json` is `{ incidents: {key: message}, public, service }` and
5 * `checkedAt` is an ISO time. Keys: `down`, `edge`, `stuck` (a rollout
6 * cannot finish), `held` (the reconciler's preflight refused a release),
7 * `oom:*`. Missing or empty means nothing is wrong; a view older than ten
8 * minutes is said to be stale, never "all fine".
9 */
10
11 import { readFileSync } from "node:fs";
12 import { join } from "node:path";
13 import pkg from "../../package.json" with { type: "json" };
14 import { health } from "../http/health.ts";
15
16 export const GUARD_STALE_MS = 10 * 60_000;
17 const MAX_INCIDENTS = 50;
18 const MAX_MESSAGE = 800;
19
20 export interface Incident {
21 readonly key: string;
22 readonly message: string;
23 }
24
25 export interface Guard {
26 readonly incidents: Incident[];
27 readonly checkedAt: string | null;
28 }
29
30 export interface Rollout {
31 readonly state: "ok" | "blocked" | "held";
32 readonly reason: string;
33 }
34
35 export interface ForgeHealth {
36 readonly ok: boolean;
37 readonly version: string;
38 readonly draining: boolean;
39 readonly rollout: Rollout;
40 readonly incidents: Incident[];
41 readonly stale: boolean;
42 readonly checkedAt: string | null;
43 }
44
45 /** Where the ConfigMap is mounted. */
46 export function guardDir(): string {
47 return process.env["FORGE_GUARD_DIR"] || "/etc/forge-guard";
48 }
49
50 function readText(path: string): string | null {
51 try {
52 return readFileSync(path, "utf8");
53 } catch {
54 return null;
55 }
56 }
57
58 /** Parse the two keys; anything malformed is treated as absent. */
59 export function parseGuard(
60 statusJson: string | null,
61 checkedAtText: string | null,
62 ): Guard {
63 const incidents: Incident[] = [];
64 if (statusJson) {
65 try {
66 const raw = (JSON.parse(statusJson) as { incidents?: unknown })
67 .incidents;
68 if (raw && typeof raw === "object" && !Array.isArray(raw)) {
69 for (const [key, message] of Object.entries(raw)) {
70 if (incidents.length >= MAX_INCIDENTS) break;
71 if (!/^[a-z][a-z0-9:._-]{0,63}$/i.test(key)) continue;
72 if (typeof message !== "string") continue;
73 incidents.push({ key, message: message.trim().slice(0, MAX_MESSAGE) });
74 }
75 }
76 } catch {
77 // a half-written file reads as nothing
78 }
79 }
80 const at = checkedAtText?.trim() ?? "";
81 const checkedAt =
82 at && Number.isFinite(Date.parse(at)) ? new Date(at).toISOString() : null;
83 return { incidents, checkedAt };
84 }
85
86 export function readGuard(dir: string = guardDir()): Guard {
87 return parseGuard(
88 readText(join(dir, "status.json")),
89 readText(join(dir, "checkedAt")),
90 );
91 }
92
93 /** The forge's view plus the platform's, as plain data. */
94 export function forgeHealth(
95 guard: Guard = readGuard(),
96 now: number = Date.now(),
97 ): ForgeHealth {
98 const stale =
99 guard.checkedAt !== null && now - Date.parse(guard.checkedAt) > GUARD_STALE_MS;
100 const find = (key: string) => guard.incidents.find((i) => i.key === key);
101 const stuck = find("stuck");
102 const held = find("held");
103 const rollout: Rollout = stuck
104 ? { state: "blocked", reason: stuck.message }
105 : held
106 ? { state: "held", reason: held.message }
107 : stale
108 ? {
109 state: "ok",
110 reason: "the platform's view is stale: its last check was " +
111 `${guard.checkedAt}`,
112 }
113 : { state: "ok", reason: "" };
114 return {
115 ok:
116 !health.draining && !stale && guard.incidents.length === 0,
117 version: pkg.version,
118 draining: health.draining,
119 rollout,
120 incidents: guard.incidents,
121 stale,
122 checkedAt: guard.checkedAt,
123 };
124 }
125
126 /** The status-line words for a rollout that is not simply fine, else null. */
127 export function rolloutLine(h: ForgeHealth): string | null {
128 if (h.rollout.state === "blocked") return `rollout blocked: ${h.rollout.reason}`;
129 if (h.rollout.state === "held") return `rollout held: ${h.rollout.reason}`;
130 return h.stale ? h.rollout.reason : null;
131 }
src/web/pages/board.ts +12 −2
@@ -24,6 +24,7 @@ import { efficiencyForRepos } from "../../core/efficiency.ts";
2424 import { standings, WINDOW_MS } from "../../core/standing.ts";
2525 import { listSubscriptions } from "../../core/subscriptions.ts";
2626 import { listApps } from "../../core/apps.ts";
27 import { forgeHealth, rolloutLine } from "../../kernel/guard.ts";
2728 import { vitals, type Vitals } from "../../kernel/vitals.ts";
2829 import { getIssue, getIssuesByIds, type Issue } from "../../core/issues.ts";
2930 import { looksForApps } from "../../core/looks.ts";
@@ -270,15 +271,24 @@ function forgeStrip(ctx: Ctx): Html | null {
270271 ? listApps(app.db).flatMap((record) => runtime.status(record))
271272 : [],
272273 });
274 const fh = forgeHealth();
275 const rollout = rolloutLine(fh);
276 const others = fh.incidents.filter(
277 (i) => i.key !== "stuck" && i.key !== "held",
278 );
279 const ok = v.ok && fh.ok;
273280 return html`<div
274281 class="board-forge"
275282 id="forge"
276283 data-live="run.*,app.*"
277 data-ok="${v.ok ? "yes" : "no"}"
284 data-ok="${ok ? "yes" : "no"}"
285 data-rollout="${fh.rollout.state}"
278286 role="status"
279287 >
288 ${rollout === null ? "" : html`<span class="board-forge-need">${rollout}</span>`}
289 ${others.map((i) => html`<span class="board-forge-need">${i.key}: ${i.message}</span>`)}
280290 ${
281 v.ok
291 ok
282292 ? html`<span class="board-forge-ok">All fine</span>`
283293 : v.needs.map((need) => {
284294 const href = needHref(need, v);
test/board.test.ts +21 −1
@@ -5,6 +5,7 @@
55 * anonymous is sent to sign in.
66 */
77 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
8 import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
89 import { mkdtemp, rm } from "node:fs/promises";
910 import { tmpdir } from "node:os";
1011 import { join } from "node:path";
@@ -371,7 +372,7 @@ describe("the board's forge strip", () => {
371372 if (username === "root") admin = u.value;
372373 cookies[username] = await signIn(username);
373374 }
374 });
375 }, 30_000);
375376
376377 afterAll(async () => {
377378 servers.stop();
@@ -402,4 +403,23 @@ describe("the board's forge strip", () => {
402403 expect(other).not.toContain("All fine");
403404 expect(other).not.toContain("stuck.");
404405 });
406
407 test("a blocked rollout shows in the status line, escaped, for an admin only", async () => {
408 const guard = mkdtempSync(join(tmpdir(), "forge-board-guard-"));
409 process.env["FORGE_GUARD_DIR"] = guard;
410 try {
411 writeFileSync(
412 join(guard, "status.json"),
413 JSON.stringify({ incidents: { stuck: "quota refused <i>8Gi</i>" } }),
414 );
415 writeFileSync(join(guard, "checkedAt"), new Date().toISOString());
416 const blocked = await board("root");
417 expect(blocked).toContain("rollout blocked: quota refused &lt;i&gt;8Gi&lt;/i&gt;");
418 expect(blocked).not.toContain("<i>8Gi</i>");
419 expect(await board("bob")).not.toContain("rollout blocked");
420 } finally {
421 delete process.env["FORGE_GUARD_DIR"];
422 rmSync(guard, { recursive: true, force: true });
423 }
424 });
405425 });
test/guard.test.ts added +102
@@ -0,0 +1,102 @@
1 /** The platform's word on the rollout: ok, blocked, held — and stale. */
2
3 import { afterAll, beforeAll, expect, test } from "bun:test";
4 import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
5 import { mkdtemp, rm } from "node:fs/promises";
6 import { tmpdir } from "node:os";
7 import { join } from "node:path";
8 import { createApp, type App } from "../src/app.ts";
9 import { serve, type Servers } from "../src/http/server.ts";
10 import {
11 forgeHealth,
12 parseGuard,
13 readGuard,
14 rolloutLine,
15 } from "../src/kernel/guard.ts";
16
17 const NOW = Date.parse("2026-10-02T12:00:00Z");
18 const fresh = "2026-10-02T11:59:00Z";
19 const status = (incidents: Record<string, string>) =>
20 JSON.stringify({ incidents, public: 200, service: 200 });
21
22 test("no ConfigMap, or an empty one, reports ok", () => {
23 const none = forgeHealth(readGuard("/nonexistent/forge-guard"), NOW);
24 expect(none.ok).toBe(true);
25 expect(none.rollout).toEqual({ state: "ok", reason: "" });
26 expect(none.checkedAt).toBeNull();
27 const empty = forgeHealth(parseGuard(status({}), fresh), NOW);
28 expect(empty.ok).toBe(true);
29 expect(empty.incidents).toEqual([]);
30 expect(rolloutLine(empty)).toBeNull();
31 });
32
33 test("a stuck incident blocks the rollout with its reason", () => {
34 const reason = "pod refused: exceeded quota; raise requests.memory to 10Gi";
35 const h = forgeHealth(parseGuard(status({ stuck: reason }), fresh), NOW);
36 expect(h.rollout).toEqual({ state: "blocked", reason });
37 expect(h.ok).toBe(false);
38 expect(rolloutLine(h)).toBe(`rollout blocked: ${reason}`);
39 });
40
41 test("a held incident holds the rollout", () => {
42 const h = forgeHealth(
43 parseGuard(status({ held: "preflight: no room on node" }), fresh),
44 NOW,
45 );
46 expect(h.rollout).toEqual({ state: "held", reason: "preflight: no room on node" });
47 expect(rolloutLine(h)).toBe("rollout held: preflight: no room on node");
48 });
49
50 test("a stale view says so instead of all fine", () => {
51 const h = forgeHealth(parseGuard(status({}), "2026-10-02T11:40:00Z"), NOW);
52 expect(h.stale).toBe(true);
53 expect(h.ok).toBe(false);
54 expect(h.rollout.reason).toContain("stale");
55 expect(rolloutLine(h)).toContain("stale");
56 });
57
58 test("malformed input reads as nothing, and odd keys are dropped", () => {
59 expect(parseGuard("{not json", "nonsense")).toEqual({ incidents: [], checkedAt: null });
60 const g = parseGuard(status({ "bad key!": "x", "oom:web": "killed", n: 5 as never }), fresh);
61 expect(g.incidents).toEqual([{ key: "oom:web", message: "killed" }]);
62 });
63
64 let dir: string;
65 let guardDir: string;
66 let app: App;
67 let servers: Servers;
68
69 beforeAll(async () => {
70 dir = await mkdtemp(join(tmpdir(), "forge-guard-"));
71 guardDir = mkdtempSync(join(tmpdir(), "forge-guard-cm-"));
72 process.env["FORGE_GUARD_DIR"] = guardDir;
73 app = createApp({ dataDir: dir, port: 0 });
74 servers = await serve(app);
75 });
76
77 afterAll(async () => {
78 delete process.env["FORGE_GUARD_DIR"];
79 servers.stop();
80 app.close();
81 await rm(dir, { recursive: true, force: true });
82 rmSync(guardDir, { recursive: true, force: true });
83 });
84
85 test("/api/v1/health answers anyone, from the mounted ConfigMap", async () => {
86 let res = await fetch(servers.url + "/api/v1/health");
87 expect(res.status).toBe(200);
88 let body = (await res.json()) as any;
89 expect(body.ok).toBe(true);
90 expect(body.rollout.state).toBe("ok");
91 expect(body.draining).toBe(false);
92 expect(body.version).toMatch(/^\d+\.\d+\.\d+/);
93
94 mkdirSync(guardDir, { recursive: true });
95 writeFileSync(join(guardDir, "status.json"), status({ stuck: "quota refused <b>" }));
96 writeFileSync(join(guardDir, "checkedAt"), new Date().toISOString());
97 res = await fetch(servers.url + "/api/v1/health");
98 body = await res.json();
99 expect(body.rollout).toEqual({ state: "blocked", reason: "quota refused <b>" });
100 expect(body.ok).toBe(false);
101 expect(body.incidents).toEqual([{ key: "stuck", message: "quota refused <b>" }]);
102 });
test/vitals.test.ts +13 −4
@@ -73,18 +73,27 @@ afterAll(async () => {
7373 await rm(dataDir, { recursive: true, force: true });
7474 });
7575
76 test("a site admin with a read-only token gets the shape; others get a 404", async () => {
76 test("a site admin with a read-only token gets the shape; others get the short one", async () => {
7777 const res = await health(admin);
7878 expect(res.status).toBe(200);
7979 const v = (await res.json()) as Record<string, any>;
8080 expect(Object.keys(v).sort()).toEqual(
81 ["apps", "at", "disk", "errors", "memory", "needs", "ok", "queue", "stuck"].sort(),
81 [
82 "apps", "at", "checkedAt", "disk", "draining", "errors", "incidents",
83 "memory", "needs", "ok", "queue", "rollout", "stale", "stuck", "version",
84 ].sort(),
8285 );
8386 expect(v.disk.totalBytes).toBeGreaterThan(0);
8487 expect(v.memory.rssBytes).toBeGreaterThan(0);
8588 expect(v.apps).toEqual({ serving: 0, asleep: 0, previews: 0 });
86 expect((await health(plain)).status).toBe(404);
87 expect((await health()).status).toBe(404);
89 // Everyone else gets the platform's short shape, never the numbers.
90 for (const token of [plain, undefined]) {
91 const res = await health(token);
92 expect(res.status).toBe(200);
93 const short = (await res.json()) as Record<string, unknown>;
94 expect(short["disk"]).toBeUndefined();
95 expect(short["rollout"]).toBeDefined();
96 }
8897 expect((await fetch(servers.url + "/healthz")).status).toBe(200);
8998 });
9099

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs