trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRuns15g rEventsFlows

The forge says how it is at /api/v1/health: disk, memory, queue, stuck runs, apps asleep and previews, as one plain answer #352

Merged scoutcrew opened this

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

scout/forge:scout/343-the-forge-says-how-it-is-at-api-v1-healt → main 2 commits fork

  • Merged. scout/343-the-forge-says-how-it-is-at-api-v1-healt was merged into main as 80ce2a0.

Review

Sign in to review this change.
6 files changed +319 −1
docs/ARCHITECTURE.md +10
@@ -486,6 +486,16 @@ budget, a deadline. Containment is the forge's, not the agent's; the agent
486486 is trusted to work, and the forge is built so that trust costs at most one
487487 run.
488488
489 ## The forge's standing
490
491 `GET /api/v1/health` (`kernel/vitals.ts`) answers a site admin, by session or
492 any token scope, with `{ ok, needs, disk, memory, queue, stuck, apps, at }`;
493 everyone else gets a 404. `needs` is one sentence per thing off: disk under
494 10% free on the `FORGE_DATA` volume, a run running with a lapsed lease (or
495 past six hours), a queue whose oldest run has waited over thirty minutes.
496 It is a fixed handful of queries however many runs and apps exist, measured
497 per request and never cached. `/healthz` stays the orchestrator's answer.
498
489499 ## Search, and where semantic search would go
490500
491501 Search (`core/search.ts`, `actions/search.ts`, `GET /api/v1/search`) holds no
forge.json +1 −1
@@ -6,7 +6,7 @@
66 "ready": 90
77 },
88 "check": {
9 "run": "bun x tsc --noEmit && bun scripts/test-shards.ts --timeout 60000 test/crew.test.ts test/crew-wrapup.test.ts test/crew-limits.test.ts test/crew-sandbox.test.ts test/steer.test.ts test/landing.test.ts test/decide.test.ts test/ledger.test.ts test/checks.test.ts test/dependencies.test.ts test/autonomy.test.ts test/studio.test.ts test/studio-words.test.ts test/forks.test.ts test/fork-flow.test.ts test/cross-repo-pull.test.ts test/pulls.test.ts test/apps.test.ts test/apps-adversarial.test.ts test/apps-busy.test.ts test/web.test.ts test/git-trust.test.ts test/helpers.test.ts test/shards.test.ts test/migrate.test.ts",
9 "run": "bun x tsc --noEmit && bun scripts/test-shards.ts --timeout 60000 test/crew.test.ts test/crew-wrapup.test.ts test/crew-limits.test.ts test/crew-sandbox.test.ts test/steer.test.ts test/landing.test.ts test/decide.test.ts test/ledger.test.ts test/checks.test.ts test/dependencies.test.ts test/autonomy.test.ts test/studio.test.ts test/studio-words.test.ts test/forks.test.ts test/fork-flow.test.ts test/cross-repo-pull.test.ts test/pulls.test.ts test/apps.test.ts test/apps-adversarial.test.ts test/apps-busy.test.ts test/web.test.ts test/git-trust.test.ts test/helpers.test.ts test/shards.test.ts test/migrate.test.ts test/vitals.test.ts",
1010 "timeout": 1800
1111 }
1212 }
src/core/runs.ts +37
@@ -789,3 +789,40 @@ export function listActiveRuns(
789789 .all(...unique.slice(0, 400))
790790 .map(toRun);
791791 }
792
793 /** What is waiting to start, by kind: one grouped query, however many runs there are. */
794 export function queueDepth(
795 db: Database,
796 now = Date.now(),
797 ): { kind: RunKind; queued: number; oldest: string }[] {
798 return db
799 .query<{ kind: RunKind; queued: number; oldest: string }, [string]>(
800 `SELECT kind, COUNT(*) AS queued, MIN(created_at) AS oldest
801 FROM run
802 WHERE state = 'queued' AND (not_before IS NULL OR not_before <= ?1)
803 GROUP BY kind`,
804 )
805 .all(new Date(now).toISOString());
806 }
807
808 /** Every running run with the repository and issue number it is about, in one query. */
809 export function listRunning(
810 db: Database,
811 ): { run: Run; repo: string; issue: number | null }[] {
812 const rows = db
813 .query<RunRow & { slug: string; issue_number: number | null }, []>(
814 `SELECT run.*, user.username || '/' || repo.name AS slug,
815 issue.number AS issue_number
816 FROM run
817 JOIN repo ON repo.id = run.repo_id
818 JOIN user ON user.id = repo.owner_id
819 LEFT JOIN issue ON issue.id = run.issue_id
820 WHERE run.state = 'running'`,
821 )
822 .all();
823 return rows.map((r) => ({
824 run: toRun(r),
825 repo: r.slug,
826 issue: r.issue_number,
827 }));
828 }
src/http/api.ts +14
@@ -213,6 +213,7 @@ import {
213213 type Run,
214214 } from "../core/runs.ts";
215215 import { resolveOid } from "../git/merge.ts";
216 import { vitals } from "../kernel/vitals.ts";
216217 import { readRunLog, subscribeRunLog } from "../runs/log.ts";
217218 import { blobForRender, commitLog, listRefs, listTree } from "../git/read.ts";
218219 import { repoDir } from "../git/repos.ts";
@@ -2679,6 +2680,19 @@ export function apiRoutes(router: Router<Handler>): void {
26792680
26802681 // ---- apps, runs, and the crew (docs/APPS.md) ----
26812682
2683 // How the forge itself is: a site admin's to read, and nobody else's to
2684 // know exists — a 404, never the numbers.
2685 router.add("GET", "/api/v1/health", (ctx) => {
2686 if (!ctx.auth.user?.isAdmin) return notFound();
2687 const runtime = appsRuntime(ctx.app);
2688 const envs = runtime
2689 ? listApps(ctx.app.db).flatMap((record) => runtime.status(record))
2690 : [];
2691 return json(
2692 vitals({ db: ctx.app.db, dataDir: ctx.app.config.dataDir, envs }),
2693 );
2694 });
2695
26822696 router.add("GET", "/api/v1/apps", (ctx) => {
26832697 const records = listApps(ctx.app.db);
26842698 const repos = getReposByIds(
src/kernel/vitals.ts added +116
@@ -0,0 +1,116 @@
1 /**
2 * The forge's standing: is it fine, and if not, what is off. Plain data in,
3 * plain data out — the disk under `FORGE_DATA`, this process's memory, the
4 * queue, runs that stopped making progress, and how many apps are serving,
5 * asleep or previewed. A fixed handful of queries however many runs and
6 * apps there are; nothing is cached, nothing shelled out.
7 */
8
9 import type { Database } from "bun:sqlite";
10 import { statfsSync } from "node:fs";
11 import { freemem, totalmem } from "node:os";
12 import { leaseLapsed, listRunning, queueDepth } from "../core/runs.ts";
13 import { STALE_AFTER_MS } from "../runs/lease.ts";
14
15 /** Free disk under this share of the volume is something to say. */
16 export const DISK_FREE_MIN = 0.1;
17 /** A run still running this long after it started is stuck even if it breathes. */
18 export const NO_PROGRESS_MS = 6 * 3_600_000;
19 /** A run that has waited this long to start is a queue that is not moving. */
20 export const QUEUE_WAIT_MAX_MS = 30 * 60_000;
21
22 export interface EnvCount {
23 readonly env: string;
24 readonly state: string;
25 }
26
27 export interface Vitals {
28 readonly ok: boolean;
29 readonly needs: string[];
30 readonly disk: { freeBytes: number; totalBytes: number };
31 readonly memory: {
32 rssBytes: number;
33 hostFreeBytes: number;
34 hostTotalBytes: number;
35 };
36 readonly queue: Record<string, number>;
37 readonly stuck: {
38 runId: string;
39 repo: string;
40 issue: number | null;
41 since: string;
42 }[];
43 readonly apps: { serving: number; asleep: number; previews: number };
44 readonly at: string;
45 }
46
47 export interface VitalsInput {
48 readonly db: Database;
49 /** The directory holding the forge's data: its volume is the one measured. */
50 readonly dataDir: string;
51 /** Every environment the runtime holds, when one is up. */
52 readonly envs?: readonly EnvCount[];
53 readonly now?: number;
54 }
55
56 export function vitals(input: VitalsInput): Vitals {
57 const now = input.now ?? Date.now();
58 const needs: string[] = [];
59
60 const fs = statfsSync(input.dataDir);
61 const disk = {
62 freeBytes: Number(fs.bavail) * Number(fs.bsize),
63 totalBytes: Number(fs.blocks) * Number(fs.bsize),
64 };
65 if (disk.totalBytes > 0 && disk.freeBytes / disk.totalBytes < DISK_FREE_MIN)
66 needs.push(
67 `Disk is nearly full: ${Math.round((disk.freeBytes / disk.totalBytes) * 100)}% free.`,
68 );
69
70 const queue: Record<string, number> = {};
71 for (const row of queueDepth(input.db, now)) {
72 queue[row.kind] = row.queued;
73 if (now - new Date(row.oldest).getTime() > QUEUE_WAIT_MAX_MS)
74 needs.push(
75 `The ${row.kind} queue has waited over ${QUEUE_WAIT_MAX_MS / 60_000} minutes.`,
76 );
77 }
78
79 const stuck: Vitals["stuck"] = [];
80 for (const { run, repo, issue } of listRunning(input.db)) {
81 const started = new Date(run.startedAt ?? run.createdAt).getTime();
82 if (leaseLapsed(run, STALE_AFTER_MS, now) || now - started > NO_PROGRESS_MS)
83 stuck.push({
84 runId: run.id,
85 repo,
86 issue,
87 since: run.heartbeatAt ?? run.startedAt ?? run.createdAt,
88 });
89 }
90 if (stuck.length > 0)
91 needs.push(
92 `${stuck.length} run${stuck.length === 1 ? " is" : "s are"} stuck.`,
93 );
94
95 const apps = { serving: 0, asleep: 0, previews: 0 };
96 for (const e of input.envs ?? []) {
97 if (e.env !== "production") apps.previews++;
98 else if (e.state === "asleep") apps.asleep++;
99 else if (e.state === "running") apps.serving++;
100 }
101
102 return {
103 ok: needs.length === 0,
104 needs,
105 disk,
106 memory: {
107 rssBytes: process.memoryUsage().rss,
108 hostFreeBytes: freemem(),
109 hostTotalBytes: totalmem(),
110 },
111 queue,
112 stuck,
113 apps,
114 at: new Date(now).toISOString(),
115 };
116 }
test/vitals.test.ts added +141
@@ -0,0 +1,141 @@
1 /** The forge's standing: its shape, the admin gate, and what it flags. */
2
3 import { afterAll, beforeAll, expect, test } from "bun:test";
4 import { mkdtemp, rm } from "node:fs/promises";
5 import { tmpdir } from "node:os";
6 import { join } from "node:path";
7 import { createApp, type App } from "../src/app.ts";
8 import { createRun } from "../src/core/runs.ts";
9 import { vitals } from "../src/kernel/vitals.ts";
10 import { serve, type Servers } from "../src/http/server.ts";
11
12 let dataDir: string;
13 let app: App;
14 let servers: Servers;
15 let admin = "";
16 let plain = "";
17 let repoId = "";
18
19 const post = (path: string, body: unknown, token?: string) =>
20 fetch(servers.url + path, {
21 method: "POST",
22 headers: {
23 "content-type": "application/json",
24 ...(token ? { authorization: `Bearer ${token}` } : {}),
25 },
26 body: JSON.stringify(body),
27 });
28 const mint = async (username: string, scope: string) =>
29 (
30 (await (
31 await post("/api/v1/tokens", {
32 username,
33 password: "a-good-passphrase",
34 name: scope,
35 scope,
36 })
37 ).json()) as { token: string }
38 ).token;
39 const health = (token?: string) =>
40 fetch(servers.url + "/api/v1/health", {
41 headers: token ? { authorization: `Bearer ${token}` } : {},
42 });
43
44 beforeAll(async () => {
45 dataDir = await mkdtemp(join(tmpdir(), "forge-vitals-"));
46 app = createApp({ dataDir, port: 0 });
47 servers = await serve(app);
48 for (const username of ["alice", "bob"])
49 await post("/api/v1/users", {
50 username,
51 email: `${username}@forge.local`,
52 password: "a-good-passphrase",
53 });
54 admin = await mint("alice", "ro");
55 plain = await mint("bob", "admin");
56 repoId = (
57 app.db.query("SELECT id FROM repo").get() as { id: string } | null
58 )?.id ?? "";
59 if (!repoId) {
60 const owner = app.db.query("SELECT id FROM user LIMIT 1").get() as {
61 id: string;
62 };
63 repoId = "repo-vitals";
64 app.db
65 .query("INSERT INTO repo (id, owner_id, name) VALUES (?1, ?2, 'r')")
66 .run(repoId, owner.id);
67 }
68 });
69
70 afterAll(async () => {
71 servers.stop();
72 app.close();
73 await rm(dataDir, { recursive: true, force: true });
74 });
75
76 test("a site admin with a read-only token gets the shape; others get a 404", async () => {
77 const res = await health(admin);
78 expect(res.status).toBe(200);
79 const v = (await res.json()) as Record<string, any>;
80 expect(Object.keys(v).sort()).toEqual(
81 ["apps", "at", "disk", "memory", "needs", "ok", "queue", "stuck"].sort(),
82 );
83 expect(v.disk.totalBytes).toBeGreaterThan(0);
84 expect(v.memory.rssBytes).toBeGreaterThan(0);
85 expect(v.apps).toEqual({ serving: 0, asleep: 0, previews: 0 });
86 expect((await health(plain)).status).toBe(404);
87 expect((await health()).status).toBe(404);
88 expect((await fetch(servers.url + "/healthz")).status).toBe(200);
89 });
90
91 test("a full queue shows in queue, and an old one in needs", () => {
92 const old = Date.now() - 3_600_000;
93 for (let i = 0; i < 3; i++) createRun(app.db, { repoId, kind: "agent" });
94 const v = vitals({ db: app.db, dataDir });
95 expect(v.queue).toEqual({ agent: 3 });
96 expect(v.ok).toBe(true);
97 const later = vitals({ db: app.db, dataDir, now: old + 2 * 3_600_000 + 3_600_000 });
98 expect(later.ok).toBe(false);
99 expect(later.needs.join(" ")).toContain("agent queue");
100 });
101
102 test("a running run with a lapsed lease is stuck, and needs says so", async () => {
103 const run = createRun(app.db, { repoId, kind: "deploy", env: "production" });
104 app.db
105 .query(
106 "UPDATE run SET state='running', started_at=?2, heartbeat_at=?2 WHERE id=?1",
107 )
108 .run(run.id, new Date(Date.now() - 10 * 60_000).toISOString());
109 const v = (await (await health(admin)).json()) as ReturnType<typeof vitals>;
110 expect(v.stuck.map((s) => s.runId)).toEqual([run.id]);
111 expect(v.stuck[0]!.repo).toBe("alice/r");
112 expect(v.ok).toBe(false);
113 expect(v.needs.some((n) => n.includes("stuck"))).toBe(true);
114 // A fresh heartbeat clears it.
115 app.db
116 .query("UPDATE run SET heartbeat_at=?2 WHERE id=?1")
117 .run(run.id, new Date().toISOString());
118 expect(vitals({ db: app.db, dataDir }).stuck).toEqual([]);
119 });
120
121 test("the number of queries does not grow with runs", () => {
122 const count = () => {
123 let n = 0;
124 const counting = new Proxy(app.db, {
125 get(t, k) {
126 const v = Reflect.get(t, k, t);
127 if (k === "query") return (...a: any[]) => (n++, (v as any).apply(t, a));
128 return typeof v === "function" ? v.bind(t) : v;
129 },
130 });
131 vitals({ db: counting, dataDir });
132 return n;
133 };
134 const before = count();
135 for (let i = 0; i < 40; i++) {
136 const r = createRun(app.db, { repoId, kind: "check" });
137 if (i % 2) app.db.query("UPDATE run SET state='running' WHERE id=?1").run(r.id);
138 }
139 expect(count()).toBe(before);
140 expect(before).toBe(2);
141 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs