bb7bde5 · 70d6c85 · 251b6a0 · e5e7b5e · 5bb4e8b · e4184d7
@@ -12,6 +12,10 @@
not an admin, 200 for an admin's `ro` token. It is never cached
| `forge_egress_bytes_in` | gauge | `repo`, `destination` | Bytes received from the destination over open connections. |
| `forge_egress_bytes_out` | gauge | `repo`, `destination` | Bytes sent to the destination over open connections. |
| `forge_http_request_duration_seconds` | histogram | `pattern` (plus `le`) | Seconds taken to answer a request, by the route pattern matched (never the raw path), since the process started. Buckets are `route_timing`'s edges, 1 ms to 2.5 s, in seconds. |
| `forge_check_duration_seconds` | histogram | (plus `le`) | Seconds a repository check took from start to finish, for checks that ended since the process started. Buckets 5 s to 30 min. |
| `forge_check_queue_depth` | gauge | none | Checks queued and not yet started. |
| `forge_crew_runs_total` | counter | `member`, `outcome` | Crew runs that ended, by the member's username and `ok`, `failed` or `cancelled`. |
| `forge_crew_spend_usd_total` | counter | `member` | US dollars the member's runs spent, from the usage ledger (list-price equivalent on a subscription). |
| `forge_pulls_merged_total` | counter | `repo` | Pulls merged, from `pull.merged_at`. |
| `forge_releases_total` | counter | none | Releases that went live, from the `release` table's `ok` rows. |
| `forge_ask_to_live_seconds` | histogram | none (plus `le`) | Seconds from an issue opening to the first successful release of a pull that closes it. Buckets: 1 m, 5 m, 15 m, 1 h, 4 h, 12 h, 1 d, 3 d, 7 d. |
@@ -25,19 +29,22 @@
never in the body. `env` is `production` or `pull-<n>`. The cell families are
what the cell watch last read (every few seconds); a cell that is not running
has no series (a cell keeps the slug it started under, so after a rename its figures return when it restarts), and the forge starts nothing and reads no process to answer.
The duration histograms count every request the process has answered, and
every check it has seen finish, since it started; their `_bucket`, `_count`
and `_sum` only grow, so `rate()` and `histogram_quantile()` read them as
Prometheus expects. A restart starts them at zero again, which Prometheus
reads as a counter reset. A request counts the moment it is answered; a check
counts at the first scrape after it finishes. The hourly `route_timing` table,
which keeps 30 days, is the place to look further back.
The delivery series are read from the `pull`, `release` and `run` tables on
every scrape, so the scrape cost grows with those tables.
every scrape, so the scrape cost grows with those tables. The check histogram
reads only the checks finished since the last scrape, through the
`run_kind_finished` index; the queue and crew series are small grouped reads.
A public repository's `repo` label is `owner/name`; a private one appears
under its id only, never its name.
The duration histogram counts every request the process has answered since
it started; its `_bucket`, `_count` and `_sum` only grow, so `rate()` and
`histogram_quantile()` read it as Prometheus expects. A restart starts it at
zero again, which Prometheus reads as a counter reset. A request counts the
moment it is answered. The hourly `route_timing` table, which keeps 30 days,
is the place to look further back.
## Prometheus
Make an admin `ro` token in your account settings, then:
@@ -1,14 +1,16 @@
/**
 * The forge's own families for `GET /metrics` (#3975, #3752): request duration
 * as a histogram per route pattern, and the apps, up or not, with each cell's
 * CPU and memory as the watch last read them. The duration is read from the
 * totals `route_timing` keeps in memory since the process started, not from
 * the hourly table, so every value only grows (#4013): Prometheus reads a drop
 * as a counter reset, and a restart resets them, which it expects. Nothing is
 * started or shelled out to read a figure. The delivery series (#4057) are
 * read from the tables that hold the facts: `pull`, `release` (successful rows
 * only) and the default branch's `run` rows. The text comes from
 * `openmetrics.ts`.
 * The forge's own families for `GET /metrics` (#3975, #3752, #3751): request
 * duration as a histogram per route pattern, the apps, up or not, with each
 * cell's CPU and memory as the watch last read them, and the checks and the
 * crew. The duration is read from the totals `route_timing` keeps in memory
 * since the process started, not from the hourly table, so every value only
 * grows (#4013): Prometheus reads a drop as a counter reset, and a restart
 * resets them, which it expects. The check histogram folds newly finished
 * checks into in-memory totals; the queue and the crew's runs and spend are
 * small queries, as are the delivery series (#4057), read from `pull`, `release`
 * (successful rows only) and the default branch's `run` rows. Nothing is started
 * or shelled out to read a figure. The text
 * comes from `openmetrics.ts`.
 */
import type { Database } from "bun:sqlite";
import type { CellCounter } from "../apps/contain.ts";
@@ -39,6 +41,108 @@
export function durationFamily(db: Database): Family {
  };
}
      
        junipercrew
        24d ago
        outdated
      
      Sliding last-hour window under a histogram type: values drop as checks age out, which Prometheus reads as a reset. Make it cumulative since process start like the request histogram.
    
/** Upper bounds, in seconds, of the check duration histogram. */
export const CHECK_EDGES_S: readonly number[] = [5, 15, 30, 60, 120, 300, 600, 1800];
/** What the check histogram has counted so far, per database: only ever added to. */
interface CheckTotals {
  mark: string;
  seen: Set<string>;
  counts: number[];
  count: number;
  sum: number;
}
const STARTED = new Date().toISOString();
const checkTotals = new WeakMap<Database, CheckTotals>();
/**
 * Check duration since the process started, started to finished, cumulative.
 * Each scrape folds in the checks finished since the last one, once each
 * (`mark` is the newest `finished_at` folded, `seen` its run ids), so nothing
 * ages out and the series only grows (#4013).
 */
export function checkDurationFamily(db: Database): Family {
  let t = checkTotals.get(db);
  if (!t) checkTotals.set(db, (t = { mark: STARTED, seen: new Set(), counts: CHECK_EDGES_S.map(() => 0), count: 0, sum: 0 }));
  const rows = db
    .query<{ id: string; at: string; s: number }, [string]>(
      
        junipercrew
        24d ago
        outdated
      
      Late-arriving checks with finished_at below mark are skipped forever. Track a monotonic id or rowid instead.
    
      `SELECT id, finished_at AS at, (julianday(finished_at) - julianday(started_at)) * 86400.0 AS s
         FROM run
        WHERE kind = 'check' AND state IN ('ok','failed')
      
        junipercrew
        24d ago
        
      
      This scans run on every scrape with no index on finished_at, so the cost grows with the table. The docs mention the scrape cost for the delivery series but not for this one.
    
      
        junipercrew
        24d ago
        
      
      A check whose finished_at is older than the newest folded mark (committed late) is silently skipped. Rare, but say so in the comment or re-read a few seconds behind the mark.
    
          AND started_at IS NOT NULL AND finished_at >= ?1
        ORDER BY finished_at, id`,
    )
    .all(t.mark);
  for (const r of rows) {
    if (r.at > t.mark) {
      t.mark = r.at;
      t.seen.clear();
    }
    if (t.seen.has(r.id)) continue;
    t.seen.add(r.id);
    const secs = Math.max(0, r.s);
    t.sum += secs;
    t.count++;
    CHECK_EDGES_S.forEach((le, i) => {
      if (secs <= le) t!.counts[i]!++;
    });
  }
  return {
    type: "histogram",
    name: "forge_check_duration_seconds",
    help: "Seconds a repository check took from start to finish, since the process started.",
    samples: [{ buckets: CHECK_EDGES_S.map((le, i) => ({ le, count: t!.counts[i]! })), count: t.count, sum: t.sum }],
  };
}
/** Checks waiting to start: the `queued` runs of kind check. */
export function checkQueueFamily(db: Database): Family {
  const row = db
    .query<{ n: number }, []>(`SELECT COUNT(*) AS n FROM run WHERE kind = 'check' AND state = 'queued'`)
    .get();
  return {
    type: "gauge",
    name: "forge_check_queue_depth",
    help: "Checks queued and not yet started.",
    samples: [{ value: row?.n ?? 0 }],
  };
}
/** Finished crew runs by member (the agent's username) and outcome. */
export function crewRunsFamily(db: Database): Family {
  const rows = db
    .query<{ member: string; outcome: string; n: number }, []>(
      `SELECT u.username AS member, r.state AS outcome, COUNT(*) AS n
         FROM run r JOIN user u ON u.id = r.actor_id
        WHERE r.kind = 'agent' AND r.state IN ('ok','failed','cancelled')
        GROUP BY u.username, r.state ORDER BY u.username, r.state`,
    )
    .all();
  return {
    type: "counter",
    name: "forge_crew_runs_total",
    help: "Crew runs that ended, by member and outcome.",
    samples: rows.map((r) => ({ labels: { member: r.member, outcome: r.outcome }, value: r.n })),
  };
}
/** What each crew member's runs have cost, in US dollars, from the usage ledger. */
export function crewSpendFamily(db: Database): Family {
  const rows = db
    .query<{ member: string; usd: number }, []>(
      `SELECT u.username AS member, SUM(g.cost_usd) AS usd
         FROM usage g JOIN user u ON u.id = g.agent_id
        GROUP BY u.username ORDER BY u.username`,
    )
    .all();
  return {
    type: "counter",
    name: "forge_crew_spend_usd_total",
    help: "US dollars the crew's runs spent, by member, from the usage ledger.",
    samples: rows.map((r) => ({ labels: { member: r.member }, value: r.usd })),
  };
}
export const PULLS_FAMILY = "forge_pulls_merged_total";
export const RELEASES_FAMILY = "forge_releases_total";
export const ASK_FAMILY = "forge_ask_to_live_seconds";
@@ -230,6 +334,10 @@
export function forgeMetrics(
): string {
  return openMetrics([
    durationFamily(db),
    checkDurationFamily(db),
    checkQueueFamily(db),
    crewRunsFamily(db),
    crewSpendFamily(db),
    ...appFamilies(sources),
    pullsMergedFamily(db),
    releasesFamily(db),
@@ -0,0 +1,117 @@
/** The check and crew families of `/metrics` move with the tables they read (#3751). */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { mkdtemp, readFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createRepo } from "../src/actions/repos.ts";
import { createApp, type App } from "../src/app.ts";
import { createRun } from "../src/core/runs.ts";
import { recordUsage } from "../src/core/usage.ts";
import { createUser } from "../src/core/users.ts";
import { forgeMetrics } from "../src/metrics/forge.ts";
let dir: string;
let app: App;
let repoId: string;
let scoutId: string;
beforeAll(async () => {
  dir = await mkdtemp(join(tmpdir(), "metrics-crew-"));
  app = createApp({ dataDir: dir, port: 0 });
  const mk = async (username: string, adminIfFirst: boolean) => {
    const u = await createUser(app.db, {
      origin: "seed",
      trusted: true,
      username,
      email: `${username}@forge.local`,
      password: "a-good-passphrase",
      adminIfFirst,
    });
    if (!u.ok) throw new Error(u.error);
    return u.value;
  };
  const alice = await mk("alice", true);
  scoutId = (await mk("scout", false)).id;
  const made = await createRepo(app, alice, { name: "one", visibility: "public" });
  if (!made.ok) throw new Error("repo");
  repoId = made.value.id;
}, 60_000);
afterAll(async () => {
  app.close();
  await rm(dir, { recursive: true, force: true });
});
const line = (text: string, re: string) => new RegExp(`^${re} (\\S+)$`, "m").exec(text)?.[1];
test("a finished check moves the duration histogram", () => {
  const now = Date.now();
  expect(line(forgeMetrics(app.db), "forge_check_duration_seconds_count")).toBe("0");
  const run = createRun(app.db, { repoId, kind: "check" });
  app.db
    .query(`UPDATE run SET state = 'ok', started_at = ?2, finished_at = ?3 WHERE id = ?1`)
    .run(run.id, new Date(now - 29_999).toISOString(), new Date(now + 1).toISOString());
  const text = forgeMetrics(app.db);
  expect(line(text, "forge_check_duration_seconds_count")).toBe("1");
  expect(line(text, 'forge_check_duration_seconds_bucket\\{le="15"\\}')).toBe("0");
  expect(line(text, 'forge_check_duration_seconds_bucket\\{le="60"\\}')).toBe("1");
  expect(Number(line(text, "forge_check_duration_seconds_sum"))).toBeCloseTo(30, 3);
});
test("the check histogram never drops: a check is counted once and stays counted", () => {
  const before = forgeMetrics(app.db);
  expect(forgeMetrics(app.db)).toBe(before);
  const run = createRun(app.db, { repoId, kind: "check" });
  const end = Date.now() + 5_000;
  app.db
    .query(`UPDATE run SET state = 'failed', started_at = ?2, finished_at = ?3 WHERE id = ?1`)
    .run(run.id, new Date(end - 4_000).toISOString(), new Date(end).toISOString());
  const n = Number(line(before, "forge_check_duration_seconds_count"));
  expect(line(forgeMetrics(app.db), "forge_check_duration_seconds_count")).toBe(String(n + 1));
  expect(line(forgeMetrics(app.db), "forge_check_duration_seconds_count")).toBe(String(n + 1));
  app.db.query(`DELETE FROM run WHERE id = ?1`).run(run.id);
  expect(line(forgeMetrics(app.db), "forge_check_duration_seconds_count")).toBe(String(n + 1));
});
test("a queued check moves the queue gauge, an agent run does not", () => {
  expect(line(forgeMetrics(app.db), "forge_check_queue_depth")).toBe("0");
  createRun(app.db, { repoId, kind: "agent", actorId: scoutId });
  expect(line(forgeMetrics(app.db), "forge_check_queue_depth")).toBe("0");
  createRun(app.db, { repoId, kind: "check" });
  expect(line(forgeMetrics(app.db), "forge_check_queue_depth")).toBe("1");
});
test("a recorded run moves runs and spend, by member", () => {
  const before = forgeMetrics(app.db);
  expect(before).not.toContain("forge_crew_runs_total{");
  const run = createRun(app.db, { repoId, kind: "agent", actorId: scoutId });
  app.db.query(`UPDATE run SET state = 'failed' WHERE id = ?1`).run(run.id);
  recordUsage(app.db, {
    runId: run.id,
    repoId,
    issueId: null,
    agentId: scoutId,
    subscriptionId: null,
    tokensIn: 10,
    tokensOut: 5,
    costUsd: 0.42,
    wallMs: 1000,
    outcome: "failed",
    at: new Date().toISOString(),
  });
  const text = forgeMetrics(app.db);
  expect(line(text, 'forge_crew_runs_total\\{member="scout",outcome="failed"\\}')).toBe("1");
  expect(line(text, 'forge_crew_spend_usd_total\\{member="scout"\\}')).toBe("0.42");
});
test("docs/METRICS.md lists the four families", async () => {
  const doc = await readFile(join(import.meta.dir, "../docs/METRICS.md"), "utf8");
  for (const name of [
    "forge_check_duration_seconds",
    "forge_check_queue_depth",
    "forge_crew_runs_total",
    "forge_crew_spend_usd_total",
  ])
    expect(doc).toContain(`\`${name}\``);
});
@@ -70,7 +70,8 @@
test("a request without credentials is answered 404, as docs/METRICS.md says", a
test("every family the route serves is listed in docs/METRICS.md", async () => {
  const text = await read();
  const doc = await readFile(join(import.meta.dir, "..", "docs", "METRICS.md"), "utf8");
  const families = [...text.matchAll(/^# TYPE (\S+) /gm)].map((m) => m[1]!);
  // A counter's family is named without `_total`; the doc lists the name it is scraped by.
  const families = [...text.matchAll(/^# TYPE (\S+) (\S+)/gm)].map((m) => (m[2] === "counter" ? `${m[1]}_total` : m[1]!));
  expect(families).toContain("forge_egress_connections");
  expect(families).toContain("forge_http_request_duration_seconds");
  // A counter's TYPE line names the family without `_total`; the doc names the sample.
		@@ -12,6 +12,10 @@ not an admin, 200 for an admin's `ro` token. It is never cached
12	12	\| `forge_egress_bytes_in` \| gauge \| `repo`, `destination` \| Bytes received from the destination over open connections. \|
13	13	\| `forge_egress_bytes_out` \| gauge \| `repo`, `destination` \| Bytes sent to the destination over open connections. \|
14	14	\| `forge_http_request_duration_seconds` \| histogram \| `pattern` (plus `le`) \| Seconds taken to answer a request, by the route pattern matched (never the raw path), since the process started. Buckets are `route_timing`'s edges, 1 ms to 2.5 s, in seconds. \|
	15	\| `forge_check_duration_seconds` \| histogram \| (plus `le`) \| Seconds a repository check took from start to finish, for checks that ended since the process started. Buckets 5 s to 30 min. \|
	16	\| `forge_check_queue_depth` \| gauge \| none \| Checks queued and not yet started. \|
	17	\| `forge_crew_runs_total` \| counter \| `member`, `outcome` \| Crew runs that ended, by the member's username and `ok`, `failed` or `cancelled`. \|
	18	\| `forge_crew_spend_usd_total` \| counter \| `member` \| US dollars the member's runs spent, from the usage ledger (list-price equivalent on a subscription). \|
15	19	\| `forge_pulls_merged_total` \| counter \| `repo` \| Pulls merged, from `pull.merged_at`. \|
16	20	\| `forge_releases_total` \| counter \| none \| Releases that went live, from the `release` table's `ok` rows. \|
17	21	\| `forge_ask_to_live_seconds` \| histogram \| none (plus `le`) \| Seconds from an issue opening to the first successful release of a pull that closes it. Buckets: 1 m, 5 m, 15 m, 1 h, 4 h, 12 h, 1 d, 3 d, 7 d. \|
		@@ -25,19 +29,22 @@ never in the body. `env` is `production` or `pull-<n>`. The cell families are
25	29	what the cell watch last read (every few seconds); a cell that is not running
26	30	has no series (a cell keeps the slug it started under, so after a rename its figures return when it restarts), and the forge starts nothing and reads no process to answer.
27	31
	32	The duration histograms count every request the process has answered, and
	33	every check it has seen finish, since it started; their `_bucket`, `_count`
	34	and `_sum` only grow, so `rate()` and `histogram_quantile()` read them as
	35	Prometheus expects. A restart starts them at zero again, which Prometheus
	36	reads as a counter reset. A request counts the moment it is answered; a check
	37	counts at the first scrape after it finishes. The hourly `route_timing` table,
	38	which keeps 30 days, is the place to look further back.
	39
28	40	The delivery series are read from the `pull`, `release` and `run` tables on
29		every scrape, so the scrape cost grows with those tables.
	41	every scrape, so the scrape cost grows with those tables. The check histogram
	42	reads only the checks finished since the last scrape, through the
	43	`run_kind_finished` index; the queue and crew series are small grouped reads.
30	44
31	45	A public repository's `repo` label is `owner/name`; a private one appears
32	46	under its id only, never its name.
33	47
34		The duration histogram counts every request the process has answered since
35		it started; its `_bucket`, `_count` and `_sum` only grow, so `rate()` and
36		`histogram_quantile()` read it as Prometheus expects. A restart starts it at
37		zero again, which Prometheus reads as a counter reset. A request counts the
38		moment it is answered. The hourly `route_timing` table, which keeps 30 days,
39		is the place to look further back.
40
41	48	## Prometheus
42	49
43	50	Make an admin `ro` token in your account settings, then:
		@@ -1,14 +1,16 @@
1	1	/**
2		* The forge's own families for `GET /metrics` (#3975, #3752): request duration
3		* as a histogram per route pattern, and the apps, up or not, with each cell's
4		* CPU and memory as the watch last read them. The duration is read from the
5		* totals `route_timing` keeps in memory since the process started, not from
6		* the hourly table, so every value only grows (#4013): Prometheus reads a drop
7		* as a counter reset, and a restart resets them, which it expects. Nothing is
8		* started or shelled out to read a figure. The delivery series (#4057) are
9		* read from the tables that hold the facts: `pull`, `release` (successful rows
10		* only) and the default branch's `run` rows. The text comes from
11		* `openmetrics.ts`.
	2	* The forge's own families for `GET /metrics` (#3975, #3752, #3751): request
	3	* duration as a histogram per route pattern, the apps, up or not, with each
	4	* cell's CPU and memory as the watch last read them, and the checks and the
	5	* crew. The duration is read from the totals `route_timing` keeps in memory
	6	* since the process started, not from the hourly table, so every value only
	7	* grows (#4013): Prometheus reads a drop as a counter reset, and a restart
	8	* resets them, which it expects. The check histogram folds newly finished
	9	* checks into in-memory totals; the queue and the crew's runs and spend are
	10	* small queries, as are the delivery series (#4057), read from `pull`, `release`
	11	* (successful rows only) and the default branch's `run` rows. Nothing is started
	12	* or shelled out to read a figure. The text
	13	* comes from `openmetrics.ts`.
12	14	*/
13	15	import type { Database } from "bun:sqlite";
14	16	import type { CellCounter } from "../apps/contain.ts";
		@@ -39,6 +41,108 @@ export function durationFamily(db: Database): Family {
39	41	};
40	42	}
41	43
		junipercrew 24d ago outdated Sliding last-hour window under a histogram type: values drop as checks age out, which Prometheus reads as a reset. Make it cumulative since process start like the request histogram.
	44	/** Upper bounds, in seconds, of the check duration histogram. */
	45	export const CHECK_EDGES_S: readonly number[] = [5, 15, 30, 60, 120, 300, 600, 1800];
	46
	47	/** What the check histogram has counted so far, per database: only ever added to. */
	48	interface CheckTotals {
	49	mark: string;
	50	seen: Set<string>;
	51	counts: number[];
	52	count: number;
	53	sum: number;
	54	}
	55	const STARTED = new Date().toISOString();
	56	const checkTotals = new WeakMap<Database, CheckTotals>();
	57
	58	/**
	59	* Check duration since the process started, started to finished, cumulative.
	60	* Each scrape folds in the checks finished since the last one, once each
	61	* (`mark` is the newest `finished_at` folded, `seen` its run ids), so nothing
	62	* ages out and the series only grows (#4013).
	63	*/
	64	export function checkDurationFamily(db: Database): Family {
	65	let t = checkTotals.get(db);
	66	if (!t) checkTotals.set(db, (t = { mark: STARTED, seen: new Set(), counts: CHECK_EDGES_S.map(() => 0), count: 0, sum: 0 }));
	67	const rows = db
	68	.query<{ id: string; at: string; s: number }, [string]>(
		junipercrew 24d ago outdated Late-arriving checks with finished_at below mark are skipped forever. Track a monotonic id or rowid instead.
	69	`SELECT id, finished_at AS at, (julianday(finished_at) - julianday(started_at)) * 86400.0 AS s
	70	FROM run
	71	WHERE kind = 'check' AND state IN ('ok','failed')
		junipercrew 24d ago This scans run on every scrape with no index on finished_at, so the cost grows with the table. The docs mention the scrape cost for the delivery series but not for this one. junipercrew 24d ago A check whose finished_at is older than the newest folded mark (committed late) is silently skipped. Rare, but say so in the comment or re-read a few seconds behind the mark.
	72	AND started_at IS NOT NULL AND finished_at >= ?1
	73	ORDER BY finished_at, id`,
	74	)
	75	.all(t.mark);
	76	for (const r of rows) {
	77	if (r.at > t.mark) {
	78	t.mark = r.at;
	79	t.seen.clear();
	80	}
	81	if (t.seen.has(r.id)) continue;
	82	t.seen.add(r.id);
	83	const secs = Math.max(0, r.s);
	84	t.sum += secs;
	85	t.count++;
	86	CHECK_EDGES_S.forEach((le, i) => {
	87	if (secs <= le) t!.counts[i]!++;
	88	});
	89	}
	90	return {
	91	type: "histogram",
	92	name: "forge_check_duration_seconds",
	93	help: "Seconds a repository check took from start to finish, since the process started.",
	94	samples: [{ buckets: CHECK_EDGES_S.map((le, i) => ({ le, count: t!.counts[i]! })), count: t.count, sum: t.sum }],
	95	};
	96	}
	97
	98	/** Checks waiting to start: the `queued` runs of kind check. */
	99	export function checkQueueFamily(db: Database): Family {
	100	const row = db
	101	.query<{ n: number }, []>(`SELECT COUNT(*) AS n FROM run WHERE kind = 'check' AND state = 'queued'`)
	102	.get();
	103	return {
	104	type: "gauge",
	105	name: "forge_check_queue_depth",
	106	help: "Checks queued and not yet started.",
	107	samples: [{ value: row?.n ?? 0 }],
	108	};
	109	}
	110
	111	/** Finished crew runs by member (the agent's username) and outcome. */
	112	export function crewRunsFamily(db: Database): Family {
	113	const rows = db
	114	.query<{ member: string; outcome: string; n: number }, []>(
	115	`SELECT u.username AS member, r.state AS outcome, COUNT(*) AS n
	116	FROM run r JOIN user u ON u.id = r.actor_id
	117	WHERE r.kind = 'agent' AND r.state IN ('ok','failed','cancelled')
	118	GROUP BY u.username, r.state ORDER BY u.username, r.state`,
	119	)
	120	.all();
	121	return {
	122	type: "counter",
	123	name: "forge_crew_runs_total",
	124	help: "Crew runs that ended, by member and outcome.",
	125	samples: rows.map((r) => ({ labels: { member: r.member, outcome: r.outcome }, value: r.n })),
	126	};
	127	}
	128
	129	/** What each crew member's runs have cost, in US dollars, from the usage ledger. */
	130	export function crewSpendFamily(db: Database): Family {
	131	const rows = db
	132	.query<{ member: string; usd: number }, []>(
	133	`SELECT u.username AS member, SUM(g.cost_usd) AS usd
	134	FROM usage g JOIN user u ON u.id = g.agent_id
	135	GROUP BY u.username ORDER BY u.username`,
	136	)
	137	.all();
	138	return {
	139	type: "counter",
	140	name: "forge_crew_spend_usd_total",
	141	help: "US dollars the crew's runs spent, by member, from the usage ledger.",
	142	samples: rows.map((r) => ({ labels: { member: r.member }, value: r.usd })),
	143	};
	144	}
	145
42	146	export const PULLS_FAMILY = "forge_pulls_merged_total";
43	147	export const RELEASES_FAMILY = "forge_releases_total";
44	148	export const ASK_FAMILY = "forge_ask_to_live_seconds";
		@@ -230,6 +334,10 @@ export function forgeMetrics(
230	334	): string {
231	335	return openMetrics([
232	336	durationFamily(db),
	337	checkDurationFamily(db),
	338	checkQueueFamily(db),
	339	crewRunsFamily(db),
	340	crewSpendFamily(db),
233	341	...appFamilies(sources),
234	342	pullsMergedFamily(db),
235	343	releasesFamily(db),
		@@ -0,0 +1,117 @@
	1	/** The check and crew families of `/metrics` move with the tables they read (#3751). */
	2
	3	import { afterAll, beforeAll, expect, test } from "bun:test";
	4	import { mkdtemp, readFile, rm } from "node:fs/promises";
	5	import { tmpdir } from "node:os";
	6	import { join } from "node:path";
	7	import { createRepo } from "../src/actions/repos.ts";
	8	import { createApp, type App } from "../src/app.ts";
	9	import { createRun } from "../src/core/runs.ts";
	10	import { recordUsage } from "../src/core/usage.ts";
	11	import { createUser } from "../src/core/users.ts";
	12	import { forgeMetrics } from "../src/metrics/forge.ts";
	13
	14	let dir: string;
	15	let app: App;
	16	let repoId: string;
	17	let scoutId: string;
	18
	19	beforeAll(async () => {
	20	dir = await mkdtemp(join(tmpdir(), "metrics-crew-"));
	21	app = createApp({ dataDir: dir, port: 0 });
	22	const mk = async (username: string, adminIfFirst: boolean) => {
	23	const u = await createUser(app.db, {
	24	origin: "seed",
	25	trusted: true,
	26	username,
	27	email: `${username}@forge.local`,
	28	password: "a-good-passphrase",
	29	adminIfFirst,
	30	});
	31	if (!u.ok) throw new Error(u.error);
	32	return u.value;
	33	};
	34	const alice = await mk("alice", true);
	35	scoutId = (await mk("scout", false)).id;
	36	const made = await createRepo(app, alice, { name: "one", visibility: "public" });
	37	if (!made.ok) throw new Error("repo");
	38	repoId = made.value.id;
	39	}, 60_000);
	40
	41	afterAll(async () => {
	42	app.close();
	43	await rm(dir, { recursive: true, force: true });
	44	});
	45
	46	const line = (text: string, re: string) => new RegExp(`^${re} (\\S+)$`, "m").exec(text)?.[1];
	47
	48	test("a finished check moves the duration histogram", () => {
	49	const now = Date.now();
	50	expect(line(forgeMetrics(app.db), "forge_check_duration_seconds_count")).toBe("0");
	51	const run = createRun(app.db, { repoId, kind: "check" });
	52	app.db
	53	.query(`UPDATE run SET state = 'ok', started_at = ?2, finished_at = ?3 WHERE id = ?1`)
	54	.run(run.id, new Date(now - 29_999).toISOString(), new Date(now + 1).toISOString());
	55	const text = forgeMetrics(app.db);
	56	expect(line(text, "forge_check_duration_seconds_count")).toBe("1");
	57	expect(line(text, 'forge_check_duration_seconds_bucket\\{le="15"\\}')).toBe("0");
	58	expect(line(text, 'forge_check_duration_seconds_bucket\\{le="60"\\}')).toBe("1");
	59	expect(Number(line(text, "forge_check_duration_seconds_sum"))).toBeCloseTo(30, 3);
	60	});
	61
	62	test("the check histogram never drops: a check is counted once and stays counted", () => {
	63	const before = forgeMetrics(app.db);
	64	expect(forgeMetrics(app.db)).toBe(before);
	65	const run = createRun(app.db, { repoId, kind: "check" });
	66	const end = Date.now() + 5_000;
	67	app.db
	68	.query(`UPDATE run SET state = 'failed', started_at = ?2, finished_at = ?3 WHERE id = ?1`)
	69	.run(run.id, new Date(end - 4_000).toISOString(), new Date(end).toISOString());
	70	const n = Number(line(before, "forge_check_duration_seconds_count"));
	71	expect(line(forgeMetrics(app.db), "forge_check_duration_seconds_count")).toBe(String(n + 1));
	72	expect(line(forgeMetrics(app.db), "forge_check_duration_seconds_count")).toBe(String(n + 1));
	73	app.db.query(`DELETE FROM run WHERE id = ?1`).run(run.id);
	74	expect(line(forgeMetrics(app.db), "forge_check_duration_seconds_count")).toBe(String(n + 1));
	75	});
	76
	77	test("a queued check moves the queue gauge, an agent run does not", () => {
	78	expect(line(forgeMetrics(app.db), "forge_check_queue_depth")).toBe("0");
	79	createRun(app.db, { repoId, kind: "agent", actorId: scoutId });
	80	expect(line(forgeMetrics(app.db), "forge_check_queue_depth")).toBe("0");
	81	createRun(app.db, { repoId, kind: "check" });
	82	expect(line(forgeMetrics(app.db), "forge_check_queue_depth")).toBe("1");
	83	});
	84
	85	test("a recorded run moves runs and spend, by member", () => {
	86	const before = forgeMetrics(app.db);
	87	expect(before).not.toContain("forge_crew_runs_total{");
	88	const run = createRun(app.db, { repoId, kind: "agent", actorId: scoutId });
	89	app.db.query(`UPDATE run SET state = 'failed' WHERE id = ?1`).run(run.id);
	90	recordUsage(app.db, {
	91	runId: run.id,
	92	repoId,
	93	issueId: null,
	94	agentId: scoutId,
	95	subscriptionId: null,
	96	tokensIn: 10,
	97	tokensOut: 5,
	98	costUsd: 0.42,
	99	wallMs: 1000,
	100	outcome: "failed",
	101	at: new Date().toISOString(),
	102	});
	103	const text = forgeMetrics(app.db);
	104	expect(line(text, 'forge_crew_runs_total\\{member="scout",outcome="failed"\\}')).toBe("1");
	105	expect(line(text, 'forge_crew_spend_usd_total\\{member="scout"\\}')).toBe("0.42");
	106	});
	107
	108	test("docs/METRICS.md lists the four families", async () => {
	109	const doc = await readFile(join(import.meta.dir, "../docs/METRICS.md"), "utf8");
	110	for (const name of [
	111	"forge_check_duration_seconds",
	112	"forge_check_queue_depth",
	113	"forge_crew_runs_total",
	114	"forge_crew_spend_usd_total",
	115	])
	116	expect(doc).toContain(`\`${name}\``);
	117	});
		@@ -70,7 +70,8 @@ test("a request without credentials is answered 404, as docs/METRICS.md says", a
70	70	test("every family the route serves is listed in docs/METRICS.md", async () => {
71	71	const text = await read();
72	72	const doc = await readFile(join(import.meta.dir, "..", "docs", "METRICS.md"), "utf8");
73		const families = [...text.matchAll(/^# TYPE (\S+) /gm)].map((m) => m[1]!);
	73	// A counter's family is named without `_total`; the doc lists the name it is scraped by.
	74	const families = [...text.matchAll(/^# TYPE (\S+) (\S+)/gm)].map((m) => (m[2] === "counter" ? `${m[1]}_total` : m[1]!));
74	75	expect(families).toContain("forge_egress_connections");
75	76	expect(families).toContain("forge_http_request_duration_seconds");
76	77	// A counter's TYPE line names the family without `_total`; the doc names the sample.