Public: anyone can read this; the crew can change it.
The existing GET /metrics also serves forge_http_request_duration_seconds, and docs/METRICS.md lists every family #3985
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/3975-the-existing-get-metrics-also-serves-for → main 1 commit fork 2 threads · 2 unresolved
4 files
changed +153
−1
docs/METRICS.md added +33
| @@ -0,0 +1,33 @@ | ||
| 1 | # Metrics | |
| 2 | ||
| 3 | `GET /metrics` answers Prometheus text (`text/plain; version=0.0.4`), one | |
| 4 | format for the whole body. It names repositories and route patterns, so it is | |
| 5 | for a site admin alone: 401 without credentials, 404 for anyone else, 200 for | |
| 6 | an admin's `ro` token. It is never cached (`cache-control: no-store`). | |
| 7 | ||
| 8 | | Family | Type | Labels | Meaning | | |
| 9 | | --- | --- | --- | --- | | |
| 10 | | `forge_egress_connections` | gauge | `repo`, `destination` | Open egress connections. | | |
| 11 | | `forge_egress_bytes_in` | gauge | `repo`, `destination` | Bytes received from the destination over open connections. | | |
| 12 | | `forge_egress_bytes_out` | gauge | `repo`, `destination` | Bytes sent to the destination over open connections. | | |
| 13 | | `forge_http_request_duration_seconds` | histogram | `pattern` (plus `le`) | Seconds taken to answer a request, by the route pattern matched (never the raw path), over the last hour. Buckets are `route_timing`'s edges, 1 ms to 2.5 s, in seconds. | | |
| 14 | ||
| 15 | The duration histogram is read from the hourly `route_timing` table, so it | |
| 16 | covers the current hour and the one before; requests are written within about | |
| 17 | a second. | |
| 18 | ||
| 19 | ## Prometheus | |
| 20 | ||
| 21 | Make an admin `ro` token in your account settings, then: | |
| 22 | ||
| 23 | yaml | |
| 24 | scrape_configs: | |
| 25 | - job_name: forge | |
| 26 | scheme: https | |
| 27 | metrics_path: /metrics | |
| 28 | authorization: | |
| 29 | type: Bearer | |
| 30 | credentials_file: /etc/prometheus/forge-token | |
| 31 | static_configs: | |
| 32 | - targets: ["forge.example.com"] | |
| 33 |
src/http/api.ts +2 −1
| @@ -433,6 +433,7 @@ import { clientAddress, clientBucket } from "./peer.ts"; | ||
| 433 | 433 | import { manifestFor } from "../actions/here.ts"; |
| 434 | 434 | import { egressSnapshot } from "../apps/egress.ts"; |
| 435 | 435 | import { egressMetrics } from "../apps/egress_metrics.ts"; |
| 436 | import { forgeMetrics } from "../metrics/forge.ts"; | |
| 436 | 437 | import { |
| 437 | 438 | acceptReport, |
| 438 | 439 | dismissReport, |
| @@ -4873,7 +4874,7 @@ export function apiRoutes(router: Router<Handler>): void { | ||
| 4873 | 4874 | router.add("GET", "/metrics", (ctx) => { |
| 4874 | 4875 | const denied = accountAdmin(ctx, "ro"); |
| 4875 | 4876 | if (denied) return denied; |
| 4876 | return new Response(egressMetrics(egressSnapshot()), { | |
| 4877 | return new Response(egressMetrics(egressSnapshot()) + forgeMetrics(ctx.app.db), { | |
| 4877 | 4878 | headers: { |
| 4878 | 4879 | "content-type": "text/plain; version=0.0.4; charset=utf-8", |
| 4879 | 4880 | "cache-control": "no-store", |
src/metrics/forge.ts added +48
| @@ -0,0 +1,48 @@ | ||
| 1 | /** | |
| 2 | * The forge's own families for `GET /metrics` (#3975): request duration as a | |
| 3 | * histogram per route pattern, read from `route_timing` over the last hour. | |
| 4 | * One indexed range query on the primary key; the text comes from | |
| 5 | * `openmetrics.ts`. | |
| 6 | */ | |
| 7 | import type { Database } from "bun:sqlite"; | |
| 8 | import { EDGES_MS, readTiming } from "../core/route_timing.ts"; | |
| 9 | import { openMetrics, type Family, type HistogramSample } from "./openmetrics.ts"; | |
| 10 | ||
| 11 | const HOUR_MS = 3_600_000; | |
| 12 | export const DURATION_FAMILY = "forge_http_request_duration_seconds"; | |
| 13 | ||
| 14 | /** The duration histogram: hour buckets from an hour ago on, cumulative, in seconds. */ | |
| 15 | export function durationFamily(db: Database, now: number = Date.now()): Family { | |
| 16 | const since = new Date(now - HOUR_MS).toISOString().slice(0, 13); | |
| 17 | const per = new Map<string, { counts: number[]; sumUs: number }>(); | |
Medium, follow-up: the issue asked for this window, but a Prometheus histogram's |
||
| 18 | for (const row of readTiming(db, since)) { | |
| 19 | let p = per.get(row.pattern); | |
| 20 | if (p === undefined) { | |
| 21 | p = { counts: new Array<number>(EDGES_MS.length + 1).fill(0), sumUs: 0 }; | |
| 22 | per.set(row.pattern, p); | |
| 23 | } | |
| 24 | p.counts[row.bucket] = (p.counts[row.bucket] ?? 0) + row.n; | |
| 25 | p.sumUs += row.sum_us; | |
| 26 | } | |
| 27 | const samples: HistogramSample[] = []; | |
| 28 | for (const [pattern, p] of [...per].sort(([a], [b]) => (a < b ? -1 : 1))) { | |
| 29 | let running = 0; | |
| 30 | const buckets = EDGES_MS.map((ms, i) => { | |
| 31 | running += p.counts[i] ?? 0; | |
| 32 | return { le: ms / 1000, count: running }; | |
| 33 | }); | |
| 34 | const count = running + (p.counts[EDGES_MS.length] ?? 0); | |
| 35 | samples.push({ labels: { pattern }, buckets, count, sum: p.sumUs / 1e6 }); | |
| 36 | } | |
| 37 | return { | |
| 38 | type: "histogram", | |
| 39 | name: DURATION_FAMILY, | |
| 40 | help: "Seconds the forge took to answer a request, by route pattern, over the last hour.", | |
| 41 | samples, | |
| 42 | }; | |
| 43 | } | |
| 44 | ||
| 45 | /** The forge's families as Prometheus text (no `# EOF`: the body is `text/plain; version=0.0.4`). */ | |
| 46 | export function forgeMetrics(db: Database, now: number = Date.now()): string { | |
| 47 | return openMetrics([durationFamily(db, now)]).replace(/# EOF\n$/, ""); | |
| 48 | } |
test/metrics-route.test.ts added +70
| @@ -0,0 +1,70 @@ | ||
| 1 | /** `GET /metrics` serves the duration histogram, and docs/METRICS.md lists every family it serves (#3975). */ | |
| 2 | ||
| 3 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 4 | import { mkdtemp, readFile, rm } from "node:fs/promises"; | |
| 5 | import { tmpdir } from "node:os"; | |
| 6 | import { join } from "node:path"; | |
| 7 | import { createApp, type App } from "../src/app.ts"; | |
| 8 | import { createRouteTiming } from "../src/core/route_timing.ts"; | |
| 9 | import { createToken } from "../src/core/tokens.ts"; | |
| 10 | import { createUser } from "../src/core/users.ts"; | |
| 11 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 12 | ||
| 13 | let dir: string; | |
| 14 | let app: App; | |
| 15 | let servers: Servers; | |
| 16 | let ro: string; | |
| 17 | ||
| 18 | beforeAll(async () => { | |
| 19 | dir = await mkdtemp(join(tmpdir(), "metrics-route-")); | |
| 20 | app = createApp({ dataDir: dir, port: 0 }); | |
| 21 | servers = await serve(app); | |
| 22 | const made = await createUser(app.db, { | |
| 23 | origin: "seed", | |
| 24 | trusted: true, | |
| 25 | username: "alice", | |
| 26 | email: "[email protected]", | |
| 27 | password: "a-good-passphrase", | |
| 28 | adminIfFirst: true, | |
| 29 | }); | |
| 30 | if (!made.ok) throw new Error(made.error); | |
| 31 | ro = createToken(app.db, made.value.id, "t", "ro").token; | |
| 32 | }, 60_000); | |
| 33 | ||
| 34 | afterAll(async () => { | |
| 35 | servers.stop(); | |
| 36 | app.close(); | |
| 37 | await rm(dir, { recursive: true, force: true }); | |
| 38 | }); | |
| 39 | ||
| 40 | const read = async () => | |
| 41 | (await fetch(`${servers.url}/metrics`, { headers: { authorization: `Bearer ${ro}` } })).text(); | |
| 42 | ||
| 43 | test("a recorded request moves its bucket, count and sum", async () => { | |
| 44 | const pattern = "/zz/metrics-test/:id"; | |
| 45 | const bucket = (text: string, le: string) => | |
| 46 | Number( | |
| 47 | new RegExp(`forge_http_request_duration_seconds_bucket\\{pattern="${pattern.replace(/[/:]/g, (c) => `\\${c}`)}",le="${le.replace(".", "\\.").replace("+", "\\+")}"\\} (\\d+)`).exec(text)?.[1] ?? 0, | |
| 48 | ); | |
| 49 | const before = await read(); | |
| 50 | expect(bucket(before, "0.005")).toBe(0); | |
| 51 | const timing = createRouteTiming(app.db); | |
| 52 | timing.record(pattern, 3); | |
| 53 | timing.flush(); | |
| 54 | const after = await read(); | |
| 55 | expect(bucket(after, "0.002")).toBe(0); | |
| 56 | expect(bucket(after, "0.005")).toBe(1); | |
| 57 | expect(bucket(after, "+Inf")).toBe(1); | |
| 58 | expect(after).toContain(`forge_http_request_duration_seconds_count{pattern="${pattern}"} 1`); | |
| 59 | expect(after).toContain(`forge_http_request_duration_seconds_sum{pattern="${pattern}"} 0.003`); | |
| 60 | expect(after).not.toContain("# EOF"); | |
| 61 | }); | |
| 62 | ||
| 63 | test("every family the route serves is listed in docs/METRICS.md", async () => { | |
| 64 | const text = await read(); | |
| 65 | const doc = await readFile(join(import.meta.dir, "..", "docs", "METRICS.md"), "utf8"); | |
| 66 | const families = [...text.matchAll(/^# TYPE (\S+) /gm)].map((m) => m[1]!); | |
| 67 | expect(families).toContain("forge_egress_connections"); | |
| 68 | expect(families).toContain("forge_http_request_duration_seconds"); | |
| 69 | for (const f of families) expect(doc).toContain(`\`${f}\``); | |
| 70 | }); |
new above
offline
Low: this line is inaccurate. Without credentials the route answers 404, not 401:
test/egress-metrics-route.test.ts:92asserts it, and the preview gives 404 to an anonymous curl. Say "404 for anyone but a site admin" so a person debugging a scrape is not sent looking for a 401.