78f82aa · 86d9299 · bc6bb89 · 44f94c4 · 7b7a9d5
@@ -12,6 +12,14 @@
not an admin, 200 for an admin's `ro` token. It is never cached
| `forge_egress_bytes_in` | gauge | `repo`, `destination` | Bytes received from the destination over open connections. |
| `forge_egress_bytes_out` | gauge | `repo`, `destination` | Bytes sent to the destination over open connections. |
| `forge_http_request_duration_seconds` | histogram | `pattern` (plus `le`) | Seconds taken to answer a request, by the route pattern matched (never the raw path), since the process started. Buckets are `route_timing`'s edges, 1 ms to 2.5 s, in seconds. |
| `forge_app_up` | gauge | `app`, `env` | 1 when the app environment is serving, 0 when it is asleep, stopped, deploying or failed. |
| `forge_cell_cpu_seconds` (samples `forge_cell_cpu_seconds_total`) | counter | `app`, `env` | CPU seconds (user plus system) used by the processes of a running app cell. |
| `forge_cell_memory_bytes` | gauge | `app`, `env` | Resident memory of the processes of a running app cell. |
`app` is the app's slug, or its id for a private app: a private app's name is
never in the body. `env` is `production` or `pull-<n>`. The cell families are
what the cell watch last read (every few seconds); a cell that is not running
has no series (a cell keeps the slug it started under, so after a rename its figures return when it restarts), and the forge starts nothing and reads no process to answer.
The duration histogram counts every request the process has answered since
it started; its `_bucket`, `_count` and `_sum` only grow, so `rate()` and
@@ -435,6 +435,7 @@
import { manifestFor } from "../actions/here.ts";
import { egressSnapshot } from "../apps/egress.ts";
import { egressMetrics } from "../apps/egress_metrics.ts";
import { forgeMetrics } from "../metrics/forge.ts";
import { appSources } from "../metrics/app_sources.ts";
import {
  acceptReport,
  dismissReport,
@@ -4981,7 +4982,7 @@
export function apiRoutes(router: Router<Handler>): void {
  router.add("GET", "/metrics", (ctx) => {
    const denied = accountAdmin(ctx, "ro");
    if (denied) return denied;
    return new Response(egressMetrics(egressSnapshot()) + forgeMetrics(ctx.app.db), {
    return new Response(egressMetrics(egressSnapshot()) + forgeMetrics(ctx.app.db, appSources(ctx.app)), {
      headers: {
        "content-type": "text/plain; version=0.0.4; charset=utf-8",
        "cache-control": "no-store",
@@ -0,0 +1,24 @@
/**
 * What the app families of `GET /metrics` read from a running forge: every
 * app's environments as the runtime already holds them, and the cells the
 * watch has already read. It starts nothing and reads no process.
 */
import type { App } from "../app.ts";
import { cellCounters } from "../apps/contain.ts";
import { appsRuntime } from "../apps/runtime.ts";
import { listApps } from "../core/apps.ts";
import type { AppSources } from "./forge.ts";
export function appSources(app: App): AppSources {
  const runtime = appsRuntime(app);
  if (runtime === undefined) return { apps: [], cells: [] };
  return {
    apps: listApps(app.db).map((a) => ({
      id: a.id,
      slug: a.slug,
      private: a.access === "private",
      
        sentinelcrew
        24d ago
        
      
      This line is the only thing that keeps a private app's slug out of /metrics, and no test runs it. Every test passes private: true directly to appFamilies. Add a test that builds appSources from a db with a private app and checks that the slug is absent from the body.
    
      envs: runtime.status(a).map((e) => ({ env: e.env, state: e.state })),
    })),
    cells: cellCounters(),
  };
}
@@ -1,15 +1,17 @@
/**
 * The forge's own families for `GET /metrics` (#3975): request duration as a
 * histogram per route pattern. It is read from the totals `route_timing`
 * keeps in memory since the process started, not from the hourly table, so
 * every value only grows (#4013): Prometheus reads a drop as a counter reset,
 * and a sliding window would make `rate()` and `histogram_quantile()` jump
 * every hour. A restart resets them, which Prometheus expects. No query runs;
 * the text comes from `openmetrics.ts`.
 * The forge's own families for `GET /metrics` (#3975, #3752): request duration
 * as a histogram per route pattern, and the apps, up or not, with each cell's
 * CPU and memory as the watch last read them. The duration is read from the
 * totals `route_timing` keeps in memory since the process started, not from
 * the hourly table, so every value only grows (#4013): Prometheus reads a drop
 * as a counter reset, and a restart resets them, which it expects. Nothing is
 * started or shelled out to read a figure. The text comes from `openmetrics.ts`.
 */
import type { Database } from "bun:sqlite";
import type { CellCounter } from "../apps/contain.ts";
import type { EnvState } from "../apps/runtime.ts";
import { EDGES_MS, readTotals } from "../core/route_timing.ts";
import { openMetrics, type Family, type HistogramSample } from "./openmetrics.ts";
import { openMetrics, type Family, type HistogramSample, type Sample } from "./openmetrics.ts";
export const DURATION_FAMILY = "forge_http_request_duration_seconds";
@@ -33,7 +35,67 @@
export function durationFamily(db: Database): Family {
  };
}
/** The forge's families as Prometheus text (no `# EOF`: the body is `text/plain; version=0.0.4`). */
export function forgeMetrics(db: Database): string {
  return openMetrics([durationFamily(db)]).replace(/# EOF\n$/, "");
export const UP_FAMILY = "forge_app_up";
export const CELL_CPU_FAMILY = "forge_cell_cpu_seconds_total";
export const CELL_MEMORY_FAMILY = "forge_cell_memory_bytes";
/** One app as the runtime holds it: who it is, whether it is private, and each environment's state. */
export interface AppReading {
  readonly id: string;
  readonly slug: string;
  readonly private: boolean;
  readonly envs: readonly { readonly env: string; readonly state: EnvState }[];
}
/** What the app families read: the runtime's recorded states and the watch's counters. */
export interface AppSources {
  readonly apps: readonly AppReading[];
  readonly cells: readonly CellCounter[];
}
/** A private app is named by its id alone, never its slug. */
const nameOf = (a: AppReading): string => (a.private ? a.id : a.slug);
/** `forge_app_up` per app and environment, and each running cell's CPU and memory. */
export function appFamilies({ apps, cells }: AppSources): Family[] {
  const sorted = [...apps].sort((a, b) => (nameOf(a) < nameOf(b) ? -1 : 1));
  const up: Sample[] = [];
  const cpu: Sample[] = [];
  const mem: Sample[] = [];
  // The watch labels a cell `app:<slug>/<env>`, and during a deploy the old
  // and new cells share a label: their figures add up to the environment's.
  const byCell = new Map<string, { cpuUsec: number; memoryBytes: number }>();
  for (const c of cells) {
    if (!c.label.startsWith("app:")) continue;
    const key = c.label.slice(4);
    const held = byCell.get(key) ?? { cpuUsec: 0, memoryBytes: 0 };
    held.cpuUsec += c.cpuUsec;
    held.memoryBytes += c.memoryBytes;
    byCell.set(key, held);
  }
  for (const a of sorted) {
    const app = nameOf(a);
    for (const e of a.envs)
      up.push({ labels: { app, env: e.env }, value: e.state === "running" ? 1 : 0 });
    for (const e of a.envs) {
      const c = byCell.get(`${a.slug}/${e.env}`);
      
        sentinelcrew
        24d ago
        
      
      Low, non-blocking. The lookup goes through a.slug, but the runtime fixes a cell's label at start (runtime.ts:1416, app:<slug>/<env>). So a cell that outlives a rename is credited to whichever app now holds the old slug. If that app is public, a private app's CPU and memory go out under the public app's name. Only admins can read this endpoint, so nothing reaches a non-admin, but the figures are attributed to the wrong app. Fix in a follow-up: put the app id in the cell label (app:<id>/<env>) and key byCell on a.id.
    
      if (c === undefined) continue;
      const labels = { app, env: e.env };
      
        sentinelcrew
        24d ago
        outdated
      
      Matching on the slug the cell started under means a cell from a renamed app is credited to whichever app holds that slug now, which can be a different and public app. At the least, this must not create a duplicate series (see the comment below). Ideally, carry the app id in the cell's label.
    
      cpu.push({ labels, value: c.cpuUsec / 1e6 });
      mem.push({ labels, value: c.memoryBytes });
      
        sentinelcrew
        24d ago
        outdated
      
      High: during a deploy, the old cell and the new cell are both watched under app:<slug>/<env>. That gives two samples with the same {app, env}, openMetrics throws duplicate series, and GET /metrics returns 500 for the whole scrape. Aggregate the cells per {app, env} (or keep the newest key) before you push samples, and add a test with two counters that share a label.
    
    }
  }
  return [
    { type: "gauge", name: UP_FAMILY, help: "1 when the app environment is serving, 0 when it is asleep, stopped, deploying or failed.", samples: up },
    { type: "counter", name: CELL_CPU_FAMILY, help: "CPU seconds, user plus system, used by the processes of an app cell since it started.", samples: cpu },
    { type: "gauge", name: CELL_MEMORY_FAMILY, help: "Resident memory in bytes of the processes of an app cell.", samples: mem },
  ];
      
        sentinelcrew
        24d ago
        outdated
      
      Non-blocking: this joins on the slug the cell's label captured when it started, not on the app id. After a rename, a still-running cell's figures attach to whichever app now holds the old slug. That can be a private app's figures under a public name, though only admins can scrape this. The durable fix is an id in the watch label in runtime.ts. Please file it as an issue rather than growing this pull.
    
}
/** The forge's families as Prometheus text (no `# EOF`: the body is `text/plain; version=0.0.4`). */
export function forgeMetrics(
  db: Database,
  sources: AppSources = { apps: [], cells: [] },
): string {
  return openMetrics([durationFamily(db), ...appFamilies(sources)]).replace(/# EOF\n$/, "");
}
@@ -0,0 +1,97 @@
/** `GET /metrics` from a real forge: a seeded running app is up, a sleeping one is 0, a private app is named by id only (#3752). */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createRepo } from "../src/actions/repos.ts";
import { createApp, type App } from "../src/app.ts";
import { AppsRuntime } from "../src/apps/runtime.ts";
import { createApp as createAppRecord, type AppRecord } from "../src/core/apps.ts";
import { createToken } from "../src/core/tokens.ts";
import { createUser } from "../src/core/users.ts";
import { serve, type Servers } from "../src/http/server.ts";
import { appSources } from "../src/metrics/app_sources.ts";
import { forgeMetrics } from "../src/metrics/forge.ts";
let dir: string;
let app: App;
let servers: Servers;
let ro: string;
let open: AppRecord;
let secret: AppRecord;
let runtime: AppsRuntime;
const record = (r: { ok: boolean; value?: AppRecord; error?: string }): AppRecord => {
  if (!r.ok) throw new Error(String(r.error));
  return r.value!;
};
beforeAll(async () => {
  dir = await mkdtemp(join(tmpdir(), "metrics-apps-"));
  app = createApp({ dataDir: dir, port: 0 });
  servers = await serve(app);
  const made = await createUser(app.db, {
    origin: "seed",
    trusted: true,
    username: "alice",
    email: "[email protected]",
    password: "a-good-passphrase",
    adminIfFirst: true,
  });
  if (!made.ok) throw new Error(made.error);
  ro = createToken(app.db, made.value.id, "t", "ro").token;
  const repos = [];
  for (const name of ["notes", "plans"]) {
    const r = await createRepo(app, made.value, { name });
    if (!r.ok) throw new Error(String(r.error));
    repos.push(r.value);
  }
  open = record(createAppRecord(app.db, { repoId: repos[0]!.id, slug: "notes", access: "public" }));
  secret = record(createAppRecord(app.db, { repoId: repos[1]!.id, slug: "secret-plans", access: "private" }));
  // An unstarted runtime holding what a started one would: no cell is spawned.
  runtime = new AppsRuntime(app, { domain: "apps.test", scheme: "http", publicPort: servers.port, forgeUrl: servers.url });
  const envs = (runtime as unknown as { envs: Map<string, unknown> }).envs;
  const live = {
    commitOid: "abc",
    handle: { port: 1 },
    runId: "r1",
    since: "2026-01-01T00:00:00Z",
    servingAt: "2026-01-01T00:00:00Z",
    createdAt: "2026-01-01T00:00:00Z",
    manifest: {},
    deployStartedAt: null,
  };
  const base = { openStreams: 0, openRequests: 0, chain: Promise.resolve(), restarts: [] };
  envs.set("a", { ...base, app: open, env: "production", live });
  envs.set("b", { ...base, app: secret, env: "production", live });
  envs.set("c", { ...base, app: open, env: "pull-7", asleep: { commitOid: "abc", runId: "r2", since: "2026-01-01T00:00:00Z" } });
}, 60_000);
afterAll(async () => {
  servers.stop();
  app.close();
  await rm(dir, { recursive: true, force: true });
});
test("appSources marks the private app private", () => {
  const bySlug = new Map(appSources(app).apps.map((a) => [a.slug, a]));
  expect(bySlug.get("notes")?.private).toBe(false);
  expect(bySlug.get("secret-plans")?.private).toBe(true);
});
test("the body names a private app by id and never by slug", () => {
  const body = forgeMetrics(app.db, appSources(app));
  expect(body).toContain(`forge_app_up{app="${secret.id}",env="production"} 1`);
  expect(body).not.toContain("secret-plans");
});
test("GET /metrics with an admin ro token carries forge_app_up for the seeded apps", async () => {
  const res = await fetch(`${servers.url}/metrics`, { headers: { authorization: `Bearer ${ro}` } });
  const body = await res.text();
  expect(res.status).toBe(200);
  expect(body).toContain('forge_app_up{app="notes",env="production"} 1');
  expect(body).toContain('forge_app_up{app="notes",env="pull-7"} 0');
  expect(body).toContain(`forge_app_up{app="${secret.id}",env="production"} 1`);
  expect(body).not.toContain("secret-plans");
});
@@ -0,0 +1,52 @@
/** `/metrics` serves the app series: `forge_app_up` moves when an app sleeps, a cell's figures appear once recorded, a private app is named by id only (#3752). */
import { expect, test } from "bun:test";
import { Database } from "bun:sqlite";
import type { CellCounter } from "../src/apps/contain.ts";
import { appFamilies, forgeMetrics, type AppReading } from "../src/metrics/forge.ts";
import { openMetrics } from "../src/metrics/openmetrics.ts";
const app = (state: "running" | "asleep"): AppReading[] => [
  { id: "app1", slug: "notes", private: false, envs: [{ env: "production", state }] },
  { id: "app2", slug: "secret-plans", private: true, envs: [{ env: "pull-3", state: "running" }] },
];
const cell = (label: string): CellCounter => ({ key: 1, label, cpuUsec: 2_500_000, memoryBytes: 4096 });
const text = (apps: AppReading[], cells: CellCounter[]) => openMetrics(appFamilies({ apps, cells }));
test("forge_app_up is 1 while running and 0 once the app sleeps", () => {
  expect(text(app("running"), [])).toContain('forge_app_up{app="notes",env="production"} 1');
  expect(text(app("asleep"), [])).toContain('forge_app_up{app="notes",env="production"} 0');
});
test("a cell's figures appear when the watch has read it, and not before", () => {
  expect(text(app("running"), [])).not.toContain("forge_cell_memory_bytes{");
  const out = text(app("running"), [cell("app:notes/production")]);
  expect(out).toContain('forge_cell_cpu_seconds_total{app="notes",env="production"} 2.5');
  expect(out).toContain('forge_cell_memory_bytes{app="notes",env="production"} 4096');
});
test("a private app appears under its id only", () => {
  const out = text(app("running"), [cell("app:secret-plans/pull-3"), cell("check:x")]);
  expect(out).toContain('forge_app_up{app="app2",env="pull-3"} 1');
  expect(out).toContain('forge_cell_memory_bytes{app="app2",env="pull-3"} 4096');
  expect(out).not.toContain("secret-plans");
  expect(out).not.toContain("check:x");
});
test("forgeMetrics carries the families as Prometheus text without a trailing EOF", () => {
  const db = new Database(":memory:");
  db.run("CREATE TABLE route_timing (hour TEXT, pattern TEXT, bucket INTEGER, n INTEGER, sum_us INTEGER)");
  const body = forgeMetrics(db, { apps: app("running"), cells: [] });
  expect(body).toContain("# TYPE forge_app_up gauge");
  expect(body).toContain("# TYPE forge_cell_cpu_seconds counter");
  expect(body).not.toContain("# EOF");
});
test("two cells under one label (a deploy's old and new) are one series, summed, and the scrape does not throw", () => {
  const db = new Database(":memory:");
  db.run("CREATE TABLE route_timing (hour TEXT, pattern TEXT, bucket INTEGER, n INTEGER, sum_us INTEGER)");
  const cells = [cell("app:notes/production"), { ...cell("app:notes/production"), key: 2 }];
  const body = forgeMetrics(db, { apps: app("running"), cells });
  expect(body).toContain('forge_cell_memory_bytes{app="notes",env="production"} 8192');
  expect(body).toContain('forge_cell_cpu_seconds_total{app="notes",env="production"} 5');
});
		@@ -12,6 +12,14 @@ not an admin, 200 for an admin's `ro` token. It is never cached
12	12	\| `forge_egress_bytes_in` \| gauge \| `repo`, `destination` \| Bytes received from the destination over open connections. \|
13	13	\| `forge_egress_bytes_out` \| gauge \| `repo`, `destination` \| Bytes sent to the destination over open connections. \|
14	14	\| `forge_http_request_duration_seconds` \| histogram \| `pattern` (plus `le`) \| Seconds taken to answer a request, by the route pattern matched (never the raw path), since the process started. Buckets are `route_timing`'s edges, 1 ms to 2.5 s, in seconds. \|
	15	\| `forge_app_up` \| gauge \| `app`, `env` \| 1 when the app environment is serving, 0 when it is asleep, stopped, deploying or failed. \|
	16	\| `forge_cell_cpu_seconds` (samples `forge_cell_cpu_seconds_total`) \| counter \| `app`, `env` \| CPU seconds (user plus system) used by the processes of a running app cell. \|
	17	\| `forge_cell_memory_bytes` \| gauge \| `app`, `env` \| Resident memory of the processes of a running app cell. \|
	18
	19	`app` is the app's slug, or its id for a private app: a private app's name is
	20	never in the body. `env` is `production` or `pull-<n>`. The cell families are
	21	what the cell watch last read (every few seconds); a cell that is not running
	22	has no series (a cell keeps the slug it started under, so after a rename its figures return when it restarts), and the forge starts nothing and reads no process to answer.
15	23
16	24	The duration histogram counts every request the process has answered since
17	25	it started; its `_bucket`, `_count` and `_sum` only grow, so `rate()` and
		@@ -435,6 +435,7 @@ import { manifestFor } from "../actions/here.ts";
435	435	import { egressSnapshot } from "../apps/egress.ts";
436	436	import { egressMetrics } from "../apps/egress_metrics.ts";
437	437	import { forgeMetrics } from "../metrics/forge.ts";
	438	import { appSources } from "../metrics/app_sources.ts";
438	439	import {
439	440	acceptReport,
440	441	dismissReport,
		@@ -4981,7 +4982,7 @@ export function apiRoutes(router: Router<Handler>): void {
4981	4982	router.add("GET", "/metrics", (ctx) => {
4982	4983	const denied = accountAdmin(ctx, "ro");
4983	4984	if (denied) return denied;
4984		return new Response(egressMetrics(egressSnapshot()) + forgeMetrics(ctx.app.db), {
	4985	return new Response(egressMetrics(egressSnapshot()) + forgeMetrics(ctx.app.db, appSources(ctx.app)), {
4985	4986	headers: {
4986	4987	"content-type": "text/plain; version=0.0.4; charset=utf-8",
4987	4988	"cache-control": "no-store",
		@@ -0,0 +1,24 @@
	1	/**
	2	* What the app families of `GET /metrics` read from a running forge: every
	3	* app's environments as the runtime already holds them, and the cells the
	4	* watch has already read. It starts nothing and reads no process.
	5	*/
	6	import type { App } from "../app.ts";
	7	import { cellCounters } from "../apps/contain.ts";
	8	import { appsRuntime } from "../apps/runtime.ts";
	9	import { listApps } from "../core/apps.ts";
	10	import type { AppSources } from "./forge.ts";
	11
	12	export function appSources(app: App): AppSources {
	13	const runtime = appsRuntime(app);
	14	if (runtime === undefined) return { apps: [], cells: [] };
	15	return {
	16	apps: listApps(app.db).map((a) => ({
	17	id: a.id,
	18	slug: a.slug,
	19	private: a.access === "private",
		sentinelcrew 24d ago This line is the only thing that keeps a private app's slug out of `/metrics`, and no test runs it. Every test passes `private: true` directly to `appFamilies`. Add a test that builds `appSources` from a db with a private app and checks that the slug is absent from the body.
	20	envs: runtime.status(a).map((e) => ({ env: e.env, state: e.state })),
	21	})),
	22	cells: cellCounters(),
	23	};
	24	}
		@@ -1,15 +1,17 @@
1	1	/**
2		* The forge's own families for `GET /metrics` (#3975): request duration as a
3		* histogram per route pattern. It is read from the totals `route_timing`
4		* keeps in memory since the process started, not from the hourly table, so
5		* every value only grows (#4013): Prometheus reads a drop as a counter reset,
6		* and a sliding window would make `rate()` and `histogram_quantile()` jump
7		* every hour. A restart resets them, which Prometheus expects. No query runs;
8		* the text comes from `openmetrics.ts`.
	2	* The forge's own families for `GET /metrics` (#3975, #3752): request duration
	3	* as a histogram per route pattern, and the apps, up or not, with each cell's
	4	* CPU and memory as the watch last read them. The duration is read from the
	5	* totals `route_timing` keeps in memory since the process started, not from
	6	* the hourly table, so every value only grows (#4013): Prometheus reads a drop
	7	* as a counter reset, and a restart resets them, which it expects. Nothing is
	8	* started or shelled out to read a figure. The text comes from `openmetrics.ts`.
9	9	*/
10	10	import type { Database } from "bun:sqlite";
	11	import type { CellCounter } from "../apps/contain.ts";
	12	import type { EnvState } from "../apps/runtime.ts";
11	13	import { EDGES_MS, readTotals } from "../core/route_timing.ts";
12		import { openMetrics, type Family, type HistogramSample } from "./openmetrics.ts";
	14	import { openMetrics, type Family, type HistogramSample, type Sample } from "./openmetrics.ts";
13	15
14	16	export const DURATION_FAMILY = "forge_http_request_duration_seconds";
15	17
		@@ -33,7 +35,67 @@ export function durationFamily(db: Database): Family {
33	35	};
34	36	}
35	37
36		/** The forge's families as Prometheus text (no `# EOF`: the body is `text/plain; version=0.0.4`). */
37		export function forgeMetrics(db: Database): string {
38		return openMetrics([durationFamily(db)]).replace(/# EOF\n$/, "");
	38	export const UP_FAMILY = "forge_app_up";
	39	export const CELL_CPU_FAMILY = "forge_cell_cpu_seconds_total";
	40	export const CELL_MEMORY_FAMILY = "forge_cell_memory_bytes";
	41
	42	/** One app as the runtime holds it: who it is, whether it is private, and each environment's state. */
	43	export interface AppReading {
	44	readonly id: string;
	45	readonly slug: string;
	46	readonly private: boolean;
	47	readonly envs: readonly { readonly env: string; readonly state: EnvState }[];
	48	}
	49
	50	/** What the app families read: the runtime's recorded states and the watch's counters. */
	51	export interface AppSources {
	52	readonly apps: readonly AppReading[];
	53	readonly cells: readonly CellCounter[];
	54	}
	55
	56	/** A private app is named by its id alone, never its slug. */
	57	const nameOf = (a: AppReading): string => (a.private ? a.id : a.slug);
	58
	59	/** `forge_app_up` per app and environment, and each running cell's CPU and memory. */
	60	export function appFamilies({ apps, cells }: AppSources): Family[] {
	61	const sorted = [...apps].sort((a, b) => (nameOf(a) < nameOf(b) ? -1 : 1));
	62	const up: Sample[] = [];
	63	const cpu: Sample[] = [];
	64	const mem: Sample[] = [];
	65	// The watch labels a cell `app:<slug>/<env>`, and during a deploy the old
	66	// and new cells share a label: their figures add up to the environment's.
	67	const byCell = new Map<string, { cpuUsec: number; memoryBytes: number }>();
	68	for (const c of cells) {
	69	if (!c.label.startsWith("app:")) continue;
	70	const key = c.label.slice(4);
	71	const held = byCell.get(key) ?? { cpuUsec: 0, memoryBytes: 0 };
	72	held.cpuUsec += c.cpuUsec;
	73	held.memoryBytes += c.memoryBytes;
	74	byCell.set(key, held);
	75	}
	76	for (const a of sorted) {
	77	const app = nameOf(a);
	78	for (const e of a.envs)
	79	up.push({ labels: { app, env: e.env }, value: e.state === "running" ? 1 : 0 });
	80	for (const e of a.envs) {
	81	const c = byCell.get(`${a.slug}/${e.env}`);
		sentinelcrew 24d ago Low, non-blocking. The lookup goes through `a.slug`, but the runtime fixes a cell's label at start (`runtime.ts:1416`, `app:<slug>/<env>`). So a cell that outlives a rename is credited to whichever app now holds the old slug. If that app is public, a private app's CPU and memory go out under the public app's name. Only admins can read this endpoint, so nothing reaches a non-admin, but the figures are attributed to the wrong app. Fix in a follow-up: put the app id in the cell label (`app:<id>/<env>`) and key `byCell` on `a.id`.
	82	if (c === undefined) continue;
	83	const labels = { app, env: e.env };
		sentinelcrew 24d ago outdated Matching on the slug the cell started under means a cell from a renamed app is credited to whichever app holds that slug now, which can be a different and public app. At the least, this must not create a duplicate series (see the comment below). Ideally, carry the app id in the cell's label.
	84	cpu.push({ labels, value: c.cpuUsec / 1e6 });
	85	mem.push({ labels, value: c.memoryBytes });
		sentinelcrew 24d ago outdated High: during a deploy, the old cell and the new cell are both watched under `app:<slug>/<env>`. That gives two samples with the same `{app, env}`, `openMetrics` throws `duplicate series`, and `GET /metrics` returns 500 for the whole scrape. Aggregate the cells per `{app, env}` (or keep the newest `key`) before you push samples, and add a test with two counters that share a label.
	86	}
	87	}
	88	return [
	89	{ type: "gauge", name: UP_FAMILY, help: "1 when the app environment is serving, 0 when it is asleep, stopped, deploying or failed.", samples: up },
	90	{ type: "counter", name: CELL_CPU_FAMILY, help: "CPU seconds, user plus system, used by the processes of an app cell since it started.", samples: cpu },
	91	{ type: "gauge", name: CELL_MEMORY_FAMILY, help: "Resident memory in bytes of the processes of an app cell.", samples: mem },
	92	];
		sentinelcrew 24d ago outdated Non-blocking: this joins on the slug the cell's label captured when it started, not on the app id. After a rename, a still-running cell's figures attach to whichever app now holds the old slug. That can be a private app's figures under a public name, though only admins can scrape this. The durable fix is an id in the watch label in `runtime.ts`. Please file it as an issue rather than growing this pull.
	93	}
	94
	95	/** The forge's families as Prometheus text (no `# EOF`: the body is `text/plain; version=0.0.4`). */
	96	export function forgeMetrics(
	97	db: Database,
	98	sources: AppSources = { apps: [], cells: [] },
	99	): string {
	100	return openMetrics([durationFamily(db), ...appFamilies(sources)]).replace(/# EOF\n$/, "");
39	101	}
		@@ -0,0 +1,97 @@
	1	/** `GET /metrics` from a real forge: a seeded running app is up, a sleeping one is 0, a private app is named by id only (#3752). */
	2
	3	import { afterAll, beforeAll, expect, test } from "bun:test";
	4	import { mkdtemp, rm } from "node:fs/promises";
	5	import { tmpdir } from "node:os";
	6	import { join } from "node:path";
	7	import { createRepo } from "../src/actions/repos.ts";
	8	import { createApp, type App } from "../src/app.ts";
	9	import { AppsRuntime } from "../src/apps/runtime.ts";
	10	import { createApp as createAppRecord, type AppRecord } from "../src/core/apps.ts";
	11	import { createToken } from "../src/core/tokens.ts";
	12	import { createUser } from "../src/core/users.ts";
	13	import { serve, type Servers } from "../src/http/server.ts";
	14	import { appSources } from "../src/metrics/app_sources.ts";
	15	import { forgeMetrics } from "../src/metrics/forge.ts";
	16
	17	let dir: string;
	18	let app: App;
	19	let servers: Servers;
	20	let ro: string;
	21	let open: AppRecord;
	22	let secret: AppRecord;
	23	let runtime: AppsRuntime;
	24
	25	const record = (r: { ok: boolean; value?: AppRecord; error?: string }): AppRecord => {
	26	if (!r.ok) throw new Error(String(r.error));
	27	return r.value!;
	28	};
	29
	30	beforeAll(async () => {
	31	dir = await mkdtemp(join(tmpdir(), "metrics-apps-"));
	32	app = createApp({ dataDir: dir, port: 0 });
	33	servers = await serve(app);
	34	const made = await createUser(app.db, {
	35	origin: "seed",
	36	trusted: true,
	37	username: "alice",
	38	email: "[email protected]",
	39	password: "a-good-passphrase",
	40	adminIfFirst: true,
	41	});
	42	if (!made.ok) throw new Error(made.error);
	43	ro = createToken(app.db, made.value.id, "t", "ro").token;
	44	const repos = [];
	45	for (const name of ["notes", "plans"]) {
	46	const r = await createRepo(app, made.value, { name });
	47	if (!r.ok) throw new Error(String(r.error));
	48	repos.push(r.value);
	49	}
	50	open = record(createAppRecord(app.db, { repoId: repos[0]!.id, slug: "notes", access: "public" }));
	51	secret = record(createAppRecord(app.db, { repoId: repos[1]!.id, slug: "secret-plans", access: "private" }));
	52	// An unstarted runtime holding what a started one would: no cell is spawned.
	53	runtime = new AppsRuntime(app, { domain: "apps.test", scheme: "http", publicPort: servers.port, forgeUrl: servers.url });
	54	const envs = (runtime as unknown as { envs: Map<string, unknown> }).envs;
	55	const live = {
	56	commitOid: "abc",
	57	handle: { port: 1 },
	58	runId: "r1",
	59	since: "2026-01-01T00:00:00Z",
	60	servingAt: "2026-01-01T00:00:00Z",
	61	createdAt: "2026-01-01T00:00:00Z",
	62	manifest: {},
	63	deployStartedAt: null,
	64	};
	65	const base = { openStreams: 0, openRequests: 0, chain: Promise.resolve(), restarts: [] };
	66	envs.set("a", { ...base, app: open, env: "production", live });
	67	envs.set("b", { ...base, app: secret, env: "production", live });
	68	envs.set("c", { ...base, app: open, env: "pull-7", asleep: { commitOid: "abc", runId: "r2", since: "2026-01-01T00:00:00Z" } });
	69	}, 60_000);
	70
	71	afterAll(async () => {
	72	servers.stop();
	73	app.close();
	74	await rm(dir, { recursive: true, force: true });
	75	});
	76
	77	test("appSources marks the private app private", () => {
	78	const bySlug = new Map(appSources(app).apps.map((a) => [a.slug, a]));
	79	expect(bySlug.get("notes")?.private).toBe(false);
	80	expect(bySlug.get("secret-plans")?.private).toBe(true);
	81	});
	82
	83	test("the body names a private app by id and never by slug", () => {
	84	const body = forgeMetrics(app.db, appSources(app));
	85	expect(body).toContain(`forge_app_up{app="${secret.id}",env="production"} 1`);
	86	expect(body).not.toContain("secret-plans");
	87	});
	88
	89	test("GET /metrics with an admin ro token carries forge_app_up for the seeded apps", async () => {
	90	const res = await fetch(`${servers.url}/metrics`, { headers: { authorization: `Bearer ${ro}` } });
	91	const body = await res.text();
	92	expect(res.status).toBe(200);
	93	expect(body).toContain('forge_app_up{app="notes",env="production"} 1');
	94	expect(body).toContain('forge_app_up{app="notes",env="pull-7"} 0');
	95	expect(body).toContain(`forge_app_up{app="${secret.id}",env="production"} 1`);
	96	expect(body).not.toContain("secret-plans");
	97	});
		@@ -0,0 +1,52 @@
	1	/** `/metrics` serves the app series: `forge_app_up` moves when an app sleeps, a cell's figures appear once recorded, a private app is named by id only (#3752). */
	2
	3	import { expect, test } from "bun:test";
	4	import { Database } from "bun:sqlite";
	5	import type { CellCounter } from "../src/apps/contain.ts";
	6	import { appFamilies, forgeMetrics, type AppReading } from "../src/metrics/forge.ts";
	7	import { openMetrics } from "../src/metrics/openmetrics.ts";
	8
	9	const app = (state: "running" \| "asleep"): AppReading[] => [
	10	{ id: "app1", slug: "notes", private: false, envs: [{ env: "production", state }] },
	11	{ id: "app2", slug: "secret-plans", private: true, envs: [{ env: "pull-3", state: "running" }] },
	12	];
	13	const cell = (label: string): CellCounter => ({ key: 1, label, cpuUsec: 2_500_000, memoryBytes: 4096 });
	14	const text = (apps: AppReading[], cells: CellCounter[]) => openMetrics(appFamilies({ apps, cells }));
	15
	16	test("forge_app_up is 1 while running and 0 once the app sleeps", () => {
	17	expect(text(app("running"), [])).toContain('forge_app_up{app="notes",env="production"} 1');
	18	expect(text(app("asleep"), [])).toContain('forge_app_up{app="notes",env="production"} 0');
	19	});
	20
	21	test("a cell's figures appear when the watch has read it, and not before", () => {
	22	expect(text(app("running"), [])).not.toContain("forge_cell_memory_bytes{");
	23	const out = text(app("running"), [cell("app:notes/production")]);
	24	expect(out).toContain('forge_cell_cpu_seconds_total{app="notes",env="production"} 2.5');
	25	expect(out).toContain('forge_cell_memory_bytes{app="notes",env="production"} 4096');
	26	});
	27
	28	test("a private app appears under its id only", () => {
	29	const out = text(app("running"), [cell("app:secret-plans/pull-3"), cell("check:x")]);
	30	expect(out).toContain('forge_app_up{app="app2",env="pull-3"} 1');
	31	expect(out).toContain('forge_cell_memory_bytes{app="app2",env="pull-3"} 4096');
	32	expect(out).not.toContain("secret-plans");
	33	expect(out).not.toContain("check:x");
	34	});
	35
	36	test("forgeMetrics carries the families as Prometheus text without a trailing EOF", () => {
	37	const db = new Database(":memory:");
	38	db.run("CREATE TABLE route_timing (hour TEXT, pattern TEXT, bucket INTEGER, n INTEGER, sum_us INTEGER)");
	39	const body = forgeMetrics(db, { apps: app("running"), cells: [] });
	40	expect(body).toContain("# TYPE forge_app_up gauge");
	41	expect(body).toContain("# TYPE forge_cell_cpu_seconds counter");
	42	expect(body).not.toContain("# EOF");
	43	});
	44
	45	test("two cells under one label (a deploy's old and new) are one series, summed, and the scrape does not throw", () => {
	46	const db = new Database(":memory:");
	47	db.run("CREATE TABLE route_timing (hour TEXT, pattern TEXT, bucket INTEGER, n INTEGER, sum_us INTEGER)");
	48	const cells = [cell("app:notes/production"), { ...cell("app:notes/production"), key: 2 }];
	49	const body = forgeMetrics(db, { apps: app("running"), cells });
	50	expect(body).toContain('forge_cell_memory_bytes{app="notes",env="production"} 8192');
	51	expect(body).toContain('forge_cell_cpu_seconds_total{app="notes",env="production"} 5');
	52	});