c2d4501 · 1855e7f · d85d378
@@ -25,6 +25,7 @@
export const METHODS = [
  "PATCH",
  "DELETE",
  "OPTIONS",
  "OTHER",
] as const;
export type UsageRole = (typeof ROLES)[number];
@@ -413,7 +413,10 @@
import { notFoundPage } from "../web/chrome.ts";
import { targetForShortCode } from "../origins/oauth.ts";
import { apiRoutes } from "./api.ts";
import { feedRoutes } from "./feed.ts";
import type { Database } from "bun:sqlite";
import { mcpRoutes } from "./mcp.ts";
import { createUsageCounter, type UsageCounter } from "../core/usage_count.ts";
import { methodOf, noteUsage, roleOf, surfaceOf, takeUsage } from "./usage_tag.ts";
import { oauthRoutes } from "./oauth.ts";
import { flushAccess, recordAccess } from "./access_log.ts";
import { carryPeer, clientAddress, notePeer, type ProxyTrust } from "./peer.ts";
@@ -1693,6 +1696,20 @@
function refusedByDatabase(error: unknown): string | undefined {
  return code === "SQLITE_CONSTRAINT_TRIGGER" ? error.message : undefined;
}
/**
 * What a thrown handler is answered with: the guard's own words and a 403
 * when a database trigger refused, else a 500 that says nothing of the fault.
 */
function failure(app: App, error: unknown): Response {
  const refused = refusedByDatabase(error);
  if (refused !== undefined) {
    app.log.warn("refused by the database", { reason: refused });
    return securityFloor(json({ error: "refused", message: refused }, 403));
  }
  app.log.error("unhandled request error", { error: String(error) });
  return securityFloor(json({ error: "internal" }, 500));
}
/** A response with one header set, copying it when its headers are frozen. */
function withHeader(response: Response, name: string, value: string): Response {
  try {
@@ -1837,6 +1854,41 @@
export async function serve(app: App): Promise<Servers> {
      client: clientAddress(request),
    });
      
        sentinelcrew
        29d ago
        outdated
      
      Medium: request.method comes from the client. Any method outside METHODS makes count throw, and the catch below logs one warning per request from anyone. Map unknown methods to a bounded OTHER bucket before counting, and add a test showing that PROPFIND / is counted and logs nothing.
    
  // One counter per database: a hosted child counts in its own. `count` is a
  // map increment; a refused event would be a programming error, not worth a 500.
  const counters = new Map<Database, UsageCounter>();
  const counted = (owner: App, request: Request, response: Response): void => {
    const tag = takeUsage(request);
    try {
      let counter = counters.get(owner.db);
      if (counter === undefined) {
        counter = createUsageCounter(owner.db);
        counters.set(owner.db, counter);
      }
      counter.count({
        pattern: tag.pattern,
        method: methodOf(request.method),
        status: response.status,
        role: tag.role,
        surface: surfaceOf(new URL(request.url).pathname),
      });
    } catch (error) {
      owner.log.warn("could not count a request", { error: String(error) });
    }
  };
  // Behind the floor, and a throw is answered here so it is counted too.
  const answered = async (
    owner: App,
    run: () => Promise<Response>,
  ): Promise<Response> => {
    try {
      return securityFloor(await run());
    } catch (error) {
      return failure(owner, error);
    }
  };
  const handle = async (
    request: Request,
    server: Bun.Server<AppSocket>,
@@ -1883,7 +1935,8 @@
export async function serve(app: App): Promise<Servers> {
          }
          notePeer(request, server.requestIP(request)?.address, trust);
          armClose(request, () => server.timeout(request, 1));
          const response = securityFloor(await http.handle(request));
          const response = await answered(child, () => http.handle(request));
          counted(child, request, response);
          logged(key, request, response, started);
          await settleBody(request);
          if (!mustClose(request)) return response;
@@ -1899,9 +1952,10 @@
export async function serve(app: App): Promise<Servers> {
      armClose(request, () => server.timeout(request, 1));
      // Everything the forge answers for itself leaves through this one
      // door, so a responder added next year cannot forget the floor.
      const response = securityFloor(
        await answer(app, router, apiRouter, limiter, request),
      const response = await answered(app, () =>
        answer(app, router, apiRouter, limiter, request),
      );
      counted(app, request, response);
      logged(app.config.dataDir, request, response, started);
      await settleBody(request);
      if (!mustClose(request)) return response;
@@ -1923,15 +1977,8 @@
export async function serve(app: App): Promise<Servers> {
    // proxy; its own live surfaces are SSE (`sse.ts`) and stay so.
    websocket: appWebSocket,
    fetch: handle,
    error(error) {
      const refused = refusedByDatabase(error);
      if (refused !== undefined) {
        app.log.warn("refused by the database", { reason: refused });
        return securityFloor(json({ error: "refused", message: refused }, 403));
      }
      app.log.error("unhandled request error", { error: String(error) });
      return securityFloor(json({ error: "internal" }, 500));
    },
    // A last resort: `handle` answers a throw itself, so it is counted.
    error: (error) => failure(app, error),
  });
  const onPort = portInsteadOfSocket();
@@ -1973,6 +2020,15 @@
export async function serve(app: App): Promise<Servers> {
    stop() {
      endOpen();
      clearInterval(sweep);
      // Counts first: a hosted child's database closes with its app.
      for (const counter of counters.values()) {
        try {
          counter.stop();
        } catch (error) {
          app.log.warn("could not flush request counts", { error: String(error) });
        }
      }
      counters.clear();
      for (const http of childHttps.values()) http.close();
      childHttps.clear();
      closeHostedChildren(app);
@@ -2017,9 +2073,14 @@
async function answerGet(
  request: Request,
): Promise<Response> {
  const url = publicUrl(request, app.config.url);
  if (url.pathname === "/healthz")
  if (url.pathname === "/healthz") {
    noteUsage(request, { pattern: "/healthz", role: "anon" });
    return liveResponse(() => app.db.query("select 1").get(), app.log);
  if (url.pathname === "/readyz") return readyResponse();
  }
  if (url.pathname === "/readyz") {
    noteUsage(request, { pattern: "/readyz", role: "anon" });
    return readyResponse();
  }
  const routeMethod = request.method === "HEAD" ? "GET" : request.method;
  // The one place a path decides its own shape of miss: anything under
  // `/api` matches only against the API's own routes, so a studio route as
@@ -2054,6 +2115,7 @@
async function answerGet(
    return noRoute(request.method, url.pathname);
  }
  const auth = await resolveAuth(app, request);
  noteUsage(request, { pattern: matched.pattern, role: roleOf(auth) });
  // From here a body's cap depends on who is asking (`http/limits.ts`), and
  // a stream this request opens is the account's, to end with it.
  if (auth.user !== undefined) {
@@ -0,0 +1,51 @@
/**
 * What the server tells the usage counter about one request: the route
 * pattern that matched, and who asked. The router notes it while it answers;
 * the door every response leaves through reads it once and counts.
 *
 * A path that matched no route is counted as `(unmatched)`, one row for
 * whatever strangers type, so the table never grows with their input.
 */
import type { Auth } from "./context.ts";
import { METHODS, type UsageRole, type UsageSurface } from "../core/usage_count.ts";
export const UNMATCHED = "(unmatched)";
export interface UsageTag {
  readonly pattern: string;
  readonly role: UsageRole;
}
const noted = new WeakMap<Request, UsageTag>();
/** The viewer's role: a token for an agent account is `agent`. */
export function roleOf(auth: Auth): UsageRole {
  const user = auth.user;
  if (user === undefined) return "anon";
  if (user.kind === "agent") return "agent";
  return user.isAdmin ? "admin" : "user";
}
/** `/api/` is `api`, the MCP route is `mcp`, everything else is `web`. */
export function surfaceOf(pathname: string): UsageSurface {
  if (pathname === "/api" || pathname.startsWith("/api/")) return "api";
  if (pathname === "/mcp") return "mcp";
  return "web";
}
/** The method counted: any token a stranger sends outside `METHODS` is `OTHER`. */
export function methodOf(method: string): string {
  return (METHODS as readonly string[]).includes(method) ? method : "OTHER";
}
export function noteUsage(request: Request, tag: UsageTag): void {
  noted.set(request, tag);
}
      
        sentinelcrew
        29d ago
        
      
      Nit, no action needed: takeUsage deletes from a WeakMap, so a request whose tag is never taken (a proxied app, an early 404) leaves nothing behind. Good.
    
/** What was noted for this request, or the unmatched, anonymous default. */
export function takeUsage(request: Request): UsageTag {
  const tag = noted.get(request);
  noted.delete(request);
  return tag ?? { pattern: UNMATCHED, role: "anon" };
}
@@ -0,0 +1,87 @@
/** A hosted child's requests are counted in the child's database (#1918). */
import { afterAll, beforeAll, expect, spyOn, test } from "bun:test";
import { Database } from "bun:sqlite";
import { mkdtemp, rm } from "node:fs/promises";
import { readFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { childDirOf, spawnForge } from "../src/actions/spawn.ts";
import { createApp, type App } from "../src/app.ts";
import { createUser } from "../src/core/users.ts";
import { readUsage } from "../src/core/usage_count.ts";
import { serve, type Servers } from "../src/http/server.ts";
import { resolveHostedChild } from "../src/spawn/host.ts";
import { hostedPlacement } from "../src/spawn/placement.ts";
import { quietLogs } from "./helpers/globals.ts";
quietLogs();
let dir = "";
let parent: App;
let servers: Servers;
beforeAll(async () => {
  dir = await mkdtemp(join(tmpdir(), "forge-usage-child-"));
  parent = createApp({ dataDir: dir, url: "https://forge.test", port: 0 });
  const made = await createUser(parent.db, {
    trusted: true,
    username: "alice",
    email: "[email protected]",
    isAdmin: true,
  });
  if (!made.ok) throw new Error("user");
  const spawned = await spawnForge(
    parent,
    made.value,
    { name: "homie", org: "homie", apps: [], carryBriefs: false, carrySubscriptions: false },
    { catalog: [], briefs: [], subscriptions: [] },
    {
      placement: hostedPlacement({ host: "forge.test" }),
      request: new Request("https://forge.test/spawn", {
        method: "POST",
        headers: { origin: "https://forge.test" },
      }),
      url: new URL("https://forge.test/spawn"),
    },
  );
  if (!spawned.ok) throw new Error(JSON.stringify(spawned.error));
  servers = await serve(parent);
});
afterAll(() => rm(dir, { recursive: true, force: true }));
test("the child's request lands in the child's usage_count, not the parent's", async () => {
  await fetch(servers.url + "/healthz", { headers: { host: "homie--forge.test" } });
  await fetch(servers.url + "/healthz", { headers: { host: "homie--forge.test" } });
  await fetch(servers.url + "/readyz", { headers: { host: "forge.test" } });
  // A throw in the child is the child's 500, counted in the child.
  const host = { headers: { host: "homie--forge.test" } };
  const kid = await resolveHostedChild(parent, new Request("http://x/", host));
  if (typeof kid === "string" || kid === undefined) throw new Error("child");
  const real = kid.db.query.bind(kid.db);
  const broken = spyOn(kid.db, "query").mockImplementation(((sql: string) => {
    if (/FROM\s+repo\b/i.test(sql)) throw new Error("disk on fire");
    return real(sql);
  }) as never);
  const boom = await fetch(servers.url + "/api/v1/repos/alice/x/issues/1", host);
  broken.mockRestore();
  expect(boom.status).toBe(500);
  servers.stop();
  const { bodyId } = JSON.parse(
    readFileSync(join(dir, "spawn", "homie.json"), "utf8"),
  ) as { bodyId: string };
  const child = new Database(join(childDirOf(parent, bodyId), "forge.db"), { readonly: true });
  const inChild = readUsage(child, "");
  const inParent = readUsage(parent.db, "");
  child.close();
  parent.close();
  expect(inChild.map((r) => [r.pattern, r.n])).toEqual([
    ["/api/v1/repos/:owner/:name/issues/:number", 1],
    ["/healthz", 2],
  ]);
  expect(inChild.find((r) => r.status === "5xx")?.pattern).toBe(
    "/api/v1/repos/:owner/:name/issues/:number",
  );
  expect(inParent.map((r) => [r.pattern, r.n])).toEqual([["/readyz", 1]]);
});
@@ -0,0 +1,143 @@
/** Every answered request is counted once, by pattern, role and surface (#1918). */
import { afterAll, beforeAll, expect, spyOn, test } from "bun:test";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { createAgent } from "../src/actions/crew.ts";
import { createToken } from "../src/core/tokens.ts";
import { createUser } from "../src/core/users.ts";
import { methodOf, roleOf, surfaceOf } from "../src/http/usage_tag.ts";
import { readUsage } from "../src/core/usage_count.ts";
import { serve, type Servers } from "../src/http/server.ts";
let dataDir: string;
let app: App;
let servers: Servers;
let agentToken: string;
let adminToken: string;
let userToken: string;
beforeAll(async () => {
  dataDir = await mkdtemp(join(tmpdir(), "forge-usage-"));
  app = createApp({ dataDir, port: 0 });
  const root = await createUser(app.db, {
    trusted: true,
    username: "alice",
    email: "[email protected]",
    password: "a-good-passphrase",
    adminIfFirst: true,
  });
  if (!root.ok) throw new Error("user");
  const agent = await createAgent(app, root.value, {
    username: "helper",
    model: "",
    brief: "",
  });
  if (!agent.ok) throw new Error(String(agent.error));
  const bob = await createUser(app.db, {
    trusted: true,
    username: "bob",
    email: "[email protected]",
    password: "a-good-passphrase",
  });
  if (!bob.ok) throw new Error("user");
  adminToken = createToken(app.db, root.value.id, "t", "ro").token;
      
        sentinelcrew
        29d ago
        outdated
      
      Extend this to cover /healthz (pattern /healthz), /mcp (surface mcp), a signed-in user and an admin (role), and a stranger's method. As it stands, removing the /healthz noteUsage or breaking surfaceOf's mcp branch would still pass.
    
  userToken = createToken(app.db, bob.value.id, "t", "ro").token;
  agentToken = createToken(app.db, agent.value.user.id, "t", "ro").token;
});
afterAll(async () => {
  app.close();
  await rm(dataDir, { recursive: true, force: true });
});
test("three ids on one route are one row; an agent's API call is agent/api; strangers share one pattern", async () => {
  servers = await serve(app);
  for (const n of [1, 2, 3])
    await fetch(`${servers.url}/api/v1/repos/alice/none/issues/${n}`, {
      headers: { authorization: `Bearer ${agentToken}` },
    });
  const miss = await fetch(`${servers.url}/api/v1/nothing/${crypto.randomUUID()}`);
  await fetch(`${servers.url}/no/such/${crypto.randomUUID()}/path/x/y`);
  // Every response keeps the floor.
  expect(miss.headers.get("x-content-type-options")).toBe("nosniff");
  servers.stop();
  const rows = readUsage(app.db, "");
  const issue = rows.filter(
    (r) => r.pattern === "/api/v1/repos/:owner/:name/issues/:number",
  );
  expect(issue).toHaveLength(1);
  expect(issue[0]).toMatchObject({ n: 3, role: "agent", surface: "api", method: "GET" });
  const unmatched = rows.filter((r) => r.pattern === "(unmatched)");
  expect(unmatched.reduce((sum, r) => sum + r.n, 0)).toBeGreaterThanOrEqual(2);
  expect(rows.some((r) => r.pattern.includes("nothing"))).toBe(false);
});
test("health routes, the mcp surface, roles and a stranger's method are each counted, and nothing logs", async () => {
  servers = await serve(app);
  const warn = spyOn(app.log, "warn");
  await fetch(`${servers.url}/healthz`);
  await fetch(`${servers.url}/readyz`);
  await fetch(`${servers.url}/mcp`, { method: "POST", body: "{}" });
  await fetch(`${servers.url}/`);
  for (const [token, n] of [[adminToken, 1], [userToken, 2]] as const)
    for (let i = 0; i < n; i++)
      await fetch(`${servers.url}/api/v1/repos/alice/none/issues/1`, {
        headers: { authorization: `Bearer ${token}` },
      });
  await fetch(`${servers.url}/`, { method: "PROPFIND" });
  await fetch(`${servers.url}/x`, { method: "MKCOL" });
  expect(warn).not.toHaveBeenCalled();
  warn.mockRestore();
  servers.stop();
  const rows = readUsage(app.db, "");
  const n = (f: (r: (typeof rows)[number]) => boolean) =>
    rows.filter(f).reduce((sum, r) => sum + r.n, 0);
  expect(n((r) => r.pattern === "/healthz" && r.role === "anon")).toBe(1);
  expect(n((r) => r.pattern === "/readyz")).toBe(1);
  expect(n((r) => r.pattern === "/mcp" && r.surface === "mcp")).toBe(1);
  expect(n((r) => r.pattern === "/" && r.surface === "web" && r.method === "GET")).toBe(1);
  expect(n((r) => r.pattern.endsWith("issues/:number") && r.role === "admin")).toBe(1);
  expect(n((r) => r.pattern.endsWith("issues/:number") && r.role === "user")).toBe(2);
  expect(n((r) => r.method === "OTHER")).toBe(2);
  expect(rows.some((r) => r.method === "PROPFIND" || r.method === "MKCOL")).toBe(false);
});
test("roleOf, surfaceOf and methodOf map as the table expects", () => {
  const who = (kind: "user" | "agent", isAdmin: boolean) =>
    ({ user: { kind, isAdmin }, scope: undefined, via: "token" }) as never;
  expect(roleOf({ user: undefined, scope: undefined, via: "none" })).toBe("anon");
  expect(roleOf(who("user", false))).toBe("user");
  expect(roleOf(who("user", true))).toBe("admin");
  expect(roleOf(who("agent", false))).toBe("agent");
  expect(surfaceOf("/mcp")).toBe("mcp");
  expect(surfaceOf("/api")).toBe("api");
  expect(surfaceOf("/api/v1/x")).toBe("api");
  expect(surfaceOf("/alice/repo")).toBe("web");
  expect(methodOf("GET")).toBe("GET");
  expect(methodOf("PROPFIND")).toBe("OTHER");
});
test("a handler that throws is answered 500 and counted 5xx under its route", async () => {
  servers = await serve(app);
  const real = app.db.query.bind(app.db);
  const broken = spyOn(app.db, "query").mockImplementation(((sql: string) => {
    if (/FROM\s+repo\b/i.test(sql)) throw new Error("disk on fire");
    return real(sql);
  }) as never);
  const res = await fetch(`${servers.url}/api/v1/repos/alice/boom/issues/9`, {
    headers: { authorization: `Bearer ${adminToken}` },
  });
  broken.mockRestore();
  expect(res.status).toBe(500);
  expect(res.headers.get("x-content-type-options")).toBe("nosniff");
  servers.stop();
  const row = readUsage(app.db, "").filter((r) => r.status === "5xx");
  expect(row.map((r) => [r.pattern, r.role, r.n])).toEqual([
    ["/api/v1/repos/:owner/:name/issues/:number", "admin", 1],
  ]);
});
		@@ -25,6 +25,7 @@ export const METHODS = [
25	25	"PATCH",
26	26	"DELETE",
27	27	"OPTIONS",
	28	"OTHER",
28	29	] as const;
29	30
30	31	export type UsageRole = (typeof ROLES)[number];
		@@ -413,7 +413,10 @@ import { notFoundPage } from "../web/chrome.ts";
413	413	import { targetForShortCode } from "../origins/oauth.ts";
414	414	import { apiRoutes } from "./api.ts";
415	415	import { feedRoutes } from "./feed.ts";
	416	import type { Database } from "bun:sqlite";
416	417	import { mcpRoutes } from "./mcp.ts";
	418	import { createUsageCounter, type UsageCounter } from "../core/usage_count.ts";
	419	import { methodOf, noteUsage, roleOf, surfaceOf, takeUsage } from "./usage_tag.ts";
417	420	import { oauthRoutes } from "./oauth.ts";
418	421	import { flushAccess, recordAccess } from "./access_log.ts";
419	422	import { carryPeer, clientAddress, notePeer, type ProxyTrust } from "./peer.ts";
		@@ -1693,6 +1696,20 @@ function refusedByDatabase(error: unknown): string \| undefined {
1693	1696	return code === "SQLITE_CONSTRAINT_TRIGGER" ? error.message : undefined;
1694	1697	}
1695	1698
	1699	/**
	1700	* What a thrown handler is answered with: the guard's own words and a 403
	1701	* when a database trigger refused, else a 500 that says nothing of the fault.
	1702	*/
	1703	function failure(app: App, error: unknown): Response {
	1704	const refused = refusedByDatabase(error);
	1705	if (refused !== undefined) {
	1706	app.log.warn("refused by the database", { reason: refused });
	1707	return securityFloor(json({ error: "refused", message: refused }, 403));
	1708	}
	1709	app.log.error("unhandled request error", { error: String(error) });
	1710	return securityFloor(json({ error: "internal" }, 500));
	1711	}
	1712
1696	1713	/** A response with one header set, copying it when its headers are frozen. */
1697	1714	function withHeader(response: Response, name: string, value: string): Response {
1698	1715	try {
		@@ -1837,6 +1854,41 @@ export async function serve(app: App): Promise<Servers> {
1837	1854	client: clientAddress(request),
1838	1855	});
1839	1856
		sentinelcrew 29d ago outdated Medium: `request.method` comes from the client. Any method outside `METHODS` makes `count` throw, and the catch below logs one warning per request from anyone. Map unknown methods to a bounded `OTHER` bucket before counting, and add a test showing that `PROPFIND /` is counted and logs nothing.
	1857	// One counter per database: a hosted child counts in its own. `count` is a
	1858	// map increment; a refused event would be a programming error, not worth a 500.
	1859	const counters = new Map<Database, UsageCounter>();
	1860	const counted = (owner: App, request: Request, response: Response): void => {
	1861	const tag = takeUsage(request);
	1862	try {
	1863	let counter = counters.get(owner.db);
	1864	if (counter === undefined) {
	1865	counter = createUsageCounter(owner.db);
	1866	counters.set(owner.db, counter);
	1867	}
	1868	counter.count({
	1869	pattern: tag.pattern,
	1870	method: methodOf(request.method),
	1871	status: response.status,
	1872	role: tag.role,
	1873	surface: surfaceOf(new URL(request.url).pathname),
	1874	});
	1875	} catch (error) {
	1876	owner.log.warn("could not count a request", { error: String(error) });
	1877	}
	1878	};
	1879
	1880	// Behind the floor, and a throw is answered here so it is counted too.
	1881	const answered = async (
	1882	owner: App,
	1883	run: () => Promise<Response>,
	1884	): Promise<Response> => {
	1885	try {
	1886	return securityFloor(await run());
	1887	} catch (error) {
	1888	return failure(owner, error);
	1889	}
	1890	};
	1891
1840	1892	const handle = async (
1841	1893	request: Request,
1842	1894	server: Bun.Server<AppSocket>,
		@@ -1883,7 +1935,8 @@ export async function serve(app: App): Promise<Servers> {
1883	1935	}
1884	1936	notePeer(request, server.requestIP(request)?.address, trust);
1885	1937	armClose(request, () => server.timeout(request, 1));
1886		const response = securityFloor(await http.handle(request));
	1938	const response = await answered(child, () => http.handle(request));
	1939	counted(child, request, response);
1887	1940	logged(key, request, response, started);
1888	1941	await settleBody(request);
1889	1942	if (!mustClose(request)) return response;
		@@ -1899,9 +1952,10 @@ export async function serve(app: App): Promise<Servers> {
1899	1952	armClose(request, () => server.timeout(request, 1));
1900	1953	// Everything the forge answers for itself leaves through this one
1901	1954	// door, so a responder added next year cannot forget the floor.
1902		const response = securityFloor(
1903		await answer(app, router, apiRouter, limiter, request),
	1955	const response = await answered(app, () =>
	1956	answer(app, router, apiRouter, limiter, request),
1904	1957	);
	1958	counted(app, request, response);
1905	1959	logged(app.config.dataDir, request, response, started);
1906	1960	await settleBody(request);
1907	1961	if (!mustClose(request)) return response;
		@@ -1923,15 +1977,8 @@ export async function serve(app: App): Promise<Servers> {
1923	1977	// proxy; its own live surfaces are SSE (`sse.ts`) and stay so.
1924	1978	websocket: appWebSocket,
1925	1979	fetch: handle,
1926		error(error) {
1927		const refused = refusedByDatabase(error);
1928		if (refused !== undefined) {
1929		app.log.warn("refused by the database", { reason: refused });
1930		return securityFloor(json({ error: "refused", message: refused }, 403));
1931		}
1932		app.log.error("unhandled request error", { error: String(error) });
1933		return securityFloor(json({ error: "internal" }, 500));
1934		},
	1980	// A last resort: `handle` answers a throw itself, so it is counted.
	1981	error: (error) => failure(app, error),
1935	1982	});
1936	1983
1937	1984	const onPort = portInsteadOfSocket();
		@@ -1973,6 +2020,15 @@ export async function serve(app: App): Promise<Servers> {
1973	2020	stop() {
1974	2021	endOpen();
1975	2022	clearInterval(sweep);
	2023	// Counts first: a hosted child's database closes with its app.
	2024	for (const counter of counters.values()) {
	2025	try {
	2026	counter.stop();
	2027	} catch (error) {
	2028	app.log.warn("could not flush request counts", { error: String(error) });
	2029	}
	2030	}
	2031	counters.clear();
1976	2032	for (const http of childHttps.values()) http.close();
1977	2033	childHttps.clear();
1978	2034	closeHostedChildren(app);
		@@ -2017,9 +2073,14 @@ async function answerGet(
2017	2073	request: Request,
2018	2074	): Promise<Response> {
2019	2075	const url = publicUrl(request, app.config.url);
2020		if (url.pathname === "/healthz")
	2076	if (url.pathname === "/healthz") {
	2077	noteUsage(request, { pattern: "/healthz", role: "anon" });
2021	2078	return liveResponse(() => app.db.query("select 1").get(), app.log);
2022		if (url.pathname === "/readyz") return readyResponse();
	2079	}
	2080	if (url.pathname === "/readyz") {
	2081	noteUsage(request, { pattern: "/readyz", role: "anon" });
	2082	return readyResponse();
	2083	}
2023	2084	const routeMethod = request.method === "HEAD" ? "GET" : request.method;
2024	2085	// The one place a path decides its own shape of miss: anything under
2025	2086	// `/api` matches only against the API's own routes, so a studio route as
		@@ -2054,6 +2115,7 @@ async function answerGet(
2054	2115	return noRoute(request.method, url.pathname);
2055	2116	}
2056	2117	const auth = await resolveAuth(app, request);
	2118	noteUsage(request, { pattern: matched.pattern, role: roleOf(auth) });
2057	2119	// From here a body's cap depends on who is asking (`http/limits.ts`), and
2058	2120	// a stream this request opens is the account's, to end with it.
2059	2121	if (auth.user !== undefined) {
		@@ -0,0 +1,51 @@
	1	/**
	2	* What the server tells the usage counter about one request: the route
	3	* pattern that matched, and who asked. The router notes it while it answers;
	4	* the door every response leaves through reads it once and counts.
	5	*
	6	* A path that matched no route is counted as `(unmatched)`, one row for
	7	* whatever strangers type, so the table never grows with their input.
	8	*/
	9
	10	import type { Auth } from "./context.ts";
	11	import { METHODS, type UsageRole, type UsageSurface } from "../core/usage_count.ts";
	12
	13	export const UNMATCHED = "(unmatched)";
	14
	15	export interface UsageTag {
	16	readonly pattern: string;
	17	readonly role: UsageRole;
	18	}
	19
	20	const noted = new WeakMap<Request, UsageTag>();
	21
	22	/** The viewer's role: a token for an agent account is `agent`. */
	23	export function roleOf(auth: Auth): UsageRole {
	24	const user = auth.user;
	25	if (user === undefined) return "anon";
	26	if (user.kind === "agent") return "agent";
	27	return user.isAdmin ? "admin" : "user";
	28	}
	29
	30	/** `/api/` is `api`, the MCP route is `mcp`, everything else is `web`. */
	31	export function surfaceOf(pathname: string): UsageSurface {
	32	if (pathname === "/api" \|\| pathname.startsWith("/api/")) return "api";
	33	if (pathname === "/mcp") return "mcp";
	34	return "web";
	35	}
	36
	37	/** The method counted: any token a stranger sends outside `METHODS` is `OTHER`. */
	38	export function methodOf(method: string): string {
	39	return (METHODS as readonly string[]).includes(method) ? method : "OTHER";
	40	}
	41
	42	export function noteUsage(request: Request, tag: UsageTag): void {
	43	noted.set(request, tag);
	44	}
		sentinelcrew 29d ago Nit, no action needed: `takeUsage` deletes from a WeakMap, so a request whose tag is never taken (a proxied app, an early 404) leaves nothing behind. Good.
	45
	46	/** What was noted for this request, or the unmatched, anonymous default. */
	47	export function takeUsage(request: Request): UsageTag {
	48	const tag = noted.get(request);
	49	noted.delete(request);
	50	return tag ?? { pattern: UNMATCHED, role: "anon" };
	51	}
		@@ -0,0 +1,87 @@
	1	/** A hosted child's requests are counted in the child's database (#1918). */
	2
	3	import { afterAll, beforeAll, expect, spyOn, test } from "bun:test";
	4	import { Database } from "bun:sqlite";
	5	import { mkdtemp, rm } from "node:fs/promises";
	6	import { readFileSync } from "node:fs";
	7	import { tmpdir } from "node:os";
	8	import { join } from "node:path";
	9	import { childDirOf, spawnForge } from "../src/actions/spawn.ts";
	10	import { createApp, type App } from "../src/app.ts";
	11	import { createUser } from "../src/core/users.ts";
	12	import { readUsage } from "../src/core/usage_count.ts";
	13	import { serve, type Servers } from "../src/http/server.ts";
	14	import { resolveHostedChild } from "../src/spawn/host.ts";
	15	import { hostedPlacement } from "../src/spawn/placement.ts";
	16	import { quietLogs } from "./helpers/globals.ts";
	17
	18	quietLogs();
	19
	20	let dir = "";
	21	let parent: App;
	22	let servers: Servers;
	23
	24	beforeAll(async () => {
	25	dir = await mkdtemp(join(tmpdir(), "forge-usage-child-"));
	26	parent = createApp({ dataDir: dir, url: "https://forge.test", port: 0 });
	27	const made = await createUser(parent.db, {
	28	trusted: true,
	29	username: "alice",
	30	email: "[email protected]",
	31	isAdmin: true,
	32	});
	33	if (!made.ok) throw new Error("user");
	34	const spawned = await spawnForge(
	35	parent,
	36	made.value,
	37	{ name: "homie", org: "homie", apps: [], carryBriefs: false, carrySubscriptions: false },
	38	{ catalog: [], briefs: [], subscriptions: [] },
	39	{
	40	placement: hostedPlacement({ host: "forge.test" }),
	41	request: new Request("https://forge.test/spawn", {
	42	method: "POST",
	43	headers: { origin: "https://forge.test" },
	44	}),
	45	url: new URL("https://forge.test/spawn"),
	46	},
	47	);
	48	if (!spawned.ok) throw new Error(JSON.stringify(spawned.error));
	49	servers = await serve(parent);
	50	});
	51
	52	afterAll(() => rm(dir, { recursive: true, force: true }));
	53
	54	test("the child's request lands in the child's usage_count, not the parent's", async () => {
	55	await fetch(servers.url + "/healthz", { headers: { host: "homie--forge.test" } });
	56	await fetch(servers.url + "/healthz", { headers: { host: "homie--forge.test" } });
	57	await fetch(servers.url + "/readyz", { headers: { host: "forge.test" } });
	58	// A throw in the child is the child's 500, counted in the child.
	59	const host = { headers: { host: "homie--forge.test" } };
	60	const kid = await resolveHostedChild(parent, new Request("http://x/", host));
	61	if (typeof kid === "string" \|\| kid === undefined) throw new Error("child");
	62	const real = kid.db.query.bind(kid.db);
	63	const broken = spyOn(kid.db, "query").mockImplementation(((sql: string) => {
	64	if (/FROM\s+repo\b/i.test(sql)) throw new Error("disk on fire");
	65	return real(sql);
	66	}) as never);
	67	const boom = await fetch(servers.url + "/api/v1/repos/alice/x/issues/1", host);
	68	broken.mockRestore();
	69	expect(boom.status).toBe(500);
	70	servers.stop();
	71	const { bodyId } = JSON.parse(
	72	readFileSync(join(dir, "spawn", "homie.json"), "utf8"),
	73	) as { bodyId: string };
	74	const child = new Database(join(childDirOf(parent, bodyId), "forge.db"), { readonly: true });
	75	const inChild = readUsage(child, "");
	76	const inParent = readUsage(parent.db, "");
	77	child.close();
	78	parent.close();
	79	expect(inChild.map((r) => [r.pattern, r.n])).toEqual([
	80	["/api/v1/repos/:owner/:name/issues/:number", 1],
	81	["/healthz", 2],
	82	]);
	83	expect(inChild.find((r) => r.status === "5xx")?.pattern).toBe(
	84	"/api/v1/repos/:owner/:name/issues/:number",
	85	);
	86	expect(inParent.map((r) => [r.pattern, r.n])).toEqual([["/readyz", 1]]);
	87	});
		@@ -0,0 +1,143 @@
	1	/** Every answered request is counted once, by pattern, role and surface (#1918). */
	2
	3	import { afterAll, beforeAll, expect, spyOn, test } from "bun:test";
	4	import { mkdtemp, rm } from "node:fs/promises";
	5	import { tmpdir } from "node:os";
	6	import { join } from "node:path";
	7	import { createApp, type App } from "../src/app.ts";
	8	import { createAgent } from "../src/actions/crew.ts";
	9	import { createToken } from "../src/core/tokens.ts";
	10	import { createUser } from "../src/core/users.ts";
	11	import { methodOf, roleOf, surfaceOf } from "../src/http/usage_tag.ts";
	12	import { readUsage } from "../src/core/usage_count.ts";
	13	import { serve, type Servers } from "../src/http/server.ts";
	14
	15	let dataDir: string;
	16	let app: App;
	17	let servers: Servers;
	18	let agentToken: string;
	19	let adminToken: string;
	20	let userToken: string;
	21
	22	beforeAll(async () => {
	23	dataDir = await mkdtemp(join(tmpdir(), "forge-usage-"));
	24	app = createApp({ dataDir, port: 0 });
	25	const root = await createUser(app.db, {
	26	trusted: true,
	27	username: "alice",
	28	email: "[email protected]",
	29	password: "a-good-passphrase",
	30	adminIfFirst: true,
	31	});
	32	if (!root.ok) throw new Error("user");
	33	const agent = await createAgent(app, root.value, {
	34	username: "helper",
	35	model: "",
	36	brief: "",
	37	});
	38	if (!agent.ok) throw new Error(String(agent.error));
	39	const bob = await createUser(app.db, {
	40	trusted: true,
	41	username: "bob",
	42	email: "[email protected]",
	43	password: "a-good-passphrase",
	44	});
	45	if (!bob.ok) throw new Error("user");
	46	adminToken = createToken(app.db, root.value.id, "t", "ro").token;
		sentinelcrew 29d ago outdated Extend this to cover `/healthz` (pattern `/healthz`), `/mcp` (surface `mcp`), a signed-in user and an admin (role), and a stranger's method. As it stands, removing the `/healthz` noteUsage or breaking `surfaceOf`'s mcp branch would still pass.
	47	userToken = createToken(app.db, bob.value.id, "t", "ro").token;
	48	agentToken = createToken(app.db, agent.value.user.id, "t", "ro").token;
	49	});
	50
	51	afterAll(async () => {
	52	app.close();
	53	await rm(dataDir, { recursive: true, force: true });
	54	});
	55
	56	test("three ids on one route are one row; an agent's API call is agent/api; strangers share one pattern", async () => {
	57	servers = await serve(app);
	58	for (const n of [1, 2, 3])
	59	await fetch(`${servers.url}/api/v1/repos/alice/none/issues/${n}`, {
	60	headers: { authorization: `Bearer ${agentToken}` },
	61	});
	62	const miss = await fetch(`${servers.url}/api/v1/nothing/${crypto.randomUUID()}`);
	63	await fetch(`${servers.url}/no/such/${crypto.randomUUID()}/path/x/y`);
	64	// Every response keeps the floor.
	65	expect(miss.headers.get("x-content-type-options")).toBe("nosniff");
	66
	67	servers.stop();
	68	const rows = readUsage(app.db, "");
	69	const issue = rows.filter(
	70	(r) => r.pattern === "/api/v1/repos/:owner/:name/issues/:number",
	71	);
	72	expect(issue).toHaveLength(1);
	73	expect(issue[0]).toMatchObject({ n: 3, role: "agent", surface: "api", method: "GET" });
	74	const unmatched = rows.filter((r) => r.pattern === "(unmatched)");
	75	expect(unmatched.reduce((sum, r) => sum + r.n, 0)).toBeGreaterThanOrEqual(2);
	76	expect(rows.some((r) => r.pattern.includes("nothing"))).toBe(false);
	77	});
	78
	79	test("health routes, the mcp surface, roles and a stranger's method are each counted, and nothing logs", async () => {
	80	servers = await serve(app);
	81	const warn = spyOn(app.log, "warn");
	82	await fetch(`${servers.url}/healthz`);
	83	await fetch(`${servers.url}/readyz`);
	84	await fetch(`${servers.url}/mcp`, { method: "POST", body: "{}" });
	85	await fetch(`${servers.url}/`);
	86	for (const [token, n] of [[adminToken, 1], [userToken, 2]] as const)
	87	for (let i = 0; i < n; i++)
	88	await fetch(`${servers.url}/api/v1/repos/alice/none/issues/1`, {
	89	headers: { authorization: `Bearer ${token}` },
	90	});
	91	await fetch(`${servers.url}/`, { method: "PROPFIND" });
	92	await fetch(`${servers.url}/x`, { method: "MKCOL" });
	93	expect(warn).not.toHaveBeenCalled();
	94	warn.mockRestore();
	95
	96	servers.stop();
	97	const rows = readUsage(app.db, "");
	98	const n = (f: (r: (typeof rows)[number]) => boolean) =>
	99	rows.filter(f).reduce((sum, r) => sum + r.n, 0);
	100	expect(n((r) => r.pattern === "/healthz" && r.role === "anon")).toBe(1);
	101	expect(n((r) => r.pattern === "/readyz")).toBe(1);
	102	expect(n((r) => r.pattern === "/mcp" && r.surface === "mcp")).toBe(1);
	103	expect(n((r) => r.pattern === "/" && r.surface === "web" && r.method === "GET")).toBe(1);
	104	expect(n((r) => r.pattern.endsWith("issues/:number") && r.role === "admin")).toBe(1);
	105	expect(n((r) => r.pattern.endsWith("issues/:number") && r.role === "user")).toBe(2);
	106	expect(n((r) => r.method === "OTHER")).toBe(2);
	107	expect(rows.some((r) => r.method === "PROPFIND" \|\| r.method === "MKCOL")).toBe(false);
	108	});
	109
	110	test("roleOf, surfaceOf and methodOf map as the table expects", () => {
	111	const who = (kind: "user" \| "agent", isAdmin: boolean) =>
	112	({ user: { kind, isAdmin }, scope: undefined, via: "token" }) as never;
	113	expect(roleOf({ user: undefined, scope: undefined, via: "none" })).toBe("anon");
	114	expect(roleOf(who("user", false))).toBe("user");
	115	expect(roleOf(who("user", true))).toBe("admin");
	116	expect(roleOf(who("agent", false))).toBe("agent");
	117	expect(surfaceOf("/mcp")).toBe("mcp");
	118	expect(surfaceOf("/api")).toBe("api");
	119	expect(surfaceOf("/api/v1/x")).toBe("api");
	120	expect(surfaceOf("/alice/repo")).toBe("web");
	121	expect(methodOf("GET")).toBe("GET");
	122	expect(methodOf("PROPFIND")).toBe("OTHER");
	123	});
	124
	125	test("a handler that throws is answered 500 and counted 5xx under its route", async () => {
	126	servers = await serve(app);
	127	const real = app.db.query.bind(app.db);
	128	const broken = spyOn(app.db, "query").mockImplementation(((sql: string) => {
	129	if (/FROM\s+repo\b/i.test(sql)) throw new Error("disk on fire");
	130	return real(sql);
	131	}) as never);
	132	const res = await fetch(`${servers.url}/api/v1/repos/alice/boom/issues/9`, {
	133	headers: { authorization: `Bearer ${adminToken}` },
	134	});
	135	broken.mockRestore();
	136	expect(res.status).toBe(500);
	137	expect(res.headers.get("x-content-type-options")).toBe("nosniff");
	138	servers.stop();
	139	const row = readUsage(app.db, "").filter((r) => r.status === "5xx");
	140	expect(row.map((r) => [r.pattern, r.role, r.n])).toEqual([
	141	["/api/v1/repos/:owner/:name/issues/:number", "admin", 1],
	142	]);
	143	});