17f1ae7 · ec56410 · 6c353a1 · cbc390e · dca5468 · 7097f34
@@ -61,7 +61,7 @@
for a tunnel and a forge that sleeps.
| `board_write` | `{ slug, board }` (person admins): write a whole board under a slug, made or replaced: `{ title, range, refresh, panels: [{ title, chart, query, w, h }] }` | yes |
| `boards` | `{ slug?, panel?, from?, to?, step? }` (person admins): without `slug`, lists the forge's boards (slug, title, range, refresh, panels); with `slug`, that board; with `panel` (place from 0, up to 23), that panel's data over the `from`/`to` window and `step` |  |
| `branches` | `{ repo, name?, action?, from?, kind? }`: without `name`, lists the branches; `kind: "tags"` lists the tags instead (as GET `/api/v1/repos/:owner/:name/tags` does, newest first; no `name` or `action`); with `name` and `action` (`create` from `from`, or `delete`) it does what the branches page does: the default branch is never deleted, only a writer may act | yes |
| `capacity` | `{ key?, value?, dryRun? }` (forge admin): the forge's capacity and guards, live (crew and checks concurrency, check shards, cell egress, the new-account guard, the sign-in edge, queue depth, child forges; only the first six are settable by `key`); with `key` and `value` (`""` restores the default), planned first (`dryRun` is true unless `false`); applying needs an admin-scope token | yes |
| `capacity` | `{ key?, value?, dryRun? }` (forge admin): the forge's capacity and guards, live (crew and checks concurrency, check shards, cell egress, the new-account guard, the sign-in edge, queue depth, child forges, `egress.npmBytesLastHour`; only the first six are settable by `key`); with `key` and `value` (`""` restores the default), planned first (`dryRun` is true unless `false`); applying needs an admin-scope token | yes |
| `claim_link` | `{ name }`: a new claim link for a child you spawned that nobody has claimed; the old one stops working. An admin's | yes |
| `close` | `{ repo, number, state?, reason?, into? }`: close an issue or a pull (`state` closed by default) or reopen it (`open`); the same rules and refusals as `PATCH …/issues/:number`: an agent closes only what the rules give it and reopens nothing; answers `state` and `closeReason` | yes |
| `cloud_account` | `{ verb, provider, name, value?, to?, allowed?, repo? }`: change a cloud account: `attach` or `rotate` a token (`value`), `verify` it, `rename` it (`to`), `remove` it, `allow_writes` (`allowed` true or false) or `audit` (`repo` owner/name); a forge admin with an admin-scope token; answers a one-line `says` | yes |
@@ -11,6 +11,8 @@
not an admin, 200 for an admin's `ro` token. It is never cached
| `forge_egress_connections` | gauge | `repo`, `destination` | Open egress connections. |
| `forge_egress_bytes_in` | gauge | `repo`, `destination` | Bytes received from the destination over open connections. |
| `forge_egress_bytes_out` | gauge | `repo`, `destination` | Bytes sent to the destination over open connections. |
| `forge_egress_received_bytes_total` | counter | `destination` | Bytes received from the destination since the process started, every app and check together. |
| `forge_egress_sent_bytes_total` | counter | `destination` | Bytes sent to the destination since the process started. |
| `forge_http_request_duration_seconds` | histogram | `pattern` (plus `le`) | Seconds taken to answer a request, by the route pattern matched (never the raw path), since the process started. Buckets are `route_timing`'s edges, 1 ms to 2.5 s, in seconds. |
| `forge_check_duration_seconds` | histogram | (plus `le`) | Seconds a repository check took from start to finish, for checks that ended since the process started. Buckets 5 s to 30 min. |
| `forge_check_queue_depth` | gauge | none | Checks queued and not yet started. |
@@ -65,3 +67,21 @@
scrape_configs:
`forge_egress_connections`, `forge_egress_bytes_in` and `forge_egress_bytes_out`
per `repo` and `destination` as Prometheus text, so a leaked upstream socket
shows as a gauge that never falls (#3719).
Beside them, `forge_egress_received_bytes_total` and `forge_egress_sent_bytes_total`
count every byte since the process started, keyed by `destination` alone (never
by run, so the series stay one per host). They show the bun cache working
(#7546): on a warm cache,
`rate(forge_egress_received_bytes_total{destination="registry.npmjs.org:443"}[1h])`
stays near zero, and a cold one is hundreds of megabytes an hour (#7541).
Without Prometheus, the same hour is read four ways:
- MCP `capacity` (forge admin) answers `egress.npmBytesLastHour` and the ten
  busiest `egress.destinations` with their totals and last hour.
- `/settings/capacity` prints it in the Egress connections card.
- The terminal's Egress connections screen (About, `c`, site admin) prints the
  same line above the open connections.
- Each check's install step in a cell logs one line, `install fetched 0 B from
  the registry` on a warm cache: what the cell's egress proxy received. A check
  outside a cell logs none: its repository's bun cache is shared, so no figure
  there would be its own.
@@ -22,6 +22,7 @@
import {
  type CapacityChange,
  type CapacityKey,
} from "../core/capacity.ts";
import { egressTotals, npmBytesLastHour, type EgressTotal } from "../apps/egress.ts";
import { passkeyIds } from "../core/passkeys.ts";
import type { User } from "../core/users.ts";
import { err, ok, type Result } from "../kernel/result.ts";
@@ -49,6 +50,24 @@
export interface CapacityRow {
  readonly changed?: { readonly at: string; readonly by: string };
}
/**
 * What the egress proxies received over the last hour (#7546), the npm
 * registry first: a warm bun cache keeps `npmBytesLastHour` near zero, so a
 * figure in the hundreds of megabytes is the cache gone cold again (#7541).
 */
export interface EgressHour {
  readonly npmBytesLastHour: number;
  /** The ten destinations that sent most this hour, with their totals since boot. */
  readonly destinations: readonly EgressTotal[];
}
export function egressHour(now = Date.now()): EgressHour {
  return {
    npmBytesLastHour: npmBytesLastHour(now),
    destinations: egressTotals(now).slice(0, 10),
  };
}
function admin(actor: User | undefined): Result<User, "not_found"> {
  return actor?.isAdmin === true && actor.kind !== "agent"
    ? ok(actor)
@@ -70,7 +89,7 @@
export function changedOf(db: Database, key: CapacityKey): { changed?: { at: str
export function listCapacity(
  app: App,
  actor: User | undefined,
): Result<{ settings: CapacityRow[]; changes: CapacityChange[] }, "not_found"> {
): Result<{ settings: CapacityRow[]; changes: CapacityChange[]; egress: EgressHour }, "not_found"> {
  const who = admin(actor);
  if (!who.ok) return who;
  return ok({
@@ -85,6 +104,7 @@
export function listCapacity(
      ...changedOf(app.db, key),
    })),
    changes: capacityChanges(app.db),
    egress: egressHour(),
  });
}
@@ -109,6 +109,121 @@
export function egressSnapshot(): EgressRow[] {
  return rows.sort((a, b) => b.open - a.open || a.label.localeCompare(b.label));
}
/**
 * Bytes moved since the forge started, per destination alone (#7546): the
 * counters `/metrics` serves, and a minute-by-minute ring of the last hour,
 * so a cold bun cache shows as registry bytes without a packet capture.
 * Keyed by `host:port` and never by run label, so it holds one entry per
 * host the apps reach, not one per run; past `MOVED_MAX` destinations the
 * rest are counted under `other`, save the registry, which always keeps its
 * own entry so the headline figure is never lost there.
 */
interface Moved {
  bytesIn: number;
  bytesOut: number;
  /** Minute `m` (ms since epoch / 60 000) lands in slot `m % 60`. */
  readonly minute: number[];
  readonly minuteIn: number[];
}
const MOVED_MAX = 256;
const HOUR_MINUTES = 60;
const moved = new Map<string, Moved>();
/** label -> bytes in, for the labels someone is metering now (`meterEgress`). */
const meters = new Map<string, { bytesIn: number; holders: number }>();
/** Count `bytesIn` received from and `bytesOut` sent to `destination`. */
export function recordEgress(
  label: string,
  destination: string,
  bytesIn: number,
  bytesOut: number,
  now = Date.now(),
): void {
  const key =
    destination === NPM_REGISTRY || moved.has(destination) || moved.size < MOVED_MAX
      ? destination
      : "other";
  let m = moved.get(key);
  if (m === undefined)
    moved.set(key, (m = { bytesIn: 0, bytesOut: 0, minute: Array(HOUR_MINUTES).fill(-1), minuteIn: Array(HOUR_MINUTES).fill(0) }));
  m.bytesIn += bytesIn;
  m.bytesOut += bytesOut;
  const minute = Math.floor(now / 60_000);
  const slot = minute % HOUR_MINUTES;
  if (m.minute[slot] !== minute) {
    m.minute[slot] = minute;
    m.minuteIn[slot] = 0;
  }
  m.minuteIn[slot] = (m.minuteIn[slot] ?? 0) + bytesIn;
  const meter = meters.get(label);
  if (meter !== undefined) meter.bytesIn += bytesIn;
}
export interface EgressTotal {
  readonly destination: string;
  /** Bytes received from it since the forge started. */
  readonly bytesIn: number;
  /** Bytes sent to it since the forge started. */
  readonly bytesOut: number;
  /** Bytes received from it in the 60 minutes up to `now`, this minute included. */
  readonly bytesInLastHour: number;
}
/** Bytes `m` received in the 60 minutes up to `minute`, that minute included. */
function lastHour(m: Moved, minute: number): number {
  let hour = 0;
  for (let i = 0; i < HOUR_MINUTES; i++) {
    const at = m.minute[i] ?? -1;
    if (at > minute - HOUR_MINUTES && at <= minute) hour += m.minuteIn[i] ?? 0;
  }
  return hour;
}
/** Every destination reached since the forge started, most received this hour first. */
export function egressTotals(now = Date.now()): EgressTotal[] {
  const minute = Math.floor(now / 60_000);
  const rows: EgressTotal[] = [];
  for (const [destination, m] of moved)
    rows.push({ destination, bytesIn: m.bytesIn, bytesOut: m.bytesOut, bytesInLastHour: lastHour(m, minute) });
  return rows.sort((a, b) => b.bytesInLastHour - a.bytesInLastHour || a.destination.localeCompare(b.destination));
}
/** The package registry every install step reaches (`apps/contain.ts` STEP_HOSTS). */
export const NPM_REGISTRY = "registry.npmjs.org:443";
/** Bytes received from the npm registry over the last hour, across every app and check. */
export function npmBytesLastHour(now = Date.now()): number {
  const m = moved.get(NPM_REGISTRY);
  return m === undefined ? 0 : lastHour(m, Math.floor(now / 60_000));
}
/**
 * Count what `label`'s proxy receives from now until `stop()`, which answers
 * the bytes in. Only labels being metered are kept, and each goes with its
 * last holder, so a run label never outlives the step that asked.
 */
export function meterEgress(label: string): { stop(): number } {
  const meter = meters.get(label) ?? { bytesIn: 0, holders: 0 };
  meters.set(label, meter);
  meter.holders++;
  const start = meter.bytesIn;
  let stopped = false;
  return {
    stop() {
      const bytes = meter.bytesIn - start;
      if (!stopped && --meter.holders === 0) meters.delete(label);
      stopped = true;
      return bytes;
    },
  };
}
/** For tests: forget every cumulative count. */
export function resetEgressTotals(): void {
  moved.clear();
}
/** The ports a declared host may be reached on when its entry names none. */
export const WEB_PORTS: readonly number[] = [80, 443];
@@ -230,6 +345,9 @@
interface Conn {
  done: boolean;
  /** The destination's counts once dialled; the data path adds to it directly. */
  tally?: Tally;
  /** Whose proxy and where to, once dialled, for the cumulative counts. */
  label: string;
  destination?: string;
  /** When bytes last moved either way (ms). */
  last: number;
  /** Frees the destination's slot and counts it closed; set once dialled. */
@@ -394,6 +512,7 @@
async function open(
  count.open++;
  client.data.tally = count;
  const label = policy.label ?? "";
  client.data.destination = destination;
  client.data.release = () => untally(label, destination);
  const ended = () => {
    if (!client.data.done) {
@@ -410,6 +529,7 @@
async function open(
          if (client.data.done) return;
          client.data.last = Date.now();
          count.bytesIn += chunk.length;
          recordEgress(label, destination, chunk.length, 0);
          send(client, client.data.toClient, chunk);
          if (queued(client.data.toClient) > (policy.queueMax ?? QUEUE_MAX)) {
            drop(client);
@@ -443,6 +563,7 @@
async function open(
    const first = concat(prefix, client.data.head.subarray(end));
    if (first.length > 0) {
      count.bytesOut += first.length;
      recordEgress(label, destination, 0, first.length);
      send(peer, client.data.toPeer, first);
    }
  } catch {
@@ -471,6 +592,7 @@
export function startEgress(
        toPeer: [],
        toClient: [],
        done: false,
        label: policy.label ?? "",
        last: Date.now(),
        release: undefined,
      };
@@ -481,6 +603,7 @@
export function startEgress(
      if (conn.peer !== undefined) {
        if (conn.done) return;
        if (conn.tally !== undefined) conn.tally.bytesOut += chunk.length;
        if (conn.destination !== undefined) recordEgress(conn.label, conn.destination, 0, chunk.length);
        send(conn.peer, conn.toPeer, chunk);
        return;
      }
@@ -1,16 +1,19 @@
/**
 * The egress tally as Prometheus text (#3719): one line per app and
 * destination for open connections and the bytes they have moved, so a leaked
 * upstream socket shows as a gauge that never falls.
 * upstream socket shows as a gauge that never falls. Beside them, counters of
 * every byte moved since the forge started, per destination alone (#7546):
 * `rate(forge_egress_received_bytes_total{destination="registry.npmjs.org:443"}[1h])`
 * is the bun cache working, or not.
 */
import type { EgressRow } from "./egress.ts";
import type { EgressRow, EgressTotal } from "./egress.ts";
/** A label value, escaped as the exposition format asks. */
function label(value: string): string {
  return value.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/\n/g, "\\n");
}
export function egressMetrics(rows: readonly EgressRow[]): string {
export function egressMetrics(rows: readonly EgressRow[], totals: readonly EgressTotal[] = []): string {
  const series: [string, string, (r: EgressRow) => number][] = [
    ["forge_egress_connections", "Open egress connections.", (r) => r.open],
    ["forge_egress_bytes_in", "Bytes received from the destination over open connections.", (r) => r.bytesIn],
@@ -22,5 +25,14 @@
export function egressMetrics(rows: readonly EgressRow[]): string {
    for (const r of rows)
      lines.push(`${name}{repo="${label(r.label)}",destination="${label(r.destination)}"} ${value(r)}`);
  }
  const counters: [string, string, (t: EgressTotal) => number][] = [
    ["forge_egress_received_bytes", "Bytes received from the destination since the forge started.", (t) => t.bytesIn],
    ["forge_egress_sent_bytes", "Bytes sent to the destination since the forge started.", (t) => t.bytesOut],
  ];
  for (const [name, help, value] of counters) {
    // A counter's family is named without `_total`; its samples carry it.
    lines.push(`# HELP ${name} ${help}`, `# TYPE ${name} counter`);
    for (const t of totals) lines.push(`${name}_total{destination="${label(t.destination)}"} ${value(t)}`);
  }
  return `${lines.join("\n")}\n`;
}
@@ -19,6 +19,7 @@
import { cpSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
import { formatBytes } from "../core/storage.ts";
import { err, ok, type Result } from "../kernel/result.ts";
/** What an install prints when the machine, not the code, failed it. */
@@ -35,6 +36,15 @@
export const LOCK_STALE_MS = 15 * 60_000;
const LOCK_POLL_MS = 50;
/**
 * The one line an install step logs about the registry (#7546): `0 B` on a
 * warm cache. A cell's figure is what its egress proxy received from the
 * registry; uncontained, nothing is logged.
 */
export function installFetchedLine(bytes: number): string {
  return `install fetched ${formatBytes(Math.max(0, bytes))} from the registry`;
}
export type InstallOutcome =
  | { readonly ok: true }
  | { readonly ok: false; readonly error: string; readonly infrastructure: boolean };
@@ -95,7 +95,15 @@
import { summariseCheckFailure } from "./summary.ts";
import type { Result } from "../kernel/result.ts";
import { placementFromEnv, type Placement } from "./placement.ts";
import { JOB_VANISHED, runJobStep } from "./job_step.ts";
import { cellInstall, checkInstallCache, installWithRetry, prefillInstallCache, withLock } from "./install.ts";
import {
  cellInstall,
  checkInstallCache,
  installFetchedLine,
  installWithRetry,
  prefillInstallCache,
  withLock,
} from "./install.ts";
import { meterEgress } from "../apps/egress.ts";
import { checkStage, type GateStage } from "./gate_reason.ts";
import { proofsInFlight } from "./proof.ts";
import { extraGates, GATE_DEADLINE_MS, gateQueued, gateRunId, gatesInFlight, watchGates } from "./queue.ts";
@@ -1071,29 +1079,41 @@
export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
              cell: revealed,
            });
          };
          const installed = await installWithRetry(
            async (log) => {
              if (shared === undefined) {
                if (warm !== undefined) {
                  const fill = await prefillInstallCache({
                    cache: warm,
                    dir,
                    env,
                    log,
                    signal,
                    timeoutMs: manifest.timeout * 1000,
                  });
                  if (!fill.ok) return fill;
                  filled = fill.value;
          // What the cell's proxy received, for the one line that shows the cache
          // working (#7546). Uncontained, nothing is logged: the shared cache
          // is not metered, and its growth is wrong under concurrent checks.
          // Stopped in `finally`, so a throw never leaves the label metered.
          const meter = cell === undefined ? undefined : meterEgress(`check:${run.id}`);
          let fetched: number | undefined;
          let installed: Awaited<ReturnType<typeof installWithRetry>>;
          try {
            installed = await installWithRetry(
              async (log) => {
                if (shared === undefined) {
                  if (warm !== undefined) {
                    const fill = await prefillInstallCache({
                      cache: warm,
                      dir,
                      env,
                      log,
                      signal,
                      timeoutMs: manifest.timeout * 1000,
                    });
                    if (!fill.ok) return fill;
                    filled = fill.value;
                  }
                  return once(log);
                }
                return once(log);
              }
              mkdirSync(shared, { recursive: true });
              const held = await withLock(`${shared}.lock`, () => once(log), { signal });
              return held.ok ? held.value : held;
            },
            { log: write, signal, retryMs: options.installRetryMs },
          );
                mkdirSync(shared, { recursive: true });
                const held = await withLock(`${shared}.lock`, () => once(log), { signal });
                return held.ok ? held.value : held;
              },
              { log: write, signal, retryMs: options.installRetryMs },
            );
          } finally {
            fetched = meter?.stop();
          }
          if (fetched !== undefined) write(installFetchedLine(fetched));
          if (!installed.ok)
            return {
              ok: false,
@@ -335,7 +335,7 @@
import { upstreamRoutes } from "./api/upstream.ts";
import type { Router } from "./router.ts";
import { clientAddress } from "./peer.ts";
import { manifestFor } from "../actions/here.ts";
import { egressSnapshot } from "../apps/egress.ts";
import { egressSnapshot, egressTotals } from "../apps/egress.ts";
import { egressMetrics } from "../apps/egress_metrics.ts";
import { forgeMetrics } from "../metrics/forge.ts";
import { appSources } from "../metrics/app_sources.ts";
@@ -3105,7 +3105,7 @@
export function apiRoutes(router: Router<Handler>): void {
  router.add("GET", "/metrics", (ctx) => {
    const denied = accountAdmin(ctx, "ro");
    if (denied) return denied;
    return new Response(egressMetrics(egressSnapshot()) + forgeMetrics(ctx.app.db, appSources(ctx.app)), {
    return new Response(egressMetrics(egressSnapshot(), egressTotals()) + forgeMetrics(ctx.app.db, appSources(ctx.app)), {
      headers: {
        "content-type": "text/plain; version=0.0.4; charset=utf-8",
        "cache-control": "no-store",
@@ -5,7 +5,7 @@
import { good, bad, type Tool } from "../shared.ts";
export const tool: Tool = {
  name: "capacity",
  doc: "`{ key?, value?, dryRun? }` (forge admin): the forge's capacity and guards, live (crew and checks concurrency, check shards, cell egress, the new-account guard, the sign-in edge, queue depth, child forges; only the first six are settable by `key`); with `key` and `value` (`\"\"` restores the default), planned first (`dryRun` is true unless `false`); applying needs an admin-scope token",
  doc: "`{ key?, value?, dryRun? }` (forge admin): the forge's capacity and guards, live (crew and checks concurrency, check shards, cell egress, the new-account guard, the sign-in edge, queue depth, child forges, `egress.npmBytesLastHour`; only the first six are settable by `key`); with `key` and `value` (`\"\"` restores the default), planned first (`dryRun` is true unless `false`); applying needs an admin-scope token",
  description:
    "Forge capacity and guards, live. With `key` and `value` (\"\" restores default), plans.",
  inputSchema: {
@@ -10,7 +10,7 @@
import {
  revokeAppEgress,
  type EgressRow,
} from "../../actions/egress_grants.ts";
import { egressSnapshot } from "../../apps/egress.ts";
import { egressSnapshot, npmBytesLastHour } from "../../apps/egress.ts";
import { bytes } from "../../system/format.ts";
import type { Repo } from "../../core/repos.ts";
import type { Ctx, Screen } from "../app/app.ts";
@@ -89,8 +89,9 @@
export function egressScreen(f: Session, ctx: Ctx, repo: Repo): Screen {
/**
 * The egress connections open right now, per app and destination: the
 * Capacity page's card, for a site admin. Read again on every draw, so it
 * moves as connections do; anyone else is offered nothing.
 * Capacity page's card, for a site admin, under the bytes the npm registry
 * sent over the last hour (#7546). Read again on every draw, so it moves as
 * connections do; anyone else is offered nothing.
 */
export function connectionsScreen(f: Session, ctx: Ctx): Screen {
  const state = new ListState();
@@ -108,7 +109,11 @@
export function connectionsScreen(f: Session, ctx: Ctx): Screen {
      const c = ctx.theme.c;
      const rows = lines();
      state.clamp(rows.length);
      const [head, list] = split(area, "v", [len(1), fill(1)]);
      const [npm, head, list] = split(area, "v", [len(1), len(1), fill(1)]);
      if (f.user?.isAdmin === true) {
        const sent = `The npm registry sent ${bytes(npmBytesLastHour())} over the last hour.`;
        s.text(npm!.x, npm!.y, truncate(sent, npm!.w), style(c.mutedFg, c.bg), npm!.x + npm!.w);
      }
      const note =
        rows.length === 0
          ? "No app holds an egress connection open."
@@ -76,7 +76,7 @@
export function capacityPage(ctx: Ctx): Response {
    listCapacity(ctx.app, ctx.auth.user),
  );
  if (!listed.ok) return notFoundPage(ctx);
  const { settings, changes } = listed.value;
  const { settings, changes, egress } = listed.value;
  const errs = timePhase(ctx.request, "errors", () =>
    ctx.app.signals.errors.recent(10),
  );
@@ -158,14 +158,19 @@
export function capacityPage(ctx: Ctx): Response {
    },
  );
  const open = egressSnapshot();
  // The bun cache at work (#7546): a warm cache keeps this near zero.
  const npm = html`The npm registry sent
    <span class="mono">${bytes(egress.npmBytesLastHour)}</span> over the last hour.`;
  const connections = card(
    open.length === 0
      ? html`<div class="card-body">
          <p class="muted small" id="egress-npm">${npm}</p>
          <p class="muted small" id="egress-open">
            No app holds an egress connection open.
          </p>
        </div>`
      : html`<ul class="list" id="egress-open">
          ${row({ id: "egress-npm", lead: icon("activity"), title: npm })}
          ${open.map((r) =>
            row({
              lead: icon("activity"),
@@ -18,7 +18,7 @@
import { addToCrew, createAgent } from "../src/actions/crew.ts";
import { assignIssue, openIssue } from "../src/actions/issues.ts";
import { commitToBranch, createRepo } from "../src/actions/repos.ts";
import { enableApp } from "../src/actions/apps.ts";
import { startEgress } from "../src/apps/egress.ts";
import { NPM_REGISTRY, recordEgress, resetEgressTotals, startEgress } from "../src/apps/egress.ts";
import { AppsRuntime } from "../src/apps/runtime.ts";
import { startChecks, type ChecksHandle } from "../src/checks/service.ts";
import {
@@ -142,6 +142,25 @@
describe("capacity: only a forge admin reads or changes it", () => {
    await rm(dataDir, { recursive: true, force: true });
  });
  test("MCP capacity and the page answer the npm registry's bytes over the last hour (#7546)", async () => {
    resetEgressTotals();
    recordEgress("check:r1", NPM_REGISTRY, 3_000_000, 4_000);
    recordEgress("check:r1", "example.com:443", 10, 0);
    const read = await mcp(aliceRo, {});
    expect(read.error).toBe(false);
    const egress = (JSON.parse(read.text) as { egress: { npmBytesLastHour: number; destinations: { destination: string }[] } }).egress;
    expect(egress.npmBytesLastHour).toBe(3_000_000);
    // Other files' proxies share this process's tally; only the order of ours is asserted.
    const named = egress.destinations.map((d) => d.destination);
    expect(named[0]).toBe(NPM_REGISTRY);
    expect(named).toContain("example.com:443");
    const cookie = await login("alice");
    const page = await (await fetch(`${servers.url}/settings/capacity`, { headers: { cookie } })).text();
    // 3,000,000 B / 1024² = 2.86 MiB.
    expect(page).toMatch(/id="egress-npm"[\s\S]*?2\.9 MiB/);
    resetEgressTotals();
  });
  test("a member, an agent and nobody are told it is not there, at every door", async () => {
    expect(listCapacity(app, bob).ok).toBe(false);
    expect(listCapacity(app, agent).ok).toBe(false);
@@ -27,6 +27,7 @@
import { INFRA_SUMMARY, listRuns, type Run } from "../src/core/runs.ts";
import { createUser } from "../src/core/users.ts";
import type { Repo } from "../src/core/repos.ts";
import { serve, type Servers } from "../src/http/server.ts";
import { readRunLog } from "../src/runs/log.ts";
import { err, ok, type Result } from "../src/kernel/result.ts";
const never = new AbortController().signal;
@@ -220,6 +221,10 @@
describe("a check whose install fails on the machine", () => {
    expect(runs[0]!.state).toBe("ok");
    // Uncontained, the repository's own cache is the one bun is pointed at.
    expect(existsSync(checkInstallCache(dataDir, flaky.id))).toBe(true);
    // Uncontained, no install line: its shared cache is not metered (#7546).
    expect(
      readRunLog(dataDir, runs[0]!.id).lines.some((l) => l.startsWith("install fetched")),
    ).toBe(false);
  });
  test("failing twice is infrastructure: queued again, not red; a second time it fails saying so", async () => {
@@ -13,7 +13,7 @@
import { commitToBranch, createRepo } from "../src/actions/repos.ts";
import { listEgressGrants } from "../src/core/egress_grants.ts";
import type { Repo } from "../src/core/repos.ts";
import { createUser, type User } from "../src/core/users.ts";
import { startEgress } from "../src/apps/egress.ts";
import { NPM_REGISTRY, recordEgress, resetEgressTotals, startEgress } from "../src/apps/egress.ts";
import { createForgeApp } from "../src/tui/forge.ts";
import { headlessTerminal } from "../src/tui/term/terminal.ts";
import { quietLogs } from "./helpers/globals.ts";
@@ -126,6 +126,8 @@
describe("egress grants in the terminal", () => {
    });
    client.write(`CONNECT 127.0.0.1:${camera.port} HTTP/1.1\r\n\r\n`);
    while (!reply.includes("200")) await Bun.sleep(10);
    resetEgressTotals();
    recordEgress("check:r", NPM_REGISTRY, 3_000_000, 0);
    const admin = boot(ada);
    const other = boot(bob);
    try {
@@ -133,11 +135,15 @@
describe("egress grants in the terminal", () => {
      await admin.press("c");
      await settleUntil(admin.tui, "app:cam/main");
      expect(admin.text()).toContain(`127.0.0.1:${camera.port} · 1 open`);
      // The registry's hour, as the Capacity page and MCP `capacity` read it (#7546).
      expect(admin.text()).toContain("The npm registry sent 2.9 MiB over the last hour.");
      await other.openAbout();
      await other.press("c");
      expect(other.text()).not.toContain("Egress connections");
      expect(other.text()).not.toContain("app:cam/main");
      expect(other.text()).not.toContain("npm registry");
    } finally {
      resetEgressTotals();
      admin.tui.stop();
      other.tui.stop();
      client.terminate();
@@ -13,3 +13,13 @@
test("every destination gets its open count and bytes, labels escaped", () => {
test("no connections leaves only the headers", () => {
  expect(egressMetrics([])).not.toContain("{");
});
test("cumulative bytes are counters keyed by destination alone (#7546)", () => {
  const text = egressMetrics([], [
    { destination: "registry.npmjs.org:443", bytesIn: 5_000, bytesOut: 70, bytesInLastHour: 5_000 },
  ]);
  expect(text).toContain("# TYPE forge_egress_received_bytes counter");
  expect(text).toContain('forge_egress_received_bytes_total{destination="registry.npmjs.org:443"} 5000');
  expect(text).toContain('forge_egress_sent_bytes_total{destination="registry.npmjs.org:443"} 70');
  expect(text).not.toContain("repo=");
});
@@ -0,0 +1,136 @@
/**
 * The egress tally keeps every byte per destination, not only what open
 * connections moved (#7546): counters that survive a closed connection, the
 * last hour minute by minute, a meter an install step reads, and a bound on
 * how many destinations it holds.
 */
import { afterEach, beforeEach, expect, test } from "bun:test";
import {
  egressSnapshot,
  egressTotals,
  meterEgress,
  NPM_REGISTRY,
  npmBytesLastHour,
  recordEgress,
  resetEgressTotals,
  startEgress,
} from "../src/apps/egress.ts";
import { installFetchedLine } from "../src/checks/install.ts";
// The tally is the process's own, and `bun test` runs many files in one
// process: a proxy test in a file before this one leaves its destinations
// behind, so each test starts from nothing, not only ends there.
beforeEach(resetEgressTotals);
afterEach(resetEgressTotals);
const T = Date.UTC(2026, 9, 27, 12, 0, 0);
const MIN = 60_000;
test("bytes add up per destination across labels", () => {
  recordEgress("check:a", NPM_REGISTRY, 1_000, 10, T);
  recordEgress("check:b", NPM_REGISTRY, 2_500, 5, T);
  expect(egressTotals(T)).toEqual([
    { destination: NPM_REGISTRY, bytesIn: 3_500, bytesOut: 15, bytesInLastHour: 3_500 },
  ]);
});
test("the last hour counts the 60 minutes up to now and drops older ones", () => {
  recordEgress("x", NPM_REGISTRY, 100, 0, T); // minute 0
  recordEgress("x", NPM_REGISTRY, 20, 0, T + 30 * MIN); // minute 30
  recordEgress("x", NPM_REGISTRY, 3, 0, T + 59 * MIN); // minute 59
  expect(npmBytesLastHour(T + 59 * MIN)).toBe(123);
  // At minute 60, minute 0 has left the window: 20 + 3.
  expect(npmBytesLastHour(T + 60 * MIN)).toBe(23);
  // Minute 60 reuses minute 0's slot and starts it from nothing.
  recordEgress("x", NPM_REGISTRY, 7, 0, T + 60 * MIN);
  expect(npmBytesLastHour(T + 60 * MIN)).toBe(30);
  expect(npmBytesLastHour(T + 200 * MIN)).toBe(0);
  // The counter never falls: 100 + 20 + 3 + 7.
  expect(egressTotals(T + 200 * MIN)[0]?.bytesIn).toBe(130);
});
test("no traffic to the registry reads 0", () => {
  recordEgress("x", "example.com:443", 9, 0, T);
  expect(npmBytesLastHour(T)).toBe(0);
});
test("destinations are bounded: past 256 the rest count as other", () => {
  for (let i = 0; i < 300; i++) recordEgress("x", `h${i}.example:443`, 1, 0, T);
  const totals = egressTotals(T);
  expect(totals.length).toBe(257);
  expect(totals.find((t) => t.destination === "other")?.bytesIn).toBe(44);
});
test("the registry keeps its own entry however many destinations came first", () => {
  for (let i = 0; i < 300; i++) recordEgress("x", `h${i}.example:443`, 1, 0, T);
  recordEgress("check:a", NPM_REGISTRY, 5_000, 0, T);
  expect(npmBytesLastHour(T)).toBe(5_000);
  expect(egressTotals(T).find((t) => t.destination === "other")?.bytesIn).toBe(44);
});
test("a meter counts its label from start to stop, and forgets the label after", () => {
  recordEgress("check:r", NPM_REGISTRY, 50, 0, T);
  const meter = meterEgress("check:r");
  recordEgress("check:r", NPM_REGISTRY, 400, 0, T);
  recordEgress("check:other", NPM_REGISTRY, 9_999, 0, T);
  expect(meter.stop()).toBe(400);
  // A second meter on the same label starts from nothing.
  const again = meterEgress("check:r");
  expect(again.stop()).toBe(0);
});
test("the install line reads 0 B on a warm cache", () => {
  expect(installFetchedLine(0)).toBe("install fetched 0 B from the registry");
  expect(installFetchedLine(12_345_678)).toBe("install fetched 12.3 MB from the registry");
  expect(installFetchedLine(-4096)).toBe("install fetched 0 B from the registry");
});
test("bytes through the proxy stay counted after the connection closes", async () => {
  const upstream = Bun.listen<undefined>({
    hostname: "127.0.0.1",
    port: 0,
    socket: {
      data(s) {
        s.end("HTTP/1.1 200 OK\r\nContent-Length: 5\r\nConnection: close\r\n\r\nhello");
      },
    },
  });
  const port = 20_000 + Math.floor(Math.random() * 20_000);
  const label = `check:totals-${port}`;
  const egress = startEgress({ tcp: port }, { hosts: [], origins: [{ host: "127.0.0.1", port: upstream.port }], label });
  const meter = meterEgress(label);
  try {
    // A raw socket: fetch goes straight to a loopback host, past any proxy.
    const answer = await new Promise<string>((resolve) => {
      let got = "";
      void Bun.connect<undefined>({
        hostname: "127.0.0.1",
        port,
        socket: {
          open(s) {
            s.write(`GET http://127.0.0.1:${upstream.port}/ HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n`);
          },
          data(_s, chunk) {
            got += new TextDecoder().decode(chunk);
          },
          close() {
            resolve(got);
          },
        },
      });
    });
    expect(answer).toEndWith("hello");
    const destination = `127.0.0.1:${upstream.port}`;
    const end = Date.now() + 2_000;
    while (Date.now() < end && egressSnapshot().some((r) => r.label === label)) await Bun.sleep(20);
    expect(egressSnapshot().some((r) => r.label === label)).toBe(false);
    const total = egressTotals().find((t) => t.destination === destination);
    expect(total?.bytesIn).toBeGreaterThan(5);
    expect(total?.bytesOut).toBeGreaterThan(0);
    expect(meter.stop()).toBe(total?.bytesIn ?? -1);
  } finally {
    egress.close();
    upstream.stop(true);
  }
});
		@@ -61,7 +61,7 @@ for a tunnel and a forge that sleeps.
61	61	\| `board_write` \| `{ slug, board }` (person admins): write a whole board under a slug, made or replaced: `{ title, range, refresh, panels: [{ title, chart, query, w, h }] }` \| yes \|
62	62	\| `boards` \| `{ slug?, panel?, from?, to?, step? }` (person admins): without `slug`, lists the forge's boards (slug, title, range, refresh, panels); with `slug`, that board; with `panel` (place from 0, up to 23), that panel's data over the `from`/`to` window and `step` \| \|
63	63	\| `branches` \| `{ repo, name?, action?, from?, kind? }`: without `name`, lists the branches; `kind: "tags"` lists the tags instead (as GET `/api/v1/repos/:owner/:name/tags` does, newest first; no `name` or `action`); with `name` and `action` (`create` from `from`, or `delete`) it does what the branches page does: the default branch is never deleted, only a writer may act \| yes \|
64		\| `capacity` \| `{ key?, value?, dryRun? }` (forge admin): the forge's capacity and guards, live (crew and checks concurrency, check shards, cell egress, the new-account guard, the sign-in edge, queue depth, child forges; only the first six are settable by `key`); with `key` and `value` (`""` restores the default), planned first (`dryRun` is true unless `false`); applying needs an admin-scope token \| yes \|
	64	\| `capacity` \| `{ key?, value?, dryRun? }` (forge admin): the forge's capacity and guards, live (crew and checks concurrency, check shards, cell egress, the new-account guard, the sign-in edge, queue depth, child forges, `egress.npmBytesLastHour`; only the first six are settable by `key`); with `key` and `value` (`""` restores the default), planned first (`dryRun` is true unless `false`); applying needs an admin-scope token \| yes \|
65	65	\| `claim_link` \| `{ name }`: a new claim link for a child you spawned that nobody has claimed; the old one stops working. An admin's \| yes \|
66	66	\| `close` \| `{ repo, number, state?, reason?, into? }`: close an issue or a pull (`state` closed by default) or reopen it (`open`); the same rules and refusals as `PATCH …/issues/:number`: an agent closes only what the rules give it and reopens nothing; answers `state` and `closeReason` \| yes \|
67	67	\| `cloud_account` \| `{ verb, provider, name, value?, to?, allowed?, repo? }`: change a cloud account: `attach` or `rotate` a token (`value`), `verify` it, `rename` it (`to`), `remove` it, `allow_writes` (`allowed` true or false) or `audit` (`repo` owner/name); a forge admin with an admin-scope token; answers a one-line `says` \| yes \|
		@@ -11,6 +11,8 @@ not an admin, 200 for an admin's `ro` token. It is never cached
11	11	\| `forge_egress_connections` \| gauge \| `repo`, `destination` \| Open egress connections. \|
12	12	\| `forge_egress_bytes_in` \| gauge \| `repo`, `destination` \| Bytes received from the destination over open connections. \|
13	13	\| `forge_egress_bytes_out` \| gauge \| `repo`, `destination` \| Bytes sent to the destination over open connections. \|
	14	\| `forge_egress_received_bytes_total` \| counter \| `destination` \| Bytes received from the destination since the process started, every app and check together. \|
	15	\| `forge_egress_sent_bytes_total` \| counter \| `destination` \| Bytes sent to the destination since the process started. \|
14	16	\| `forge_http_request_duration_seconds` \| histogram \| `pattern` (plus `le`) \| Seconds taken to answer a request, by the route pattern matched (never the raw path), since the process started. Buckets are `route_timing`'s edges, 1 ms to 2.5 s, in seconds. \|
15	17	\| `forge_check_duration_seconds` \| histogram \| (plus `le`) \| Seconds a repository check took from start to finish, for checks that ended since the process started. Buckets 5 s to 30 min. \|
16	18	\| `forge_check_queue_depth` \| gauge \| none \| Checks queued and not yet started. \|
		@@ -65,3 +67,21 @@ scrape_configs:
65	67	`forge_egress_connections`, `forge_egress_bytes_in` and `forge_egress_bytes_out`
66	68	per `repo` and `destination` as Prometheus text, so a leaked upstream socket
67	69	shows as a gauge that never falls (#3719).
	70
	71	Beside them, `forge_egress_received_bytes_total` and `forge_egress_sent_bytes_total`
	72	count every byte since the process started, keyed by `destination` alone (never
	73	by run, so the series stay one per host). They show the bun cache working
	74	(#7546): on a warm cache,
	75	`rate(forge_egress_received_bytes_total{destination="registry.npmjs.org:443"}[1h])`
	76	stays near zero, and a cold one is hundreds of megabytes an hour (#7541).
	77	Without Prometheus, the same hour is read four ways:
	78
	79	- MCP `capacity` (forge admin) answers `egress.npmBytesLastHour` and the ten
	80	busiest `egress.destinations` with their totals and last hour.
	81	- `/settings/capacity` prints it in the Egress connections card.
	82	- The terminal's Egress connections screen (About, `c`, site admin) prints the
	83	same line above the open connections.
	84	- Each check's install step in a cell logs one line, `install fetched 0 B from
	85	the registry` on a warm cache: what the cell's egress proxy received. A check
	86	outside a cell logs none: its repository's bun cache is shared, so no figure
	87	there would be its own.
		@@ -22,6 +22,7 @@ import {
22	22	type CapacityChange,
23	23	type CapacityKey,
24	24	} from "../core/capacity.ts";
	25	import { egressTotals, npmBytesLastHour, type EgressTotal } from "../apps/egress.ts";
25	26	import { passkeyIds } from "../core/passkeys.ts";
26	27	import type { User } from "../core/users.ts";
27	28	import { err, ok, type Result } from "../kernel/result.ts";
		@@ -49,6 +50,24 @@ export interface CapacityRow {
49	50	readonly changed?: { readonly at: string; readonly by: string };
50	51	}
51	52
	53	/**
	54	* What the egress proxies received over the last hour (#7546), the npm
	55	* registry first: a warm bun cache keeps `npmBytesLastHour` near zero, so a
	56	* figure in the hundreds of megabytes is the cache gone cold again (#7541).
	57	*/
	58	export interface EgressHour {
	59	readonly npmBytesLastHour: number;
	60	/** The ten destinations that sent most this hour, with their totals since boot. */
	61	readonly destinations: readonly EgressTotal[];
	62	}
	63
	64	export function egressHour(now = Date.now()): EgressHour {
	65	return {
	66	npmBytesLastHour: npmBytesLastHour(now),
	67	destinations: egressTotals(now).slice(0, 10),
	68	};
	69	}
	70
52	71	function admin(actor: User \| undefined): Result<User, "not_found"> {
53	72	return actor?.isAdmin === true && actor.kind !== "agent"
54	73	? ok(actor)
		@@ -70,7 +89,7 @@ export function changedOf(db: Database, key: CapacityKey): { changed?: { at: str
70	89	export function listCapacity(
71	90	app: App,
72	91	actor: User \| undefined,
73		): Result<{ settings: CapacityRow[]; changes: CapacityChange[] }, "not_found"> {
	92	): Result<{ settings: CapacityRow[]; changes: CapacityChange[]; egress: EgressHour }, "not_found"> {
74	93	const who = admin(actor);
75	94	if (!who.ok) return who;
76	95	return ok({
		@@ -85,6 +104,7 @@ export function listCapacity(
85	104	...changedOf(app.db, key),
86	105	})),
87	106	changes: capacityChanges(app.db),
	107	egress: egressHour(),
88	108	});
89	109	}
90	110
		@@ -109,6 +109,121 @@ export function egressSnapshot(): EgressRow[] {
109	109	return rows.sort((a, b) => b.open - a.open \|\| a.label.localeCompare(b.label));
110	110	}
111	111
	112	/**
	113	* Bytes moved since the forge started, per destination alone (#7546): the
	114	* counters `/metrics` serves, and a minute-by-minute ring of the last hour,
	115	* so a cold bun cache shows as registry bytes without a packet capture.
	116	* Keyed by `host:port` and never by run label, so it holds one entry per
	117	* host the apps reach, not one per run; past `MOVED_MAX` destinations the
	118	* rest are counted under `other`, save the registry, which always keeps its
	119	* own entry so the headline figure is never lost there.
	120	*/
	121	interface Moved {
	122	bytesIn: number;
	123	bytesOut: number;
	124	/** Minute `m` (ms since epoch / 60 000) lands in slot `m % 60`. */
	125	readonly minute: number[];
	126	readonly minuteIn: number[];
	127	}
	128
	129	const MOVED_MAX = 256;
	130	const HOUR_MINUTES = 60;
	131	const moved = new Map<string, Moved>();
	132	/** label -> bytes in, for the labels someone is metering now (`meterEgress`). */
	133	const meters = new Map<string, { bytesIn: number; holders: number }>();
	134
	135	/** Count `bytesIn` received from and `bytesOut` sent to `destination`. */
	136	export function recordEgress(
	137	label: string,
	138	destination: string,
	139	bytesIn: number,
	140	bytesOut: number,
	141	now = Date.now(),
	142	): void {
	143	const key =
	144	destination === NPM_REGISTRY \|\| moved.has(destination) \|\| moved.size < MOVED_MAX
	145	? destination
	146	: "other";
	147	let m = moved.get(key);
	148	if (m === undefined)
	149	moved.set(key, (m = { bytesIn: 0, bytesOut: 0, minute: Array(HOUR_MINUTES).fill(-1), minuteIn: Array(HOUR_MINUTES).fill(0) }));
	150	m.bytesIn += bytesIn;
	151	m.bytesOut += bytesOut;
	152	const minute = Math.floor(now / 60_000);
	153	const slot = minute % HOUR_MINUTES;
	154	if (m.minute[slot] !== minute) {
	155	m.minute[slot] = minute;
	156	m.minuteIn[slot] = 0;
	157	}
	158	m.minuteIn[slot] = (m.minuteIn[slot] ?? 0) + bytesIn;
	159	const meter = meters.get(label);
	160	if (meter !== undefined) meter.bytesIn += bytesIn;
	161	}
	162
	163	export interface EgressTotal {
	164	readonly destination: string;
	165	/** Bytes received from it since the forge started. */
	166	readonly bytesIn: number;
	167	/** Bytes sent to it since the forge started. */
	168	readonly bytesOut: number;
	169	/** Bytes received from it in the 60 minutes up to `now`, this minute included. */
	170	readonly bytesInLastHour: number;
	171	}
	172
	173	/** Bytes `m` received in the 60 minutes up to `minute`, that minute included. */
	174	function lastHour(m: Moved, minute: number): number {
	175	let hour = 0;
	176	for (let i = 0; i < HOUR_MINUTES; i++) {
	177	const at = m.minute[i] ?? -1;
	178	if (at > minute - HOUR_MINUTES && at <= minute) hour += m.minuteIn[i] ?? 0;
	179	}
	180	return hour;
	181	}
	182
	183	/** Every destination reached since the forge started, most received this hour first. */
	184	export function egressTotals(now = Date.now()): EgressTotal[] {
	185	const minute = Math.floor(now / 60_000);
	186	const rows: EgressTotal[] = [];
	187	for (const [destination, m] of moved)
	188	rows.push({ destination, bytesIn: m.bytesIn, bytesOut: m.bytesOut, bytesInLastHour: lastHour(m, minute) });
	189	return rows.sort((a, b) => b.bytesInLastHour - a.bytesInLastHour \|\| a.destination.localeCompare(b.destination));
	190	}
	191
	192	/** The package registry every install step reaches (`apps/contain.ts` STEP_HOSTS). */
	193	export const NPM_REGISTRY = "registry.npmjs.org:443";
	194
	195	/** Bytes received from the npm registry over the last hour, across every app and check. */
	196	export function npmBytesLastHour(now = Date.now()): number {
	197	const m = moved.get(NPM_REGISTRY);
	198	return m === undefined ? 0 : lastHour(m, Math.floor(now / 60_000));
	199	}
	200
	201	/**
	202	* Count what `label`'s proxy receives from now until `stop()`, which answers
	203	* the bytes in. Only labels being metered are kept, and each goes with its
	204	* last holder, so a run label never outlives the step that asked.
	205	*/
	206	export function meterEgress(label: string): { stop(): number } {
	207	const meter = meters.get(label) ?? { bytesIn: 0, holders: 0 };
	208	meters.set(label, meter);
	209	meter.holders++;
	210	const start = meter.bytesIn;
	211	let stopped = false;
	212	return {
	213	stop() {
	214	const bytes = meter.bytesIn - start;
	215	if (!stopped && --meter.holders === 0) meters.delete(label);
	216	stopped = true;
	217	return bytes;
	218	},
	219	};
	220	}
	221
	222	/** For tests: forget every cumulative count. */
	223	export function resetEgressTotals(): void {
	224	moved.clear();
	225	}
	226
112	227	/** The ports a declared host may be reached on when its entry names none. */
113	228	export const WEB_PORTS: readonly number[] = [80, 443];
114	229
		@@ -230,6 +345,9 @@ interface Conn {
230	345	done: boolean;
231	346	/** The destination's counts once dialled; the data path adds to it directly. */
232	347	tally?: Tally;
	348	/** Whose proxy and where to, once dialled, for the cumulative counts. */
	349	label: string;
	350	destination?: string;
233	351	/** When bytes last moved either way (ms). */
234	352	last: number;
235	353	/** Frees the destination's slot and counts it closed; set once dialled. */
		@@ -394,6 +512,7 @@ async function open(
394	512	count.open++;
395	513	client.data.tally = count;
396	514	const label = policy.label ?? "";
	515	client.data.destination = destination;
397	516	client.data.release = () => untally(label, destination);
398	517	const ended = () => {
399	518	if (!client.data.done) {
		@@ -410,6 +529,7 @@ async function open(
410	529	if (client.data.done) return;
411	530	client.data.last = Date.now();
412	531	count.bytesIn += chunk.length;
	532	recordEgress(label, destination, chunk.length, 0);
413	533	send(client, client.data.toClient, chunk);
414	534	if (queued(client.data.toClient) > (policy.queueMax ?? QUEUE_MAX)) {
415	535	drop(client);
		@@ -443,6 +563,7 @@ async function open(
443	563	const first = concat(prefix, client.data.head.subarray(end));
444	564	if (first.length > 0) {
445	565	count.bytesOut += first.length;
	566	recordEgress(label, destination, 0, first.length);
446	567	send(peer, client.data.toPeer, first);
447	568	}
448	569	} catch {
		@@ -471,6 +592,7 @@ export function startEgress(
471	592	toPeer: [],
472	593	toClient: [],
473	594	done: false,
	595	label: policy.label ?? "",
474	596	last: Date.now(),
475	597	release: undefined,
476	598	};
		@@ -481,6 +603,7 @@ export function startEgress(
481	603	if (conn.peer !== undefined) {
482	604	if (conn.done) return;
483	605	if (conn.tally !== undefined) conn.tally.bytesOut += chunk.length;
	606	if (conn.destination !== undefined) recordEgress(conn.label, conn.destination, 0, chunk.length);
484	607	send(conn.peer, conn.toPeer, chunk);
485	608	return;
486	609	}
		@@ -1,16 +1,19 @@
1	1	/**
2	2	* The egress tally as Prometheus text (#3719): one line per app and
3	3	* destination for open connections and the bytes they have moved, so a leaked
4		* upstream socket shows as a gauge that never falls.
	4	* upstream socket shows as a gauge that never falls. Beside them, counters of
	5	* every byte moved since the forge started, per destination alone (#7546):
	6	* `rate(forge_egress_received_bytes_total{destination="registry.npmjs.org:443"}[1h])`
	7	* is the bun cache working, or not.
5	8	*/
6		import type { EgressRow } from "./egress.ts";
	9	import type { EgressRow, EgressTotal } from "./egress.ts";
7	10
8	11	/** A label value, escaped as the exposition format asks. */
9	12	function label(value: string): string {
10	13	return value.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/\n/g, "\\n");
11	14	}
12	15
13		export function egressMetrics(rows: readonly EgressRow[]): string {
	16	export function egressMetrics(rows: readonly EgressRow[], totals: readonly EgressTotal[] = []): string {
14	17	const series: [string, string, (r: EgressRow) => number][] = [
15	18	["forge_egress_connections", "Open egress connections.", (r) => r.open],
16	19	["forge_egress_bytes_in", "Bytes received from the destination over open connections.", (r) => r.bytesIn],
		@@ -22,5 +25,14 @@ export function egressMetrics(rows: readonly EgressRow[]): string {
22	25	for (const r of rows)
23	26	lines.push(`${name}{repo="${label(r.label)}",destination="${label(r.destination)}"} ${value(r)}`);
24	27	}
	28	const counters: [string, string, (t: EgressTotal) => number][] = [
	29	["forge_egress_received_bytes", "Bytes received from the destination since the forge started.", (t) => t.bytesIn],
	30	["forge_egress_sent_bytes", "Bytes sent to the destination since the forge started.", (t) => t.bytesOut],
	31	];
	32	for (const [name, help, value] of counters) {
	33	// A counter's family is named without `_total`; its samples carry it.
	34	lines.push(`# HELP ${name} ${help}`, `# TYPE ${name} counter`);
	35	for (const t of totals) lines.push(`${name}_total{destination="${label(t.destination)}"} ${value(t)}`);
	36	}
25	37	return `${lines.join("\n")}\n`;
26	38	}
		@@ -19,6 +19,7 @@
19	19	import { cpSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
20	20	import { tmpdir } from "node:os";
21	21	import { dirname, join } from "node:path";
	22	import { formatBytes } from "../core/storage.ts";
22	23	import { err, ok, type Result } from "../kernel/result.ts";
23	24
24	25	/** What an install prints when the machine, not the code, failed it. */
		@@ -35,6 +36,15 @@ export const LOCK_STALE_MS = 15 * 60_000;
35	36
36	37	const LOCK_POLL_MS = 50;
37	38
	39	/**
	40	* The one line an install step logs about the registry (#7546): `0 B` on a
	41	* warm cache. A cell's figure is what its egress proxy received from the
	42	* registry; uncontained, nothing is logged.
	43	*/
	44	export function installFetchedLine(bytes: number): string {
	45	return `install fetched ${formatBytes(Math.max(0, bytes))} from the registry`;
	46	}
	47
38	48	export type InstallOutcome =
39	49	\| { readonly ok: true }
40	50	\| { readonly ok: false; readonly error: string; readonly infrastructure: boolean };
		@@ -95,7 +95,15 @@ import { summariseCheckFailure } from "./summary.ts";
95	95	import type { Result } from "../kernel/result.ts";
96	96	import { placementFromEnv, type Placement } from "./placement.ts";
97	97	import { JOB_VANISHED, runJobStep } from "./job_step.ts";
98		import { cellInstall, checkInstallCache, installWithRetry, prefillInstallCache, withLock } from "./install.ts";
	98	import {
	99	cellInstall,
	100	checkInstallCache,
	101	installFetchedLine,
	102	installWithRetry,
	103	prefillInstallCache,
	104	withLock,
	105	} from "./install.ts";
	106	import { meterEgress } from "../apps/egress.ts";
99	107	import { checkStage, type GateStage } from "./gate_reason.ts";
100	108	import { proofsInFlight } from "./proof.ts";
101	109	import { extraGates, GATE_DEADLINE_MS, gateQueued, gateRunId, gatesInFlight, watchGates } from "./queue.ts";
		@@ -1071,29 +1079,41 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
1071	1079	cell: revealed,
1072	1080	});
1073	1081	};
1074		const installed = await installWithRetry(
1075		async (log) => {
1076		if (shared === undefined) {
1077		if (warm !== undefined) {
1078		const fill = await prefillInstallCache({
1079		cache: warm,
1080		dir,
1081		env,
1082		log,
1083		signal,
1084		timeoutMs: manifest.timeout * 1000,
1085		});
1086		if (!fill.ok) return fill;
1087		filled = fill.value;
	1082	// What the cell's proxy received, for the one line that shows the cache
	1083	// working (#7546). Uncontained, nothing is logged: the shared cache
	1084	// is not metered, and its growth is wrong under concurrent checks.
	1085	// Stopped in `finally`, so a throw never leaves the label metered.
	1086	const meter = cell === undefined ? undefined : meterEgress(`check:${run.id}`);
	1087	let fetched: number \| undefined;
	1088	let installed: Awaited<ReturnType<typeof installWithRetry>>;
	1089	try {
	1090	installed = await installWithRetry(
	1091	async (log) => {
	1092	if (shared === undefined) {
	1093	if (warm !== undefined) {
	1094	const fill = await prefillInstallCache({
	1095	cache: warm,
	1096	dir,
	1097	env,
	1098	log,
	1099	signal,
	1100	timeoutMs: manifest.timeout * 1000,
	1101	});
	1102	if (!fill.ok) return fill;
	1103	filled = fill.value;
	1104	}
	1105	return once(log);
1088	1106	}
1089		return once(log);
1090		}
1091		mkdirSync(shared, { recursive: true });
1092		const held = await withLock(`${shared}.lock`, () => once(log), { signal });
1093		return held.ok ? held.value : held;
1094		},
1095		{ log: write, signal, retryMs: options.installRetryMs },
1096		);
	1107	mkdirSync(shared, { recursive: true });
	1108	const held = await withLock(`${shared}.lock`, () => once(log), { signal });
	1109	return held.ok ? held.value : held;
	1110	},
	1111	{ log: write, signal, retryMs: options.installRetryMs },
	1112	);
	1113	} finally {
	1114	fetched = meter?.stop();
	1115	}
	1116	if (fetched !== undefined) write(installFetchedLine(fetched));
1097	1117	if (!installed.ok)
1098	1118	return {
1099	1119	ok: false,
		@@ -335,7 +335,7 @@ import { upstreamRoutes } from "./api/upstream.ts";
335	335	import type { Router } from "./router.ts";
336	336	import { clientAddress } from "./peer.ts";
337	337	import { manifestFor } from "../actions/here.ts";
338		import { egressSnapshot } from "../apps/egress.ts";
	338	import { egressSnapshot, egressTotals } from "../apps/egress.ts";
339	339	import { egressMetrics } from "../apps/egress_metrics.ts";
340	340	import { forgeMetrics } from "../metrics/forge.ts";
341	341	import { appSources } from "../metrics/app_sources.ts";
		@@ -3105,7 +3105,7 @@ export function apiRoutes(router: Router<Handler>): void {
3105	3105	router.add("GET", "/metrics", (ctx) => {
3106	3106	const denied = accountAdmin(ctx, "ro");
3107	3107	if (denied) return denied;
3108		return new Response(egressMetrics(egressSnapshot()) + forgeMetrics(ctx.app.db, appSources(ctx.app)), {
	3108	return new Response(egressMetrics(egressSnapshot(), egressTotals()) + forgeMetrics(ctx.app.db, appSources(ctx.app)), {
3109	3109	headers: {
3110	3110	"content-type": "text/plain; version=0.0.4; charset=utf-8",
3111	3111	"cache-control": "no-store",
		@@ -5,7 +5,7 @@ import { good, bad, type Tool } from "../shared.ts";
5	5
6	6	export const tool: Tool = {
7	7	name: "capacity",
8		doc: "`{ key?, value?, dryRun? }` (forge admin): the forge's capacity and guards, live (crew and checks concurrency, check shards, cell egress, the new-account guard, the sign-in edge, queue depth, child forges; only the first six are settable by `key`); with `key` and `value` (`\"\"` restores the default), planned first (`dryRun` is true unless `false`); applying needs an admin-scope token",
	8	doc: "`{ key?, value?, dryRun? }` (forge admin): the forge's capacity and guards, live (crew and checks concurrency, check shards, cell egress, the new-account guard, the sign-in edge, queue depth, child forges, `egress.npmBytesLastHour`; only the first six are settable by `key`); with `key` and `value` (`\"\"` restores the default), planned first (`dryRun` is true unless `false`); applying needs an admin-scope token",
9	9	description:
10	10	"Forge capacity and guards, live. With `key` and `value` (\"\" restores default), plans.",
11	11	inputSchema: {
		@@ -10,7 +10,7 @@ import {
10	10	revokeAppEgress,
11	11	type EgressRow,
12	12	} from "../../actions/egress_grants.ts";
13		import { egressSnapshot } from "../../apps/egress.ts";
	13	import { egressSnapshot, npmBytesLastHour } from "../../apps/egress.ts";
14	14	import { bytes } from "../../system/format.ts";
15	15	import type { Repo } from "../../core/repos.ts";
16	16	import type { Ctx, Screen } from "../app/app.ts";
		@@ -89,8 +89,9 @@ export function egressScreen(f: Session, ctx: Ctx, repo: Repo): Screen {
89	89
90	90	/**
91	91	* The egress connections open right now, per app and destination: the
92		* Capacity page's card, for a site admin. Read again on every draw, so it
93		* moves as connections do; anyone else is offered nothing.
	92	* Capacity page's card, for a site admin, under the bytes the npm registry
	93	* sent over the last hour (#7546). Read again on every draw, so it moves as
	94	* connections do; anyone else is offered nothing.
94	95	*/
95	96	export function connectionsScreen(f: Session, ctx: Ctx): Screen {
96	97	const state = new ListState();
		@@ -108,7 +109,11 @@ export function connectionsScreen(f: Session, ctx: Ctx): Screen {
108	109	const c = ctx.theme.c;
109	110	const rows = lines();
110	111	state.clamp(rows.length);
111		const [head, list] = split(area, "v", [len(1), fill(1)]);
	112	const [npm, head, list] = split(area, "v", [len(1), len(1), fill(1)]);
	113	if (f.user?.isAdmin === true) {
	114	const sent = `The npm registry sent ${bytes(npmBytesLastHour())} over the last hour.`;
	115	s.text(npm!.x, npm!.y, truncate(sent, npm!.w), style(c.mutedFg, c.bg), npm!.x + npm!.w);
	116	}
112	117	const note =
113	118	rows.length === 0
114	119	? "No app holds an egress connection open."