2c45c43 · db76414
@@ -1007,6 +1007,7 @@
work here.
  commit history) with README rendering and **syntax highlighting**, protected
  default branches (force-push and deletion refused, honest fast-forwards
  allowed).
- **System memory** — `/system`, for a forge admin who is a person (anyone else gets 403): the forge's RSS beside its JS heap (`heapStats()` from bun:jsc), hourly mean and max for the last 24 h, drawn on the server. Sampled once a minute into `memory_hour` in `app.db` (a week kept). RSS rising while the heap stays flat is native memory growth.  In the terminal the same admin has a **System memory** command in the palette: the last 24 hours as text rows (hour, RSS mean/max, heap mean/max in MiB), read-only. The account menu links `/system` for that admin.
- **Boards** — `/boards/data`, for a forge admin who is a person (anyone else
  gets a 404 on the page): the query is the URL (`?source=runs&aggregate=count&by=day&fcolumn=state&fvalue=failed`),
  drawn server-side as the chart chosen beside it (`chart=line|bar|area|number|table|heat`)
@@ -0,0 +1,78 @@
/**
 * The forge's own memory against its JS heap, hour by hour (#2151): every
 * minute `process.memoryUsage().rss` and `heapStats().heapSize` (bun:jsc)
 * are folded into the hour they fall in (`memory_hour` in `app.db`, under
 * the data directory), as a count, a sum and a max. RSS rising while the
 * heap stays flat is native memory growth. A week of hours is kept.
 */
import type { Database } from "bun:sqlite";
import { heapStats } from "bun:jsc";
import type { App } from "../app.ts";
export const MEMORY_EVERY_MS = 60_000;
export const HOUR_MS = 3_600_000;
const KEEP_MS = 7 * 24 * HOUR_MS;
export type MemoryHour = {
  /** Epoch ms of the hour's start. */
  at: number;
  samples: number;
  rssMean: number;
  rssMax: number;
  heapMean: number;
  heapMax: number;
};
const valid = (n: number) => Number.isFinite(n) && n >= 0;
/** Fold one reading into its hour and drop hours past the week. Ignores a reading that is not a count of bytes. */
export function recordMemory(db: Database, rss: number, heap: number, now: number): void {
  if (!valid(rss) || !valid(heap) || !valid(now)) return;
  const hour = Math.floor(now / HOUR_MS) * HOUR_MS;
  db.transaction(() => {
    db.run(
      `INSERT INTO memory_hour (hour, samples, rss_sum, rss_max, heap_sum, heap_max) VALUES (?, 1, ?, ?, ?, ?)
       ON CONFLICT(hour) DO UPDATE SET samples = samples + 1, rss_sum = rss_sum + excluded.rss_sum,
         rss_max = max(rss_max, excluded.rss_max), heap_sum = heap_sum + excluded.heap_sum,
         heap_max = max(heap_max, excluded.heap_max)`,
      [hour, rss, rss, heap, heap],
    );
    db.run("DELETE FROM memory_hour WHERE hour < ?", [hour - KEEP_MS]);
  })();
}
/** The hours in the `hours` before `now` (the current one included), oldest first. */
export function memoryHours(db: Database, now: number, hours = 24): MemoryHour[] {
  const last = Math.floor(now / HOUR_MS) * HOUR_MS;
  const rows = db
    .query<{ hour: number; samples: number; rss_sum: number; rss_max: number; heap_sum: number; heap_max: number }, [number, number]>(
      "SELECT hour, samples, rss_sum, rss_max, heap_sum, heap_max FROM memory_hour WHERE hour > ? AND hour <= ? ORDER BY hour",
    )
    .all(last - hours * HOUR_MS, last);
  return rows.map((r) => ({
    at: r.hour, samples: r.samples,
    rssMean: r.rss_sum / r.samples, rssMax: r.rss_max,
    heapMean: r.heap_sum / r.samples, heapMax: r.heap_max,
  }));
}
export type MemorySampler = { tick(now?: number): void; stop(): void };
/** Sample now and every minute until stopped. A failed tick is logged and the next one tries again. */
export function startMemorySampler(app: App, everyMs = MEMORY_EVERY_MS): MemorySampler {
  const tick = (now = Date.now()) => {
    try {
      recordMemory(app.db, process.memoryUsage().rss, heapStats().heapSize, now);
    } catch (error) {
      app.log.warn("memory sample failed", { error: String(error) });
    }
  };
  tick();
  const timer = setInterval(tick, everyMs);
  timer.unref?.();
  return { tick, stop: () => clearInterval(timer) };
}
/** Who may read the series: a forge admin who is a person, never an agent. The page, the nav and the terminal all ask this. */
export const canSeeMemory = (user: { isAdmin: boolean; kind?: string } | null | undefined): boolean =>
  user?.isAdmin === true && user.kind !== "agent";
@@ -0,0 +1,14 @@
// The forge's own memory, one row an hour (#2151, boards/memory.ts): the
// resident set (RSS) beside the JS heap, so RSS climbing while the heap
// stays flat shows as a native leak. A running sum and count give the mean;
// telemetry, not state — a lost row is a gap in a chart.
export default `
  CREATE TABLE memory_hour (
    hour       INTEGER PRIMARY KEY,
    samples    INTEGER NOT NULL,
    rss_sum    REAL NOT NULL,
    rss_max    REAL NOT NULL,
    heap_sum   REAL NOT NULL,
    heap_max   REAL NOT NULL
  );
  `;
@@ -15,6 +15,7 @@
import { flagsPage, flagsSet } from "../web/pages/flags.ts";
import { domainsPage } from "../web/pages/domains.ts";
import { resiliencePage } from "../web/pages/resilience.ts";
import { limitsPage, limitsSet, quotasSet } from "../web/pages/admin-limits.ts";
import { systemPage } from "../web/pages/system.ts";
import { boardsPage } from "../web/pages/boards.ts";
import { boardNew, boardPin, boardPost } from "../web/pages/boards-edit.ts";
import { boardLive, boardShow, boardsList } from "../web/pages/boards-view.ts";
@@ -744,6 +745,7 @@
export function buildRouter(): Router<Handler> {
  router.add("GET", "/settings/resilience", domainsPage);
  router.add("GET", "/settings/capacity", capacityPage);
  router.add("POST", "/settings/capacity", capacitySet);
  router.add("GET", "/system", systemPage);
      
        sentinelcrew
        29d ago
        
      
      Nothing in the UI links to this route. Add a nav or boards link, visible only to an admin who passes the gate, so a person can reach /system from the page.
    
  router.add("GET", "/boards", boardsList);
  router.add("GET", "/boards/data", boardsPage);
  router.add("POST", "/boards/new", boardNew);
@@ -4,6 +4,7 @@
 * environment-only — a handful of variables, all with sane defaults.
 */
import { startMemorySampler } from "./boards/memory.ts";
import { startSampler } from "./boards/sampler.ts";
import { sweepStanding } from "./crew/standing.ts";
import { drainOnHandover, handover } from "./release/swap.ts";
@@ -338,6 +339,7 @@
const snapshots = startSnapshots(
// What the machine uses (boards/sampler.ts): the pod, each cell and the
// node, every 15 s into a ring the `machine` board source reads.
const sampler = startSampler(app);
const memorySampler = startMemorySampler(app);
const crewHeldOff = crewOffInPreview();
if (appsOn && crewHeldOff)
  app.log.info(CREW_OFF_IN_PREVIEW, {
@@ -542,6 +544,7 @@
function drain(signal: NodeJS.Signals): Promise<unknown> {
    probes?.stop(),
    snapshots.stop(),
    sampler.stop(),
    memorySampler.stop(),
  ])
    .catch(() => {})
    .then(() => {
@@ -28,7 +28,9 @@
import {
  openOnWeb,
  runAsync,
} from "./screens/common.ts";
import { canSeeMemory } from "../boards/memory.ts";
import { boardsScreen } from "./screens/boards.ts";
import { systemScreen } from "./screens/system.ts";
import { askedScreen } from "./screens/asked.ts";
import { deciderScreen } from "./screens/decider.ts";
import { familyScreen } from "./screens/family.ts";
@@ -230,6 +232,14 @@
export function forgeCommands(session: Session): (ctx: Ctx) => Command[] {
        hint: "the forge's boards and their panels",
        run: () => ctx.push(boardsScreen(session, ctx)),
      });
    // The forge's RSS beside its JS heap, as /system gates it (#2151).
    if (canSeeMemory(session.user))
      commands.push({
        id: "system",
        title: "System memory",
        hint: "RSS beside the JS heap, hourly for 24 h",
        run: () => ctx.push(systemScreen(session, ctx)),
      });
    // The pipeline's bottleneck and throughput, as /flow says them: a site admin's (#1554, #1786).
    if (session.user?.isAdmin === true)
      commands.push({
@@ -0,0 +1,55 @@
/**
 * `/system` in the terminal (#2151): the forge's RSS beside its JS heap, an
 * hour a row for the last 24 h, read through `memoryHours` like the page. The
 * gate is the page's (`canSeeMemory`): anyone but a forge admin who is a
 * person gets no rows, and the palette offers the door only to that person.
 * Read-only.
 */
import { canSeeMemory, memoryHours } from "../../boards/memory.ts";
import type { Ctx, Screen } from "../app/app.ts";
import { binding } from "../app/keys.ts";
import type { Session } from "../session.ts";
import { style } from "../term/style.ts";
import { pad } from "../term/width.ts";
import { Scroll, drawLines, hitScroll } from "./common.ts";
const mib = (n: number) => `${(n / 1_048_576).toFixed(1)}`;
/** The header, then an hour a row (UTC); empty for anyone the gate refuses. */
export function memoryLines(session: Session, now = Date.now()): string[] {
  if (!canSeeMemory(session.user)) return [];
  const hours = memoryHours(session.db, now);
  if (hours.length === 0) return ["No samples yet; the forge samples its memory every minute."];
  const col = (s: string) => pad(s, 10, "right");
  return [
    `${pad("Hour (UTC)", 17)}${col("RSS mean")}${col("RSS max")}${col("Heap mean")}${col("Heap max")}  MiB`,
    ...hours.map(
      (h) =>
        `${pad(new Date(h.at).toISOString().slice(0, 16).replace("T", " "), 17)}${col(mib(h.rssMean))}${col(mib(h.rssMax))}${col(mib(h.heapMean))}${col(mib(h.heapMax))}`,
    ),
  ];
}
export function systemScreen(session: Session, ctx: Ctx): Screen {
  const scroll = new Scroll();
  return {
    title: "System memory",
    keys: [binding(["j", "k"], "scroll"), binding(["esc", "q"], "back")],
    render(area, s) {
      const c = ctx.theme.c;
      const lines = memoryLines(session);
      const spans = lines.map((text, i) => [{ text, style: style(i === 0 ? c.fg : c.mutedFg, c.bg) }]);
      drawLines(s, area, c, scroll, spans);
      hitScroll(ctx, area, scroll, lines.length);
    },
    key(e) {
      if (scroll.key(e, memoryLines(session).length)) return true;
      if (e.key === "esc" || e.key === "q") {
        ctx.pop();
        return true;
      }
      return false;
    },
  };
}
@@ -218,7 +218,10 @@
function userMenu(ctx: Ctx, user: User, signOut: MenuItem): Html {
          { label: "Crew", href: "/settings/crew", icon: "bot" as const },
          ...(user.kind === "agent"
            ? []
            : [{ label: "Boards", href: "/boards", icon: "activity" as const }]),
            : [
                { label: "Boards", href: "/boards", icon: "activity" as const },
                { label: "System memory", href: "/system", icon: "activity" as const },
      
        sentinelcrew
        29d ago
        
      
      Minor: the menu applies the gate itself (user.isAdmin and kind !== "agent") and does not call canSeeMemory(user), although the module doc says the nav asks it. The two match today. Calling canSeeMemory(user) here would keep them from drifting apart.
    
              ]),
        ]
      : []),
    { separator: true, label: "" },
@@ -0,0 +1,49 @@
/**
 * GET /system, for a forge admin who is a person: the forge's resident
 * memory (RSS) beside its JS heap, hourly for the last 24 h (#2151). RSS
 * climbing while the heap stays flat is native memory growth. Drawn on the
 * server as SVG, with the hours under it as a table. Anyone else gets 403.
 */
import { drawChart } from "../../boards/chart.ts";
import { canSeeMemory, memoryHours } from "../../boards/memory.ts";
import type { Ctx } from "../../http/context.ts";
import { html, raw } from "../html.ts";
import { page } from "../layout.ts";
import { btn, card, empty } from "../ui.ts";
const mib = (n: number) => `${(n / 1_048_576).toFixed(1)} MiB`;
const hhmm = (at: number) => new Date(at).toISOString().slice(0, 16).replace("T", " ");
const forbiddenPage = (ctx: Ctx) =>
  page(
    {
      
        sentinelcrew
        29d ago
        outdated
      
      The terminal needs a door with this same gate: a read-only memory view in src/tui/screens (next to boards.ts), built on memoryHours and given only to an admin who is a person. Add a test for both the admin and the refused cases.
    
      ctx,
      title: "Forbidden",
      body: html`<div class="page-head"><h1>403</h1></div>
        ${empty({ icon: "lock", title: "Admins only", note: "System memory is for a forge admin who is a person.", action: btn("Go home", { href: "/", variant: "primary" }) })}`,
    },
    403,
  );
export async function systemPage(ctx: Ctx): Promise<Response> {
  if (!canSeeMemory(ctx.auth.user)) return forbiddenPage(ctx);
  const hours = memoryHours(ctx.app.db, Date.now());
  const series = (title: string, pick: (h: (typeof hours)[number]) => number) =>
    card(
      html`<div class="chart-box">${raw(drawChart("line", hours.map((h) => ({ label: hhmm(h.at).slice(11), value: pick(h) / 1_048_576 })), title))}</div>`,
      { head: html`<h2>${title}</h2>` },
    );
  const body =
    hours.length === 0
      ? empty({ icon: "activity", title: "No samples yet", note: "The forge samples its memory every minute; the first hour appears shortly." })
      : html`<div class="sys-pair">${series("RSS (MiB, hourly mean)", (h) => h.rssMean)}${series("JS heap (MiB, hourly mean)", (h) => h.heapMean)}</div>
        <table class="table">
          <thead><tr><th scope="col">Hour (UTC)</th><th scope="col" class="num">RSS mean</th><th scope="col" class="num">RSS max</th><th scope="col" class="num">Heap mean</th><th scope="col" class="num">Heap max</th></tr></thead>
          <tbody>${hours.map((h) => html`<tr><td class="mono">${hhmm(h.at)}</td><td class="num mono">${mib(h.rssMean)}</td><td class="num mono">${mib(h.rssMax)}</td><td class="num mono">${mib(h.heapMean)}</td><td class="num mono">${mib(h.heapMax)}</td></tr>`)}</tbody>
        </table>`;
  return page({
    ctx,
    title: "System",
    body: html`<div class="page-head"><h1>System memory</h1></div>${body}`,
  });
}
@@ -383,6 +383,8 @@
a.origin-chip:hover { border-color: var(--border-strong); color: var(--fg); }
.bp-h2 .chart-box.is-table { height: 400px; }
.bp-h3 .chart-box.is-table { height: 540px; }
.boards-result .table { margin-top: 12px; }
.sys-pair { display: grid; grid-template-columns: repeat(auto-fit, minmax(320px, 1fr)); gap: 16px; margin-bottom: 16px; }
.sys-pair .chart-box { max-width: none; }
.bgrid { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 12px; margin-bottom: 16px; }
.bp-w2 { grid-column: span 2; }
.bp-w3 { grid-column: span 3; }
@@ -0,0 +1,105 @@
/**
 * /system (#2151): fixed samples roll up into hours (mean, max, count),
 * old hours are dropped, and the page draws RSS and the heap for an admin
 * and is 403 for anyone else.
 */
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp } from "../src/app.ts";
import { HOUR_MS, memoryHours, recordMemory } from "../src/boards/memory.ts";
import { createToken } from "../src/core/tokens.ts";
import { createUser } from "../src/core/users.ts";
import { serve } from "../src/http/server.ts";
const MiB = 1_048_576;
let dir: string;
let app: ReturnType<typeof createApp>;
let servers: Awaited<ReturnType<typeof serve>>;
const tok: Record<string, string> = {};
const get = (who: string | null) =>
  fetch(servers.url + "/system", { headers: who ? { authorization: `Bearer ${tok[who]}` } : {} });
beforeAll(async () => {
  dir = await mkdtemp(join(tmpdir(), "forge-system-"));
  app = createApp({ dataDir: dir, port: 0 });
  servers = await serve(app);
  for (const [username, first] of [["alice", true], ["bob", false]] as const) {
    const r = await createUser(app.db, {
      trusted: true, username, email: `${username}@forge.local`,
      password: "a-good-passphrase", ...(first ? { adminIfFirst: true } : {}),
    });
    if (!r.ok) throw new Error(r.error);
    tok[username] = createToken(app.db, r.value.id, "t", "admin").token;
  }
});
afterAll(async () => {
  await servers.stop();
  await rm(dir, { recursive: true, force: true });
});
describe("hourly roll-up", () => {
  test("mean, max and count per hour from fixed samples", () => {
    const h0 = 1_000 * HOUR_MS;
    recordMemory(app.db, 100 * MiB, 10 * MiB, h0);
    recordMemory(app.db, 300 * MiB, 30 * MiB, h0 + 60_000);
    recordMemory(app.db, 200 * MiB, 20 * MiB, h0 + 120_000);
    recordMemory(app.db, 500 * MiB, 10 * MiB, h0 + HOUR_MS + 5);
    const hours = memoryHours(app.db, h0 + HOUR_MS + 10);
    expect(hours).toEqual([
      { at: h0, samples: 3, rssMean: 200 * MiB, rssMax: 300 * MiB, heapMean: 20 * MiB, heapMax: 30 * MiB },
      { at: h0 + HOUR_MS, samples: 1, rssMean: 500 * MiB, rssMax: 500 * MiB, heapMean: 10 * MiB, heapMax: 10 * MiB },
    ]);
  });
  test("a bad reading is ignored and hours past a week are dropped", () => {
    const now = 5_000 * HOUR_MS;
    recordMemory(app.db, Number.NaN, 1, now);
    recordMemory(app.db, -1, 1, now);
    recordMemory(app.db, 1, 1, now - 8 * 24 * HOUR_MS);
    recordMemory(app.db, MiB, MiB, now);
    expect(memoryHours(app.db, now, 24 * 30).map((h) => h.at)).toEqual([now]);
  });
  test("only the last 24 hours are returned", () => {
    const now = 9_000 * HOUR_MS;
    recordMemory(app.db, MiB, MiB, now - 30 * HOUR_MS);
    recordMemory(app.db, MiB, MiB, now);
    expect(memoryHours(app.db, now).map((h) => h.at)).toEqual([now]);
  });
});
describe("GET /system", () => {
  test("an admin sees both series, server-rendered", async () => {
    recordMemory(app.db, 120 * MiB, 40 * MiB, Date.now());
    const res = await get("alice");
    expect(res.status).toBe(200);
    const body = await res.text();
    expect(body).toContain("RSS (MiB, hourly mean)");
    expect(body).toContain("JS heap (MiB, hourly mean)");
    expect(body.match(/<svg/g)?.length).toBeGreaterThanOrEqual(2);
    expect(body).toContain("120.0 MiB");
    expect(body).toContain("40.0 MiB");
  });
  test("a non-admin and an anonymous visitor get 403", async () => {
    expect((await get("bob")).status).toBe(403);
    expect((await get(null)).status).toBe(403);
  });
});
describe("reaching /system", () => {
  test("the account menu links it for an admin only", async () => {
    const withCookie = async (who: string) =>
      
        sentinelcrew
        29d ago
        
      
      Nit: the helper is called withCookie but it sends a bearer token. Something like homeFor would say what it does.
    
      (await (await fetch(servers.url + "/", { headers: { authorization: `Bearer ${tok[who]}` } })).text());
    expect(await withCookie("alice")).toContain('href="/system"');
    expect(await withCookie("bob")).not.toContain('href="/system"');
  });
  test("a refused visitor gets a page, not bare text", async () => {
    const res = await get(null);
    expect(res.status).toBe(403);
    expect(await res.text()).toContain("<h1>403</h1>");
  });
});
@@ -0,0 +1,100 @@
/**
 * /system in the terminal (#2151): a forge admin who is a person reads the
 * hourly RSS and heap rows and is offered the door; an agent, a member and a
 * guest get neither.
 */
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createAgent } from "../src/actions/crew.ts";
import { createApp, type App } from "../src/app.ts";
import { recordMemory } from "../src/boards/memory.ts";
import { createUser, type User } from "../src/core/users.ts";
import { createForgeApp, forgeCommands, Session } from "../src/tui/forge.ts";
import { memoryLines, systemScreen } from "../src/tui/screens/system.ts";
import { headlessTerminal } from "../src/tui/term/terminal.ts";
import { quietLogs } from "./helpers/globals.ts";
import { settleApp } from "./helpers/tui.ts";
quietLogs();
const MiB = 1_048_576;
let dir: string;
let app: App;
let carol: User;
let bob: User;
let bot: User;
const identity = (user?: User) => ({
  sshUsername: user?.username ?? "git",
  remoteAddress: "127.0.0.1",
  fingerprint: "SHA256:test",
  user,
});
async function screenText(user?: User): Promise<string> {
  const terminal = headlessTerminal(120, 30);
  const { app: tui, session } = createForgeApp(app, identity(user), { terminal, exitHooks: false });
  tui.start();
  tui.ctx.push(systemScreen(session, tui.ctx));
  await settleApp(tui);
  const text = tui.screen.toText();
  tui.stop?.();
  return text;
}
const doors = (user?: User): string[] => {
  const { app: tui, session } = createForgeApp(app, identity(user), {
    terminal: headlessTerminal(80, 24),
    exitHooks: false,
  });
  const ids = forgeCommands(session)(tui.ctx).map((c) => c.id);
  tui.stop?.();
  return ids;
};
beforeAll(async () => {
  dir = await mkdtemp(join(tmpdir(), "forge-tui-system-"));
  app = createApp({ dataDir: dir, port: 0 });
  const mk = async (username: string, first = false) => {
    const r = await createUser(app.db, {
      trusted: true, username, email: `${username}@forge.local`,
      password: "a-good-passphrase", adminIfFirst: first,
    });
    if (!r.ok) throw new Error(r.error);
    return r.value;
  };
  carol = await mk("carol", true);
  bob = await mk("bob");
  const made = await createAgent(app, carol, { username: "sysbot", brief: "x" });
  if (!made.ok) throw new Error(String(made.error));
  app.db.query("UPDATE user SET is_admin = 1 WHERE id = ?1").run(made.value.user.id);
  bot = { ...made.value.user, isAdmin: true };
  recordMemory(app.db, 123 * MiB, 45 * MiB, Date.now());
}, 30_000);
afterAll(async () => {
  app.close();
  await rm(dir, { recursive: true, force: true });
});
describe("tui system memory", () => {
  test("an admin person gets the rows and the door", async () => {
    const lines = memoryLines(new Session(app, identity(carol)));
    expect(lines[0]).toContain("RSS mean");
    expect(lines.join("\n")).toContain("123.0");
    const text = await screenText(carol);
    expect(text).toContain("123.0");
    expect(text).toContain("45.0");
    expect(doors(carol)).toContain("system");
  });
  test("nobody else has rows or the door", async () => {
    for (const who of [bob, bot, undefined]) {
      expect(memoryLines(new Session(app, identity(who)))).toEqual([]);
      expect(doors(who)).not.toContain("system");
      expect(await screenText(who)).not.toContain("123.0");
    }
  });
});
		@@ -1007,6 +1007,7 @@ work here.
1007	1007	commit history) with README rendering and syntax highlighting, protected
1008	1008	default branches (force-push and deletion refused, honest fast-forwards
1009	1009	allowed).
	1010	- System memory — `/system`, for a forge admin who is a person (anyone else gets 403): the forge's RSS beside its JS heap (`heapStats()` from bun:jsc), hourly mean and max for the last 24 h, drawn on the server. Sampled once a minute into `memory_hour` in `app.db` (a week kept). RSS rising while the heap stays flat is native memory growth. In the terminal the same admin has a System memory command in the palette: the last 24 hours as text rows (hour, RSS mean/max, heap mean/max in MiB), read-only. The account menu links `/system` for that admin.
1010	1011	- Boards — `/boards/data`, for a forge admin who is a person (anyone else
1011	1012	gets a 404 on the page): the query is the URL (`?source=runs&aggregate=count&by=day&fcolumn=state&fvalue=failed`),
1012	1013	drawn server-side as the chart chosen beside it (`chart=line\|bar\|area\|number\|table\|heat`)
		@@ -0,0 +1,78 @@
	1	/**
	2	* The forge's own memory against its JS heap, hour by hour (#2151): every
	3	* minute `process.memoryUsage().rss` and `heapStats().heapSize` (bun:jsc)
	4	* are folded into the hour they fall in (`memory_hour` in `app.db`, under
	5	* the data directory), as a count, a sum and a max. RSS rising while the
	6	* heap stays flat is native memory growth. A week of hours is kept.
	7	*/
	8	import type { Database } from "bun:sqlite";
	9	import { heapStats } from "bun:jsc";
	10	import type { App } from "../app.ts";
	11
	12	export const MEMORY_EVERY_MS = 60_000;
	13	export const HOUR_MS = 3_600_000;
	14	const KEEP_MS = 7 * 24 * HOUR_MS;
	15
	16	export type MemoryHour = {
	17	/** Epoch ms of the hour's start. */
	18	at: number;
	19	samples: number;
	20	rssMean: number;
	21	rssMax: number;
	22	heapMean: number;
	23	heapMax: number;
	24	};
	25
	26	const valid = (n: number) => Number.isFinite(n) && n >= 0;
	27
	28	/** Fold one reading into its hour and drop hours past the week. Ignores a reading that is not a count of bytes. */
	29	export function recordMemory(db: Database, rss: number, heap: number, now: number): void {
	30	if (!valid(rss) \|\| !valid(heap) \|\| !valid(now)) return;
	31	const hour = Math.floor(now / HOUR_MS) * HOUR_MS;
	32	db.transaction(() => {
	33	db.run(
	34	`INSERT INTO memory_hour (hour, samples, rss_sum, rss_max, heap_sum, heap_max) VALUES (?, 1, ?, ?, ?, ?)
	35	ON CONFLICT(hour) DO UPDATE SET samples = samples + 1, rss_sum = rss_sum + excluded.rss_sum,
	36	rss_max = max(rss_max, excluded.rss_max), heap_sum = heap_sum + excluded.heap_sum,
	37	heap_max = max(heap_max, excluded.heap_max)`,
	38	[hour, rss, rss, heap, heap],
	39	);
	40	db.run("DELETE FROM memory_hour WHERE hour < ?", [hour - KEEP_MS]);
	41	})();
	42	}
	43
	44	/** The hours in the `hours` before `now` (the current one included), oldest first. */
	45	export function memoryHours(db: Database, now: number, hours = 24): MemoryHour[] {
	46	const last = Math.floor(now / HOUR_MS) * HOUR_MS;
	47	const rows = db
	48	.query<{ hour: number; samples: number; rss_sum: number; rss_max: number; heap_sum: number; heap_max: number }, [number, number]>(
	49	"SELECT hour, samples, rss_sum, rss_max, heap_sum, heap_max FROM memory_hour WHERE hour > ? AND hour <= ? ORDER BY hour",
	50	)
	51	.all(last - hours * HOUR_MS, last);
	52	return rows.map((r) => ({
	53	at: r.hour, samples: r.samples,
	54	rssMean: r.rss_sum / r.samples, rssMax: r.rss_max,
	55	heapMean: r.heap_sum / r.samples, heapMax: r.heap_max,
	56	}));
	57	}
	58
	59	export type MemorySampler = { tick(now?: number): void; stop(): void };
	60
	61	/** Sample now and every minute until stopped. A failed tick is logged and the next one tries again. */
	62	export function startMemorySampler(app: App, everyMs = MEMORY_EVERY_MS): MemorySampler {
	63	const tick = (now = Date.now()) => {
	64	try {
	65	recordMemory(app.db, process.memoryUsage().rss, heapStats().heapSize, now);
	66	} catch (error) {
	67	app.log.warn("memory sample failed", { error: String(error) });
	68	}
	69	};
	70	tick();
	71	const timer = setInterval(tick, everyMs);
	72	timer.unref?.();
	73	return { tick, stop: () => clearInterval(timer) };
	74	}
	75
	76	/** Who may read the series: a forge admin who is a person, never an agent. The page, the nav and the terminal all ask this. */
	77	export const canSeeMemory = (user: { isAdmin: boolean; kind?: string } \| null \| undefined): boolean =>
	78	user?.isAdmin === true && user.kind !== "agent";
		@@ -0,0 +1,14 @@
	1	// The forge's own memory, one row an hour (#2151, boards/memory.ts): the
	2	// resident set (RSS) beside the JS heap, so RSS climbing while the heap
	3	// stays flat shows as a native leak. A running sum and count give the mean;
	4	// telemetry, not state — a lost row is a gap in a chart.
	5	export default `
	6	CREATE TABLE memory_hour (
	7	hour INTEGER PRIMARY KEY,
	8	samples INTEGER NOT NULL,
	9	rss_sum REAL NOT NULL,
	10	rss_max REAL NOT NULL,
	11	heap_sum REAL NOT NULL,
	12	heap_max REAL NOT NULL
	13	);
	14	`;
		@@ -15,6 +15,7 @@ import { flagsPage, flagsSet } from "../web/pages/flags.ts";
15	15	import { domainsPage } from "../web/pages/domains.ts";
16	16	import { resiliencePage } from "../web/pages/resilience.ts";
17	17	import { limitsPage, limitsSet, quotasSet } from "../web/pages/admin-limits.ts";
	18	import { systemPage } from "../web/pages/system.ts";
18	19	import { boardsPage } from "../web/pages/boards.ts";
19	20	import { boardNew, boardPin, boardPost } from "../web/pages/boards-edit.ts";
20	21	import { boardLive, boardShow, boardsList } from "../web/pages/boards-view.ts";
		@@ -744,6 +745,7 @@ export function buildRouter(): Router<Handler> {
744	745	router.add("GET", "/settings/resilience", domainsPage);
745	746	router.add("GET", "/settings/capacity", capacityPage);
746	747	router.add("POST", "/settings/capacity", capacitySet);
	748	router.add("GET", "/system", systemPage);
		sentinelcrew 29d ago Nothing in the UI links to this route. Add a nav or boards link, visible only to an admin who passes the gate, so a person can reach `/system` from the page.
747	749	router.add("GET", "/boards", boardsList);
748	750	router.add("GET", "/boards/data", boardsPage);
749	751	router.add("POST", "/boards/new", boardNew);
		@@ -4,6 +4,7 @@
4	4	* environment-only — a handful of variables, all with sane defaults.
5	5	*/
6	6
	7	import { startMemorySampler } from "./boards/memory.ts";
7	8	import { startSampler } from "./boards/sampler.ts";
8	9	import { sweepStanding } from "./crew/standing.ts";
9	10	import { drainOnHandover, handover } from "./release/swap.ts";
		@@ -338,6 +339,7 @@ const snapshots = startSnapshots(
338	339	// What the machine uses (boards/sampler.ts): the pod, each cell and the
339	340	// node, every 15 s into a ring the `machine` board source reads.
340	341	const sampler = startSampler(app);
	342	const memorySampler = startMemorySampler(app);
341	343	const crewHeldOff = crewOffInPreview();
342	344	if (appsOn && crewHeldOff)
343	345	app.log.info(CREW_OFF_IN_PREVIEW, {
		@@ -542,6 +544,7 @@ function drain(signal: NodeJS.Signals): Promise<unknown> {
542	544	probes?.stop(),
543	545	snapshots.stop(),
544	546	sampler.stop(),
	547	memorySampler.stop(),
545	548	])
546	549	.catch(() => {})
547	550	.then(() => {
		@@ -28,7 +28,9 @@ import {
28	28	openOnWeb,
29	29	runAsync,
30	30	} from "./screens/common.ts";
	31	import { canSeeMemory } from "../boards/memory.ts";
31	32	import { boardsScreen } from "./screens/boards.ts";
	33	import { systemScreen } from "./screens/system.ts";
32	34	import { askedScreen } from "./screens/asked.ts";
33	35	import { deciderScreen } from "./screens/decider.ts";
34	36	import { familyScreen } from "./screens/family.ts";
		@@ -230,6 +232,14 @@ export function forgeCommands(session: Session): (ctx: Ctx) => Command[] {
230	232	hint: "the forge's boards and their panels",
231	233	run: () => ctx.push(boardsScreen(session, ctx)),
232	234	});
	235	// The forge's RSS beside its JS heap, as /system gates it (#2151).
	236	if (canSeeMemory(session.user))
	237	commands.push({
	238	id: "system",
	239	title: "System memory",
	240	hint: "RSS beside the JS heap, hourly for 24 h",
	241	run: () => ctx.push(systemScreen(session, ctx)),
	242	});
233	243	// The pipeline's bottleneck and throughput, as /flow says them: a site admin's (#1554, #1786).
234	244	if (session.user?.isAdmin === true)
235	245	commands.push({
		@@ -0,0 +1,55 @@
	1	/**
	2	* `/system` in the terminal (#2151): the forge's RSS beside its JS heap, an
	3	* hour a row for the last 24 h, read through `memoryHours` like the page. The
	4	* gate is the page's (`canSeeMemory`): anyone but a forge admin who is a
	5	* person gets no rows, and the palette offers the door only to that person.
	6	* Read-only.
	7	*/
	8
	9	import { canSeeMemory, memoryHours } from "../../boards/memory.ts";
	10	import type { Ctx, Screen } from "../app/app.ts";
	11	import { binding } from "../app/keys.ts";
	12	import type { Session } from "../session.ts";
	13	import { style } from "../term/style.ts";
	14	import { pad } from "../term/width.ts";
	15	import { Scroll, drawLines, hitScroll } from "./common.ts";
	16
	17	const mib = (n: number) => `${(n / 1_048_576).toFixed(1)}`;
	18
	19	/** The header, then an hour a row (UTC); empty for anyone the gate refuses. */
	20	export function memoryLines(session: Session, now = Date.now()): string[] {
	21	if (!canSeeMemory(session.user)) return [];
	22	const hours = memoryHours(session.db, now);
	23	if (hours.length === 0) return ["No samples yet; the forge samples its memory every minute."];
	24	const col = (s: string) => pad(s, 10, "right");
	25	return [
	26	`${pad("Hour (UTC)", 17)}${col("RSS mean")}${col("RSS max")}${col("Heap mean")}${col("Heap max")} MiB`,
	27	...hours.map(
	28	(h) =>
	29	`${pad(new Date(h.at).toISOString().slice(0, 16).replace("T", " "), 17)}${col(mib(h.rssMean))}${col(mib(h.rssMax))}${col(mib(h.heapMean))}${col(mib(h.heapMax))}`,
	30	),
	31	];
	32	}
	33
	34	export function systemScreen(session: Session, ctx: Ctx): Screen {
	35	const scroll = new Scroll();
	36	return {
	37	title: "System memory",
	38	keys: [binding(["j", "k"], "scroll"), binding(["esc", "q"], "back")],
	39	render(area, s) {
	40	const c = ctx.theme.c;
	41	const lines = memoryLines(session);
	42	const spans = lines.map((text, i) => [{ text, style: style(i === 0 ? c.fg : c.mutedFg, c.bg) }]);
	43	drawLines(s, area, c, scroll, spans);
	44	hitScroll(ctx, area, scroll, lines.length);
	45	},
	46	key(e) {
	47	if (scroll.key(e, memoryLines(session).length)) return true;
	48	if (e.key === "esc" \|\| e.key === "q") {
	49	ctx.pop();
	50	return true;
	51	}
	52	return false;
	53	},
	54	};
	55	}
		@@ -218,7 +218,10 @@ function userMenu(ctx: Ctx, user: User, signOut: MenuItem): Html {
218	218	{ label: "Crew", href: "/settings/crew", icon: "bot" as const },
219	219	...(user.kind === "agent"
220	220	? []
221		: [{ label: "Boards", href: "/boards", icon: "activity" as const }]),
	221	: [
	222	{ label: "Boards", href: "/boards", icon: "activity" as const },
	223	{ label: "System memory", href: "/system", icon: "activity" as const },
		sentinelcrew 29d ago Minor: the menu applies the gate itself (`user.isAdmin` and `kind !== "agent"`) and does not call `canSeeMemory(user)`, although the module doc says the nav asks it. The two match today. Calling `canSeeMemory(user)` here would keep them from drifting apart.
	224	]),
222	225	]
223	226	: []),
224	227	{ separator: true, label: "" },
		@@ -0,0 +1,49 @@
	1	/**
	2	* GET /system, for a forge admin who is a person: the forge's resident
	3	* memory (RSS) beside its JS heap, hourly for the last 24 h (#2151). RSS
	4	* climbing while the heap stays flat is native memory growth. Drawn on the
	5	* server as SVG, with the hours under it as a table. Anyone else gets 403.
	6	*/
	7	import { drawChart } from "../../boards/chart.ts";
	8	import { canSeeMemory, memoryHours } from "../../boards/memory.ts";
	9	import type { Ctx } from "../../http/context.ts";
	10	import { html, raw } from "../html.ts";
	11	import { page } from "../layout.ts";
	12	import { btn, card, empty } from "../ui.ts";
	13
	14	const mib = (n: number) => `${(n / 1_048_576).toFixed(1)} MiB`;
	15	const hhmm = (at: number) => new Date(at).toISOString().slice(0, 16).replace("T", " ");
	16
	17	const forbiddenPage = (ctx: Ctx) =>
	18	page(
	19	{
		sentinelcrew 29d ago outdated The terminal needs a door with this same gate: a read-only memory view in `src/tui/screens` (next to `boards.ts`), built on `memoryHours` and given only to an admin who is a person. Add a test for both the admin and the refused cases.
	20	ctx,
	21	title: "Forbidden",
	22	body: html`<div class="page-head"><h1>403</h1></div>
	23	${empty({ icon: "lock", title: "Admins only", note: "System memory is for a forge admin who is a person.", action: btn("Go home", { href: "/", variant: "primary" }) })}`,
	24	},
	25	403,
	26	);
	27
	28	export async function systemPage(ctx: Ctx): Promise<Response> {
	29	if (!canSeeMemory(ctx.auth.user)) return forbiddenPage(ctx);
	30	const hours = memoryHours(ctx.app.db, Date.now());
	31	const series = (title: string, pick: (h: (typeof hours)[number]) => number) =>
	32	card(
	33	html`<div class="chart-box">${raw(drawChart("line", hours.map((h) => ({ label: hhmm(h.at).slice(11), value: pick(h) / 1_048_576 })), title))}</div>`,
	34	{ head: html`<h2>${title}</h2>` },
	35	);
	36	const body =
	37	hours.length === 0
	38	? empty({ icon: "activity", title: "No samples yet", note: "The forge samples its memory every minute; the first hour appears shortly." })
	39	: html`<div class="sys-pair">${series("RSS (MiB, hourly mean)", (h) => h.rssMean)}${series("JS heap (MiB, hourly mean)", (h) => h.heapMean)}</div>
	40	<table class="table">
	41	<thead><tr><th scope="col">Hour (UTC)</th><th scope="col" class="num">RSS mean</th><th scope="col" class="num">RSS max</th><th scope="col" class="num">Heap mean</th><th scope="col" class="num">Heap max</th></tr></thead>
	42	<tbody>${hours.map((h) => html`<tr><td class="mono">${hhmm(h.at)}</td><td class="num mono">${mib(h.rssMean)}</td><td class="num mono">${mib(h.rssMax)}</td><td class="num mono">${mib(h.heapMean)}</td><td class="num mono">${mib(h.heapMax)}</td></tr>`)}</tbody>
	43	</table>`;
	44	return page({
	45	ctx,
	46	title: "System",
	47	body: html`<div class="page-head"><h1>System memory</h1></div>${body}`,
	48	});
	49	}
		@@ -383,6 +383,8 @@ a.origin-chip:hover { border-color: var(--border-strong); color: var(--fg); }
383	383	.bp-h2 .chart-box.is-table { height: 400px; }
384	384	.bp-h3 .chart-box.is-table { height: 540px; }
385	385	.boards-result .table { margin-top: 12px; }
	386	.sys-pair { display: grid; grid-template-columns: repeat(auto-fit, minmax(320px, 1fr)); gap: 16px; margin-bottom: 16px; }
	387	.sys-pair .chart-box { max-width: none; }
386	388	.bgrid { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 12px; margin-bottom: 16px; }
387	389	.bp-w2 { grid-column: span 2; }
388	390	.bp-w3 { grid-column: span 3; }