Public: anyone can read this; the crew can change it.
The admin profiler leaks memory until the forge crashes: it brought production down for 2.5 minutes #12456
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/12455-the-admin-profiler-leaks-memory-until-th → main 7 commits fork 1 thread · 1 unresolved
6 files
changed +135
−4
docs/features/performance-watcher.md +12
| @@ -5,3 +5,15 @@ | ||
| 5 | 5 | `history.ts` (#11477): every tick the duty folds the record's new samples into an in-memory ledger under the hour each sample happened in (so 10:50 traffic is never stamped 11:00) and upserts one row per route pattern (route, hour, n, p50, p95) into `route_hour`, so a row is the hour's p50 and p95, not the last 15 minutes'. At most 5,000 samples per route and hour are kept (past it the row stands on the first), at most 200 routes (the busiest) and 168 hours; older rows go in the same write. A restart starts an empty ledger: an empty record writes nothing, and a row is never replaced by one with fewer requests, so the hours before the restart keep the fuller row. Admins read them at `GET /api/v1/telemetry/routes?hours=N` (1 to 168, default 24; else 400). |
| 6 | 6 | |
| 7 | 7 | Not yet: a synthetic probe for routes with no traffic (a client-written label would let anyone hide requests; it needs an admin-only door first), route-hour stats in the terminal (findings reach it as issues; the MCP `system` `kind: routes` reads the live routes), browser timings, the board page, release before/after. |
| 8 | ||
| 9 | ## Sampling the main thread: `FORGE_PROFILE=1` (#12455) | |
| 10 | ||
| 11 | `GET /api/v1/admin/profile?seconds=N` samples what the main thread is doing | |
| 12 | (`src/perf/sample.ts`). Bun cannot stop the sampling profiler once started, and | |
| 13 | its traces piled up until the forge ran out of memory, so a forge that serves | |
| 14 | people answers an admin `403` with code `refused` and the sentence | |
| 15 | "Profiling is off on a forge that serves people…". To allow it, start the | |
| 16 | process with `FORGE_PROFILE=1` — meant for the stage rig, not production. Once | |
| 17 | a sample has run the profiler stays attached until the process restarts, and | |
| 18 | its traces are drained and discarded every two seconds by the `profiler-drain` | |
| 19 | job; a window being sampled keeps its traces until it ends. |
src/http/api/admin.ts +2 −1
| @@ -13,7 +13,7 @@ import { probeViews } from "../../perf/views_probe.ts"; | ||
| 13 | 13 | import { daysOf as visitorDaysOf, probeVisitors, visitorsReport } from "../../perf/visitor_report.ts"; |
| 14 | 14 | import { getReturning } from "../../settings/returning.ts"; |
| 15 | 15 | import { kpiProbe } from "../../web/studio/kpi_probe.ts"; |
| 16 | import { MAX_SECONDS, sampleStacks } from "../../perf/sample.ts"; | |
| 16 | import { MAX_SECONDS, REFUSED, profilingAllowed, sampleStacks } from "../../perf/sample.ts"; | |
| 17 | 17 | import { listAccounts, removeAccount, retireAndRemoveAccount, retirementPlan, suspendAccount, unsuspendAccount, verifyAccount } from "../../actions/accounts.ts"; |
| 18 | 18 | import { ACCESS_WORDS, accessUrl, endAllSessions, resetAccess } from "../../actions/account_access.ts"; |
| 19 | 19 | import { ADMIN_WORDS } from "../../actions/admin_words.ts"; |
| @@ -131,6 +131,7 @@ export function adminRoutes(router: Router<Handler>): void { | ||
| 131 | 131 | notePhase(ctx.request, "session", takeSessionTime(ctx.request)); |
| 132 | 132 | const denied = accountAdmin(ctx, "ro"); |
| 133 | 133 | if (denied) return denied; |
| 134 | if (!profilingAllowed()) return apiError("refused", 403, REFUSED); | |
| 134 | 135 | // Asked for by name: a sample holds the profiler on, so a bare GET never starts one. |
| 135 | 136 | const asked = Number(ctx.url.searchParams.get("seconds") ?? Number.NaN); |
| 136 | 137 | if (!Number.isFinite(asked)) return apiError("invalid", 422, `seconds is a number from 1 to ${MAX_SECONDS}`); |
src/perf/sample.ts +33 −2
| @@ -6,6 +6,7 @@ | ||
| 6 | 6 | */ |
| 7 | 7 | |
| 8 | 8 | import * as jsc from "bun:jsc"; |
| 9 | import { during } from "./jobs.ts"; | |
| 9 | 10 | |
| 10 | 11 | export const MAX_SECONDS = 30; |
| 11 | 12 | export const DEFAULT_SECONDS = 10; |
| @@ -45,17 +46,47 @@ export function foldStacks(profile: Pick<Profile, "traces">, depth = 8): Sampled | ||
| 45 | 46 | .slice(0, TOP); |
| 46 | 47 | } |
| 47 | 48 | |
| 48 | const NOTE = "the profiler stays attached until this process restarts; a wait on another process's database lock may fail early meanwhile"; | |
| 49 | const NOTE = "the profiler stays attached until this process restarts (Bun cannot stop it), so its traces are drained and discarded every few seconds; a wait on another process's database lock may fail early meanwhile"; | |
| 49 | 50 | |
| 51 | /** | |
| 52 | * How often the attached sampler's traces are thrown away: they pile up at | |
| 53 | * ~500 MiB a minute under load (#12455). | |
| 54 | */ | |
| 55 | export const DRAIN_EVERY_MS = 2_000; | |
| 50 | 56 | let sampling = false; |
| 57 | let drain: ReturnType<typeof setInterval> | undefined; | |
| 58 | /** Whether the drain timer is running: from the first sample until this process ends. */ | |
| 59 | export const draining = (): boolean => drain !== undefined; | |
| 60 | ||
| 61 | /** | |
| 62 | * Bun has no stop for the sampler, so for as long as it is attached its traces | |
| 63 | * are drained on this named timer. | |
| 64 | */ | |
| 65 | function drainWhileAttached(every: number): void { | |
| 66 | if (drain !== undefined) return; | |
| 67 | // A window being sampled keeps its traces: the drain waits for it to end. | |
| 68 | drain = setInterval( | |
| 69 | () => sampling || during("profiler-drain", () => void profiler.samplingProfilerStackTraces()), | |
| 70 | every, | |
| 71 | ); | |
| 72 | drain.unref(); | |
| 73 | } | |
| 74 | ||
| 75 | /** The sentence an admin is told when a forge that serves people refuses to profile. */ | |
| 76 | export const REFUSED = "Profiling is off on a forge that serves people: the sampler cannot be stopped in this Bun and once brought production down (#12455); the stage rig allows it with FORGE_PROFILE=1."; | |
| 77 | export const profilingAllowed = (env: Record<string, string | undefined> = process.env): boolean => env.FORGE_PROFILE === "1"; | |
| 51 | 78 | |
| 52 | 79 | /** Sample for `seconds` (1 to 30), or undefined while another sample runs. */ |
| 53 | export async function sampleStacks(seconds: number = DEFAULT_SECONDS): Promise<Sampled | undefined> { | |
| 80 | export async function sampleStacks( | |
| 81 | seconds: number = DEFAULT_SECONDS, | |
| 82 | drainEvery: number = DRAIN_EVERY_MS, | |
| 83 | ): Promise<Sampled | undefined> { | |
| 54 | 84 | if (sampling) return undefined; |
| 55 | 85 | sampling = true; |
| 56 | 86 | try { |
| 57 | 87 | const secs = Math.min(MAX_SECONDS, Math.max(1, Math.floor(seconds))); |
| 58 | 88 | profiler.startSamplingProfiler(); |
| 89 | drainWhileAttached(drainEvery); | |
| 59 | 90 | profiler.samplingProfilerStackTraces(); // drop what ran before this window |
| 60 | 91 | await new Promise((r) => setTimeout(r, secs * 1000)); |
| 61 | 92 | const profile = profiler.samplingProfilerStackTraces(); |
test/job-slices.test.ts +1
| @@ -272,6 +272,7 @@ const UNTIMED: Record<string, string> = { | ||
| 272 | 272 | "release-keeper": "cluster only (FORGE_RELEASE_KEEPER=on): talks to the cluster's API", |
| 273 | 273 | "roll-probe": "cluster only (FORGE_RELEASE_KEEPER=on): one HTTP probe", |
| 274 | 274 | "apps-sweep": "AppsRuntime's sweep over running app processes; the fixture has no apps", |
| 275 | "profiler-drain": "drains the sampling profiler's stack traces on a timer (perf/sample.ts) so they do not pile up; runs only where the profiler is on, and the fixture samples nothing", | |
| 275 | 276 | "cell-watch": "watches running app cells' cgroups; needs cells, skipped in this sandbox mode", |
| 276 | 277 | "cell-probe-deadline": "a cell's start probe running out of time: needs cells, skipped in this sandbox mode", |
| 277 | 278 | "cell-kill-root": "kills a cell's root process after its grace: needs cells, skipped in this sandbox mode", |
test/profile-guard.test.ts added +64
| @@ -0,0 +1,64 @@ | ||
| 1 | /** | |
| 2 | * #12455: the profiler cannot be stopped in this Bun, so a forge that serves people | |
| 3 | * refuses to start it unless FORGE_PROFILE=1. | |
| 4 | */ | |
| 5 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 6 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 7 | import { tmpdir } from "node:os"; | |
| 8 | import { join } from "node:path"; | |
| 9 | import { createApp, type App } from "../src/app.ts"; | |
| 10 | import { createUser } from "../src/core/users.ts"; | |
| 11 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 12 | import { REFUSED, profilingAllowed } from "../src/perf/sample.ts"; | |
| 13 | ||
| 14 | let dataDir: string; | |
| 15 | let app: App; | |
| 16 | let servers: Servers; | |
| 17 | let token = ""; | |
| 18 | const saved = process.env.FORGE_PROFILE; | |
| 19 | ||
| 20 | beforeAll(async () => { | |
| 21 | delete process.env.FORGE_PROFILE; | |
| 22 | dataDir = await mkdtemp(join(tmpdir(), "forge-profile-")); | |
| 23 | app = createApp({ dataDir, port: 0 }); | |
| 24 | servers = await serve(app); | |
| 25 | const u = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", isAdmin: true }); | |
| 26 | if (!u.ok) throw new Error(u.error); | |
| 27 | const res = await fetch(`${servers.url}/api/v1/tokens`, { | |
| 28 | method: "POST", | |
| 29 | headers: { "content-type": "application/json" }, | |
| 30 | body: JSON.stringify({ adminConfirm: true, username: "alice", password: "a-good-passphrase", name: "t", scope: "admin" }), | |
| 31 | }); | |
| 32 | token = ((await res.json()) as { token: string }).token; | |
| 33 | }); | |
| 34 | ||
| 35 | afterAll(async () => { | |
| 36 | if (saved === undefined) delete process.env.FORGE_PROFILE; | |
| 37 | else process.env.FORGE_PROFILE = saved; | |
| 38 | servers.stop(); | |
| 39 | app.close(); | |
| 40 | await rm(dataDir, { recursive: true, force: true }); | |
| 41 | }); | |
| 42 | ||
| 43 | const profile = (headers: Record<string, string> = {}) => | |
| 44 | fetch(`${servers.url}/api/v1/admin/profile?seconds=1`, { headers }); | |
| 45 | ||
| 46 | test("an admin without FORGE_PROFILE is refused with the sentence, before any sample starts", async () => { | |
| 47 | const res = await profile({ authorization: `Bearer ${token}` }); | |
| 48 | expect(res.status).toBe(403); | |
| 49 | const body = (await res.json()) as { code?: string; error?: string; message?: string }; | |
| 50 | expect(JSON.stringify(body)).toContain("refused"); | |
| 51 | expect(JSON.stringify(body)).toContain(JSON.stringify(REFUSED).slice(1, -1)); | |
| 52 | }); | |
| 53 | ||
| 54 | test("a guest still gets the admin denial, not the profiling sentence", async () => { | |
| 55 | const res = await profile(); | |
| 56 | expect(res.status).not.toBe(200); | |
| 57 | expect(await res.text()).not.toContain("Profiling is off"); | |
| 58 | }); | |
| 59 | ||
| 60 | test("only FORGE_PROFILE=1 allows profiling", () => { | |
| 61 | expect(profilingAllowed({ FORGE_PROFILE: "1" })).toBe(true); | |
| 62 | expect(profilingAllowed({})).toBe(false); | |
| 63 | expect(profilingAllowed({ FORGE_PROFILE: "0" })).toBe(false); | |
| 64 | }); |
test/sample.test.ts +23 −1
| @@ -17,7 +17,7 @@ test("a sample sees the busy function, and a second at once is refused", async ( | ||
| 17 | 17 | const script = ` |
| 18 | 18 | import { sampleStacks } from ${JSON.stringify(new URL("../src/perf/sample.ts", import.meta.url).pathname)}; |
| 19 | 19 | const first = sampleStacks(1); |
| 20 | const second = await sampleStacks(1); | |
| 20 | const second = await sampleStacks(1, 200); | |
| 21 | 21 | const stop = Date.now() + 700; |
| 22 | 22 | let x = 0; |
| 23 | 23 | const spin = () => { while (Date.now() < stop) x += Math.sqrt(x + 1); }; |
| @@ -34,3 +34,25 @@ test("a sample sees the busy function, and a second at once is refused", async ( | ||
| 34 | 34 | expect(out.first.samples).toBeGreaterThan(0); |
| 35 | 35 | expect(out.first.stacks.some((s) => s.stack.includes("spin"))).toBe(true); |
| 36 | 36 | }); |
| 37 | ||
| 38 | test("after the window the traces are still drained, so they do not pile up", async () => { | |
| 39 | const script = ` | |
| 40 | import * as jsc from "bun:jsc"; | |
| 41 | import { sampleStacks, draining } from ${JSON.stringify(new URL("../src/perf/sample.ts", import.meta.url).pathname)}; | |
| 42 | await sampleStacks(1, 200); | |
| 43 | const spin = (ms) => { const stop = Date.now() + ms; let x = 0; while (Date.now() < stop) x += Math.sqrt(x + 1); }; | |
| 44 | spin(600); // traces pile up while nothing reads them... | |
| 45 | // the real drain timer fires within a few intervals: poll, bounded, instead of one fixed sleep | |
| 46 | const until = Date.now() + 5000; | |
| 47 | let left = Infinity; | |
| 48 | while (Date.now() < until) { | |
| 49 | await new Promise((r) => setTimeout(r, 50)); // real-timer: the child process must let its real drain timer fire | |
| 50 | left = jsc.samplingProfilerStackTraces().traces.length; | |
| 51 | if (left < 50) break; | |
| 52 | } | |
| 53 | console.log(JSON.stringify({ draining: draining(), left }));`; | |
| 54 | const child = Bun.spawn(["bun", "-e", script], { stdout: "pipe" }); | |
| 55 | const out = JSON.parse(await new Response(child.stdout).text()) as { draining: boolean; left: number }; | |
| 56 | expect(out.draining).toBe(true); | |
| 57 | expect(out.left).toBeLessThan(50); | |
| 58 | }); |
new above
offline
This 403 is the behaviour the pull adds and no test reaches it. Add a route test that fails without this line.