Public: anyone can read this; the crew can change it.
An outliving cell's egress bytes reach the host's gauges, counted in the supervisor and sent back once #13755 lands #13929
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/13922-an-outliving-cell-s-egress-bytes-reach-t → main 8 commits fork
5 files
changed +166
−9
src/apps/egress.ts +15
| @@ -268,6 +268,20 @@ function recordNpmRepo(repoId: string, bytesIn: number, minute: number): void { | ||
| 268 | 268 | addMinute(m, minute, bytesIn); |
| 269 | 269 | } |
| 270 | 270 | |
| 271 | /** Where this process's egress bytes also go (`tapEgress`); undefined in the host. */ | |
| 272 | let tap: ((label: string, destination: string, bytesIn: number, bytesOut: number) => void) | undefined; | |
| 273 | ||
| 274 | /** | |
| 275 | * Hand every byte `recordEgress` counts to `fn` as well: a cell's supervisor (#13922) sums them and | |
| 276 | * sends them back to the host, whose gauges are the ones served. Returns the undo. | |
| 277 | */ | |
| 278 | export function tapEgress(fn: (label: string, destination: string, bytesIn: number, bytesOut: number) => void): () => void { | |
| 279 | tap = fn; | |
| 280 | return () => { | |
| 281 | if (tap === fn) tap = undefined; | |
| 282 | }; | |
| 283 | } | |
| 284 | ||
| 271 | 285 | /** Count `bytesIn` received from and `bytesOut` sent to `destination`. */ |
| 272 | 286 | export function recordEgress( |
| 273 | 287 | label: string, |
| @@ -276,6 +290,7 @@ export function recordEgress( | ||
| 276 | 290 | bytesOut: number, |
| 277 | 291 | now = Date.now(), |
| 278 | 292 | ): void { |
| 293 | tap?.(label, destination, bytesIn, bytesOut); | |
| 279 | 294 | const key = |
| 280 | 295 | destination === NPM_REGISTRY || moved.has(destination) || moved.size < MOVED_MAX |
| 281 | 296 | ? destination |
src/spawn/cell.ts +2 −2
| @@ -37,8 +37,8 @@ import { clearRecord, leaveRecord, readRecords, type CellRecord } from "./cell_r | ||
| 37 | 37 | * record (`cell_record.ts`) that the next host adopts if the cell is still its supervisor's and |
| 38 | 38 | * answers, and stops if it does not answer (#13756); its beat door (`beat-door.ts`) is this |
| 39 | 39 | * process's, reached through the supervisor's relay while this process lives and answered by the |
| 40 | * supervisor's held door after (`held_door.ts`, #13906), and its egress bytes are counted in the | |
| 41 | * supervisor, out of this process's gauges (#13921). | |
| 40 | * supervisor's held door after (`held_door.ts`, #13906), and its egress bytes are summed in the | |
| 41 | * supervisor and reach this process's gauges as events (#13922). | |
| 42 | 42 | */ |
| 43 | 43 | export const cellsOutlive = (env: Record<string, string | undefined> = process.env): boolean => env["FORGE_CELLS_OUTLIVE"] === "on"; |
| 44 | 44 |
src/spawn/supervisor.ts +57 −7
| @@ -12,7 +12,8 @@ | ||
| 12 | 12 | * cell.log the cell's stdout and stderr, as a file the cell holds open |
| 13 | 13 | * allowed.json egress entries allowed since the cell started (host rewrites when they change) |
| 14 | 14 | * beat.held the cell's latest beat once no host holds its door (supervisor writes; #13906) |
| 15 | * events.jsonl refusals and undeclared hosts (supervisor appends; the host tails) | |
| 15 | * events.jsonl refusals, undeclared hosts and egress byte sums (supervisor appends; the host | |
| 16 | * tails) | |
| 16 | 17 | * |
| 17 | 18 | * It ends when the cell's root process is gone, on SIGTERM, or when no cell appears in a minute. |
| 18 | 19 | * The host side of this file is `startSupervisor`; `bun supervisor.ts <ctl dir>` is the other. |
| @@ -20,7 +21,7 @@ | ||
| 20 | 21 | import { appendFileSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, readSync, rmSync, statSync, truncateSync, writeFileSync } from "node:fs"; |
| 21 | 22 | import { join } from "node:path"; |
| 22 | 23 | import { during } from "../perf/jobs.ts"; |
| 23 | import { PROXY_VARS, relay, startEgress, type Egress } from "../apps/egress.ts"; | |
| 24 | import { PROXY_VARS, recordEgress, relay, startEgress, tapEgress, type Egress } from "../apps/egress.ts"; | |
| 24 | 25 | import { proxyOptions, type CellPolicy } from "../apps/cell_policy.ts"; |
| 25 | 26 | import { openHeldDoor } from "./held_door.ts"; |
| 26 | 27 | |
| @@ -30,6 +31,8 @@ export interface SupervisorConfig extends CellPolicy { | ||
| 30 | 31 | readonly sockets: string; |
| 31 | 32 | /** Tests: the proxy listens on this loopback port, where a unix socket cannot be bound. */ |
| 32 | 33 | readonly outPort?: number; |
| 34 | /** Tests: how often summed egress bytes are sent back, in place of `EGRESS_FLUSH_MS`. */ | |
| 35 | readonly flushMs?: number; | |
| 33 | 36 | /** |
| 34 | 37 | * The port the cell knows its parent's beat door by (`FORGE_PARENT_LOCAL`, #13906): its loop is |
| 35 | 38 | * relayed to the host's door while the host lives, and to the supervisor's held door |
| @@ -44,7 +47,8 @@ export interface SupervisorConfig extends CellPolicy { | ||
| 44 | 47 | export type SupervisorEvent = |
| 45 | 48 | | { e: "refused"; host: string; port: number; private: boolean } |
| 46 | 49 | | { e: "undeclared"; host: string; port: number } |
| 47 | | { e: "log"; line: string }; | |
| 50 | | { e: "log"; line: string } | |
| 51 | | { e: "egress-bytes"; label: string; destination: string; in: number; out: number }; | |
| 48 | 52 | |
| 49 | 53 | const WAIT_FOR_CELL_MS = 60_000; |
| 50 | 54 | const TICK_MS = 1_000; |
| @@ -56,6 +60,38 @@ export const LOG_MAX_BYTES = 4_000_000; | ||
| 56 | 60 | export const EVENTS_MAX_BYTES = 1_000_000; |
| 57 | 61 | /** Events the supervisor writes in a minute; past it they are dropped. */ |
| 58 | 62 | export const EVENTS_PER_MINUTE = 300; |
| 63 | /** How often the supervisor sends its summed egress bytes back: never once per chunk. */ | |
| 64 | export const EGRESS_FLUSH_MS = 5_000; | |
| 65 | /** The most destinations one batch holds; past it the rest are summed under `other`. */ | |
| 66 | const BATCH_MAX = 64; | |
| 67 | ||
| 68 | /** Egress bytes summed per label and destination between flushes (#13922). */ | |
| 69 | export function egressBatch() { | |
| 70 | const sums = new Map<string, { label: string; destination: string; in: number; out: number }>(); | |
| 71 | return { | |
| 72 | add(label: string, destination: string, bytesIn: number, bytesOut: number): void { | |
| 73 | const at = sums.size < BATCH_MAX || sums.has(`${label}\n${destination}`) ? destination : "other"; | |
| 74 | const key = `${label}\n${at}`; | |
| 75 | let sum = sums.get(key); | |
| 76 | if (sum === undefined) sums.set(key, (sum = { label, destination: at, in: 0, out: 0 })); | |
| 77 | sum.in += bytesIn; | |
| 78 | sum.out += bytesOut; | |
| 79 | }, | |
| 80 | /** | |
| 81 | * Hand each sum to `send`; a sum `send` refuses (the events budget is spent) stays for the | |
| 82 | * next. | |
| 83 | */ | |
| 84 | flush(send: (event: SupervisorEvent) => boolean): void { | |
| 85 | for (const [key, sum] of sums) { | |
| 86 | if (sum.in === 0 && sum.out === 0) sums.delete(key); | |
| 87 | else if (send({ e: "egress-bytes", ...sum })) sums.delete(key); | |
| 88 | else return; | |
| 89 | } | |
| 90 | }, | |
| 91 | }; | |
| 92 | } | |
| 93 | ||
| 94 | const bytes = (v: unknown): v is number => Number.isSafeInteger(v) && (v as number) >= 0; | |
| 59 | 95 | |
| 60 | 96 | /** |
| 61 | 97 | * Start `path` over when it is past `max` bytes: a writer holding it open in append mode goes on. |
| @@ -178,6 +214,7 @@ export function parseConfig(text: string): SupervisorConfig | undefined { | ||
| 178 | 214 | if (!Array.isArray(v["loops"]) || !v["loops"].every(isPort)) return undefined; |
| 179 | 215 | if (v["listen"] !== undefined && !isPort(v["listen"])) return undefined; |
| 180 | 216 | if (v["outPort"] !== undefined && !isPort(v["outPort"])) return undefined; |
| 217 | if (v["flushMs"] !== undefined && !(Number.isInteger(v["flushMs"]) && (v["flushMs"] as number) >= 50 && (v["flushMs"] as number) <= EGRESS_FLUSH_MS)) return undefined; | |
| 181 | 218 | if (v["door"] !== undefined && !isPort(v["door"])) return undefined; |
| 182 | 219 | if (v["doorAt"] !== undefined && !isPort(v["doorAt"])) return undefined; |
| 183 | 220 | const origins = v["origins"]; |
| @@ -194,6 +231,12 @@ export function parseEvent(line: string): SupervisorEvent | undefined { | ||
| 194 | 231 | try { |
| 195 | 232 | const v = JSON.parse(line) as Record<string, unknown>; |
| 196 | 233 | if (v["e"] === "log" && typeof v["line"] === "string") return { e: "log", line: v["line"].slice(0, 500) }; |
| 234 | if (v["e"] === "egress-bytes") { | |
| 235 | const { label, destination } = v; | |
| 236 | if (typeof label !== "string" || typeof destination !== "string" || label.length > 200 || destination.length > 300) return undefined; | |
| 237 | if (!bytes(v["in"]) || !bytes(v["out"])) return undefined; | |
| 238 | return { e: "egress-bytes", label, destination, in: v["in"], out: v["out"] }; | |
| 239 | } | |
| 197 | 240 | if (typeof v["host"] !== "string" || typeof v["port"] !== "number") return undefined; |
| 198 | 241 | if (v["e"] === "undeclared") return { e: "undeclared", host: v["host"], port: v["port"] }; |
| 199 | 242 | if (v["e"] === "refused") return { e: "refused", host: v["host"], port: v["port"], private: v["private"] === true }; |
| @@ -209,13 +252,14 @@ export function parseEvent(line: string): SupervisorEvent | undefined { | ||
| 209 | 252 | export function runSupervisor(ctl: string, config: SupervisorConfig, wait = WAIT_FOR_CELL_MS): { done: Promise<void>; stop: () => void } { |
| 210 | 253 | const f = files(ctl); |
| 211 | 254 | const eventAllowed = perMinute(EVENTS_PER_MINUTE); |
| 212 | const say = (event: SupervisorEvent): void => { | |
| 213 | if (!eventAllowed()) return; | |
| 255 | const say = (event: SupervisorEvent): boolean => { | |
| 256 | if (!eventAllowed()) return false; | |
| 214 | 257 | try { |
| 215 | 258 | appendFileSync(f.events, `${JSON.stringify(event)}\n`); |
| 216 | 259 | } catch { |
| 217 | 260 | // the control directory is gone: the host closed it |
| 218 | 261 | } |
| 262 | return true; | |
| 219 | 263 | }; |
| 220 | 264 | let cache: { at: number; list: string[] } = { at: -1, list: [] }; |
| 221 | 265 | const allowed = (): readonly string[] => { |
| @@ -228,8 +272,10 @@ export function runSupervisor(ctl: string, config: SupervisorConfig, wait = WAIT | ||
| 228 | 272 | return cache.list; |
| 229 | 273 | }; |
| 230 | 274 | const opened: Egress[] = []; |
| 231 | // TODO(#13921): `recordEgress` counts this proxy's bytes in this process, so the host's egress | |
| 232 | // gauges miss an outliving cell's traffic until they are summed here and sent back as events. | |
| 275 | // The proxy's bytes are counted in this process; the host's gauges get them summed, as events. | |
| 276 | const batch = egressBatch(); | |
| 277 | const untap = tapEgress(batch.add); | |
| 278 | const flusher = setInterval(() => void during("cell egress flush", () => batch.flush(say)), config.flushMs ?? EGRESS_FLUSH_MS); | |
| 233 | 279 | const policy = proxyOptions(config, { |
| 234 | 280 | allowed, |
| 235 | 281 | refused: (host, port, isPrivate) => say({ e: "refused", host, port, private: isPrivate }), |
| @@ -254,7 +300,10 @@ export function runSupervisor(ctl: string, config: SupervisorConfig, wait = WAIT | ||
| 254 | 300 | if (closed) return; |
| 255 | 301 | closed = true; |
| 256 | 302 | clearInterval(timer); |
| 303 | clearInterval(flusher); | |
| 257 | 304 | for (const o of opened) o.close(); |
| 305 | untap(); | |
| 306 | batch.flush(say); | |
| 258 | 307 | rmSync(config.sockets, { recursive: true, force: true }); |
| 259 | 308 | // The host reads the last of the log and events when it is there; with no host, nobody will. |
| 260 | 309 | const host = readPid(f.host); |
| @@ -387,6 +436,7 @@ export function startSupervisor(ctl: string, config: SupervisorConfig, hooks: Su | ||
| 387 | 436 | const event = parseEvent(text); |
| 388 | 437 | if (event === undefined) return; |
| 389 | 438 | if (event.e === "log") hooks.log(event.line); |
| 439 | else if (event.e === "egress-bytes") recordEgress(event.label, event.destination, event.in, event.out); | |
| 390 | 440 | else if (event.e === "refused") hooks.onRefused?.(event.host, event.port, event.private); |
| 391 | 441 | else hooks.onUndeclared?.(event.host, event.port); |
| 392 | 442 | }); |
test/cell-outlives-host.test.ts +91
| @@ -321,6 +321,97 @@ test("stopping the host leaves an outliving cell running, and stops a default on | ||
| 321 | 321 | expect(await stopsWith(false)).toBe(true); |
| 322 | 322 | }); |
| 323 | 323 | |
| 324 | /** A CONNECT through the proxy at `port` to `destination`, then one GET: everything read back. */ | |
| 325 | const viaProxy = (port: number, destination: string): Promise<string> => | |
| 326 | new Promise((resolve, reject) => { | |
| 327 | let got = ""; | |
| 328 | let tunnelled = false; | |
| 329 | Bun.connect({ | |
| 330 | hostname: "127.0.0.1", | |
| 331 | port, | |
| 332 | socket: { | |
| 333 | open: (s) => void s.write(`CONNECT ${destination} HTTP/1.1\r\nHost: ${destination}\r\n\r\n`), | |
| 334 | data: (s, chunk) => { | |
| 335 | got += new TextDecoder().decode(chunk); | |
| 336 | if (!tunnelled && got.includes("\r\n\r\n")) { | |
| 337 | tunnelled = true; | |
| 338 | s.write(`GET / HTTP/1.1\r\nHost: ${destination}\r\nConnection: close\r\n\r\n`); | |
| 339 | } | |
| 340 | }, | |
| 341 | close: () => resolve(got), | |
| 342 | error: (_s, e) => reject(e), | |
| 343 | connectError: (_s, e) => reject(e), | |
| 344 | }, | |
| 345 | }).catch(reject); | |
| 346 | }); | |
| 347 | ||
| 348 | test("an outliving cell's egress bytes are summed in its supervisor and reach the host's gauges (#13922)", async () => { | |
| 349 | const { egressTotals } = await import("../src/apps/egress.ts"); | |
| 350 | const { startSupervisor } = await import("../src/spawn/supervisor.ts"); | |
| 351 | const body = "b".repeat(20_000); | |
| 352 | const server = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch: () => new Response(body) }); | |
| 353 | const port = server.port ?? 0; | |
| 354 | const destination = `127.0.0.1:${port}`; | |
| 355 | const dir = mkdtempSync(join(tmpdir(), "outlive-")); | |
| 356 | const outPort = freePort(); | |
| 357 | const sup = startSupervisor(join(dir, "ctl"), { sockets: join(dir, "sockets"), hosts: [], origins: [{ host: "127.0.0.1", port }], label: "app:bytes", report: false, loops: [], outPort, flushMs: 300 }, { log: () => {} }); | |
| 358 | killed.push(sup.pid); | |
| 359 | const cell = Bun.spawn(["sleep", "30"], { stdout: "ignore", stderr: "ignore" }); | |
| 360 | killed.push(cell.pid); | |
| 361 | sup.adopt(cell.pid); | |
| 362 | const seen = (): number => egressTotals().find((t) => t.destination === destination)?.bytesIn ?? 0; | |
| 363 | try { | |
| 364 | let up = false; | |
| 365 | for (let i = 0; i < 100 && !up; i++) { | |
| 366 | up = await accepts(outPort); | |
| 367 | if (!up) await Bun.sleep(50); // real-timer: polling a real supervisor's proxy | |
| 368 | } | |
| 369 | expect(up).toBe(true); | |
| 370 | // The cell's request: through its supervisor's proxy, so this process's proxy counts nothing. | |
| 371 | for (let i = 0; i < 3; i++) { | |
| 372 | const got = await viaProxy(outPort, destination); | |
| 373 | expect(got).toContain(body); | |
| 374 | } | |
| 375 | expect(seen()).toBe(0); | |
| 376 | // Summed and sent back each flush (300 ms here, 5 s in a cell), one event per destination. | |
| 377 | expect(await until(() => seen() >= 3 * body.length, 5_000)).toBe(true); | |
| 378 | const lines = readFileSync(files(sup.ctl).events, "utf8").split("\n").filter((l) => l.includes("egress-bytes")); | |
| 379 | expect(lines.length).toBeGreaterThan(0); | |
| 380 | expect(lines.length).toBeLessThanOrEqual(3); // at most one a flush; the unit test below sums chunks | |
| 381 | expect(parseEvent(lines[0] ?? "")).toMatchObject({ e: "egress-bytes", label: "app:bytes", destination }); | |
| 382 | } finally { | |
| 383 | cell.kill("SIGKILL"); | |
| 384 | sup.close(); | |
| 385 | server.stop(true); | |
| 386 | } | |
| 387 | }, 20_000); | |
| 388 | ||
| 389 | test("an egress-bytes event is plain data: counts are whole and not negative", () => { | |
| 390 | expect(parseEvent('{"e":"egress-bytes","label":"a","destination":"h:443","in":5,"out":2}')).toEqual({ e: "egress-bytes", label: "a", destination: "h:443", in: 5, out: 2 }); | |
| 391 | expect(parseEvent('{"e":"egress-bytes","label":"a","destination":"h:443","in":-5,"out":2}')).toBeUndefined(); | |
| 392 | expect(parseEvent('{"e":"egress-bytes","label":"a","destination":"h:443","in":"5","out":2}')).toBeUndefined(); | |
| 393 | expect(parseEvent('{"e":"egress-bytes","label":1,"destination":"h:443","in":5,"out":2}')).toBeUndefined(); | |
| 394 | }); | |
| 395 | ||
| 396 | test("a batch sums chunks per label and destination, and keeps what the events budget refused", async () => { | |
| 397 | const { egressBatch } = await import("../src/spawn/supervisor.ts"); | |
| 398 | const b = egressBatch(); | |
| 399 | for (let i = 0; i < 1_000; i++) b.add("app:x", "h:443", 10, 1); | |
| 400 | b.add("app:y", "h:443", 3, 0); | |
| 401 | const sent: unknown[] = []; | |
| 402 | b.flush((e) => (sent.push(e), false)); | |
| 403 | expect(sent).toHaveLength(1); | |
| 404 | sent.length = 0; | |
| 405 | b.flush((e) => (sent.push(e), true)); | |
| 406 | expect(sent).toEqual([ | |
| 407 | { e: "egress-bytes", label: "app:x", destination: "h:443", in: 10_000, out: 1_000 }, | |
| 408 | { e: "egress-bytes", label: "app:y", destination: "h:443", in: 3, out: 0 }, | |
| 409 | ]); | |
| 410 | sent.length = 0; | |
| 411 | b.flush((e) => (sent.push(e), true)); | |
| 412 | expect(sent).toEqual([]); | |
| 413 | }); | |
| 414 | ||
| 324 | 415 | test("kill the host: the cell's beat still answers, from the supervisor's held door, never through the dead host's port", async () => { |
| 325 | 416 | const dir = mkdtempSync(join(tmpdir(), "outlive-")); |
| 326 | 417 | const ctl = join(dir, "ctl"); |
test/job-slices.test.ts +1
| @@ -294,6 +294,7 @@ const UNTIMED: Record<string, string> = { | ||
| 294 | 294 | "apps-sweep": "AppsRuntime's sweep over running app processes; the fixture has no apps", |
| 295 | 295 | "profiler-drain": "drains the sampling profiler's stack traces on a timer (perf/sample.ts) so they do not pile up; runs only where the profiler is on, and the fixture samples nothing", |
| 296 | 296 | "cell output tail": "runs in a cell's supervisor process, not the host: reads a bounded tail of the child's output file and trims it; needs a supervised cell", |
| 297 | "cell egress flush": "runs in a cell's supervisor process, not the host: appends at most 64 summed byte lines to the events file every 5 s; needs a supervised cell", | |
| 297 | 298 | "cell allowed sync": "runs in a cell's supervisor process, not the host: one small file write of the allowed hosts; needs a supervised cell", |
| 298 | 299 | "cell-watch": "watches running app cells' cgroups; needs cells, skipped in this sandbox mode", |
| 299 | 300 | "cell-probe-deadline": "a cell's start probe running out of time: needs cells, skipped in this sandbox mode", |
new above
offline