Public: anyone can read this; the crew can change it.

An outliving cell's egress bytes reach the host's gauges, counted in the supervisor and sent back once #13755 lands #13929

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/13922-an-outliving-cell-s-egress-bytes-reach-t → main 8 commits fork

  • Merged. scout/13922-an-outliving-cell-s-egress-bytes-reach-t was merged into main as 312a256.

Review

Sign in to review this change.
5 files changed +166 −9
src/apps/egress.ts +15
@@ -268,6 +268,20 @@ function recordNpmRepo(repoId: string, bytesIn: number, minute: number): void {
268268 addMinute(m, minute, bytesIn);
269269 }
270270
271 /** Where this process's egress bytes also go (`tapEgress`); undefined in the host. */
272 let tap: ((label: string, destination: string, bytesIn: number, bytesOut: number) => void) | undefined;
273
274 /**
275 * Hand every byte `recordEgress` counts to `fn` as well: a cell's supervisor (#13922) sums them and
276 * sends them back to the host, whose gauges are the ones served. Returns the undo.
277 */
278 export function tapEgress(fn: (label: string, destination: string, bytesIn: number, bytesOut: number) => void): () => void {
279 tap = fn;
280 return () => {
281 if (tap === fn) tap = undefined;
282 };
283 }
284
271285 /** Count `bytesIn` received from and `bytesOut` sent to `destination`. */
272286 export function recordEgress(
273287 label: string,
@@ -276,6 +290,7 @@ export function recordEgress(
276290 bytesOut: number,
277291 now = Date.now(),
278292 ): void {
293 tap?.(label, destination, bytesIn, bytesOut);
279294 const key =
280295 destination === NPM_REGISTRY || moved.has(destination) || moved.size < MOVED_MAX
281296 ? destination
src/spawn/cell.ts +2 −2
@@ -37,8 +37,8 @@ import { clearRecord, leaveRecord, readRecords, type CellRecord } from "./cell_r
3737 * record (`cell_record.ts`) that the next host adopts if the cell is still its supervisor's and
3838 * answers, and stops if it does not answer (#13756); its beat door (`beat-door.ts`) is this
3939 * process's, reached through the supervisor's relay while this process lives and answered by the
40 * supervisor's held door after (`held_door.ts`, #13906), and its egress bytes are counted in the
41 * supervisor, out of this process's gauges (#13921).
40 * supervisor's held door after (`held_door.ts`, #13906), and its egress bytes are summed in the
41 * supervisor and reach this process's gauges as events (#13922).
4242 */
4343 export const cellsOutlive = (env: Record<string, string | undefined> = process.env): boolean => env["FORGE_CELLS_OUTLIVE"] === "on";
4444
src/spawn/supervisor.ts +57 −7
@@ -12,7 +12,8 @@
1212 * cell.log the cell's stdout and stderr, as a file the cell holds open
1313 * allowed.json egress entries allowed since the cell started (host rewrites when they change)
1414 * beat.held the cell's latest beat once no host holds its door (supervisor writes; #13906)
15 * events.jsonl refusals and undeclared hosts (supervisor appends; the host tails)
15 * events.jsonl refusals, undeclared hosts and egress byte sums (supervisor appends; the host
16 * tails)
1617 *
1718 * It ends when the cell's root process is gone, on SIGTERM, or when no cell appears in a minute.
1819 * The host side of this file is `startSupervisor`; `bun supervisor.ts <ctl dir>` is the other.
@@ -20,7 +21,7 @@
2021 import { appendFileSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, readSync, rmSync, statSync, truncateSync, writeFileSync } from "node:fs";
2122 import { join } from "node:path";
2223 import { during } from "../perf/jobs.ts";
23 import { PROXY_VARS, relay, startEgress, type Egress } from "../apps/egress.ts";
24 import { PROXY_VARS, recordEgress, relay, startEgress, tapEgress, type Egress } from "../apps/egress.ts";
2425 import { proxyOptions, type CellPolicy } from "../apps/cell_policy.ts";
2526 import { openHeldDoor } from "./held_door.ts";
2627
@@ -30,6 +31,8 @@ export interface SupervisorConfig extends CellPolicy {
3031 readonly sockets: string;
3132 /** Tests: the proxy listens on this loopback port, where a unix socket cannot be bound. */
3233 readonly outPort?: number;
34 /** Tests: how often summed egress bytes are sent back, in place of `EGRESS_FLUSH_MS`. */
35 readonly flushMs?: number;
3336 /**
3437 * The port the cell knows its parent's beat door by (`FORGE_PARENT_LOCAL`, #13906): its loop is
3538 * relayed to the host's door while the host lives, and to the supervisor's held door
@@ -44,7 +47,8 @@ export interface SupervisorConfig extends CellPolicy {
4447 export type SupervisorEvent =
4548 | { e: "refused"; host: string; port: number; private: boolean }
4649 | { e: "undeclared"; host: string; port: number }
47 | { e: "log"; line: string };
50 | { e: "log"; line: string }
51 | { e: "egress-bytes"; label: string; destination: string; in: number; out: number };
4852
4953 const WAIT_FOR_CELL_MS = 60_000;
5054 const TICK_MS = 1_000;
@@ -56,6 +60,38 @@ export const LOG_MAX_BYTES = 4_000_000;
5660 export const EVENTS_MAX_BYTES = 1_000_000;
5761 /** Events the supervisor writes in a minute; past it they are dropped. */
5862 export const EVENTS_PER_MINUTE = 300;
63 /** How often the supervisor sends its summed egress bytes back: never once per chunk. */
64 export const EGRESS_FLUSH_MS = 5_000;
65 /** The most destinations one batch holds; past it the rest are summed under `other`. */
66 const BATCH_MAX = 64;
67
68 /** Egress bytes summed per label and destination between flushes (#13922). */
69 export function egressBatch() {
70 const sums = new Map<string, { label: string; destination: string; in: number; out: number }>();
71 return {
72 add(label: string, destination: string, bytesIn: number, bytesOut: number): void {
73 const at = sums.size < BATCH_MAX || sums.has(`${label}\n${destination}`) ? destination : "other";
74 const key = `${label}\n${at}`;
75 let sum = sums.get(key);
76 if (sum === undefined) sums.set(key, (sum = { label, destination: at, in: 0, out: 0 }));
77 sum.in += bytesIn;
78 sum.out += bytesOut;
79 },
80 /**
81 * Hand each sum to `send`; a sum `send` refuses (the events budget is spent) stays for the
82 * next.
83 */
84 flush(send: (event: SupervisorEvent) => boolean): void {
85 for (const [key, sum] of sums) {
86 if (sum.in === 0 && sum.out === 0) sums.delete(key);
87 else if (send({ e: "egress-bytes", ...sum })) sums.delete(key);
88 else return;
89 }
90 },
91 };
92 }
93
94 const bytes = (v: unknown): v is number => Number.isSafeInteger(v) && (v as number) >= 0;
5995
6096 /**
6197 * Start `path` over when it is past `max` bytes: a writer holding it open in append mode goes on.
@@ -178,6 +214,7 @@ export function parseConfig(text: string): SupervisorConfig | undefined {
178214 if (!Array.isArray(v["loops"]) || !v["loops"].every(isPort)) return undefined;
179215 if (v["listen"] !== undefined && !isPort(v["listen"])) return undefined;
180216 if (v["outPort"] !== undefined && !isPort(v["outPort"])) return undefined;
217 if (v["flushMs"] !== undefined && !(Number.isInteger(v["flushMs"]) && (v["flushMs"] as number) >= 50 && (v["flushMs"] as number) <= EGRESS_FLUSH_MS)) return undefined;
181218 if (v["door"] !== undefined && !isPort(v["door"])) return undefined;
182219 if (v["doorAt"] !== undefined && !isPort(v["doorAt"])) return undefined;
183220 const origins = v["origins"];
@@ -194,6 +231,12 @@ export function parseEvent(line: string): SupervisorEvent | undefined {
194231 try {
195232 const v = JSON.parse(line) as Record<string, unknown>;
196233 if (v["e"] === "log" && typeof v["line"] === "string") return { e: "log", line: v["line"].slice(0, 500) };
234 if (v["e"] === "egress-bytes") {
235 const { label, destination } = v;
236 if (typeof label !== "string" || typeof destination !== "string" || label.length > 200 || destination.length > 300) return undefined;
237 if (!bytes(v["in"]) || !bytes(v["out"])) return undefined;
238 return { e: "egress-bytes", label, destination, in: v["in"], out: v["out"] };
239 }
197240 if (typeof v["host"] !== "string" || typeof v["port"] !== "number") return undefined;
198241 if (v["e"] === "undeclared") return { e: "undeclared", host: v["host"], port: v["port"] };
199242 if (v["e"] === "refused") return { e: "refused", host: v["host"], port: v["port"], private: v["private"] === true };
@@ -209,13 +252,14 @@ export function parseEvent(line: string): SupervisorEvent | undefined {
209252 export function runSupervisor(ctl: string, config: SupervisorConfig, wait = WAIT_FOR_CELL_MS): { done: Promise<void>; stop: () => void } {
210253 const f = files(ctl);
211254 const eventAllowed = perMinute(EVENTS_PER_MINUTE);
212 const say = (event: SupervisorEvent): void => {
213 if (!eventAllowed()) return;
255 const say = (event: SupervisorEvent): boolean => {
256 if (!eventAllowed()) return false;
214257 try {
215258 appendFileSync(f.events, `${JSON.stringify(event)}\n`);
216259 } catch {
217260 // the control directory is gone: the host closed it
218261 }
262 return true;
219263 };
220264 let cache: { at: number; list: string[] } = { at: -1, list: [] };
221265 const allowed = (): readonly string[] => {
@@ -228,8 +272,10 @@ export function runSupervisor(ctl: string, config: SupervisorConfig, wait = WAIT
228272 return cache.list;
229273 };
230274 const opened: Egress[] = [];
231 // TODO(#13921): `recordEgress` counts this proxy's bytes in this process, so the host's egress
232 // gauges miss an outliving cell's traffic until they are summed here and sent back as events.
275 // The proxy's bytes are counted in this process; the host's gauges get them summed, as events.
276 const batch = egressBatch();
277 const untap = tapEgress(batch.add);
278 const flusher = setInterval(() => void during("cell egress flush", () => batch.flush(say)), config.flushMs ?? EGRESS_FLUSH_MS);
233279 const policy = proxyOptions(config, {
234280 allowed,
235281 refused: (host, port, isPrivate) => say({ e: "refused", host, port, private: isPrivate }),
@@ -254,7 +300,10 @@ export function runSupervisor(ctl: string, config: SupervisorConfig, wait = WAIT
254300 if (closed) return;
255301 closed = true;
256302 clearInterval(timer);
303 clearInterval(flusher);
257304 for (const o of opened) o.close();
305 untap();
306 batch.flush(say);
258307 rmSync(config.sockets, { recursive: true, force: true });
259308 // The host reads the last of the log and events when it is there; with no host, nobody will.
260309 const host = readPid(f.host);
@@ -387,6 +436,7 @@ export function startSupervisor(ctl: string, config: SupervisorConfig, hooks: Su
387436 const event = parseEvent(text);
388437 if (event === undefined) return;
389438 if (event.e === "log") hooks.log(event.line);
439 else if (event.e === "egress-bytes") recordEgress(event.label, event.destination, event.in, event.out);
390440 else if (event.e === "refused") hooks.onRefused?.(event.host, event.port, event.private);
391441 else hooks.onUndeclared?.(event.host, event.port);
392442 });
test/cell-outlives-host.test.ts +91
@@ -321,6 +321,97 @@ test("stopping the host leaves an outliving cell running, and stops a default on
321321 expect(await stopsWith(false)).toBe(true);
322322 });
323323
324 /** A CONNECT through the proxy at `port` to `destination`, then one GET: everything read back. */
325 const viaProxy = (port: number, destination: string): Promise<string> =>
326 new Promise((resolve, reject) => {
327 let got = "";
328 let tunnelled = false;
329 Bun.connect({
330 hostname: "127.0.0.1",
331 port,
332 socket: {
333 open: (s) => void s.write(`CONNECT ${destination} HTTP/1.1\r\nHost: ${destination}\r\n\r\n`),
334 data: (s, chunk) => {
335 got += new TextDecoder().decode(chunk);
336 if (!tunnelled && got.includes("\r\n\r\n")) {
337 tunnelled = true;
338 s.write(`GET / HTTP/1.1\r\nHost: ${destination}\r\nConnection: close\r\n\r\n`);
339 }
340 },
341 close: () => resolve(got),
342 error: (_s, e) => reject(e),
343 connectError: (_s, e) => reject(e),
344 },
345 }).catch(reject);
346 });
347
348 test("an outliving cell's egress bytes are summed in its supervisor and reach the host's gauges (#13922)", async () => {
349 const { egressTotals } = await import("../src/apps/egress.ts");
350 const { startSupervisor } = await import("../src/spawn/supervisor.ts");
351 const body = "b".repeat(20_000);
352 const server = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch: () => new Response(body) });
353 const port = server.port ?? 0;
354 const destination = `127.0.0.1:${port}`;
355 const dir = mkdtempSync(join(tmpdir(), "outlive-"));
356 const outPort = freePort();
357 const sup = startSupervisor(join(dir, "ctl"), { sockets: join(dir, "sockets"), hosts: [], origins: [{ host: "127.0.0.1", port }], label: "app:bytes", report: false, loops: [], outPort, flushMs: 300 }, { log: () => {} });
358 killed.push(sup.pid);
359 const cell = Bun.spawn(["sleep", "30"], { stdout: "ignore", stderr: "ignore" });
360 killed.push(cell.pid);
361 sup.adopt(cell.pid);
362 const seen = (): number => egressTotals().find((t) => t.destination === destination)?.bytesIn ?? 0;
363 try {
364 let up = false;
365 for (let i = 0; i < 100 && !up; i++) {
366 up = await accepts(outPort);
367 if (!up) await Bun.sleep(50); // real-timer: polling a real supervisor's proxy
368 }
369 expect(up).toBe(true);
370 // The cell's request: through its supervisor's proxy, so this process's proxy counts nothing.
371 for (let i = 0; i < 3; i++) {
372 const got = await viaProxy(outPort, destination);
373 expect(got).toContain(body);
374 }
375 expect(seen()).toBe(0);
376 // Summed and sent back each flush (300 ms here, 5 s in a cell), one event per destination.
377 expect(await until(() => seen() >= 3 * body.length, 5_000)).toBe(true);
378 const lines = readFileSync(files(sup.ctl).events, "utf8").split("\n").filter((l) => l.includes("egress-bytes"));
379 expect(lines.length).toBeGreaterThan(0);
380 expect(lines.length).toBeLessThanOrEqual(3); // at most one a flush; the unit test below sums chunks
381 expect(parseEvent(lines[0] ?? "")).toMatchObject({ e: "egress-bytes", label: "app:bytes", destination });
382 } finally {
383 cell.kill("SIGKILL");
384 sup.close();
385 server.stop(true);
386 }
387 }, 20_000);
388
389 test("an egress-bytes event is plain data: counts are whole and not negative", () => {
390 expect(parseEvent('{"e":"egress-bytes","label":"a","destination":"h:443","in":5,"out":2}')).toEqual({ e: "egress-bytes", label: "a", destination: "h:443", in: 5, out: 2 });
391 expect(parseEvent('{"e":"egress-bytes","label":"a","destination":"h:443","in":-5,"out":2}')).toBeUndefined();
392 expect(parseEvent('{"e":"egress-bytes","label":"a","destination":"h:443","in":"5","out":2}')).toBeUndefined();
393 expect(parseEvent('{"e":"egress-bytes","label":1,"destination":"h:443","in":5,"out":2}')).toBeUndefined();
394 });
395
396 test("a batch sums chunks per label and destination, and keeps what the events budget refused", async () => {
397 const { egressBatch } = await import("../src/spawn/supervisor.ts");
398 const b = egressBatch();
399 for (let i = 0; i < 1_000; i++) b.add("app:x", "h:443", 10, 1);
400 b.add("app:y", "h:443", 3, 0);
401 const sent: unknown[] = [];
402 b.flush((e) => (sent.push(e), false));
403 expect(sent).toHaveLength(1);
404 sent.length = 0;
405 b.flush((e) => (sent.push(e), true));
406 expect(sent).toEqual([
407 { e: "egress-bytes", label: "app:x", destination: "h:443", in: 10_000, out: 1_000 },
408 { e: "egress-bytes", label: "app:y", destination: "h:443", in: 3, out: 0 },
409 ]);
410 sent.length = 0;
411 b.flush((e) => (sent.push(e), true));
412 expect(sent).toEqual([]);
413 });
414
324415 test("kill the host: the cell's beat still answers, from the supervisor's held door, never through the dead host's port", async () => {
325416 const dir = mkdtempSync(join(tmpdir(), "outlive-"));
326417 const ctl = join(dir, "ctl");
test/job-slices.test.ts +1
@@ -294,6 +294,7 @@ const UNTIMED: Record<string, string> = {
294294 "apps-sweep": "AppsRuntime's sweep over running app processes; the fixture has no apps",
295295 "profiler-drain": "drains the sampling profiler's stack traces on a timer (perf/sample.ts) so they do not pile up; runs only where the profiler is on, and the fixture samples nothing",
296296 "cell output tail": "runs in a cell's supervisor process, not the host: reads a bounded tail of the child's output file and trims it; needs a supervised cell",
297 "cell egress flush": "runs in a cell's supervisor process, not the host: appends at most 64 summed byte lines to the events file every 5 s; needs a supervised cell",
297298 "cell allowed sync": "runs in a cell's supervisor process, not the host: one small file write of the allowed hosts; needs a supervised cell",
298299 "cell-watch": "watches running app cells' cgroups; needs cells, skipped in this sandbox mode",
299300 "cell-probe-deadline": "a cell's start probe running out of time: needs cells, skipped in this sandbox mode",

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs