Public: anyone can read this; the crew can change it.

An outliving cell's egress bytes are counted in the host's gauges (#13921) #15277

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/15254-an-outliving-cell-s-egress-bytes-are-cou → main 1 commit fork

  • Merged. scout/15254-an-outliving-cell-s-egress-bytes-are-cou was merged into main as 0e1969c.

Review

Sign in to review this change.
4 files changed +66 −5
docs/features/cells-outlive.md +1 −1
@@ -13,7 +13,7 @@ On when the host boots (#13750), so a production release leaves a hosted forge's
1313 - the port must answer. A cell that is its supervisor's but silent is stopped and its record cleared, since nothing else would reap it;
1414 - otherwise the cell is held in the registry as if this host had started it. A start for the same body waits for adoption to finish and finds the adopted cell, never a second one. The record is cleared when the cell goes.
1515
16 Not yet: the beat door still runs in the host, egress bytes of an outliving cell are not in the host's gauges (#13921).
16 Not yet: the beat door still runs in the host (#13921).
1717
1818 **What the host did is shown.** For this host's life it keeps each cell it adopted or dropped at boot (`spawn/adopted.ts`: name, pid, when, why). /system lists them under "Child forges across the restart" ("kept running across the restart", "stopped: did not answer"), and nothing when there were none; the MCP `system` tool (`kind: adopted`) and the terminal's System screen read the same rows.
1919
src/spawn/cell.ts +5 −2
@@ -27,7 +27,7 @@ import { childModelHosts, childPaymentHosts, childRegistryHosts } from "../core/
2727 import { liveChildDisk, type ChildDisk } from "./disk_mount.ts";
2828 import { oneAtATime, sourcesFor, startFailure, startFirstThatWorks, useTree, type CodeSource } from "./own_code.ts";
2929 import { childGate, type ChildGate } from "./child_reach.ts";
30 import { files, readListen, readStartedPid } from "./supervisor.ts";
30 import { files, followEgress, readListen, readStartedPid } from "./supervisor.ts";
3131 import { adoptionRows, type Adoption, type AdoptionRow } from "./adopted.ts";
3232 import { clearRecord, dropLegacyRecords, leaveRecord, readRecords, type CellRecord } from "./cell_record.ts";
3333
@@ -40,7 +40,7 @@ import { clearRecord, dropLegacyRecords, leaveRecord, readRecords, type CellReco
4040 * (#13756); its beat door (`beat-door.ts`) is this process's, reached through the supervisor's
4141 * relay while this process lives and answered by the supervisor's held door after (`held_door.ts`,
4242 * #13906), and its egress bytes are summed in the supervisor and reach this process's gauges as
43 * events (#13922).
43 * events (#13922), an adopted cell's as well (`followEgress`, #15254).
4444 */
4545 export const cellsOutlive = (env: Record<string, string | undefined> = process.env): boolean => env["FORGE_CELLS_OUTLIVE"] === "on";
4646
@@ -499,7 +499,10 @@ export function makeRegistry(parent: App, launcher?: Launcher): CellChildren {
499499 startedAt.set(record.bodyId, record.startedAt);
500500 boot.push({ name: record.name, pid: record.pid, at: Date.now(), outcome: "kept", why: null });
501501 parent.log.info("child forge adopted", { name: record.name, pid: record.pid });
502 // Its supervisor still sums the cell's egress bytes: they reach this host's gauges too.
503 const unfollow = followEgress(record.supervisor);
502504 void gone.then(() => {
505 unfollow();
503506 drop();
504507 if (held.get(record.bodyId) === child) {
505508 held.delete(record.bodyId);
src/spawn/supervisor.ts +21 −2
@@ -375,10 +375,16 @@ export interface Supervised {
375375
376376 /**
377377 * Call `line` for each new line appended to `path`, polling; the returned function stops, after a
378 * last read.
378 * last read. `fromEnd` skips what the file already holds: lines another reader has had.
379379 */
380 export function tailLines(path: string, line: (text: string) => void, everyMs = 250, max = LOG_MAX_BYTES): () => void {
380 export function tailLines(path: string, line: (text: string) => void, everyMs = 250, max = LOG_MAX_BYTES, fromEnd = false): () => void {
381381 let offset = 0;
382 if (fromEnd)
383 try {
384 offset = statSync(path).size;
385 } catch {
386 // not there yet: read it from its top when it is
387 }
382388 let rest = "";
383389 const read = (): void => {
384390 let fd: number;
@@ -420,6 +426,19 @@ export function tailLines(path: string, line: (text: string) => void, everyMs =
420426 };
421427 }
422428
429 /**
430 * The egress bytes of a cell a previous host started, from its supervisor's `events.jsonl`, into
431 * this process's gauges as for a cell it started (#15254). Read from the end: what is already in
432 * the file was the previous host's to count. Only bytes are taken; what egress allows is the
433 * supervisor's, unchanged.
434 */
435 export function followEgress(ctl: string, everyMs = 250): () => void {
436 return tailLines(files(ctl).events, (text) => {
437 const event = parseEvent(text);
438 if (event?.e === "egress-bytes") recordEgress(event.label, event.destination, event.in, event.out);
439 }, everyMs, LOG_MAX_BYTES, true);
440 }
441
423442 /**
424443 * The environment a supervisor needs: where to find tools, and the way out a child forge's cell
425444 * has.
test/cell-adopted-on-start.test.ts +39
@@ -325,3 +325,42 @@ test("a cell.pid with no start time is no cell's: the process it names is never
325325 app.close();
326326 }
327327 });
328
329 test("an adopted cell's egress bytes reach the host's gauges, and what its file held before is not counted again", async () => {
330 const { appendFileSync } = await import("node:fs");
331 const { egressTotals, resetEgressTotals, tapEgress } = await import("../src/apps/egress.ts");
332 const { egressBatch } = await import("../src/spawn/supervisor.ts");
333 resetEgressTotals();
334 const { app, dir } = setup();
335 const server = Bun.serve({ port: 0, hostname: "127.0.0.1", fetch: () => new Response("ok") });
336 const destination = "adopted.example:443";
337 // The supervisor's own way of saying its sums: one line each in events.jsonl.
338 const ctl = join(dir, "ctl");
339 const say = (event: object): boolean => (appendFileSync(files(ctl).events, `${JSON.stringify(event)}\n`), true);
340 try {
341 supervised(ctl, process.pid, server.port as number);
342 // Bytes the previous host already counted before it went.
343 say({ e: "egress-bytes", label: "child:a", destination, in: 999, out: 999 });
344 writeRecord(dir, recordFor({ port: server.port as number, supervisor: ctl }));
345 const cells = makeRegistry(app);
346 await cells.adopted;
347 expect(cells.get("cell-a")).toBeDefined();
348 // The host counts the line when it reads it: the tap hears that count.
349 const counted = Promise.withResolvers<void>();
350 const untap = tapEgress((_label, at) => at === destination && counted.resolve());
351 // The proxy relays N bytes after adoption, summed as the supervisor sums them.
352 const batch = egressBatch();
353 batch.add("child:a", destination, 1_000, 200);
354 batch.add("child:a", destination, 234, 34);
355 batch.flush(say);
356 const moved = (): { bytesIn: number; bytesOut: number } | undefined => egressTotals().find((t) => t.destination === destination);
357 await counted.promise;
358 untap();
359 expect(moved()?.bytesIn).toBe(1_234);
360 expect(moved()?.bytesOut).toBe(234);
361 } finally {
362 server.stop(true);
363 app.close();
364 resetEgressTotals();
365 }
366 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs