Public: anyone can read this; the crew can change it.
An outliving cell's egress bytes are counted in the host's gauges (#13921) #15277
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/15254-an-outliving-cell-s-egress-bytes-are-cou → main 1 commit fork
4 files
changed +66
−5
docs/features/cells-outlive.md +1 −1
| @@ -13,7 +13,7 @@ On when the host boots (#13750), so a production release leaves a hosted forge's | ||
| 13 | 13 | - the port must answer. A cell that is its supervisor's but silent is stopped and its record cleared, since nothing else would reap it; |
| 14 | 14 | - otherwise the cell is held in the registry as if this host had started it. A start for the same body waits for adoption to finish and finds the adopted cell, never a second one. The record is cleared when the cell goes. |
| 15 | 15 | |
| 16 | Not yet: the beat door still runs in the host, egress bytes of an outliving cell are not in the host's gauges (#13921). | |
| 16 | Not yet: the beat door still runs in the host (#13921). | |
| 17 | 17 | |
| 18 | 18 | **What the host did is shown.** For this host's life it keeps each cell it adopted or dropped at boot (`spawn/adopted.ts`: name, pid, when, why). /system lists them under "Child forges across the restart" ("kept running across the restart", "stopped: did not answer"), and nothing when there were none; the MCP `system` tool (`kind: adopted`) and the terminal's System screen read the same rows. |
| 19 | 19 |
src/spawn/cell.ts +5 −2
| @@ -27,7 +27,7 @@ import { childModelHosts, childPaymentHosts, childRegistryHosts } from "../core/ | ||
| 27 | 27 | import { liveChildDisk, type ChildDisk } from "./disk_mount.ts"; |
| 28 | 28 | import { oneAtATime, sourcesFor, startFailure, startFirstThatWorks, useTree, type CodeSource } from "./own_code.ts"; |
| 29 | 29 | import { childGate, type ChildGate } from "./child_reach.ts"; |
| 30 | import { files, readListen, readStartedPid } from "./supervisor.ts"; | |
| 30 | import { files, followEgress, readListen, readStartedPid } from "./supervisor.ts"; | |
| 31 | 31 | import { adoptionRows, type Adoption, type AdoptionRow } from "./adopted.ts"; |
| 32 | 32 | import { clearRecord, dropLegacyRecords, leaveRecord, readRecords, type CellRecord } from "./cell_record.ts"; |
| 33 | 33 | |
| @@ -40,7 +40,7 @@ import { clearRecord, dropLegacyRecords, leaveRecord, readRecords, type CellReco | ||
| 40 | 40 | * (#13756); its beat door (`beat-door.ts`) is this process's, reached through the supervisor's |
| 41 | 41 | * relay while this process lives and answered by the supervisor's held door after (`held_door.ts`, |
| 42 | 42 | * #13906), and its egress bytes are summed in the supervisor and reach this process's gauges as |
| 43 | * events (#13922). | |
| 43 | * events (#13922), an adopted cell's as well (`followEgress`, #15254). | |
| 44 | 44 | */ |
| 45 | 45 | export const cellsOutlive = (env: Record<string, string | undefined> = process.env): boolean => env["FORGE_CELLS_OUTLIVE"] === "on"; |
| 46 | 46 | |
| @@ -499,7 +499,10 @@ export function makeRegistry(parent: App, launcher?: Launcher): CellChildren { | ||
| 499 | 499 | startedAt.set(record.bodyId, record.startedAt); |
| 500 | 500 | boot.push({ name: record.name, pid: record.pid, at: Date.now(), outcome: "kept", why: null }); |
| 501 | 501 | parent.log.info("child forge adopted", { name: record.name, pid: record.pid }); |
| 502 | // Its supervisor still sums the cell's egress bytes: they reach this host's gauges too. | |
| 503 | const unfollow = followEgress(record.supervisor); | |
| 502 | 504 | void gone.then(() => { |
| 505 | unfollow(); | |
| 503 | 506 | drop(); |
| 504 | 507 | if (held.get(record.bodyId) === child) { |
| 505 | 508 | held.delete(record.bodyId); |
src/spawn/supervisor.ts +21 −2
| @@ -375,10 +375,16 @@ export interface Supervised { | ||
| 375 | 375 | |
| 376 | 376 | /** |
| 377 | 377 | * Call `line` for each new line appended to `path`, polling; the returned function stops, after a |
| 378 | * last read. | |
| 378 | * last read. `fromEnd` skips what the file already holds: lines another reader has had. | |
| 379 | 379 | */ |
| 380 | export function tailLines(path: string, line: (text: string) => void, everyMs = 250, max = LOG_MAX_BYTES): () => void { | |
| 380 | export function tailLines(path: string, line: (text: string) => void, everyMs = 250, max = LOG_MAX_BYTES, fromEnd = false): () => void { | |
| 381 | 381 | let offset = 0; |
| 382 | if (fromEnd) | |
| 383 | try { | |
| 384 | offset = statSync(path).size; | |
| 385 | } catch { | |
| 386 | // not there yet: read it from its top when it is | |
| 387 | } | |
| 382 | 388 | let rest = ""; |
| 383 | 389 | const read = (): void => { |
| 384 | 390 | let fd: number; |
| @@ -420,6 +426,19 @@ export function tailLines(path: string, line: (text: string) => void, everyMs = | ||
| 420 | 426 | }; |
| 421 | 427 | } |
| 422 | 428 | |
| 429 | /** | |
| 430 | * The egress bytes of a cell a previous host started, from its supervisor's `events.jsonl`, into | |
| 431 | * this process's gauges as for a cell it started (#15254). Read from the end: what is already in | |
| 432 | * the file was the previous host's to count. Only bytes are taken; what egress allows is the | |
| 433 | * supervisor's, unchanged. | |
| 434 | */ | |
| 435 | export function followEgress(ctl: string, everyMs = 250): () => void { | |
| 436 | return tailLines(files(ctl).events, (text) => { | |
| 437 | const event = parseEvent(text); | |
| 438 | if (event?.e === "egress-bytes") recordEgress(event.label, event.destination, event.in, event.out); | |
| 439 | }, everyMs, LOG_MAX_BYTES, true); | |
| 440 | } | |
| 441 | ||
| 423 | 442 | /** |
| 424 | 443 | * The environment a supervisor needs: where to find tools, and the way out a child forge's cell |
| 425 | 444 | * has. |
test/cell-adopted-on-start.test.ts +39
| @@ -325,3 +325,42 @@ test("a cell.pid with no start time is no cell's: the process it names is never | ||
| 325 | 325 | app.close(); |
| 326 | 326 | } |
| 327 | 327 | }); |
| 328 | ||
| 329 | test("an adopted cell's egress bytes reach the host's gauges, and what its file held before is not counted again", async () => { | |
| 330 | const { appendFileSync } = await import("node:fs"); | |
| 331 | const { egressTotals, resetEgressTotals, tapEgress } = await import("../src/apps/egress.ts"); | |
| 332 | const { egressBatch } = await import("../src/spawn/supervisor.ts"); | |
| 333 | resetEgressTotals(); | |
| 334 | const { app, dir } = setup(); | |
| 335 | const server = Bun.serve({ port: 0, hostname: "127.0.0.1", fetch: () => new Response("ok") }); | |
| 336 | const destination = "adopted.example:443"; | |
| 337 | // The supervisor's own way of saying its sums: one line each in events.jsonl. | |
| 338 | const ctl = join(dir, "ctl"); | |
| 339 | const say = (event: object): boolean => (appendFileSync(files(ctl).events, `${JSON.stringify(event)}\n`), true); | |
| 340 | try { | |
| 341 | supervised(ctl, process.pid, server.port as number); | |
| 342 | // Bytes the previous host already counted before it went. | |
| 343 | say({ e: "egress-bytes", label: "child:a", destination, in: 999, out: 999 }); | |
| 344 | writeRecord(dir, recordFor({ port: server.port as number, supervisor: ctl })); | |
| 345 | const cells = makeRegistry(app); | |
| 346 | await cells.adopted; | |
| 347 | expect(cells.get("cell-a")).toBeDefined(); | |
| 348 | // The host counts the line when it reads it: the tap hears that count. | |
| 349 | const counted = Promise.withResolvers<void>(); | |
| 350 | const untap = tapEgress((_label, at) => at === destination && counted.resolve()); | |
| 351 | // The proxy relays N bytes after adoption, summed as the supervisor sums them. | |
| 352 | const batch = egressBatch(); | |
| 353 | batch.add("child:a", destination, 1_000, 200); | |
| 354 | batch.add("child:a", destination, 234, 34); | |
| 355 | batch.flush(say); | |
| 356 | const moved = (): { bytesIn: number; bytesOut: number } | undefined => egressTotals().find((t) => t.destination === destination); | |
| 357 | await counted.promise; | |
| 358 | untap(); | |
| 359 | expect(moved()?.bytesIn).toBe(1_234); | |
| 360 | expect(moved()?.bytesOut).toBe(234); | |
| 361 | } finally { | |
| 362 | server.stop(true); | |
| 363 | app.close(); | |
| 364 | resetEgressTotals(); | |
| 365 | } | |
| 366 | }); |
new above
offline