Public: anyone can read this; the crew can change it.
After every release a cell child cannot wake for ten minutes: the boot opens every hosted child and fills FORGE_CHILD_AWAKE_MAX ("the forge is full") #10606
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/10502-after-every-release-a-cell-child-cannot → main 2 commits fork 2 threads · 2 unresolved
3 files
changed +124
−1
AGENTS.md +3
| @@ -19,3 +19,6 @@ State shared between modules through process memory, not the database, is named | ||
| 19 | 19 | `src/perf/loop_worst.ts`: `loopWorst`, the worst event-loop lag of the last 10 s in ten one-second slots, fed by one unref'd 50 ms timer that the first `serve` starts (`startLoopWorst`), and read by every response's `lag;dur` and by `/healthz` asked for JSON (`loopLagMs`). Tests listing a request's phases read `requestTiming` (`test/helpers/request-timing.ts`), which leaves `lag` out. |
| 20 | 20 | |
| 21 | 21 | `src/core/transfer.ts`: the old owner/name pairs a transfer left behind (`repo_redirect`), held per database after the first request reads them and dropped by `transferRepo`, so a request under a path nothing moved from costs no statement (#569). |
| 22 | ||
| 23 | ## Awake ceiling and the boot | |
| 24 | `FORGE_CHILD_AWAKE_MAX` counts forges in use. Hosted children the boot opens (so their apps come back) are exempt from `awakeCount` until a credentialed request reaches them or they sleep (`bootOpened` in `src/spawn/host.ts`). The exempt set holds only body ids of existing finished children and is emptied when the parent closes, so it is bounded by the number of children. Cell children are always counted; they are not opened by the boot. |
src/spawn/host.ts +16 −1
| @@ -119,11 +119,16 @@ function touch(parent: App, ledger: { bodyId: string; name: string }, now = Date | ||
| 119 | 119 | |
| 120 | 120 | /** Forges awake now: hosted children open in this process plus children running in a cell. */ |
| 121 | 121 | export function awakeCount(parent: App): number { |
| 122 | const awake = (caches.get(parent)?.size ?? 0) + cellChildren(parent).list().length; | |
| 122 | // A child boot opened and nobody has asked for is not in use: it never crowds out a person's wake. | |
| 123 | const unasked = bootOpened.get(parent)?.size ?? 0; | |
| 124 | const awake = Math.max(0, (caches.get(parent)?.size ?? 0) - unasked) + cellChildren(parent).list().length; | |
| 123 | 125 | // A wake granted but not finished holds its slot, so two at once cannot both pass the ceiling. |
| 124 | 126 | return awake + reservedCount(parent); |
| 125 | 127 | } |
| 126 | 128 | |
| 129 | /** Hosted children the boot opened that no request has reached since, by body id. */ | |
| 130 | const bootOpened = new WeakMap<App, Set<string>>(); | |
| 131 | ||
| 127 | 132 | /** Wakes granted and not yet awake, by body id, with when. */ |
| 128 | 133 | const waking = new WeakMap<App, Map<string, number>>(); |
| 129 | 134 | const RESERVE_MS = 30_000; |
| @@ -220,6 +225,7 @@ export function childAwake(parent: App, name: string): boolean { | ||
| 220 | 225 | |
| 221 | 226 | async function closeOne(parent: App, cache: Map<string, App>, bodyId: string, child: App, name: string): Promise<string> { |
| 222 | 227 | cache.delete(bodyId); |
| 228 | bootOpened.get(parent)?.delete(bodyId); | |
| 223 | 229 | touched.get(parent)?.delete(bodyId); |
| 224 | 230 | waking.get(parent)?.delete(bodyId); |
| 225 | 231 | for (const onClose of closeWatchers.get(parent) ?? []) { |
| @@ -262,6 +268,7 @@ export function wakeGate(parent: App, request: Request): Response | undefined { | ||
| 262 | 268 | const known = carriesCredentials(parent, ledger.bodyId, request); |
| 263 | 269 | if (isAwakeBody(parent, ledger.bodyId)) { |
| 264 | 270 | if (known) { |
| 271 | bootOpened.get(parent)?.delete(ledger.bodyId); | |
| 265 | 272 | touch(parent, ledger); |
This clearing on a credentialed request is behaviour with no test. Add one that fails if this line is removed: after it, awakeCount must count the child again. |
||
| 266 | 273 | cellChildren(parent).touch(ledger.bodyId); |
| 267 | 274 | } |
| @@ -295,6 +302,7 @@ export function closeHostedChildren(parent: App): void { | ||
| 295 | 302 | if (sweeper !== undefined) clearInterval(sweeper); |
| 296 | 303 | sweepers.delete(parent); |
| 297 | 304 | touched.delete(parent); |
| 305 | bootOpened.delete(parent); | |
| 298 | 306 | waking.delete(parent); |
| 299 | 307 | inFlight.delete(parent); |
| 300 | 308 | const m = caches.get(parent); |
| @@ -459,7 +467,14 @@ export async function bootHostedChildren(parent: App): Promise<void> { | ||
| 459 | 467 | if (!f.endsWith(".json")) continue; |
| 460 | 468 | const name = f.slice(0, -".json".length); |
| 461 | 469 | if (parseHostedChildName(`${name}${childHostSuffix(childHost(parent))}`, childHost(parent)) !== name) continue; |
| 470 | const ledger = readLedgerSync(parent, name); | |
| 471 | const was = ledger !== undefined && caches.get(parent)?.has(ledger.bodyId) === true; | |
| 462 | 472 | await bootHostedChild(parent, name); |
| 473 | if (ledger !== undefined && !was && caches.get(parent)?.has(ledger.bodyId) === true) { | |
| 474 | let set = bootOpened.get(parent); | |
| 475 | if (!set) bootOpened.set(parent, (set = new Set())); | |
| 476 | set.add(ledger.bodyId); | |
| 477 | } | |
| 463 | 478 | } |
| 464 | 479 | } |
| 465 | 480 |
test/boot-idle-children.test.ts added +105
| @@ -0,0 +1,105 @@ | ||
| 1 | /** | |
| 2 | * Children the boot opened so their apps come back, and nobody has asked for, | |
| 3 | * do not count against the awake ceiling: a person's wake is never refused | |
| 4 | * because of forges nobody is using. | |
| 5 | */ | |
| 6 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 7 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 8 | import { tmpdir } from "node:os"; | |
| 9 | import { join } from "node:path"; | |
| 10 | import { spawnForge } from "../src/actions/spawn.ts"; | |
| 11 | import { createApp, type App } from "../src/app.ts"; | |
| 12 | import { createSession } from "../src/core/sessions.ts"; | |
| 13 | import { createUser } from "../src/core/users.ts"; | |
| 14 | import { awakeCount, bootHostedChildren, childAwake, resolveHostedChild, sleepChild, sleepIdleChildren, wakeGate } from "../src/spawn/host.ts"; | |
| 15 | import { hostedPlacement } from "../src/spawn/placement.ts"; | |
| 16 | import { quietLogs } from "./helpers/globals.ts"; | |
| 17 | ||
| 18 | quietLogs(); | |
| 19 | ||
| 20 | let dir = ""; | |
| 21 | let parent: App; | |
| 22 | let was: string | undefined; | |
| 23 | ||
| 24 | beforeAll(async () => { | |
| 25 | was = process.env["FORGE_CHILD_AWAKE_MAX"]; | |
| 26 | dir = await mkdtemp(join(tmpdir(), "boot-idle-")); | |
| 27 | parent = createApp({ dataDir: dir, url: "https://forge.test", port: 0 }); | |
| 28 | const made = await createUser(parent.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", isAdmin: true }); | |
| 29 | if (!made.ok) throw new Error("user"); | |
| 30 | for (const name of ["idle-a", "idle-b", "sleeper"]) { | |
| 31 | const spawned = await spawnForge( | |
| 32 | parent, | |
| 33 | made.value, | |
| 34 | { name, org: name, apps: [], carryBriefs: false, carrySubscriptions: false }, | |
| 35 | { catalog: [], briefs: [], subscriptions: [] }, | |
| 36 | { | |
| 37 | placement: hostedPlacement({ host: "forge.test" }), | |
| 38 | request: new Request("https://forge.test/spawn", { method: "POST", headers: { origin: "https://forge.test" } }), | |
| 39 | url: new URL("https://forge.test/spawn"), | |
| 40 | }, | |
| 41 | ); | |
| 42 | if (!spawned.ok) throw new Error(JSON.stringify(spawned.error)); | |
| 43 | } | |
| 44 | }); | |
| 45 | ||
| 46 | afterAll(async () => { | |
| 47 | if (was === undefined) delete process.env["FORGE_CHILD_AWAKE_MAX"]; | |
| 48 | else process.env["FORGE_CHILD_AWAKE_MAX"] = was; | |
| 49 | parent.close(); | |
| 50 | await rm(dir, { recursive: true, force: true }); | |
| 51 | }); | |
| 52 | ||
| 53 | test("a wake is not refused for children the boot opened and nobody asked for", async () => { | |
| 54 | await sleepIdleChildren(parent, 0, Date.now() + 3_600_000); | |
| 55 | process.env["FORGE_CHILD_AWAKE_MAX"] = "2"; | |
| 56 | await bootHostedChildren(parent); | |
| 57 | expect(childAwake(parent, "idle-a")).toBe(true); | |
| 58 | expect(childAwake(parent, "idle-b")).toBe(true); | |
| 59 | await sleepChild(parent, "sleeper"); | |
| 60 | expect(childAwake(parent, "sleeper")).toBe(false); | |
| 61 | expect(awakeCount(parent)).toBe(0); | |
| 62 | const refused = wakeGate(parent, new Request("https://forge.test/", { headers: { host: "sleeper--forge.test" } })); | |
| 63 | expect(refused).toBeUndefined(); | |
| 64 | }); | |
| 65 | ||
| 66 | const hostOf = (name: string) => `${name}--forge.test`; | |
| 67 | const gate = (name: string, headers: Record<string, string> = {}) => | |
| 68 | wakeGate(parent, new Request("https://forge.test/", { headers: { host: hostOf(name), ...headers } })); | |
| 69 | const reset = async () => { | |
| 70 | await sleepIdleChildren(parent, 0, Date.now() + 3_600_000); | |
| 71 | process.env["FORGE_CHILD_AWAKE_MAX"] = "2"; | |
| 72 | await bootHostedChildren(parent); | |
| 73 | }; | |
| 74 | ||
| 75 | test("a boot-opened child a credentialed request reaches counts again, and the ceiling then holds", async () => { | |
| 76 | await reset(); | |
| 77 | expect(awakeCount(parent)).toBe(0); | |
| 78 | const child = (await resolveHostedChild(parent, new Request("https://forge.test/healthz", { headers: { host: hostOf("idle-a") } }))) as App; | |
| 79 | const made = await createUser(child.db, { origin: "seed", trusted: true, username: "carol", email: "[email protected]", isAdmin: false }); | |
| 80 | if (!made.ok) throw new Error("user"); | |
| 81 | const { token } = createSession(child.db, made.value.id); | |
| 82 | expect(gate("idle-a", { cookie: `forge_session=${token}` })).toBeUndefined(); | |
| 83 | expect(awakeCount(parent)).toBe(1); | |
| 84 | // A made-up cookie asks for nothing: idle-b stays uncounted. | |
| 85 | expect(gate("idle-b", { cookie: "forge_session=made-up" })).toBeUndefined(); | |
| 86 | expect(awakeCount(parent)).toBe(1); | |
| 87 | await sleepChild(parent, "sleeper"); | |
| 88 | await sleepChild(parent, "idle-b"); | |
| 89 | // One asked-for forge is awake and one wake is granted: the ceiling of 2 is full. | |
| 90 | expect(gate("sleeper")).toBeUndefined(); | |
| 91 | expect(awakeCount(parent)).toBe(2); | |
| 92 | const refused = gate("idle-b"); | |
| 93 | expect(refused?.status).toBe(429); | |
| 94 | }); | |
| 95 | ||
| 96 | test("an unasked boot-opened child put to sleep leaves the exempt set, so its next wake counts", async () => { | |
| 97 | await reset(); | |
| 98 | expect(awakeCount(parent)).toBe(0); | |
| 99 | await sleepChild(parent, "idle-b"); | |
| 100 | await resolveHostedChild(parent, new Request("https://forge.test/healthz", { headers: { host: hostOf("idle-b") } })); | |
| 101 | expect(childAwake(parent, "idle-b")).toBe(true); | |
| 102 | expect(awakeCount(parent)).toBe(1); // reopened by a request, not by the boot: counted | |
| 103 | await sleepChild(parent, "idle-b"); | |
| 104 | expect(awakeCount(parent)).toBe(0); | |
| 105 | }); |
Exempting all boot-opened children from the ceiling means real open apps are uncounted. Cover with a test that the ceiling still holds once a child is asked for, and consider capping the exempt set.