Public: anyone can read this; the crew can change it.
A release starts every hosted forge again, one after another, whatever the awake ceiling #13830
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/13816-a-release-starts-every-hosted-forge-agai → main 1 commit fork 1 thread · 1 unresolved
5 files
changed +227
−30
AGENTS.md +1 −1
| @@ -48,7 +48,7 @@ The check queue hands work to a runner through `src/checks/runner.ts` (#13652, R | ||
| 48 | 48 | `src/spawn/child_reach.ts`: `said`, when each hosted forge's refusal of each name was last written to the log (one `[egress] <child> was refused <host>:<port>` line an hour; pruned past 2,000 keys), and `calls`, each child's signed calls in the last minute; both gone on restart, and neither is state (asks and grants are rows: `child_reach_ask`, `child_reach_grant`). `src/actions/ask_host.ts` on a hosted forge: `looking` (the one refresh in flight), `lookedAt` (when the host was last asked, so a page is heard at most every 3 s) and `watching` (the one timer that looks every 10 s while an ask is under an hour old); a restart loses nothing, since every ask is a row (`reach_ask`) and the heartbeat looks again. The terminal door is `src/tui/screens/hostreach.ts` (palette entry for admins; `a` allows for that forge, `e` for every hosted forge, `n` not now, `x` takes a grant back; `allowReach`, `putAsideReach` and `revokeReach` in `actions/child_reach.ts` give the pages and the terminal the same gate and the `forge.reach_*` events). |
| 49 | 49 | |
| 50 | 50 | ## Awake ceiling and the boot |
| 51 | `FORGE_CHILD_AWAKE_MAX` counts forges in use. Hosted children the boot opens (so their apps come back) are exempt from `awakeCount` until a credentialed request reaches them or they sleep (`bootOpened` in `src/spawn/host.ts`). The exempt set holds only body ids of existing finished children and is emptied when the parent closes, so it is bounded by the number of children. Cell children are always counted; they are not opened by the boot. | |
| 51 | `FORGE_CHILD_AWAKE_MAX` counts forges in use. Hosted children the boot opens (so their apps come back) are exempt from `awakeCount` until a credentialed request reaches them or they sleep (`bootOpened` in `src/spawn/host.ts`). The exempt set holds only body ids of existing finished children and is emptied when the parent closes, so it is bounded by the number of children. Cell children are always counted. The boot starts only the forges that were awake and in use when the parent stopped (`awake-at-stop.json` under the data directory, written by `closeHostedChildren`, read and removed once by `bootHostedChildren`, `src/spawn/awake_at_stop.ts`), up to the ceiling; a crash leaves no file, so nothing starts and forges wake on their next request. A sleeping cell forge with a run due has its wake armed. The idle sweeper starts at boot. | |
| 52 | 52 | |
| 53 | 53 | An app token's requests go behind people's and repeat reads are 304s (`src/http/app_turn.ts`, `repeat_reads.ts`, `perf/asking.ts`): see `docs/features/app-asking.md`. |
| 54 | 54 | `src/core/reports.ts`: `closedSet`, the ids of repositories whose owner closed reports from outside (#12321), read once per database and kept by `setReportsClosed`, the only writer, so every repository page's head link costs no statement; at most one id per closed repository, gone on restart (the rows are `report_closed`). `src/core/contribute_token.ts`: `openSet`, the ids of repositories that allow outside agents, read once per database and kept by `setContributeOpen`, the only writer, at most one id per allowing repository, gone on restart; a raw SQL write to `contribute_open` bypasses it. The terminal door for reports is `r` on the outside agents screen (`setOutsideReports`, admins). |
src/spawn/awake_at_stop.ts added +42
| @@ -0,0 +1,42 @@ | ||
| 1 | /** | |
| 2 | * Which forges were awake and in use when the parent stopped, so the next boot starts those and | |
| 3 | * leaves the rest asleep (rule 4: a forge asleep before a release is asleep after it). One small | |
| 4 | * file under the parent's data directory, written on a clean stop and read once, then removed: a | |
| 5 | * crash leaves none, and a boot with none starts no forge (they wake on their next request). | |
| 6 | */ | |
| 7 | import { readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; | |
| 8 | import { join } from "node:path"; | |
| 9 | ||
| 10 | const FILE = "awake-at-stop.json"; | |
| 11 | const NAME = /^[a-z0-9][a-z0-9-]{0,62}$/; | |
| 12 | ||
| 13 | /** Record the names of the forges in use at this stop. Never throws. */ | |
| 14 | export function rememberAwake(dataDir: string, names: readonly string[]): void { | |
| 15 | try { | |
| 16 | const tmp = join(dataDir, `${FILE}.tmp`); | |
| 17 | writeFileSync(tmp, JSON.stringify({ names: [...new Set(names)].sort() })); | |
| 18 | renameSync(tmp, join(dataDir, FILE)); | |
| 19 | } catch { | |
| 20 | // the next boot starts nothing: forges wake on their next request | |
| 21 | } | |
| 22 | } | |
| 23 | ||
| 24 | /** | |
| 25 | * The names recorded at the last stop, once: the file is removed as it is read. Empty when none. | |
| 26 | */ | |
| 27 | export function takeAwake(dataDir: string): Set<string> { | |
| 28 | const path = join(dataDir, FILE); | |
| 29 | const out = new Set<string>(); | |
| 30 | try { | |
| 31 | const raw = JSON.parse(readFileSync(path, "utf8")) as { names?: unknown }; | |
| 32 | if (Array.isArray(raw.names)) for (const n of raw.names) if (typeof n === "string" && NAME.test(n)) out.add(n); | |
| 33 | } catch { | |
| 34 | // none, or damaged: nothing was awake | |
| 35 | } | |
| 36 | try { | |
| 37 | rmSync(path, { force: true }); | |
| 38 | } catch { | |
| 39 | // read once is enough | |
| 40 | } | |
| 41 | return out; | |
| 42 | } |
src/spawn/host.ts +95 −27
| @@ -33,6 +33,7 @@ import { limitValue } from "../settings/limits.ts"; | ||
| 33 | 33 | import { cancelRestarts, clearStop, lastStop, noteStop, stopSentence, tryAgainAt, useRestart } from "./stopped.ts"; |
| 34 | 34 | import { readOwnCode } from "./own_code.ts"; |
| 35 | 35 | import { awakeRefusal, idleAfterMs, WakeRate } from "./sleep.ts"; |
| 36 | import { rememberAwake, takeAwake } from "./awake_at_stop.ts"; | |
| 36 | 37 | import { wakingResponse } from "../web/pages/waking.ts"; |
| 37 | 38 | import { lapsedResponse, payAgainRoute, PAY_AGAIN_PATH } from "../web/pages/lapsed.ts"; |
| 38 | 39 | import { lapseFor, type Lapse } from "../billing/lapse.ts"; |
| @@ -121,7 +122,10 @@ export function hostedChildOpen(parent: App, child: App): boolean { | ||
| 121 | 122 | return false; |
| 122 | 123 | } |
| 123 | 124 | |
| 124 | /** The hosted children open in this process now, by the name each goes by; a sleeping child or one in a cell is not here. */ | |
| 125 | /** | |
| 126 | * The hosted children open in this process now, by the name each goes by; a sleeping child or one | |
| 127 | * in a cell is not here. | |
| 128 | */ | |
| 125 | 129 | export function openChildren(parent: App): { name: string; app: App }[] { |
| 126 | 130 | const out: { name: string; app: App }[] = []; |
| 127 | 131 | for (const [bodyId, child] of caches.get(parent) ?? []) { |
| @@ -147,7 +151,8 @@ function touch(parent: App, ledger: { bodyId: string; name: string }, now = Date | ||
| 147 | 151 | |
| 148 | 152 | /** Forges awake now: hosted children open in this process plus children running in a cell. */ |
| 149 | 153 | export function awakeCount(parent: App): number { |
| 150 | // A child boot opened and nobody has asked for is not in use: it never crowds out a person's wake. | |
| 154 | // A child boot opened and nobody has asked for is not in use: it never crowds out a person's | |
| 155 | // wake. | |
| 151 | 156 | const unasked = bootOpened.get(parent)?.size ?? 0; |
| 152 | 157 | const awake = Math.max(0, (caches.get(parent)?.size ?? 0) - unasked) + cellChildren(parent).list().length; |
| 153 | 158 | // A wake granted but not finished holds its slot, so two at once cannot both pass the ceiling. |
| @@ -194,7 +199,10 @@ export function holdChild(parent: App, child: App): () => void { | ||
| 194 | 199 | }; |
| 195 | 200 | } |
| 196 | 201 | |
| 197 | /** Whether the child is mid-request or has crew work queued or running: it is not idle however quiet its visitors. */ | |
| 202 | /** | |
| 203 | * Whether the child is mid-request or has crew work queued or running: it is not idle however quiet | |
| 204 | * its visitors. | |
| 205 | */ | |
| 198 | 206 | function working(parent: App, child: App): boolean { |
| 199 | 207 | if ((inFlight.get(parent)?.get(child.config.dataDir) ?? 0) > 0) return true; |
| 200 | 208 | try { |
| @@ -252,7 +260,9 @@ export function onInProcessSleep(parent: App, hook: () => void): void { | ||
| 252 | 260 | |
| 253 | 261 | /** Wakes armed for cell children asleep with a run paused on a limit, by body id. */ |
| 254 | 262 | const dueWakes = new WeakMap<App, Map<string, ReturnType<typeof setTimeout>>>(); |
| 255 | /** A due wake the awake ceiling or the disk floor turned away is tried again this much later, ms. */ | |
| 263 | /** | |
| 264 | * A due wake the awake ceiling or the disk floor turned away is tried again this much later, ms. | |
| 265 | */ | |
| 256 | 266 | const DUE_RETRY_MS = 60_000; |
| 257 | 267 | /** A forge the host stopped may take the room of one quiet this long, ms. */ |
| 258 | 268 | const STOPPED_EVICT_IDLE_MS = 60_000; |
| @@ -294,8 +304,8 @@ async function wakeDue(parent: App, name: string): Promise<void> { | ||
| 294 | 304 | const ledger = readLedgerSync(parent, name); |
| 295 | 305 | if (ledger === undefined || !ledger.done || ledger.placement !== "cell" || ledger.moving) return; |
| 296 | 306 | if (isAwakeBody(parent, ledger.bodyId) || lapsedNone(parent, name) !== undefined) return; |
| 297 | // A forge the host stopped for a limit does not wait for an idle one's ten minutes: forges quiet for | |
| 298 | // a minute and with no crew at work are put to sleep to make its room. | |
| 307 | // A forge the host stopped for a limit does not wait for an idle one's ten minutes: forges quiet | |
| 308 | // for a minute and with no crew at work are put to sleep to make its room. | |
| 299 | 309 | if (lastStop(parent, ledger.bodyId) !== undefined && awakeRefusal(awakeCount(parent)) !== undefined) |
| 300 | 310 | await sleepIdleChildren(parent, STOPPED_EVICT_IDLE_MS); |
| 301 | 311 | if (floorRefusal(parent.config.dataDir) !== undefined || awakeRefusal(awakeCount(parent)) !== undefined) { |
| @@ -320,7 +330,8 @@ async function restartStopped(parent: App, name: string): Promise<boolean> { | ||
| 320 | 330 | const ledger = readLedgerSync(parent, name); |
| 321 | 331 | if (ledger === undefined) return false; |
| 322 | 332 | if (isAwakeBody(parent, ledger.bodyId) || dueWakes.get(parent)?.has(ledger.bodyId) === true) return true; |
| 323 | // The cell did not come up (or a start was refused): the host tries again a minute on, for the hour the stop is shown. | |
| 333 | // The cell did not come up (or a start was refused): the host tries again a minute on, for the | |
| 334 | // hour the stop is shown. | |
| 324 | 335 | const stop = lastStop(parent, ledger.bodyId); |
| 325 | 336 | if (stop === undefined || ledger.placement !== "cell" || lapsedNone(parent, name) !== undefined || startRefused(parent, ledger) !== undefined) return false; |
| 326 | 337 | wakeWhenDue(parent, ledger, Date.now() + DUE_RETRY_MS); |
| @@ -408,13 +419,18 @@ export function keepWatch(parent: App): void { | ||
| 408 | 419 | /** Whether this process has the child open as an App of its own (not a cell's). */ |
| 409 | 420 | export const openHere = (parent: App, bodyId: string): boolean => caches.get(parent)?.has(bodyId) === true || opening.get(parent)?.has(bodyId) === true; |
| 410 | 421 | |
| 411 | /** A child's lapse when its billing is `none`: it sleeps and does not wake until paid (#11393); else undefined. */ | |
| 422 | /** | |
| 423 | * A child's lapse when its billing is `none`: it sleeps and does not wake until paid (#11393); else | |
| 424 | * undefined. | |
| 425 | */ | |
| 412 | 426 | export function lapsedNone(parent: App, name: string): Extract<Lapse, { state: "none" }> | undefined { |
| 413 | 427 | const lapse = lapseFor(parent.db, name); |
| 414 | 428 | return lapse.state === "none" ? lapse : undefined; |
| 415 | 429 | } |
| 416 | 430 | |
| 417 | /** Put to sleep, through `sleepChild`, every awake child whose billing is `none`. Answers the names. */ | |
| 431 | /** | |
| 432 | * Put to sleep, through `sleepChild`, every awake child whose billing is `none`. Answers the names. | |
| 433 | */ | |
| 418 | 434 | export async function sleepLapsed(parent: App): Promise<string[]> { |
| 419 | 435 | const names = new Set<string>([...(touched.get(parent)?.values() ?? [])].map((m) => m.name)); |
| 420 | 436 | for (const c of cellChildren(parent).list()) names.add(c.name); |
| @@ -462,7 +478,10 @@ function keepSweeping(parent: App): void { | ||
| 462 | 478 | sweepers.set(parent, timer); |
| 463 | 479 | } |
| 464 | 480 | |
| 465 | /** Whether the child itself vouches for the request's session or token: a header a client merely wrote is not enough. */ | |
| 481 | /** | |
| 482 | * Whether the child itself vouches for the request's session or token: a header a client merely | |
| 483 | * wrote is not enough. | |
| 484 | */ | |
| 466 | 485 | const carriesCredentials = (parent: App, bodyId: string, request: Request): Promise<boolean> => |
| 467 | 486 | vouched(childDirOf(parent, bodyId), request, caches.get(parent)?.get(bodyId)?.db); |
| 468 | 487 | |
| @@ -526,7 +545,8 @@ export async function wakeGate(parent: App, request: Request): Promise<Response | ||
| 526 | 545 | if (isAwakeBody(parent, ledger.bodyId)) void sleepChild(parent, name).catch(() => {}); |
| 527 | 546 | return lapsedResponse(parent, request, name, lapsed); |
| 528 | 547 | } |
| 529 | // Asked off this thread when the child is a cell's: its database is its own to write, and the answer has a deadline. | |
| 548 | // Asked off this thread when the child is a cell's: its database is its own to write, and the | |
| 549 | // answer has a deadline. | |
| 530 | 550 | const known = await carriesCredentials(parent, ledger.bodyId, request); |
| 531 | 551 | if (isAwakeBody(parent, ledger.bodyId)) { |
| 532 | 552 | clearStop(parent, ledger.bodyId); |
| @@ -537,7 +557,8 @@ export async function wakeGate(parent: App, request: Request): Promise<Response | ||
| 537 | 557 | } |
| 538 | 558 | return undefined; |
| 539 | 559 | } |
| 540 | // Asleep, so answering would start it: not where it has no filesystem of its own and one is required. | |
| 560 | // Asleep, so answering would start it: not where it has no filesystem of its own and one is | |
| 561 | // required. | |
| 541 | 562 | const refused = startRefused(parent, ledger); |
| 542 | 563 | if (refused !== undefined) return refusedResponse(refused); |
| 543 | 564 | // A wake already granted (a cell takes a moment to answer) is one wake, not one per request. |
| @@ -578,6 +599,11 @@ export async function payAgainGate(parent: App, request: Request): Promise<Respo | ||
| 578 | 599 | |
| 579 | 600 | /** Drop every hosted child App this parent has open (call when the parent stops). */ |
| 580 | 601 | export function closeHostedChildren(parent: App): void { |
| 602 | // The forges in use now are the ones the next boot starts; the rest stay asleep through a | |
| 603 | // release. | |
| 604 | const now = Date.now(); | |
| 605 | const inUse = [...(touched.get(parent)?.values() ?? [])].filter((m) => now - m.at < idleAfterMs()).map((m) => m.name); | |
| 606 | if (inUse.length > 0 || touched.has(parent)) rememberAwake(parent.config.dataDir, inUse); | |
| 581 | 607 | const sweeper = sweepers.get(parent); |
| 582 | 608 | if (sweeper !== undefined) clearInterval(sweeper); |
| 583 | 609 | sweepers.delete(parent); |
| @@ -656,7 +682,8 @@ export async function resolveHostedChild( | ||
| 656 | 682 | // it, and while its cell is not up `answerFromCell` says so at once. |
| 657 | 683 | if (ledger.placement === "cell") { |
| 658 | 684 | if (cellChildren(parent).get(ledger.bodyId) !== undefined) return "cell"; |
| 659 | // A host that holds no cell at all is known without waiting; a child that ran its own code is never opened here. | |
| 685 | // A host that holds no cell at all is known without waiting; a child that ran its own code is | |
| 686 | // never opened here. | |
| 660 | 687 | const here = await hostedHere(parent, ledger.name); |
| 661 | 688 | if (here.ok) return openOnce(parent, demoted(parent, ledger, here.why)); |
| 662 | 689 | void settleCell(parent, ledger); |
| @@ -699,7 +726,10 @@ function settleCell(parent: App, ledger: ChildLedger): Promise<CellChild | undef | ||
| 699 | 726 | return flight; |
| 700 | 727 | } |
| 701 | 728 | |
| 702 | /** Whether this host holds cells: the machine's own answer, unless a test stands in for it. Never read from the environment. */ | |
| 729 | /** | |
| 730 | * Whether this host holds cells: the machine's own answer, unless a test stands in for it. Never | |
| 731 | * read from the environment. | |
| 732 | */ | |
| 703 | 733 | let holdsCells: typeof cellsAvailable = cellsAvailable; |
| 704 | 734 | export function provideCellsForTest(stand?: typeof cellsAvailable): void { |
| 705 | 735 | holdsCells = stand ?? cellsAvailable; |
| @@ -711,16 +741,25 @@ export function provideCellsForTest(stand?: typeof cellsAvailable): void { | ||
| 711 | 741 | */ |
| 712 | 742 | export const cellsHere = (): ReturnType<typeof cellsAvailable> => holdsCells(); |
| 713 | 743 | |
| 714 | /** How a ledger's `placementNote` begins for a cell child whose cell has not started; `/forges` lists it. */ | |
| 744 | /** | |
| 745 | * How a ledger's `placementNote` begins for a cell child whose cell has not started; `/forges` | |
| 746 | * lists it. | |
| 747 | */ | |
| 715 | 748 | export const NOT_STARTED = "own cell, not started: "; |
| 716 | 749 | |
| 717 | /** Whether a child has ever run code of its own (`spawn/own_code.ts`): such a child runs in a cell or not at all. */ | |
| 750 | /** | |
| 751 | * Whether a child has ever run code of its own (`spawn/own_code.ts`): such a child runs in a cell | |
| 752 | * or not at all. | |
| 753 | */ | |
| 718 | 754 | export function ranOwnCode(parent: App, name: string): boolean { |
| 719 | 755 | const own = readOwnCode(parent, name); |
| 720 | 756 | return own.on || own.ran === true || own.good !== undefined || own.failed !== undefined || own.rolledBack !== undefined; |
| 721 | 757 | } |
| 722 | 758 | |
| 723 | /** Whether this host may run `name` in its own process: it holds no cell at all, and the child never ran code of its own. */ | |
| 759 | /** | |
| 760 | * Whether this host may run `name` in its own process: it holds no cell at all, and the child never | |
| 761 | * ran code of its own. | |
| 762 | */ | |
| 724 | 763 | async function hostedHere(parent: App, name: string): Promise<{ ok: true; why: string } | { ok: false; why: string }> { |
| 725 | 764 | const cells = await holdsCells(); |
| 726 | 765 | if (cells.ok) return { ok: false, why: "this host holds cells" }; |
| @@ -747,7 +786,8 @@ async function cellDidNotStart(parent: App, ledger: ChildLedger, why: string): P | ||
| 747 | 786 | const note = `${NOT_STARTED}${(/cannot make a cell/.test(here.why) ? here.why : said).slice(0, 400)}`; |
| 748 | 787 | noteLedger(parent, ledger.name, (raw) => (raw["placementNote"] === note ? undefined : { placementNote: note })); |
| 749 | 788 | parent.log.warn("child forge's cell did not start; it stays a cell child and the host tries again", { name: ledger.name, why }); |
| 750 | // One stop per failure, not one per request that found it failed: the wait and the count are the stop's. | |
| 789 | // One stop per failure, not one per request that found it failed: the wait and the count are the | |
| 790 | // stop's. | |
| 751 | 791 | if (lastStop(parent, ledger.bodyId) === undefined) noteStop(parent, ledger.bodyId, ledger.name, said); |
| 752 | 792 | } |
| 753 | 793 | |
| @@ -758,7 +798,10 @@ function demoted(parent: App, ledger: ChildLedger, why: string): ChildLedger { | ||
| 758 | 798 | return { ...ledger, placement: "hosted" }; |
| 759 | 799 | } |
| 760 | 800 | |
| 761 | /** Change fields of a child's ledger (the parent's own file); `change` answers nothing when there is nothing to write. */ | |
| 801 | /** | |
| 802 | * Change fields of a child's ledger (the parent's own file); `change` answers nothing when there is | |
| 803 | * nothing to write. | |
| 804 | */ | |
| 762 | 805 | export function noteLedger(parent: App, name: string, change: (raw: Record<string, unknown>) => Record<string, unknown> | undefined): void { |
| 763 | 806 | const path = join(parent.config.dataDir, "spawn", `${name}.json`); |
| 764 | 807 | try { |
| @@ -817,9 +860,15 @@ export async function bootHostedChild(parent: App, name: string): Promise<void> | ||
| 817 | 860 | } |
| 818 | 861 | } |
| 819 | 862 | |
| 820 | /** Open every finished child, on boot: their apps come back with the parent's. */ | |
| 821 | export async function bootHostedChildren(parent: App): Promise<void> { | |
| 863 | /** | |
| 864 | * On boot, start only the forges that were awake and in use when the parent stopped | |
| 865 | * (`awake_at_stop.ts`, or `wasAwake` when given), up to the awake ceiling; a forge asleep before | |
| 866 | * a release is asleep after it and wakes on its next request. A sleeping forge with a run due | |
| 867 | * gets its wake armed. The idle sweeper starts here, so nothing started stays awake unused. | |
| 868 | */ | |
| 869 | export async function bootHostedChildren(parent: App, wasAwake: ReadonlySet<string> = takeAwake(parent.config.dataDir)): Promise<void> { | |
| 822 | 870 | keepWatchingFloor(parent); |
| 871 | keepSweeping(parent); | |
| 823 | 872 | let files: string[] = []; |
| 824 | 873 | try { |
| 825 | 874 | files = readdirSync(join(parent.config.dataDir, "spawn")); |
| @@ -831,9 +880,20 @@ export async function bootHostedChildren(parent: App): Promise<void> { | ||
| 831 | 880 | const name = f.slice(0, -".json".length); |
| 832 | 881 | if (parseHostedChildName(`${name}${childHostSuffix(childHost(parent))}`, childHost(parent)) !== name) continue; |
| 833 | 882 | const ledger = readLedgerSync(parent, name); |
| 834 | const was = ledger !== undefined && caches.get(parent)?.has(ledger.bodyId) === true; | |
| 883 | if (ledger === undefined || !ledger.done || ledger.moving) continue; | |
| 884 | if (!wasAwake.has(name)) { | |
| 885 | // Asleep stays asleep, but a run that falls due still wakes it. | |
| 886 | if (ledger.placement === "cell") { | |
| 887 | const due = await childNextRunDue(childDirOf(parent, ledger.bodyId)); | |
| 888 | if (due !== undefined) wakeWhenDue(parent, ledger, due); | |
| 889 | } | |
| 890 | continue; | |
| 891 | } | |
| 892 | // Boot-opened forges are exempt from `awakeCount`, so the boot counts its own. | |
| 893 | if (awakeRefusal(awakeCount(parent) + (bootOpened.get(parent)?.size ?? 0)) !== undefined) continue; | |
| 894 | const was = caches.get(parent)?.has(ledger.bodyId) === true; | |
| 835 | 895 | await bootHostedChild(parent, name); |
| 836 | if (ledger !== undefined && !was && caches.get(parent)?.has(ledger.bodyId) === true) { | |
| 896 | if (!was && caches.get(parent)?.has(ledger.bodyId) === true) { | |
| 837 | 897 | let set = bootOpened.get(parent); |
| 838 | 898 | if (!set) bootOpened.set(parent, (set = new Set())); |
| 839 | 899 | set.add(ledger.bodyId); |
| @@ -852,7 +912,10 @@ export interface RefusedHit { | ||
| 852 | 912 | readonly refused: string; |
| 853 | 913 | } |
| 854 | 914 | |
| 855 | /** A child app in a child's own cell: its runtime is in that process, so the request is carried there. */ | |
| 915 | /** | |
| 916 | * A child app in a child's own cell: its runtime is in that process, so the request is carried | |
| 917 | * there. | |
| 918 | */ | |
| 856 | 919 | export interface CellAppHit { |
| 857 | 920 | readonly cell: string; |
| 858 | 921 | } |
| @@ -931,7 +994,8 @@ export async function answerFromCell(parent: App, request: Request, named?: stri | ||
| 931 | 994 | // held until the cell is up (wakeLimits.waitMs) so a submit is never lost; only a wake |
| 932 | 995 | // that takes longer is answered with the "waking up" page, which tries again by itself. |
| 933 | 996 | const starting = settleCell(parent, ledger); |
| 934 | // Only so many wait on one cell: the rest are told it is waking at once, so a crowd cannot pin connections and bodies. | |
| 997 | // Only so many wait on one cell: the rest are told it is waking at once, so a crowd cannot pin | |
| 998 | // connections and bodies. | |
| 935 | 999 | const waiting = waiters.get(ledger.bodyId) ?? 0; |
| 936 | 1000 | if (waiting >= wakeLimits.waiters) return wakingResponse(parent, request, ledger.name); |
| 937 | 1001 | waiters.set(ledger.bodyId, waiting + 1); |
| @@ -951,7 +1015,8 @@ export async function answerFromCell(parent: App, request: Request, named?: stri | ||
| 951 | 1015 | release(); |
| 952 | 1016 | } |
| 953 | 1017 | } |
| 954 | // A cell that did not start says so, with when the host tries again: never "waking up" without end. | |
| 1018 | // A cell that did not start says so, with when the host tries again: never "waking up" without | |
| 1019 | // end. | |
| 955 | 1020 | const stop = cells.failure(ledger.bodyId) === undefined ? undefined : lastStop(parent, ledger.bodyId); |
| 956 | 1021 | return wakingResponse(parent, request, ledger.name, stop === undefined ? undefined : stopSentence(stop)); |
| 957 | 1022 | } |
| @@ -962,7 +1027,10 @@ export async function answerFromCell(parent: App, request: Request, named?: stri | ||
| 962 | 1027 | */ |
| 963 | 1028 | const waitsToMove = (name: string): string => `${name} is waiting to be moved into its own sandbox: ${SEALED_WORDS}. Its host moves it; nothing of it is lost.`; |
| 964 | 1029 | |
| 965 | /** How long a request waits for a sleeping cell forge before it is told it is waking, and how many may wait on one cell. */ | |
| 1030 | /** | |
| 1031 | * How long a request waits for a sleeping cell forge before it is told it is waking, and how many | |
| 1032 | * may wait on one cell. | |
| 1033 | */ | |
| 966 | 1034 | export const wakeLimits = { waitMs: 20_000, waiters: 16 }; |
| 967 | 1035 | |
| 968 | 1036 | /** Requests held for a waking cell now, by body id. */ |
test/boot-idle-children.test.ts +5 −2
| @@ -57,7 +57,7 @@ afterAll(async () => { | ||
| 57 | 57 | test("a wake is not refused for children the boot opened and nobody asked for", async () => { |
| 58 | 58 | await sleepIdleChildren(parent, 0, Date.now() + 3_600_000); |
| 59 | 59 | process.env["FORGE_CHILD_AWAKE_MAX"] = "2"; |
| 60 | await bootHostedChildren(parent); | |
| 60 | await bootHostedChildren(parent, new Set(["idle-a", "idle-b"])); | |
| 61 | 61 | expect(childAwake(parent, "idle-a")).toBe(true); |
| 62 | 62 | expect(childAwake(parent, "idle-b")).toBe(true); |
| 63 | 63 | await sleepChild(parent, "sleeper"); |
| @@ -71,9 +71,12 @@ const hostOf = (name: string) => `${name}--forge.test`; | ||
| 71 | 71 | const gate = (name: string, headers: Record<string, string> = {}) => |
| 72 | 72 | wakeGate(parent, new Request("https://forge.test/", { headers: { host: hostOf(name), ...headers } })); |
| 73 | 73 | const reset = async () => { |
| 74 | // A wake a test granted and never finished holds a slot; a boot used to settle it by opening the | |
| 75 | // forge. | |
| 76 | await resolveHostedChild(parent, new Request("https://forge.test/healthz", { headers: { host: hostOf("sleeper") } })); | |
| 74 | 77 | await sleepIdleChildren(parent, 0, Date.now() + 3_600_000); |
| 75 | 78 | process.env["FORGE_CHILD_AWAKE_MAX"] = "2"; |
| 76 | await bootHostedChildren(parent); | |
| 79 | await bootHostedChildren(parent, new Set(["idle-a", "idle-b"])); | |
| 77 | 80 | }; |
| 78 | 81 | |
| 79 | 82 | test("a boot-opened child a credentialed request reaches counts again, and the ceiling then holds", async () => { |
test/boot-only-awake.test.ts added +84
| @@ -0,0 +1,84 @@ | ||
| 1 | /** | |
| 2 | * A release starts only the forges that were awake and in use when the parent stopped (#13816): | |
| 3 | * the rest stay asleep and wake on their next request. | |
| 4 | */ | |
| 5 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 6 | import { existsSync } from "node:fs"; | |
| 7 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 8 | import { tmpdir } from "node:os"; | |
| 9 | import { join } from "node:path"; | |
| 10 | import { spawnForge } from "../src/actions/spawn.ts"; | |
| 11 | import { createApp, type App } from "../src/app.ts"; | |
| 12 | import { createUser } from "../src/core/users.ts"; | |
| 13 | import { rememberAwake, takeAwake } from "../src/spawn/awake_at_stop.ts"; | |
| 14 | import { bootHostedChildren, childAwake, closeHostedChildren, resolveHostedChild } from "../src/spawn/host.ts"; | |
| 15 | import { hostedPlacement } from "../src/spawn/placement.ts"; | |
| 16 | import { quietLogs } from "./helpers/globals.ts"; | |
| 17 | ||
| 18 | quietLogs(); | |
| 19 | ||
| 20 | let dir = ""; | |
| 21 | let parent: App; | |
| 22 | ||
| 23 | beforeAll(async () => { | |
| 24 | dir = await mkdtemp(join(tmpdir(), "boot-only-")); | |
| 25 | parent = createApp({ dataDir: dir, url: "https://forge.test", port: 0 }); | |
| 26 | const made = await createUser(parent.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", isAdmin: true }); | |
| 27 | if (!made.ok) throw new Error("user"); | |
| 28 | for (const name of ["used", "quiet"]) { | |
| 29 | const spawned = await spawnForge( | |
| 30 | parent, | |
| 31 | made.value, | |
| 32 | { name, org: name, apps: [], carryBriefs: false, carrySubscriptions: false }, | |
| 33 | { catalog: [], briefs: [], subscriptions: [] }, | |
| 34 | { | |
| 35 | placement: hostedPlacement({ host: "forge.test" }), | |
| 36 | request: new Request("https://forge.test/spawn", { method: "POST", headers: { origin: "https://forge.test" } }), | |
| 37 | url: new URL("https://forge.test/spawn"), | |
| 38 | }, | |
| 39 | ); | |
| 40 | if (!spawned.ok) throw new Error(JSON.stringify(spawned.error)); | |
| 41 | } | |
| 42 | }); | |
| 43 | ||
| 44 | afterAll(async () => { | |
| 45 | parent.close(); | |
| 46 | await rm(dir, { recursive: true, force: true }); | |
| 47 | }); | |
| 48 | ||
| 49 | test("the names recorded at a stop are read once, and only well-formed ones", () => { | |
| 50 | rememberAwake(dir, ["b", "a", "a", "../x"]); | |
| 51 | expect([...takeAwake(dir)].sort()).toEqual(["a", "b"]); | |
| 52 | expect(takeAwake(dir).size).toBe(0); | |
| 53 | }); | |
| 54 | ||
| 55 | test("a boot with nothing recorded starts no forge", async () => { | |
| 56 | await bootHostedChildren(parent); | |
| 57 | expect(childAwake(parent, "used")).toBe(false); | |
| 58 | expect(childAwake(parent, "quiet")).toBe(false); | |
| 59 | }); | |
| 60 | ||
| 61 | test("a stop records the forges in use; the next boot starts those and no other", async () => { | |
| 62 | await resolveHostedChild(parent, new Request("https://forge.test/healthz", { headers: { host: "used--forge.test" } })); | |
| 63 | expect(childAwake(parent, "used")).toBe(true); | |
| 64 | closeHostedChildren(parent); | |
| 65 | expect(childAwake(parent, "used")).toBe(false); | |
| 66 | expect(existsSync(join(dir, "awake-at-stop.json"))).toBe(true); | |
| 67 | await bootHostedChildren(parent); | |
| 68 | expect(childAwake(parent, "used")).toBe(true); | |
| 69 | expect(childAwake(parent, "quiet")).toBe(false); | |
| 70 | expect(existsSync(join(dir, "awake-at-stop.json"))).toBe(false); | |
| 71 | }); | |
| 72 | ||
| 73 | test("the awake ceiling holds on boot", async () => { | |
| 74 | closeHostedChildren(parent); | |
| 75 | const was = process.env["FORGE_CHILD_AWAKE_MAX"]; | |
| 76 | process.env["FORGE_CHILD_AWAKE_MAX"] = "1"; | |
| 77 | try { | |
| 78 | await bootHostedChildren(parent, new Set(["used", "quiet"])); | |
| 79 | expect(["used", "quiet"].filter((n) => childAwake(parent, n))).toHaveLength(1); | |
| 80 | } finally { | |
| 81 | if (was === undefined) delete process.env["FORGE_CHILD_AWAKE_MAX"]; | |
| 82 | else process.env["FORGE_CHILD_AWAKE_MAX"] = was; | |
| 83 | } | |
| 84 | }); |
new above
offline
Next: this condition is hard to read, and a stale-file case is only safe because takeAwake removes the file at boot. Write unconditionally or explain why.