Public: anyone can read this; the crew can change it.
The release proof removes its probe forge without logging an ERROR: recording a placement for a forge already removed is not a fault #13770
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/13768-the-release-proof-removes-its-probe-forg → main 2 commits fork
2 files
changed +118
−23
src/spawn/host.ts +71 −23
| @@ -121,7 +121,10 @@ export function hostedChildOpen(parent: App, child: App): boolean { | ||
| 121 | 121 | return false; |
| 122 | 122 | } |
| 123 | 123 | |
| 124 | /** The hosted children open in this process now, by the name each goes by; a sleeping child or one in a cell is not here. */ | |
| 124 | /** | |
| 125 | * The hosted children open in this process now, by the name each goes by; a sleeping child or one | |
| 126 | * in a cell is not here. | |
| 127 | */ | |
| 125 | 128 | export function openChildren(parent: App): { name: string; app: App }[] { |
| 126 | 129 | const out: { name: string; app: App }[] = []; |
| 127 | 130 | for (const [bodyId, child] of caches.get(parent) ?? []) { |
| @@ -147,7 +150,8 @@ function touch(parent: App, ledger: { bodyId: string; name: string }, now = Date | ||
| 147 | 150 | |
| 148 | 151 | /** Forges awake now: hosted children open in this process plus children running in a cell. */ |
| 149 | 152 | export function awakeCount(parent: App): number { |
| 150 | // A child boot opened and nobody has asked for is not in use: it never crowds out a person's wake. | |
| 153 | // A child boot opened and nobody has asked for is not in use: it never crowds out a person's | |
| 154 | // wake. | |
| 151 | 155 | const unasked = bootOpened.get(parent)?.size ?? 0; |
| 152 | 156 | const awake = Math.max(0, (caches.get(parent)?.size ?? 0) - unasked) + cellChildren(parent).list().length; |
| 153 | 157 | // A wake granted but not finished holds its slot, so two at once cannot both pass the ceiling. |
| @@ -194,7 +198,10 @@ export function holdChild(parent: App, child: App): () => void { | ||
| 194 | 198 | }; |
| 195 | 199 | } |
| 196 | 200 | |
| 197 | /** Whether the child is mid-request or has crew work queued or running: it is not idle however quiet its visitors. */ | |
| 201 | /** | |
| 202 | * Whether the child is mid-request or has crew work queued or running: it is not idle however quiet | |
| 203 | * its visitors. | |
| 204 | */ | |
| 198 | 205 | function working(parent: App, child: App): boolean { |
| 199 | 206 | if ((inFlight.get(parent)?.get(child.config.dataDir) ?? 0) > 0) return true; |
| 200 | 207 | try { |
| @@ -252,7 +259,9 @@ export function onInProcessSleep(parent: App, hook: () => void): void { | ||
| 252 | 259 | |
| 253 | 260 | /** Wakes armed for cell children asleep with a run paused on a limit, by body id. */ |
| 254 | 261 | const dueWakes = new WeakMap<App, Map<string, ReturnType<typeof setTimeout>>>(); |
| 255 | /** A due wake the awake ceiling or the disk floor turned away is tried again this much later, ms. */ | |
| 262 | /** | |
| 263 | * A due wake the awake ceiling or the disk floor turned away is tried again this much later, ms. | |
| 264 | */ | |
| 256 | 265 | const DUE_RETRY_MS = 60_000; |
| 257 | 266 | /** A forge the host stopped may take the room of one quiet this long, ms. */ |
| 258 | 267 | const STOPPED_EVICT_IDLE_MS = 60_000; |
| @@ -294,8 +303,8 @@ async function wakeDue(parent: App, name: string): Promise<void> { | ||
| 294 | 303 | const ledger = readLedgerSync(parent, name); |
| 295 | 304 | if (ledger === undefined || !ledger.done || ledger.placement !== "cell" || ledger.moving) return; |
| 296 | 305 | if (isAwakeBody(parent, ledger.bodyId) || lapsedNone(parent, name) !== undefined) return; |
| 297 | // A forge the host stopped for a limit does not wait for an idle one's ten minutes: forges quiet for | |
| 298 | // a minute and with no crew at work are put to sleep to make its room. | |
| 306 | // A forge the host stopped for a limit does not wait for an idle one's ten minutes: forges quiet | |
| 307 | // for a minute and with no crew at work are put to sleep to make its room. | |
| 299 | 308 | if (lastStop(parent, ledger.bodyId) !== undefined && awakeRefusal(awakeCount(parent)) !== undefined) |
| 300 | 309 | await sleepIdleChildren(parent, STOPPED_EVICT_IDLE_MS); |
| 301 | 310 | if (floorRefusal(parent.config.dataDir) !== undefined || awakeRefusal(awakeCount(parent)) !== undefined) { |
| @@ -320,7 +329,8 @@ async function restartStopped(parent: App, name: string): Promise<boolean> { | ||
| 320 | 329 | const ledger = readLedgerSync(parent, name); |
| 321 | 330 | if (ledger === undefined) return false; |
| 322 | 331 | if (isAwakeBody(parent, ledger.bodyId) || dueWakes.get(parent)?.has(ledger.bodyId) === true) return true; |
| 323 | // The cell did not come up (or a start was refused): the host tries again a minute on, for the hour the stop is shown. | |
| 332 | // The cell did not come up (or a start was refused): the host tries again a minute on, for the | |
| 333 | // hour the stop is shown. | |
| 324 | 334 | const stop = lastStop(parent, ledger.bodyId); |
| 325 | 335 | if (stop === undefined || ledger.placement !== "cell" || lapsedNone(parent, name) !== undefined || startRefused(parent, ledger) !== undefined) return false; |
| 326 | 336 | wakeWhenDue(parent, ledger, Date.now() + DUE_RETRY_MS); |
| @@ -408,13 +418,18 @@ export function keepWatch(parent: App): void { | ||
| 408 | 418 | /** Whether this process has the child open as an App of its own (not a cell's). */ |
| 409 | 419 | export const openHere = (parent: App, bodyId: string): boolean => caches.get(parent)?.has(bodyId) === true || opening.get(parent)?.has(bodyId) === true; |
| 410 | 420 | |
| 411 | /** A child's lapse when its billing is `none`: it sleeps and does not wake until paid (#11393); else undefined. */ | |
| 421 | /** | |
| 422 | * A child's lapse when its billing is `none`: it sleeps and does not wake until paid (#11393); else | |
| 423 | * undefined. | |
| 424 | */ | |
| 412 | 425 | export function lapsedNone(parent: App, name: string): Extract<Lapse, { state: "none" }> | undefined { |
| 413 | 426 | const lapse = lapseFor(parent.db, name); |
| 414 | 427 | return lapse.state === "none" ? lapse : undefined; |
| 415 | 428 | } |
| 416 | 429 | |
| 417 | /** Put to sleep, through `sleepChild`, every awake child whose billing is `none`. Answers the names. */ | |
| 430 | /** | |
| 431 | * Put to sleep, through `sleepChild`, every awake child whose billing is `none`. Answers the names. | |
| 432 | */ | |
| 418 | 433 | export async function sleepLapsed(parent: App): Promise<string[]> { |
| 419 | 434 | const names = new Set<string>([...(touched.get(parent)?.values() ?? [])].map((m) => m.name)); |
| 420 | 435 | for (const c of cellChildren(parent).list()) names.add(c.name); |
| @@ -462,7 +477,10 @@ function keepSweeping(parent: App): void { | ||
| 462 | 477 | sweepers.set(parent, timer); |
| 463 | 478 | } |
| 464 | 479 | |
| 465 | /** Whether the child itself vouches for the request's session or token: a header a client merely wrote is not enough. */ | |
| 480 | /** | |
| 481 | * Whether the child itself vouches for the request's session or token: a header a client merely | |
| 482 | * wrote is not enough. | |
| 483 | */ | |
| 466 | 484 | const carriesCredentials = (parent: App, bodyId: string, request: Request): Promise<boolean> => |
| 467 | 485 | vouched(childDirOf(parent, bodyId), request, caches.get(parent)?.get(bodyId)?.db); |
| 468 | 486 | |
| @@ -526,7 +544,8 @@ export async function wakeGate(parent: App, request: Request): Promise<Response | ||
| 526 | 544 | if (isAwakeBody(parent, ledger.bodyId)) void sleepChild(parent, name).catch(() => {}); |
| 527 | 545 | return lapsedResponse(parent, request, name, lapsed); |
| 528 | 546 | } |
| 529 | // Asked off this thread when the child is a cell's: its database is its own to write, and the answer has a deadline. | |
| 547 | // Asked off this thread when the child is a cell's: its database is its own to write, and the | |
| 548 | // answer has a deadline. | |
| 530 | 549 | const known = await carriesCredentials(parent, ledger.bodyId, request); |
| 531 | 550 | if (isAwakeBody(parent, ledger.bodyId)) { |
| 532 | 551 | clearStop(parent, ledger.bodyId); |
| @@ -537,7 +556,8 @@ export async function wakeGate(parent: App, request: Request): Promise<Response | ||
| 537 | 556 | } |
| 538 | 557 | return undefined; |
| 539 | 558 | } |
| 540 | // Asleep, so answering would start it: not where it has no filesystem of its own and one is required. | |
| 559 | // Asleep, so answering would start it: not where it has no filesystem of its own and one is | |
| 560 | // required. | |
| 541 | 561 | const refused = startRefused(parent, ledger); |
| 542 | 562 | if (refused !== undefined) return refusedResponse(refused); |
| 543 | 563 | // A wake already granted (a cell takes a moment to answer) is one wake, not one per request. |
| @@ -656,7 +676,8 @@ export async function resolveHostedChild( | ||
| 656 | 676 | // it, and while its cell is not up `answerFromCell` says so at once. |
| 657 | 677 | if (ledger.placement === "cell") { |
| 658 | 678 | if (cellChildren(parent).get(ledger.bodyId) !== undefined) return "cell"; |
| 659 | // A host that holds no cell at all is known without waiting; a child that ran its own code is never opened here. | |
| 679 | // A host that holds no cell at all is known without waiting; a child that ran its own code is | |
| 680 | // never opened here. | |
| 660 | 681 | const here = await hostedHere(parent, ledger.name); |
| 661 | 682 | if (here.ok) return openOnce(parent, demoted(parent, ledger, here.why)); |
| 662 | 683 | void settleCell(parent, ledger); |
| @@ -699,7 +720,10 @@ function settleCell(parent: App, ledger: ChildLedger): Promise<CellChild | undef | ||
| 699 | 720 | return flight; |
| 700 | 721 | } |
| 701 | 722 | |
| 702 | /** Whether this host holds cells: the machine's own answer, unless a test stands in for it. Never read from the environment. */ | |
| 723 | /** | |
| 724 | * Whether this host holds cells: the machine's own answer, unless a test stands in for it. Never | |
| 725 | * read from the environment. | |
| 726 | */ | |
| 703 | 727 | let holdsCells: typeof cellsAvailable = cellsAvailable; |
| 704 | 728 | export function provideCellsForTest(stand?: typeof cellsAvailable): void { |
| 705 | 729 | holdsCells = stand ?? cellsAvailable; |
| @@ -711,16 +735,25 @@ export function provideCellsForTest(stand?: typeof cellsAvailable): void { | ||
| 711 | 735 | */ |
| 712 | 736 | export const cellsHere = (): ReturnType<typeof cellsAvailable> => holdsCells(); |
| 713 | 737 | |
| 714 | /** How a ledger's `placementNote` begins for a cell child whose cell has not started; `/forges` lists it. */ | |
| 738 | /** | |
| 739 | * How a ledger's `placementNote` begins for a cell child whose cell has not started; `/forges` | |
| 740 | * lists it. | |
| 741 | */ | |
| 715 | 742 | export const NOT_STARTED = "own cell, not started: "; |
| 716 | 743 | |
| 717 | /** Whether a child has ever run code of its own (`spawn/own_code.ts`): such a child runs in a cell or not at all. */ | |
| 744 | /** | |
| 745 | * Whether a child has ever run code of its own (`spawn/own_code.ts`): such a child runs in a cell | |
| 746 | * or not at all. | |
| 747 | */ | |
| 718 | 748 | export function ranOwnCode(parent: App, name: string): boolean { |
| 719 | 749 | const own = readOwnCode(parent, name); |
| 720 | 750 | return own.on || own.ran === true || own.good !== undefined || own.failed !== undefined || own.rolledBack !== undefined; |
| 721 | 751 | } |
| 722 | 752 | |
| 723 | /** Whether this host may run `name` in its own process: it holds no cell at all, and the child never ran code of its own. */ | |
| 753 | /** | |
| 754 | * Whether this host may run `name` in its own process: it holds no cell at all, and the child never | |
| 755 | * ran code of its own. | |
| 756 | */ | |
| 724 | 757 | async function hostedHere(parent: App, name: string): Promise<{ ok: true; why: string } | { ok: false; why: string }> { |
| 725 | 758 | const cells = await holdsCells(); |
| 726 | 759 | if (cells.ok) return { ok: false, why: "this host holds cells" }; |
| @@ -747,7 +780,8 @@ async function cellDidNotStart(parent: App, ledger: ChildLedger, why: string): P | ||
| 747 | 780 | const note = `${NOT_STARTED}${(/cannot make a cell/.test(here.why) ? here.why : said).slice(0, 400)}`; |
| 748 | 781 | noteLedger(parent, ledger.name, (raw) => (raw["placementNote"] === note ? undefined : { placementNote: note })); |
| 749 | 782 | parent.log.warn("child forge's cell did not start; it stays a cell child and the host tries again", { name: ledger.name, why }); |
| 750 | // One stop per failure, not one per request that found it failed: the wait and the count are the stop's. | |
| 783 | // One stop per failure, not one per request that found it failed: the wait and the count are the | |
| 784 | // stop's. | |
| 751 | 785 | if (lastStop(parent, ledger.bodyId) === undefined) noteStop(parent, ledger.bodyId, ledger.name, said); |
| 752 | 786 | } |
| 753 | 787 | |
| @@ -758,7 +792,10 @@ function demoted(parent: App, ledger: ChildLedger, why: string): ChildLedger { | ||
| 758 | 792 | return { ...ledger, placement: "hosted" }; |
| 759 | 793 | } |
| 760 | 794 | |
| 761 | /** Change fields of a child's ledger (the parent's own file); `change` answers nothing when there is nothing to write. */ | |
| 795 | /** | |
| 796 | * Change fields of a child's ledger (the parent's own file); `change` answers nothing when there is | |
| 797 | * nothing to write. | |
| 798 | */ | |
| 762 | 799 | export function noteLedger(parent: App, name: string, change: (raw: Record<string, unknown>) => Record<string, unknown> | undefined): void { |
| 763 | 800 | const path = join(parent.config.dataDir, "spawn", `${name}.json`); |
| 764 | 801 | try { |
| @@ -768,6 +805,9 @@ export function noteLedger(parent: App, name: string, change: (raw: Record<strin | ||
| 768 | 805 | writeFileSync(`${path}.tmp`, JSON.stringify({ ...raw, ...next }), { mode: 0o600 }); |
| 769 | 806 | renameSync(`${path}.tmp`, path); |
| 770 | 807 | } catch (error) { |
| 808 | // A forge already removed has nothing to record: not a fault. Checked after the fact, not | |
| 809 | // before, so a removal between the read and the rename lands here too. | |
| 810 | if ((error as NodeJS.ErrnoException).code === "ENOENT") return; | |
| 771 | 811 | parent.log.error("could not record the placement", { name, error: String(error) }); |
| 772 | 812 | } |
| 773 | 813 | } |
| @@ -852,7 +892,10 @@ export interface RefusedHit { | ||
| 852 | 892 | readonly refused: string; |
| 853 | 893 | } |
| 854 | 894 | |
| 855 | /** A child app in a child's own cell: its runtime is in that process, so the request is carried there. */ | |
| 895 | /** | |
| 896 | * A child app in a child's own cell: its runtime is in that process, so the request is carried | |
| 897 | * there. | |
| 898 | */ | |
| 856 | 899 | export interface CellAppHit { |
| 857 | 900 | readonly cell: string; |
| 858 | 901 | } |
| @@ -931,7 +974,8 @@ export async function answerFromCell(parent: App, request: Request, named?: stri | ||
| 931 | 974 | // held until the cell is up (wakeLimits.waitMs) so a submit is never lost; only a wake |
| 932 | 975 | // that takes longer is answered with the "waking up" page, which tries again by itself. |
| 933 | 976 | const starting = settleCell(parent, ledger); |
| 934 | // Only so many wait on one cell: the rest are told it is waking at once, so a crowd cannot pin connections and bodies. | |
| 977 | // Only so many wait on one cell: the rest are told it is waking at once, so a crowd cannot pin | |
| 978 | // connections and bodies. | |
| 935 | 979 | const waiting = waiters.get(ledger.bodyId) ?? 0; |
| 936 | 980 | if (waiting >= wakeLimits.waiters) return wakingResponse(parent, request, ledger.name); |
| 937 | 981 | waiters.set(ledger.bodyId, waiting + 1); |
| @@ -951,7 +995,8 @@ export async function answerFromCell(parent: App, request: Request, named?: stri | ||
| 951 | 995 | release(); |
| 952 | 996 | } |
| 953 | 997 | } |
| 954 | // A cell that did not start says so, with when the host tries again: never "waking up" without end. | |
| 998 | // A cell that did not start says so, with when the host tries again: never "waking up" without | |
| 999 | // end. | |
| 955 | 1000 | const stop = cells.failure(ledger.bodyId) === undefined ? undefined : lastStop(parent, ledger.bodyId); |
| 956 | 1001 | return wakingResponse(parent, request, ledger.name, stop === undefined ? undefined : stopSentence(stop)); |
| 957 | 1002 | } |
| @@ -962,7 +1007,10 @@ export async function answerFromCell(parent: App, request: Request, named?: stri | ||
| 962 | 1007 | */ |
| 963 | 1008 | const waitsToMove = (name: string): string => `${name} is waiting to be moved into its own sandbox: ${SEALED_WORDS}. Its host moves it; nothing of it is lost.`; |
| 964 | 1009 | |
| 965 | /** How long a request waits for a sleeping cell forge before it is told it is waking, and how many may wait on one cell. */ | |
| 1010 | /** | |
| 1011 | * How long a request waits for a sleeping cell forge before it is told it is waking, and how many | |
| 1012 | * may wait on one cell. | |
| 1013 | */ | |
| 966 | 1014 | export const wakeLimits = { waitMs: 20_000, waiters: 16 }; |
| 967 | 1015 | |
| 968 | 1016 | /** Requests held for a waking cell now, by body id. */ |
test/note-ledger-removed.test.ts added +47
| @@ -0,0 +1,47 @@ | ||
| 1 | /** Recording a placement for a forge already removed is a no-op, not an ERROR. */ | |
| 2 | ||
| 3 | import { afterAll, expect, test } from "bun:test"; | |
| 4 | import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; | |
| 5 | import { tmpdir } from "node:os"; | |
| 6 | import { join } from "node:path"; | |
| 7 | import { createApp } from "../src/app.ts"; | |
| 8 | import { noteLedger } from "../src/spawn/host.ts"; | |
| 9 | ||
| 10 | const tmp = mkdtempSync(join(tmpdir(), "note-ledger-")); | |
| 11 | afterAll(() => rmSync(tmp, { recursive: true, force: true })); | |
| 12 | ||
| 13 | test("a removed forge's placement is not recorded and not an error", () => { | |
| 14 | const parent = createApp({ dataDir: tmp }); | |
| 15 | const errors: unknown[] = []; | |
| 16 | const original = parent.log.error.bind(parent.log); | |
| 17 | parent.log.error = ((...args: unknown[]) => { | |
| 18 | errors.push(args); | |
| 19 | return (original as (...a: unknown[]) => unknown)(...args); | |
| 20 | }) as typeof parent.log.error; | |
| 21 | let called = false; | |
| 22 | noteLedger(parent, "probe-gone", () => ((called = true), { placement: "hosted" })); | |
| 23 | expect(called).toBe(false); | |
| 24 | expect(errors).toEqual([]); | |
| 25 | parent.close(); | |
| 26 | }); | |
| 27 | ||
| 28 | test("a record that exists is changed, and a real fault still logs an ERROR", () => { | |
| 29 | const dir = mkdtempSync(join(tmpdir(), "note-ledger-")); | |
| 30 | mkdirSync(join(dir, "spawn")); | |
| 31 | const parent = createApp({ dataDir: dir }); | |
| 32 | const errors: unknown[] = []; | |
| 33 | const original = parent.log.error.bind(parent.log); | |
| 34 | parent.log.error = ((...args: unknown[]) => { | |
| 35 | errors.push(args); | |
| 36 | return (original as (...a: unknown[]) => unknown)(...args); | |
| 37 | }) as typeof parent.log.error; | |
| 38 | writeFileSync(join(dir, "spawn", "kept.json"), JSON.stringify({ a: 1 })); | |
| 39 | noteLedger(parent, "kept", () => ({ placement: "hosted" })); | |
| 40 | expect(JSON.parse(readFileSync(join(dir, "spawn", "kept.json"), "utf8"))).toEqual({ a: 1, placement: "hosted" }); | |
| 41 | expect(errors).toEqual([]); | |
| 42 | writeFileSync(join(dir, "spawn", "broken.json"), "{not json"); | |
| 43 | noteLedger(parent, "broken", () => ({ placement: "hosted" })); | |
| 44 | expect(errors.length).toBe(1); | |
| 45 | parent.close(); | |
| 46 | rmSync(dir, { recursive: true, force: true }); | |
| 47 | }); |
new above
offline