Public: anyone can read this; the crew can change it.

The release proof removes its probe forge without logging an ERROR: recording a placement for a forge already removed is not a fault #13770

Merged opened this in answer to

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/13768-the-release-proof-removes-its-probe-forg → main 2 commits fork

  • Merged. scout/13768-the-release-proof-removes-its-probe-forg was merged into main as 1c4e782.

Review

Sign in to review this change.
2 files changed +118 −23
src/spawn/host.ts +71 −23
@@ -121,7 +121,10 @@ export function hostedChildOpen(parent: App, child: App): boolean {
121121 return false;
122122 }
123123
124 /** The hosted children open in this process now, by the name each goes by; a sleeping child or one in a cell is not here. */
124 /**
125 * The hosted children open in this process now, by the name each goes by; a sleeping child or one
126 * in a cell is not here.
127 */
125128 export function openChildren(parent: App): { name: string; app: App }[] {
126129 const out: { name: string; app: App }[] = [];
127130 for (const [bodyId, child] of caches.get(parent) ?? []) {
@@ -147,7 +150,8 @@ function touch(parent: App, ledger: { bodyId: string; name: string }, now = Date
147150
148151 /** Forges awake now: hosted children open in this process plus children running in a cell. */
149152 export function awakeCount(parent: App): number {
150 // A child boot opened and nobody has asked for is not in use: it never crowds out a person's wake.
153 // A child boot opened and nobody has asked for is not in use: it never crowds out a person's
154 // wake.
151155 const unasked = bootOpened.get(parent)?.size ?? 0;
152156 const awake = Math.max(0, (caches.get(parent)?.size ?? 0) - unasked) + cellChildren(parent).list().length;
153157 // A wake granted but not finished holds its slot, so two at once cannot both pass the ceiling.
@@ -194,7 +198,10 @@ export function holdChild(parent: App, child: App): () => void {
194198 };
195199 }
196200
197 /** Whether the child is mid-request or has crew work queued or running: it is not idle however quiet its visitors. */
201 /**
202 * Whether the child is mid-request or has crew work queued or running: it is not idle however quiet
203 * its visitors.
204 */
198205 function working(parent: App, child: App): boolean {
199206 if ((inFlight.get(parent)?.get(child.config.dataDir) ?? 0) > 0) return true;
200207 try {
@@ -252,7 +259,9 @@ export function onInProcessSleep(parent: App, hook: () => void): void {
252259
253260 /** Wakes armed for cell children asleep with a run paused on a limit, by body id. */
254261 const dueWakes = new WeakMap<App, Map<string, ReturnType<typeof setTimeout>>>();
255 /** A due wake the awake ceiling or the disk floor turned away is tried again this much later, ms. */
262 /**
263 * A due wake the awake ceiling or the disk floor turned away is tried again this much later, ms.
264 */
256265 const DUE_RETRY_MS = 60_000;
257266 /** A forge the host stopped may take the room of one quiet this long, ms. */
258267 const STOPPED_EVICT_IDLE_MS = 60_000;
@@ -294,8 +303,8 @@ async function wakeDue(parent: App, name: string): Promise<void> {
294303 const ledger = readLedgerSync(parent, name);
295304 if (ledger === undefined || !ledger.done || ledger.placement !== "cell" || ledger.moving) return;
296305 if (isAwakeBody(parent, ledger.bodyId) || lapsedNone(parent, name) !== undefined) return;
297 // A forge the host stopped for a limit does not wait for an idle one's ten minutes: forges quiet for
298 // a minute and with no crew at work are put to sleep to make its room.
306 // A forge the host stopped for a limit does not wait for an idle one's ten minutes: forges quiet
307 // for a minute and with no crew at work are put to sleep to make its room.
299308 if (lastStop(parent, ledger.bodyId) !== undefined && awakeRefusal(awakeCount(parent)) !== undefined)
300309 await sleepIdleChildren(parent, STOPPED_EVICT_IDLE_MS);
301310 if (floorRefusal(parent.config.dataDir) !== undefined || awakeRefusal(awakeCount(parent)) !== undefined) {
@@ -320,7 +329,8 @@ async function restartStopped(parent: App, name: string): Promise<boolean> {
320329 const ledger = readLedgerSync(parent, name);
321330 if (ledger === undefined) return false;
322331 if (isAwakeBody(parent, ledger.bodyId) || dueWakes.get(parent)?.has(ledger.bodyId) === true) return true;
323 // The cell did not come up (or a start was refused): the host tries again a minute on, for the hour the stop is shown.
332 // The cell did not come up (or a start was refused): the host tries again a minute on, for the
333 // hour the stop is shown.
324334 const stop = lastStop(parent, ledger.bodyId);
325335 if (stop === undefined || ledger.placement !== "cell" || lapsedNone(parent, name) !== undefined || startRefused(parent, ledger) !== undefined) return false;
326336 wakeWhenDue(parent, ledger, Date.now() + DUE_RETRY_MS);
@@ -408,13 +418,18 @@ export function keepWatch(parent: App): void {
408418 /** Whether this process has the child open as an App of its own (not a cell's). */
409419 export const openHere = (parent: App, bodyId: string): boolean => caches.get(parent)?.has(bodyId) === true || opening.get(parent)?.has(bodyId) === true;
410420
411 /** A child's lapse when its billing is `none`: it sleeps and does not wake until paid (#11393); else undefined. */
421 /**
422 * A child's lapse when its billing is `none`: it sleeps and does not wake until paid (#11393); else
423 * undefined.
424 */
412425 export function lapsedNone(parent: App, name: string): Extract<Lapse, { state: "none" }> | undefined {
413426 const lapse = lapseFor(parent.db, name);
414427 return lapse.state === "none" ? lapse : undefined;
415428 }
416429
417 /** Put to sleep, through `sleepChild`, every awake child whose billing is `none`. Answers the names. */
430 /**
431 * Put to sleep, through `sleepChild`, every awake child whose billing is `none`. Answers the names.
432 */
418433 export async function sleepLapsed(parent: App): Promise<string[]> {
419434 const names = new Set<string>([...(touched.get(parent)?.values() ?? [])].map((m) => m.name));
420435 for (const c of cellChildren(parent).list()) names.add(c.name);
@@ -462,7 +477,10 @@ function keepSweeping(parent: App): void {
462477 sweepers.set(parent, timer);
463478 }
464479
465 /** Whether the child itself vouches for the request's session or token: a header a client merely wrote is not enough. */
480 /**
481 * Whether the child itself vouches for the request's session or token: a header a client merely
482 * wrote is not enough.
483 */
466484 const carriesCredentials = (parent: App, bodyId: string, request: Request): Promise<boolean> =>
467485 vouched(childDirOf(parent, bodyId), request, caches.get(parent)?.get(bodyId)?.db);
468486
@@ -526,7 +544,8 @@ export async function wakeGate(parent: App, request: Request): Promise<Response
526544 if (isAwakeBody(parent, ledger.bodyId)) void sleepChild(parent, name).catch(() => {});
527545 return lapsedResponse(parent, request, name, lapsed);
528546 }
529 // Asked off this thread when the child is a cell's: its database is its own to write, and the answer has a deadline.
547 // Asked off this thread when the child is a cell's: its database is its own to write, and the
548 // answer has a deadline.
530549 const known = await carriesCredentials(parent, ledger.bodyId, request);
531550 if (isAwakeBody(parent, ledger.bodyId)) {
532551 clearStop(parent, ledger.bodyId);
@@ -537,7 +556,8 @@ export async function wakeGate(parent: App, request: Request): Promise<Response
537556 }
538557 return undefined;
539558 }
540 // Asleep, so answering would start it: not where it has no filesystem of its own and one is required.
559 // Asleep, so answering would start it: not where it has no filesystem of its own and one is
560 // required.
541561 const refused = startRefused(parent, ledger);
542562 if (refused !== undefined) return refusedResponse(refused);
543563 // A wake already granted (a cell takes a moment to answer) is one wake, not one per request.
@@ -656,7 +676,8 @@ export async function resolveHostedChild(
656676 // it, and while its cell is not up `answerFromCell` says so at once.
657677 if (ledger.placement === "cell") {
658678 if (cellChildren(parent).get(ledger.bodyId) !== undefined) return "cell";
659 // A host that holds no cell at all is known without waiting; a child that ran its own code is never opened here.
679 // A host that holds no cell at all is known without waiting; a child that ran its own code is
680 // never opened here.
660681 const here = await hostedHere(parent, ledger.name);
661682 if (here.ok) return openOnce(parent, demoted(parent, ledger, here.why));
662683 void settleCell(parent, ledger);
@@ -699,7 +720,10 @@ function settleCell(parent: App, ledger: ChildLedger): Promise<CellChild | undef
699720 return flight;
700721 }
701722
702 /** Whether this host holds cells: the machine's own answer, unless a test stands in for it. Never read from the environment. */
723 /**
724 * Whether this host holds cells: the machine's own answer, unless a test stands in for it. Never
725 * read from the environment.
726 */
703727 let holdsCells: typeof cellsAvailable = cellsAvailable;
704728 export function provideCellsForTest(stand?: typeof cellsAvailable): void {
705729 holdsCells = stand ?? cellsAvailable;
@@ -711,16 +735,25 @@ export function provideCellsForTest(stand?: typeof cellsAvailable): void {
711735 */
712736 export const cellsHere = (): ReturnType<typeof cellsAvailable> => holdsCells();
713737
714 /** How a ledger's `placementNote` begins for a cell child whose cell has not started; `/forges` lists it. */
738 /**
739 * How a ledger's `placementNote` begins for a cell child whose cell has not started; `/forges`
740 * lists it.
741 */
715742 export const NOT_STARTED = "own cell, not started: ";
716743
717 /** Whether a child has ever run code of its own (`spawn/own_code.ts`): such a child runs in a cell or not at all. */
744 /**
745 * Whether a child has ever run code of its own (`spawn/own_code.ts`): such a child runs in a cell
746 * or not at all.
747 */
718748 export function ranOwnCode(parent: App, name: string): boolean {
719749 const own = readOwnCode(parent, name);
720750 return own.on || own.ran === true || own.good !== undefined || own.failed !== undefined || own.rolledBack !== undefined;
721751 }
722752
723 /** Whether this host may run `name` in its own process: it holds no cell at all, and the child never ran code of its own. */
753 /**
754 * Whether this host may run `name` in its own process: it holds no cell at all, and the child never
755 * ran code of its own.
756 */
724757 async function hostedHere(parent: App, name: string): Promise<{ ok: true; why: string } | { ok: false; why: string }> {
725758 const cells = await holdsCells();
726759 if (cells.ok) return { ok: false, why: "this host holds cells" };
@@ -747,7 +780,8 @@ async function cellDidNotStart(parent: App, ledger: ChildLedger, why: string): P
747780 const note = `${NOT_STARTED}${(/cannot make a cell/.test(here.why) ? here.why : said).slice(0, 400)}`;
748781 noteLedger(parent, ledger.name, (raw) => (raw["placementNote"] === note ? undefined : { placementNote: note }));
749782 parent.log.warn("child forge's cell did not start; it stays a cell child and the host tries again", { name: ledger.name, why });
750 // One stop per failure, not one per request that found it failed: the wait and the count are the stop's.
783 // One stop per failure, not one per request that found it failed: the wait and the count are the
784 // stop's.
751785 if (lastStop(parent, ledger.bodyId) === undefined) noteStop(parent, ledger.bodyId, ledger.name, said);
752786 }
753787
@@ -758,7 +792,10 @@ function demoted(parent: App, ledger: ChildLedger, why: string): ChildLedger {
758792 return { ...ledger, placement: "hosted" };
759793 }
760794
761 /** Change fields of a child's ledger (the parent's own file); `change` answers nothing when there is nothing to write. */
795 /**
796 * Change fields of a child's ledger (the parent's own file); `change` answers nothing when there is
797 * nothing to write.
798 */
762799 export function noteLedger(parent: App, name: string, change: (raw: Record<string, unknown>) => Record<string, unknown> | undefined): void {
763800 const path = join(parent.config.dataDir, "spawn", `${name}.json`);
764801 try {
@@ -768,6 +805,9 @@ export function noteLedger(parent: App, name: string, change: (raw: Record<strin
768805 writeFileSync(`${path}.tmp`, JSON.stringify({ ...raw, ...next }), { mode: 0o600 });
769806 renameSync(`${path}.tmp`, path);
770807 } catch (error) {
808 // A forge already removed has nothing to record: not a fault. Checked after the fact, not
809 // before, so a removal between the read and the rename lands here too.
810 if ((error as NodeJS.ErrnoException).code === "ENOENT") return;
771811 parent.log.error("could not record the placement", { name, error: String(error) });
772812 }
773813 }
@@ -852,7 +892,10 @@ export interface RefusedHit {
852892 readonly refused: string;
853893 }
854894
855 /** A child app in a child's own cell: its runtime is in that process, so the request is carried there. */
895 /**
896 * A child app in a child's own cell: its runtime is in that process, so the request is carried
897 * there.
898 */
856899 export interface CellAppHit {
857900 readonly cell: string;
858901 }
@@ -931,7 +974,8 @@ export async function answerFromCell(parent: App, request: Request, named?: stri
931974 // held until the cell is up (wakeLimits.waitMs) so a submit is never lost; only a wake
932975 // that takes longer is answered with the "waking up" page, which tries again by itself.
933976 const starting = settleCell(parent, ledger);
934 // Only so many wait on one cell: the rest are told it is waking at once, so a crowd cannot pin connections and bodies.
977 // Only so many wait on one cell: the rest are told it is waking at once, so a crowd cannot pin
978 // connections and bodies.
935979 const waiting = waiters.get(ledger.bodyId) ?? 0;
936980 if (waiting >= wakeLimits.waiters) return wakingResponse(parent, request, ledger.name);
937981 waiters.set(ledger.bodyId, waiting + 1);
@@ -951,7 +995,8 @@ export async function answerFromCell(parent: App, request: Request, named?: stri
951995 release();
952996 }
953997 }
954 // A cell that did not start says so, with when the host tries again: never "waking up" without end.
998 // A cell that did not start says so, with when the host tries again: never "waking up" without
999 // end.
9551000 const stop = cells.failure(ledger.bodyId) === undefined ? undefined : lastStop(parent, ledger.bodyId);
9561001 return wakingResponse(parent, request, ledger.name, stop === undefined ? undefined : stopSentence(stop));
9571002 }
@@ -962,7 +1007,10 @@ export async function answerFromCell(parent: App, request: Request, named?: stri
9621007 */
9631008 const waitsToMove = (name: string): string => `${name} is waiting to be moved into its own sandbox: ${SEALED_WORDS}. Its host moves it; nothing of it is lost.`;
9641009
965 /** How long a request waits for a sleeping cell forge before it is told it is waking, and how many may wait on one cell. */
1010 /**
1011 * How long a request waits for a sleeping cell forge before it is told it is waking, and how many
1012 * may wait on one cell.
1013 */
9661014 export const wakeLimits = { waitMs: 20_000, waiters: 16 };
9671015
9681016 /** Requests held for a waking cell now, by body id. */
test/note-ledger-removed.test.ts added +47
@@ -0,0 +1,47 @@
1 /** Recording a placement for a forge already removed is a no-op, not an ERROR. */
2
3 import { afterAll, expect, test } from "bun:test";
4 import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
5 import { tmpdir } from "node:os";
6 import { join } from "node:path";
7 import { createApp } from "../src/app.ts";
8 import { noteLedger } from "../src/spawn/host.ts";
9
10 const tmp = mkdtempSync(join(tmpdir(), "note-ledger-"));
11 afterAll(() => rmSync(tmp, { recursive: true, force: true }));
12
13 test("a removed forge's placement is not recorded and not an error", () => {
14 const parent = createApp({ dataDir: tmp });
15 const errors: unknown[] = [];
16 const original = parent.log.error.bind(parent.log);
17 parent.log.error = ((...args: unknown[]) => {
18 errors.push(args);
19 return (original as (...a: unknown[]) => unknown)(...args);
20 }) as typeof parent.log.error;
21 let called = false;
22 noteLedger(parent, "probe-gone", () => ((called = true), { placement: "hosted" }));
23 expect(called).toBe(false);
24 expect(errors).toEqual([]);
25 parent.close();
26 });
27
28 test("a record that exists is changed, and a real fault still logs an ERROR", () => {
29 const dir = mkdtempSync(join(tmpdir(), "note-ledger-"));
30 mkdirSync(join(dir, "spawn"));
31 const parent = createApp({ dataDir: dir });
32 const errors: unknown[] = [];
33 const original = parent.log.error.bind(parent.log);
34 parent.log.error = ((...args: unknown[]) => {
35 errors.push(args);
36 return (original as (...a: unknown[]) => unknown)(...args);
37 }) as typeof parent.log.error;
38 writeFileSync(join(dir, "spawn", "kept.json"), JSON.stringify({ a: 1 }));
39 noteLedger(parent, "kept", () => ({ placement: "hosted" }));
40 expect(JSON.parse(readFileSync(join(dir, "spawn", "kept.json"), "utf8"))).toEqual({ a: 1, placement: "hosted" });
41 expect(errors).toEqual([]);
42 writeFileSync(join(dir, "spawn", "broken.json"), "{not json");
43 noteLedger(parent, "broken", () => ({ placement: "hosted" }));
44 expect(errors.length).toBe(1);
45 parent.close();
46 rmSync(dir, { recursive: true, force: true });
47 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs