Public: anyone can read this; the crew can change it.

A Small forge (1 CPU, 2 GiB) builds a first app: today the host stops it for… #11976

Merged opened this in answer to

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/11975-a-small-forge-1-cpu-2-gib-builds-a-first → main 4 commits fork 4 threads · 4 unresolved

  • Merged. scout/11975-a-small-forge-1-cpu-2-gib-builds-a-first was merged into main as f8930ce.

Review

Sign in to review this change.
8 files changed +248 −4
docs/features/child-stopped.md added +3
@@ -0,0 +1,3 @@
1 # A child forge the host stopped
2
3 `src/spawn/stopped.ts` records a stop (one state object per parent; the host passes in how a forge starts again) and arms a restart after a backoff (5 s, doubling per stop in the hour, to 5 min). The restart goes through the due-wake path, so the awake ceiling and the disk floor still apply. A stop is forgotten when the forge wakes, when the restart is skipped (lapsed, refused), and after an hour. While it stands, the wake page and the terminal's forges list say "ran out of memory and was stopped at HH:MM UTC; the host starts it again by itself" instead of "the forge is full".
src/spawn/cell.ts +5 −1
@@ -22,6 +22,7 @@ import { openBeatDoor, parentLocal, type BeatDoor } from "./beat-door.ts";
2222 import { hostedPlacement, parentOrigin, childPublicUrl, type BodyStore, type Placement } from "./placement.ts";
2323 import { SIZE_IO, SIZES, type Size } from "./sizes.ts";
2424 import { floorRefusal } from "./disk_floor.ts";
25 import { noteStop } from "./stopped.ts";
2526 import { CGROUP_DIR_ENV, SIZE_ENV } from "./usage.ts";
2627 import { childModelHosts, childPaymentHosts, childRegistryHosts } from "../core/capacity.ts";
2728 import { liveChildDisk, type ChildDisk } from "./disk_mount.ts";
@@ -403,7 +404,10 @@ export function makeRegistry(parent: App, launcher?: Launcher): CellChildren {
403404 });
404405 const exited = proc.exited.then((code) => {
405406 const why = cell.breach();
406 if (why !== undefined) parent.log.warn("child forge stopped", { name: input.name, why });
407 if (why !== undefined) {
408 parent.log.warn("child forge stopped", { name: input.name, why });
409 noteStop(parent, input.bodyId, input.name, why);
410 }
407411 cell.close();
408412 door.close();
409413 release();
src/spawn/clients.ts +11
@@ -23,6 +23,7 @@ import { reachOf, reachWords, type ChildReach } from "./reach.ts";
2323 import type { Size } from "./sizes.ts";
2424 import { firstScreenLine } from "../core/next-steps.ts";
2525 import { lapseFor } from "../billing/lapse.ts";
26 import { lastStop, stopSentence } from "./stopped.ts";
2627 import { forgeUse, type ForgeUse } from "./usage.ts";
2728
2829 /** What a spawned forge says about itself to anyone: nothing more. */
@@ -46,6 +47,8 @@ export interface ClientRow {
4647 readonly lastActivityAt: string | null;
4748 /** Whether its process is running now; a sleeping forge costs nothing and wakes on its next request. */
4849 readonly awake?: boolean;
50 /** Why the host stopped it, while that explains its state (spawn/stopped.ts); a sentence. */
51 readonly stopped?: string | null;
4952 /** The apps this parent runs for the child; null when it runs none here. */
5053 readonly running: ChildAppsSummary | null;
5154 /** What a visitor got from its public addresses when last asked (`spawn/reach.ts`); null before it was. */
@@ -243,6 +246,7 @@ export async function clientRows(app: App): Promise<ClientRow[]> {
243246 claimed: states[i]?.claimed ?? null,
244247 lastActivityAt: states[i]?.lastActivityAt ?? null,
245248 awake: childAwake(app, c.name),
249 stopped: stoppedWords(app, c),
246250 running: hostedChildApps(app, c.name) ?? null,
247251 reach: reachOf(app, c.name) ?? null,
248252 spendTodayUsd: beats[i]?.beat.spendTodayUsd ?? null,
@@ -413,3 +417,10 @@ function graceView(app: App, forge: string, now: number): { graceUntil?: number
413417 return {};
414418 }
415419 }
420
421 /** The stop sentence for a forge that is asleep because the host stopped it. */
422 function stoppedWords(app: App, c: { bodyId: string; name: string }): string | null {
423 if (childAwake(app, c.name)) return null;
424 const stop = lastStop(app, c.bodyId);
425 return stop === undefined ? null : stopSentence(stop);
426 }
src/spawn/host.ts +19 −1
@@ -29,6 +29,7 @@ import { childNeedsOwnFilesystem } from "../core/capacity.ts";
2929 import { liveChildDisk, startRefusal } from "./disk_mount.ts";
3030 import { floorRefusal } from "./disk_floor.ts";
3131 import { limitValue } from "../settings/limits.ts";
32 import { cancelRestarts, clearStop, lastStop, stopSentence, useRestart } from "./stopped.ts";
3233 import { awakeRefusal, idleAfterMs, WakeRate } from "./sleep.ts";
3334 import { wakingResponse } from "../web/pages/waking.ts";
3435 import { lapsedResponse, payAgainRoute, PAY_AGAIN_PATH } from "../web/pages/lapsed.ts";
@@ -265,6 +266,16 @@ async function wakeDue(parent: App, name: string): Promise<void> {
265266 await settleCell(parent, ledger);
266267 }
267268
269 /**
270 * A forge the host stopped for a limit starts again through the due-wake path, which keeps
271 * the awake ceiling and the disk floor. False when nothing will start it (refused, lapsed).
272 */
273 async function restartStopped(parent: App, name: string): Promise<boolean> {
274 await wakeDue(parent, name);
275 const ledger = readLedgerSync(parent, name);
276 return ledger !== undefined && (isAwakeBody(parent, ledger.bodyId) || dueWakes.get(parent)?.has(ledger.bodyId) === true);
277 }
278
268279 /**
269280 * Below the disk floor (`spawn/disk_floor.ts`) no child forge stays awake:
270281 * every hosted child closes and every cell stops, busy or not, so nothing
@@ -432,6 +443,7 @@ export function wakeGate(parent: App, request: Request): Response | undefined {
432443 }
433444 const known = carriesCredentials(parent, ledger.bodyId, request);
434445 if (isAwakeBody(parent, ledger.bodyId)) {
446 clearStop(parent, ledger.bodyId);
435447 if (known) {
436448 bootOpened.get(parent)?.delete(ledger.bodyId);
437449 touch(parent, ledger);
@@ -456,9 +468,13 @@ export function wakeGate(parent: App, request: Request): Response | undefined {
456468 const wait = ledger.ownerId === undefined ? 0 : people.waitS(ledger.ownerId, limitValue(parent.db, "forges.wakes_per_person_hour"), at);
457469 if (wait > 0) return wakingResponse(parent, request, name, `waking too often, try in ${wait} s`, wait);
458470 const why = awakeRefusal(awakeCount(parent)) ?? (known ? undefined : rates.refusal(ledger.bodyId, at));
459 if (why !== undefined) return wakingResponse(parent, request, name, why);
471 if (why !== undefined) {
472 const stop = lastStop(parent, ledger.bodyId);
473 return wakingResponse(parent, request, name, stop === undefined ? why : stopSentence(stop));
474 }
460475 if (!known) rates.note(ledger.bodyId, at);
461476 if (ledger.ownerId !== undefined) people.note(ledger.ownerId, at);
477 clearStop(parent, ledger.bodyId); // the wake is granted: the stop no longer explains anything
462478 reserved.set(ledger.bodyId, Date.now());
463479 touch(parent, ledger);
464480 return undefined;
@@ -482,6 +498,7 @@ export function closeHostedChildren(parent: App): void {
482498 const watcher = floorWatchers.get(parent);
483499 if (watcher !== undefined) clearInterval(watcher);
484500 floorWatchers.delete(parent);
501 cancelRestarts(parent);
485502 touched.delete(parent);
486503 bootOpened.delete(parent);
487504 waking.delete(parent);
@@ -567,6 +584,7 @@ const settling = new WeakMap<App, Map<string, Promise<CellChild | undefined>>>()
567584 * `/forges` lists), and the next request opens it in this process.
568585 */
569586 function settleCell(parent: App, ledger: ChildLedger): Promise<CellChild | undefined> {
587 useRestart(parent, restartStopped);
570588 let flights = settling.get(parent);
571589 if (!flights) settling.set(parent, (flights = new Map()));
572590 let flight = flights.get(ledger.bodyId);
src/spawn/stopped.ts added +101
@@ -0,0 +1,101 @@
1 /**
2 * A child forge its host stopped for going over a limit (#11975): what
3 * happened, kept in memory per parent, so the forge starts again after a
4 * backoff and its pages say so instead of "the forge is full".
5 */
6 import type { App } from "../app.ts";
7
8 export type Stop = { name: string; at: number; why: string; stops: number };
9
10 /**
11 * Starts a stopped forge again; true when it is awake or a retry is armed, false when the
12 * start was skipped.
13 */
14 export type Restart = (parent: App, name: string) => Promise<boolean>;
15
16 /** First wait before the restart, ms; it doubles per stop in the hour, to the ceiling. */
17 export const RESTART_BASE_MS = 5_000;
outdated

Module-global restarter set by an import side effect in host.ts; pass it in or hold it in the per-parent state.

18 export const RESTART_MAX_MS = 5 * 60_000;
19 export const HOUR_MS = 3_600_000;
20
21 /** Everything kept for one parent. */
22 type State = {
23 restart: Restart | undefined;
24 stops: Map<string, Stop>;
25 times: Map<string, number[]>;
26 timers: Map<string, ReturnType<typeof setTimeout>>;
27 };
28 const states = new WeakMap<App, State>();
29
30 function stateOf(parent: App): State {
31 let s = states.get(parent);
32 if (!s) states.set(parent, (s = { restart: undefined, stops: new Map(), times: new Map(), timers: new Map() }));
33 return s;
34 }
35
36 /** The host says how a stopped forge is started again (this module imports no host). */
37 export function useRestart(parent: App, restart: Restart): void {
38 stateOf(parent).restart = restart;
39 }
40
41 /** The wait before the restart after the `n`th stop within the hour. */
42 export const backoffMs = (n: number): number => Math.min(RESTART_MAX_MS, RESTART_BASE_MS * 2 ** Math.max(0, n - 1));
43
44 /** Record a stop and arm the restart. */
45 export function noteStop(parent: App, bodyId: string, name: string, why: string, now = Date.now(), waitMs?: number): Stop {
46 const s = stateOf(parent);
47 const recent = (s.times.get(bodyId) ?? []).filter((x) => now - x < HOUR_MS);
48 recent.push(now);
49 s.times.set(bodyId, recent);
50 const stop: Stop = { name, at: now, why, stops: recent.length };
51 s.stops.set(bodyId, stop);
52 const was = s.timers.get(bodyId);
53 if (was !== undefined) clearTimeout(was);
54 const timer = setTimeout(() => {
55 s.timers.delete(bodyId);
outdated

lastStop never expires; a stop from hours ago still explains a later full-host refusal. Expire it.

56 const restart = s.restart;
57 if (restart === undefined) return clearStop(parent, bodyId);
58 restart(parent, name).then(
59 (coming) => coming || clearStop(parent, bodyId),
60 () => clearStop(parent, bodyId),
61 );
62 }, waitMs ?? backoffMs(stop.stops));
63 timer.unref?.();
64 s.timers.set(bodyId, timer);
65 return stop;
66 }

Wording: say 'was stopped at', not 'was restarted at'; the restart has not happened yet when this is shown.

67
68 /** The stop to explain a refusal with; one older than an hour is forgotten, not shown. */
69 export function lastStop(parent: App, bodyId: string, now = Date.now()): Stop | undefined {
70 const s = states.get(parent);
71 const stop = s?.stops.get(bodyId);
72 if (stop === undefined) return undefined;
73 if (now - stop.at >= HOUR_MS) {
74 s!.stops.delete(bodyId);
75 return undefined;
76 }
77 return stop;
78 }
79
80 /**
81 * The forge is running again, or will not be started: forget the stop (the count in the
82 * hour stays).
83 */
84 export function clearStop(parent: App, bodyId: string): void {
85 states.get(parent)?.stops.delete(bodyId);
86 }
87
88 const hhmm = (at: number): string => new Date(at).toISOString().slice(11, 16);
89
90 /** One sentence for a stopped forge's pages; the sentence for a full host is a different one. */
91 export function stopSentence(stop: Stop): string {
92 const what = /memory/.test(stop.why) ? "ran out of memory" : "went over a limit";
93 const again = stop.stops > 1 ? ` It has been stopped ${stop.stops} times in the last hour; a larger plan would fit it.` : "";
94 return `This forge ${what} and was stopped at ${hhmm(stop.at)} UTC; the host starts it again by itself.${again}`;
95 }
96
97 /** Drop everything kept for a parent and cancel its pending restarts. */
98 export function cancelRestarts(parent: App): void {
99 for (const t of states.get(parent)?.timers.values() ?? []) clearTimeout(t);
100 states.delete(parent);
101 }
src/tui/screens/forges.ts +1 −1
@@ -26,7 +26,7 @@ export function forgeLines(rows: readonly ClientRow[]): [string, string][] {
2626 if (rows.length === 0) return [["", "You have not sent a forge yet"]];
2727 return rows.flatMap((r): [string, string][] => [[
2828 r.name,
29 [r.made ? (r.awake === true ? "awake" : "asleep") : null, r.attention === null ? null : ATTENTION_WORDS[r.attention], r.for === null ? null : `for ${r.for}`, CLAIM_WORDS[claimState(r)], linkWords(r.link), runningWords(r), r.stage?.words ?? null, r.nextStep.words, spendWords(r)?.words ?? null, creditWords(r), graceWords(r), useLine(r.size, r.use), r.made ? diskWords(r.disk) : null, r.lastHeartbeatAt === null ? "no heartbeat yet" : `heartbeat ${timeAgo(r.lastHeartbeatAt)}`]
29 [r.made ? (r.awake === true ? "awake" : "asleep") : null, r.stopped ?? null, r.attention === null ? null : ATTENTION_WORDS[r.attention], r.for === null ? null : `for ${r.for}`, CLAIM_WORDS[claimState(r)], linkWords(r.link), runningWords(r), r.stage?.words ?? null, r.nextStep.words, spendWords(r)?.words ?? null, creditWords(r), graceWords(r), useLine(r.size, r.use), r.made ? diskWords(r.disk) : null, r.lastHeartbeatAt === null ? "no heartbeat yet" : `heartbeat ${timeAgo(r.lastHeartbeatAt)}`]
3030 .filter((m): m is string => m !== null)
3131 .join(" · "),
3232 ], ...firstScreenLines(r), ...addressLines(r)]);
test/boot-idle-children.test.ts +43 −1
@@ -4,6 +4,7 @@
44 * because of forges nobody is using.
55 */
66 import { afterAll, beforeAll, expect, test } from "bun:test";
7 import { readFileSync } from "node:fs";
78 import { mkdtemp, rm } from "node:fs/promises";
89 import { tmpdir } from "node:os";
910 import { join } from "node:path";
@@ -11,7 +12,10 @@ import { spawnForge } from "../src/actions/spawn.ts";
1112 import { createApp, type App } from "../src/app.ts";
1213 import { createSession } from "../src/core/sessions.ts";
1314 import { createUser } from "../src/core/users.ts";
14 import { awakeCount, bootHostedChildren, childAwake, resolveHostedChild, sleepChild, sleepIdleChildren, wakeGate } from "../src/spawn/host.ts";
15 import { clientRows } from "../src/spawn/clients.ts";
16 import { lastStop, noteStop } from "../src/spawn/stopped.ts";
17 import { forgeLines } from "../src/tui/screens/forges.ts";
18 import { awakeCount, closeHostedChildren, bootHostedChildren, childAwake, resolveHostedChild, sleepChild, sleepIdleChildren, wakeGate } from "../src/spawn/host.ts";
1519 import { hostedPlacement } from "../src/spawn/placement.ts";
1620 import { quietLogs } from "./helpers/globals.ts";
1721
@@ -103,3 +107,41 @@ test("an unasked boot-opened child put to sleep leaves the exempt set, so its ne
103107 await sleepChild(parent, "idle-b");
104108 expect(awakeCount(parent)).toBe(0);
105109 });
110
111 const bodyOf = (name: string): string => JSON.parse(readFileSync(join(dir, "spawn", `${name}.json`), "utf8")).bodyId;
112 const say = async (name: string): Promise<string> => (await gate(name, { accept: "text/plain" })!.text()) ?? "";
113
114 test("a refused wake of a forge the host stopped says why it stopped, not that the host is full; a woken forge forgets it", async () => {
115 await reset();
116 process.env["FORGE_CHILD_AWAKE_MAX"] = "1";
117 await sleepChild(parent, "sleeper");
118 await sleepChild(parent, "idle-b");
119 await sleepChild(parent, "idle-a");
120 expect(gate("sleeper")).toBeUndefined(); // the ceiling of 1 is now full
121 const full = await say("idle-b");
122 expect(full).toContain("the forge is full");
123 noteStop(parent, bodyOf("idle-b"), "idle-b", "used 2374 MiB of memory", Date.now(), 3_600_000);
124 const stopped = await say("idle-b");
125 expect(stopped).toContain("ran out of memory and was stopped at");
126 expect(stopped).not.toContain("the forge is full");
127 // Room appears and the forge wakes: the stop is forgotten, and a later full host says so again.
128 process.env["FORGE_CHILD_AWAKE_MAX"] = "3";
129 expect(gate("idle-b")).toBeUndefined();
130 expect(lastStop(parent, bodyOf("idle-b"))).toBeUndefined();
131 await resolveHostedChild(parent, new Request("https://forge.test/healthz", { headers: { host: hostOf("idle-b") } }));
132 await sleepChild(parent, "idle-b");
133 process.env["FORGE_CHILD_AWAKE_MAX"] = "1";
134 expect(await say("idle-b")).toContain("the forge is full");
135 // The forges list for the terminal carries the sentence while it is asleep and stopped.
136 noteStop(parent, bodyOf("idle-b"), "idle-b", "used 2374 MiB of memory", Date.now(), 3_600_000);
137 const rows = await clientRows(parent);
138 expect(rows.find((r) => r.name === "idle-b")?.stopped).toContain("ran out of memory");
139 expect(forgeLines(rows).find(([n]) => n === "idle-b")![1]).toContain("ran out of memory");
140 expect(rows.find((r) => r.name === "sleeper")?.stopped).toBeNull();
141 });
142
143 test("closing the hosted children forgets stops and cancels the restarts", () => {
144 noteStop(parent, "zz", "zz", "memory", Date.now(), 3_600_000);
145 closeHostedChildren(parent);
146 expect(lastStop(parent, "zz")).toBeUndefined();
147 });
test/spawn-stopped.test.ts added +65
@@ -0,0 +1,65 @@
1 import { expect, test } from "bun:test";
2 import type { App } from "../src/app.ts";
3 import { backoffMs, cancelRestarts, clearStop, HOUR_MS, lastStop, noteStop, RESTART_MAX_MS, stopSentence, useRestart } from "../src/spawn/stopped.ts";
4
5 test("the wait doubles per stop and stops at the ceiling", () => {
6 expect(backoffMs(1)).toBe(5_000);
7 expect(backoffMs(2)).toBe(10_000);
8 expect(backoffMs(30)).toBe(RESTART_MAX_MS);
9 });
10
11 test("a stop is restarted through the parent's own restarter, and the sentence says stopped, not restarted", async () => {
12 const parent = {} as App;
13 const other = {} as App;
14 const restarted: string[] = [];
15 let fired!: () => void;
16 const done = new Promise<void>((r) => (fired = r));
17 useRestart(parent, async (_p, name) => {
18 restarted.push(name);
19 fired();
20 return true;
21 });
22 const at = Date.parse("2026-11-13T06:33:09Z");
23 const stop = noteStop(parent, "b1", "probe", "used 2374 MiB of memory, over its 2048 MiB limit", at, 5);
24 expect(lastStop(parent, "b1", at)).toEqual(stop);
25 expect(lastStop(other, "b1", at)).toBeUndefined();
26 expect(stopSentence(stop)).toBe("This forge ran out of memory and was stopped at 06:33 UTC; the host starts it again by itself.");
27 await done;
28 expect(restarted).toEqual(["probe"]);
29 expect(noteStop(parent, "b1", "probe", "memory", at + 1000, 60_000).stops).toBe(2);
30 expect(stopSentence(lastStop(parent, "b1", at + 1000)!)).toContain("stopped 2 times");
31 cancelRestarts(parent);
32 expect(lastStop(parent, "b1", at + 1000)).toBeUndefined();
33 });
34
35 test("a stop older than an hour is not shown, and stops older than an hour leave the count", () => {
36 const parent = {} as App;
37 const at = 1_000_000;
38 noteStop(parent, "b", "k", "memory", at, 60_000);
39 expect(lastStop(parent, "b", at + HOUR_MS - 1)).toBeDefined();
40 expect(lastStop(parent, "b", at + HOUR_MS)).toBeUndefined();
41 expect(noteStop(parent, "b", "k", "memory", at + HOUR_MS + 1, 60_000).stops).toBe(1);
42 cancelRestarts(parent);
43 });
44
45 test("a restart that is skipped drops the stop; one that is coming keeps it", async () => {
46 for (const coming of [false, true]) {
47 const parent = {} as App;
48 let fired!: () => void;
49 const done = new Promise<void>((r) => (fired = r));
50 useRestart(parent, async () => (fired(), coming));
51 noteStop(parent, "b", "k", "memory", Date.now(), 1);
52 await done;
53 await new Promise((r) => setImmediate(r)); // let the restart's answer settle
54 expect(lastStop(parent, "b") !== undefined).toBe(coming);
55 cancelRestarts(parent);
56 }
57 });
58
59 test("clearStop forgets the stop", () => {
60 const parent = {} as App;
61 noteStop(parent, "b", "k", "memory", Date.now(), 60_000);
62 clearStop(parent, "b");
63 expect(lastStop(parent, "b")).toBeUndefined();
64 cancelRestarts(parent);
65 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs