trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRunsg rEventsFlows

When the parent's disk crosses its floor, running children stop writing, new cells are refused, and the owner hears once #10988

Merged torquecrew opened this

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

torque/forge:torque/10873-when-the-parent-s-disk-crosses-its-floor → main 4 commits fork

  • Merged. torque/10873-when-the-parent-s-disk-crosses-its-floor was merged into main as ce509aa.

Review

Sign in to review this change.
7 files changed +242 −8
src/apps/process.ts +15
@@ -24,6 +24,7 @@ import { spellings, stripForgeDirs } from "../runs/paths.ts";
2424 import { seconds } from "./timing.ts";
2525 import { toolchainUse, type ToolchainUse } from "./toolchain.ts";
2626 import { notFoundHint, shellCommand } from "./run_command.ts";
27 import { floorRefusal } from "../spawn/disk_floor.ts";
2728 import { cellInstall, meteredInstall, prefillInstallCache, withLock } from "../checks/install.ts";
2829 import {
2930 commandPid,
@@ -333,6 +334,14 @@ export async function runStep(
333334 build?: Limits | undefined;
334335 },
335336 ): Promise<Result<void, string>> {
337 // Below the parent's disk floor no step's cell starts: it would write into the last room (#10873).
338 if (options.cell !== undefined) {
339 const floor = floorRefusal(options.cwd);
340 if (floor !== undefined) {
341 options.log(`${name} not started: ${floor}`);
342 return err(floor);
343 }
344 }
336345 options.log(`$ ${command}`);
337346 // A step is background work: the forge's own pages outrank it (#3319).
338347 const cell: Cell | undefined =
@@ -550,6 +559,12 @@ export async function startProcess(
550559 input: StartInput,
551560 ): Promise<Result<StartedProcess, StartError>> {
552561 const { manifest, log } = input;
562 // Below the parent's disk floor no app starts, contained or not (#10873).
563 const floor = floorRefusal(input.dataDir);
564 if (floor !== undefined) {
565 log(`not started: ${floor}`);
566 return err({ stage: "start", message: floor });
567 }
553568 const scratch = input.cell?.home ?? input.dir;
554569 const used = toolchainUse(manifest.toolchain, scratch);
555570 if (!used.ok) return err({ stage: "install", message: used.error });
src/crew/incidents.ts +7 −1
@@ -48,7 +48,7 @@ export const HOLD_MS = 5 * 60_000;
4848
4949 /** The vitals causes this pager owns; the rest are paged elsewhere or are settings. */
5050 export function owned(cause: string): boolean {
51 return cause === "errors" || cause === "internal" || cause === "disk" || cause === "release" || cause.startsWith("queue:");
51 return cause === "errors" || cause === "internal" || cause === "disk" || cause === "disk-floor" || cause === "release" || cause.startsWith("queue:");
5252 }
5353
5454 /** One owned cause being watched. */
@@ -78,6 +78,12 @@ export function observe(
7878 for (const p of present) {
7979 if (!owned(p.cause)) continue;
8080 here.add(p.cause);
81 // A disk nearly full is still under the floor (`kernel/vitals.ts` names only the
82 // worse): a floor already watched holds, unpaged anew and not recovered (#10873).
83 if (p.cause === "disk" && watch.has("disk-floor")) {
84 here.add("disk-floor");
85 delete watch.get("disk-floor")!.absentSince;
86 }
8187 const w = watch.get(p.cause);
8288 if (w === undefined) watch.set(p.cause, { since: now, sentence: p.sentence, paged: seed });
8389 else delete w.absentSince;
src/kernel/vitals.ts +1 −1
@@ -343,7 +343,7 @@ export function vitals(input: VitalsInput): Vitals {
343343 else if (disk.totalBytes > 0 && disk.freeBytes / disk.totalBytes < DISK_FLOOR)
344344 need(
345345 "disk-floor",
346 `Disk is under the ${DISK_FLOOR * 100}% the forge keeps for itself: ${Math.round((disk.freeBytes / disk.totalBytes) * 100)}% free. No child forge is made or started until room is made.`,
346 `Disk is under the ${DISK_FLOOR * 100}% the forge keeps for itself: ${Math.round((disk.freeBytes / disk.totalBytes) * 100)}% free. Child forges sleep, and no child forge, app or check starts, until room is made.`,
347347 undefined,
348348 "/settings/capacity",
349349 );
src/spawn/disk_floor.ts +6 −2
@@ -4,8 +4,12 @@
44 * data; below the floor no new child forge is made (`spawn/quota.ts`
55 * `quotaRefusal`, which the New forge page and menu also show) and no child
66 * cell starts (`spawn/cell.ts`; a sleeping one stays asleep), so what room
7 * is left is the parent's to recover with. The owner's line is the vitals
8 * need `disk-floor` (`kernel/vitals.ts`).
7 * is left is the parent's to recover with. Below it, too (#10873), a child
8 * already awake is put to sleep and kept asleep with this module's one line
9 * (`spawn/host.ts` `sleepBelowFloor`, `wakeGate`), and no app or check cell
10 * starts (`apps/process.ts` `startProcess`, `runStep`). The owner's line is
11 * the vitals need `disk-floor` (`kernel/vitals.ts`), which the pager owns
12 * (`crew/incidents.ts`): paged once per crossing, Recovered once.
913 */
1014
1115 import { readDisk, type DiskReading } from "../kernel/disk_reading.ts";
src/spawn/host.ts +55
@@ -27,6 +27,7 @@ import { cellChildren, forward, forwardSocket, type CellChild } from "./cell.ts"
2727 import { cellsAvailable } from "../apps/contain.ts";
2828 import { childNeedsOwnFilesystem } from "../core/capacity.ts";
2929 import { liveChildDisk, startRefusal } from "./disk_mount.ts";
30 import { floorRefusal } from "./disk_floor.ts";
3031 import { limitValue } from "../settings/limits.ts";
3132 import { awakeRefusal, idleAfterMs, WakeRate } from "./sleep.ts";
3233 import { wakingResponse } from "../web/pages/waking.ts";
@@ -201,6 +202,42 @@ export async function sleepIdleChildren(parent: App, idleMs = idleAfterMs(), now
201202 return asleep;
202203 }
203204
205 /**
206 * Below the disk floor (`spawn/disk_floor.ts`) no child forge stays awake:
207 * every hosted child closes and every cell stops, busy or not, so nothing
208 * they write takes the parent's last room. Their data stays on disk; the
209 * wake gate keeps them asleep, answering the floor's one line, until room
210 * is made (#10873). Answers the names put to sleep; none when above it.
211 */
212 export async function sleepBelowFloor(parent: App): Promise<string[]> {
213 const floor = floorRefusal(parent.config.dataDir);
214 if (floor === undefined) return [];
215 const asleep: string[] = [];
216 const cache = caches.get(parent);
217 for (const [bodyId, child] of [...(cache ?? [])])
218 asleep.push(await closeOne(parent, cache!, bodyId, child, touched.get(parent)?.get(bodyId)?.name ?? bodyId));
219 const cells = cellChildren(parent);
220 for (const child of [...cells.list()]) {
221 touched.get(parent)?.delete(child.bodyId);
222 waking.get(parent)?.delete(child.bodyId);
223 await cells.stop(child.bodyId);
224 asleep.push(child.name);
225 }
226 for (const name of asleep) parent.log.warn("child forge asleep", { name, why: floor });
227 return asleep;
228 }
229
230 /** How often the floor is looked at while children may be awake, ms. */
231 export const FLOOR_EVERY_MS = 30_000;
232 const floorWatchers = new WeakMap<App, ReturnType<typeof setInterval>>();
233
234 function keepWatchingFloor(parent: App): void {
235 if (floorWatchers.has(parent)) return;
236 const timer = setInterval(() => void sleepBelowFloor(parent).catch(() => {}), FLOOR_EVERY_MS);
237 timer.unref?.();
238 floorWatchers.set(parent, timer);
239 }
240
204241 /** Put one named child to sleep now (its owner stopped it); true when it was awake. */
205242 export async function sleepChild(parent: App, name: string): Promise<boolean> {
206243 const ledger = readLedgerSync(parent, name);
@@ -261,6 +298,12 @@ const carriesCredentials = (parent: App, bodyId: string, request: Request): bool
261298 * off a start costs no mountinfo read.
262299 */
263300 export function startRefused(parent: App, ledger: { readonly name: string; readonly bodyId: string }): string | undefined {
301 // Below the disk floor nothing wakes, whatever its filesystem (#10873).
302 const floor = floorRefusal(parent.config.dataDir);
303 if (floor !== undefined) {
304 parent.log.warn("child forge not started", { name: ledger.name, why: floor });
305 return floor;
306 }
264307 if (!childNeedsOwnFilesystem(parent.db)) return undefined;
265308 const why = startRefusal(ledger.name, liveChildDisk(parent.config.dataDir, ledger.bodyId), true);
266309 if (why !== undefined) parent.log.warn("child forge not started", { name: ledger.name, why });
@@ -288,6 +331,14 @@ export function wakeGate(parent: App, request: Request): Response | undefined {
288331 const ledger = readLedgerSync(parent, name);
289332 if (ledger === undefined || !ledger.done) return undefined;
290333 keepSweeping(parent);
334 keepWatchingFloor(parent);
335 // Below the disk floor an awake child is put to sleep now, not at the next look,
336 // and this request is answered with why: it writes nothing (#10873).
337 const floor = floorRefusal(parent.config.dataDir);
338 if (floor !== undefined) {
339 if (isAwakeBody(parent, ledger.bodyId)) void sleepChild(parent, name).catch(() => {});
340 return refusedResponse(floor);
341 }
291342 const known = carriesCredentials(parent, ledger.bodyId, request);
292343 if (isAwakeBody(parent, ledger.bodyId)) {
293344 if (known) {
@@ -327,6 +378,9 @@ export function closeHostedChildren(parent: App): void {
327378 const sweeper = sweepers.get(parent);
328379 if (sweeper !== undefined) clearInterval(sweeper);
329380 sweepers.delete(parent);
381 const watcher = floorWatchers.get(parent);
382 if (watcher !== undefined) clearInterval(watcher);
383 floorWatchers.delete(parent);
330384 touched.delete(parent);
331385 bootOpened.delete(parent);
332386 waking.delete(parent);
@@ -484,6 +538,7 @@ export async function bootHostedChild(parent: App, name: string): Promise<void>
484538
485539 /** Open every finished child, on boot: their apps come back with the parent's. */
486540 export async function bootHostedChildren(parent: App): Promise<void> {
541 keepWatchingFloor(parent);
487542 let files: string[] = [];
488543 try {
489544 files = readdirSync(join(parent.config.dataDir, "spawn"));
test/disk-floor.test.ts +157 −3
@@ -1,20 +1,34 @@
11 /**
22 * #10850 item 5: the parent protects itself. Below 15% free on its own
33 * volume no child forge is made, no child cell starts, and the owner reads
4 * one line in vitals. Each is tried against a disk this test hands in.
4 * one line in vitals. #10873: a child already awake is put to sleep and kept
5 * asleep with the floor's one line, no app or check cell starts, and the
6 * owner is paged once per crossing. Each is tried against a disk this test
7 * hands in.
58 */
69
710 import { afterAll, afterEach, describe, expect, test } from "bun:test";
811 import { mkdtempSync, rmSync } from "node:fs";
912 import { tmpdir } from "node:os";
1013 import { join } from "node:path";
11 import { createApp } from "../src/app.ts";
14 import { createApp, type App } from "../src/app.ts";
15 import type { CellSpec } from "../src/apps/contain.ts";
16 import { parseManifest } from "../src/apps/manifest.ts";
17 import { childEnv, runStep, startProcess, type StartInput } from "../src/apps/process.ts";
18 import { spawnForge } from "../src/actions/spawn.ts";
19 import { createUser } from "../src/core/users.ts";
20 import { observe, type Watch } from "../src/crew/incidents.ts";
21 import { bootHostedChild, childAwake, closeHostedChildren, sleepBelowFloor, wakeGate } from "../src/spawn/host.ts";
22 import { hostedPlacement } from "../src/spawn/placement.ts";
23 import { quietLogs } from "./helpers/globals.ts";
1224 import { provideDiskForTest, type DiskReading } from "../src/kernel/disk_reading.ts";
1325 import { vitals } from "../src/kernel/vitals.ts";
1426 import { cellChildren, stopCellChildren } from "../src/spawn/cell.ts";
1527 import { DISK_FLOOR, floorRefusal } from "../src/spawn/disk_floor.ts";
1628 import { quotaFull, quotaRefusal } from "../src/spawn/quota.ts";
1729
30 quietLogs();
31
1832 const tmp = mkdtempSync(join(tmpdir(), "disk-floor-"));
1933 const GiB = 1024 ** 3;
2034 const reading = (freeGiB: number, totalGiB = 100): DiskReading => ({ freeBytes: freeGiB * GiB, totalBytes: totalGiB * GiB });
@@ -78,7 +92,7 @@ describe("below the floor", () => {
7892 try {
7993 const v = vitals({ db: app.db, signals: app.signals, dataDir: app.config.dataDir, disk: () => ({ freeBytes: 12, totalBytes: 100 }) });
8094 expect(v.degraded.map((d) => d.cause)).toEqual(["disk-floor"]);
81 expect(v.needs).toEqual(["Disk is under the 15% the forge keeps for itself: 12% free. No child forge is made or started until room is made."]);
95 expect(v.needs).toEqual(["Disk is under the 15% the forge keeps for itself: 12% free. Child forges sleep, and no child forge, app or check starts, until room is made."]);
8296 const at15 = vitals({ db: app.db, signals: app.signals, dataDir: app.config.dataDir, disk: () => ({ freeBytes: 15, totalBytes: 100 }) });
8397 expect(at15.degraded.map((d) => d.cause)).toEqual([]);
8498 } finally {
@@ -86,3 +100,143 @@ describe("below the floor", () => {
86100 }
87101 });
88102 });
103
104 describe("children already awake (#10873)", () => {
105 const made: App[] = [];
106 afterAll(() => {
107 for (const app of made) {
108 closeHostedChildren(app);
109 app.close();
110 }
111 });
112
113 const spawned = async (name: string): Promise<App> => {
114 const parent = createApp({ dataDir: join(tmp, name), url: "https://forge.test", port: 0 });
115 made.push(parent);
116 const admin = await createUser(parent.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", isAdmin: true });
117 if (!admin.ok) throw new Error("user");
118 const out = await spawnForge(
119 parent,
120 admin.value,
121 { name: "kid", org: "kid", apps: [], carryBriefs: false, carrySubscriptions: false },
122 { catalog: [], briefs: [], subscriptions: [] },
123 {
124 placement: hostedPlacement({ host: "forge.test" }),
125 request: new Request("https://forge.test/spawn", { method: "POST", headers: { origin: "https://forge.test" } }),
126 url: new URL("https://forge.test/spawn"),
127 },
128 );
129 if (!out.ok) throw new Error(JSON.stringify(out.error));
130 await bootHostedChild(parent, "kid");
131 return parent;
132 };
133 const visit = (parent: App, method = "GET") =>
134 wakeGate(parent, new Request("https://kid--forge.test/", { method, headers: { host: "kid--forge.test" } }));
135
136 test("above the floor the sweep leaves an awake child alone", async () => {
137 const parent = await spawned("awake-roomy");
138 expect(childAwake(parent, "kid")).toBe(true);
139 expect(await sleepBelowFloor(parent)).toEqual([]);
140 expect(childAwake(parent, "kid")).toBe(true);
141 });
142
143 test("below it the sweep puts the child to sleep, and its forge answers one plain line until room is made", async () => {
144 const parent = await spawned("awake-floor");
145 expect(childAwake(parent, "kid")).toBe(true);
146 provideDiskForTest(() => reading(12));
147 expect(await sleepBelowFloor(parent)).toEqual(["kid"]);
148 expect(childAwake(parent, "kid")).toBe(false);
149 for (const method of ["GET", "POST"]) {
150 const answer = visit(parent, method);
151 expect(answer?.status).toBe(503);
152 expect(answer?.headers.get("content-type")).toContain("text/plain");
153 const said = await answer!.text();
154 expect(said.trim().split("\n")).toHaveLength(1);
155 expect(said).toContain("12.0% free");
156 }
157 expect(childAwake(parent, "kid")).toBe(false);
158 provideDiskForTest(roomy);
159 expect(visit(parent)).toBeUndefined();
160 });
161
162 test("a request reaching a child awake below the floor puts it to sleep at once, not at the next look", async () => {
163 const parent = await spawned("awake-request");
164 provideDiskForTest(() => reading(10));
165 expect(visit(parent)?.status).toBe(503);
166 await Bun.sleep(20);
167 expect(childAwake(parent, "kid")).toBe(false);
168 });
169 });
170
171 describe("app and check cells inside the parent (#10873)", () => {
172 const manifest = (() => {
173 const m = parseManifest(JSON.stringify({ app: { run: "bun server.ts" } }), undefined);
174 if (!m.ok) throw new Error("manifest");
175 return m.value;
176 })();
177
178 test("an app does not start below the floor, and says why", async () => {
179 provideDiskForTest(() => reading(14));
180 const lines: string[] = [];
181 // The floor is read before anything else: the rest of the input is never reached.
182 const input = { dir: tmp, dataDir: tmp, manifest, log: (l: string) => lines.push(l), signal: new AbortController().signal } as unknown as StartInput;
183 const started = await startProcess(input);
184 expect(started.ok).toBe(false);
185 if (started.ok) return;
186 expect(started.error.stage).toBe("start");
187 expect(started.error.message).toBe(floorRefusal(tmp)!);
188 expect(lines.join("\n")).toContain("14.0% free");
189 });
190
191 test("a check's cell does not start below the floor, with the same line; a plain step above it runs", async () => {
192 const step = (cell?: CellSpec) =>
193 runStep("check", "true", { cwd: tmp, env: childEnv({}), log: () => {}, signal: new AbortController().signal, cell });
194 provideDiskForTest(() => reading(14));
195 // Refused before the cell is opened, so its spec is never read.
196 const refused = await step({} as CellSpec);
197 expect(refused.ok).toBe(false);
198 if (!refused.ok) expect(refused.error).toBe(floorRefusal(tmp)!);
199 provideDiskForTest(roomy);
200 expect((await step()).ok).toBe(true);
201 });
202 });
203
204 describe("the owner hears once per crossing (#10873)", () => {
205 const MIN = 60_000;
206 const floor = { cause: "disk-floor", sentence: "Disk is under the 15% the forge keeps for itself: 12% free." };
207 const full = { cause: "disk", sentence: "Disk is nearly full: 5% free." };
208
209 test("one page held five minutes, none while it stays, none when it gets worse and better, Recovered once, one page on a new crossing", () => {
210 const watch: Watch = new Map();
211 let t = 0;
212 const look = (present: { cause: string; sentence: string }[]) => observe(watch, present, t);
213 const pages: string[] = [];
214 const recovered: string[] = [];
215 const step = (minutes: number, present: { cause: string; sentence: string }[]) => {
216 t += minutes * MIN;
217 const due = look(present);
218 for (const k of due.pages) watch.get(k)!.paged = true;
219 for (const k of due.recovered) watch.delete(k);
220 pages.push(...due.pages);
221 recovered.push(...due.recovered);
222 };
223 step(0, [floor]);
224 step(6, [floor]);
225 expect(pages).toEqual(["disk-floor"]);
226 for (let i = 0; i < 12; i++) step(5, [floor]);
227 expect(pages).toEqual(["disk-floor"]);
228 // Worse: vitals names only "disk"; the floor holds, unpaged anew and not recovered.
229 step(1, [full]);
230 step(10, [full]);
231 step(10, [floor]);
232 step(6, [floor]);
233 expect(pages).toEqual(["disk-floor", "disk"]);
234 expect(recovered).toEqual(["disk"]);
235 step(1, []);
236 step(6, []);
237 expect(recovered).toEqual(["disk", "disk-floor"]);
238 step(60, [floor]);
239 step(6, [floor]);
240 expect(pages).toEqual(["disk-floor", "disk", "disk-floor"]);
241 });
242 });
test/notify-causes.test.ts +1 −1
@@ -20,7 +20,7 @@ const MIN = 60_000;
2020
2121 describe("owned causes", () => {
2222 test("an explicit list", () => {
23 for (const c of ["errors", "internal", "disk", "release", "queue:check", "queue:run"])
23 for (const c of ["errors", "internal", "disk", "disk-floor", "release", "queue:check", "queue:run"])
2424 expect(owned(c)).toBe(true);
2525 for (const c of ["drill", "mirror", "stuck:r1", "red:r1", "snapshots", "budget:r1", "shared:r:t"])
2626 expect(owned(c)).toBe(false);

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs