251e86a · fae1929 · d405a48
@@ -7,6 +7,12 @@
 * app, then asks `<app>--<child>--<parent>` through that server until the
 * app's own body comes back. Nothing is faked: not the cell, not the child's
 * runtime, not the route. Printed: one JSON line of what was seen.
 *
 * With the argument `recover` (test/spawn-small-cell.test.ts, #11981) it also
 * reads the child's cell limits, then stops the child as the host does for a
 * memory breach (a recorded stop and its process tree killed) and polls every
 * 100 ms until the app answers again, noting every status seen from the
 * first request on, so a 429 at any moment is reported.
 */
import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
@@ -14,13 +20,14 @@
import { join } from "node:path";
import { createRepo } from "../../src/actions/repos.ts";
import { spawnForge } from "../../src/actions/spawn.ts";
import { createApp } from "../../src/app.ts";
import { cellsAvailable } from "../../src/apps/contain.ts";
import { cellsAvailable, killTree } from "../../src/apps/contain.ts";
import { createUser, type User } from "../../src/core/users.ts";
import { repoDir } from "../../src/git/repos.ts";
import { commitFiles } from "../../src/git/write.ts";
import { serve } from "../../src/http/server.ts";
import { cellPlacement, stopCellChildren } from "../../src/spawn/cell.ts";
import { cellChildren, cellPlacement, stopCellChildren } from "../../src/spawn/cell.ts";
import { bootHostedChild } from "../../src/spawn/host.ts";
import { noteStop } from "../../src/spawn/stopped.ts";
import { memoryBodies } from "../../src/spawn/placement.ts";
import "./roomy-disk.ts";
@@ -29,7 +36,9 @@
import "./roomy-disk.ts";
// and a run that ran out says which phase did. The test's timeout is their sum
// plus slack (see test/spawn-cell-e2e.test.ts).
const SPAWN_MS = 120_000;
const POLL_MS = 150_000;
const POLL_MS = 60_000;
const RECOVER_MS = 60_000;
const recover = process.argv[2] === "recover";
// A killed run (the test's timeout sends SIGTERM) still stops its cell children.
let cleanup: () => Promise<void> = async () => {};
// The spawn and boot work, kept so a teardown after its budget ran out waits
@@ -105,21 +114,36 @@
try {
  const spawning = spawnAndBoot(alice.value);
  pending = spawning;
  const made = await within(SPAWN_MS, "spawn and boot", spawning);
  let status = 0;
  let body = "";
  let app: string | null = null;
  const until = Date.now() + POLL_MS;
  while (Date.now() < until) {
    const res = await fetch(`${servers.url}/`, { headers: { host: "bakery--kid--forge.test" }, redirect: "manual" }).catch(() => undefined);
    if (res !== undefined) {
      status = res.status;
      body = await res.text();
      app = res.headers.get("x-forge-app");
      if (status === 200 && app === "bakery") break;
  const seen = new Set<number>();
  // Asks the app's address until it answers 200 from the app, every `everyMs`; every status is
  // noted.
  async function pollApp(budgetMs: number, everyMs: number) {
    let last = { status: 0, body: "", app: null as string | null };
    const until = Date.now() + budgetMs;
    while (Date.now() < until) {
      const res = await fetch(`${servers.url}/`, { headers: { host: "bakery--kid--forge.test" }, redirect: "manual" }).catch(() => undefined);
      if (res !== undefined) {
        last = { status: res.status, body: await res.text(), app: res.headers.get("x-forge-app") };
        seen.add(last.status);
        if (last.status === 200 && last.app === "bakery") break;
      }
      await Bun.sleep(everyMs);
    }
    await Bun.sleep(1000);
    return last;
  }
  console.log(JSON.stringify({ placement: made.value.placement, status, body: body.slice(0, 200), app }));
  const first = await pollApp(POLL_MS, recover ? 100 : 1000);
  let limits = null;
  let after = null;
  if (recover) {
    const held = cellChildren(parent).list()[0];
    limits = held?.limits ?? null;
    if (held !== undefined) {
      noteStop(parent, held.bodyId, "kid", "used 2374 MiB of memory, over its 2048 MiB limit", Date.now(), 1_000);
      killTree(held.pid);
      after = (await pollApp(RECOVER_MS, 100)).status;
    }
  }
  console.log(JSON.stringify({ placement: made.value.placement, status: first.status, body: first.body.slice(0, 200), app: first.app, ...(recover ? { limits, after, statuses: [...seen] } : {}) }));
} finally {
  await cleanup();
  parent.close();
@@ -0,0 +1,44 @@
/**
 * Recovery of a Small child in a real cell (#11981, part of #11975). It proves
 * three things and no more: the child's cell holds the Small limits (2048 MiB,
 * 1 core), its app is served through the parent, and a child stopped as the
 * host stops one over its memory limit comes back by itself with no request,
 * from boot to recovery, answered 429 "the forge is full". The stop is
 * simulated (a recorded stop and the process tree killed), not a real memory
 * breach, and the app is a committed one-file app: building a first app,
 * passing its tests and landing a second change are not covered here.
 *
 * Runs where bwrap and socat are, else skips. Expected runtime: about a
 * minute (spawn and boot, a restart after a 1 s backoff); the 3 minute
 * ceiling is the worst case.
 */
import { test } from "bun:test";
import { join } from "node:path";
import { portInsteadOfSocket } from "../src/git/hook-address.ts";
import { missing } from "./helpers/tools.ts";
import { failedWith } from "./helpers/machine-note.ts";
import { resultLine } from "./helpers/result-line.ts";
const cellTools = process.platform === "linux" && !portInsteadOfSocket() && !missing("bwrap", "socat");
test.skipIf(!cellTools)("a Small child in a real cell holds its limits and, once stopped, comes back with no 429", async () => {
  const env: Record<string, string> = {};
  for (const [k, v] of Object.entries(process.env)) if (v !== undefined && k !== "FORGE_CELLS") env[k] = v;
  const proc = Bun.spawn([process.execPath, join(import.meta.dir, "helpers", "real-spawn-e2e.ts"), "recover"], { env, stdout: "pipe", stderr: "pipe", timeout: 230_000 });
  const [out, errText] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited]);
  let seen: Record<string, unknown>;
  try {
    seen = JSON.parse(resultLine(out)) as Record<string, unknown>;
  } catch {
    throw new Error(failedWith("the recovery run printed no result", out.slice(-800), errText));
  }
  if (typeof seen["skipped"] === "string") {
    console.log(`SKIPPING: this host refuses a cell inside the test (${seen["skipped"]})`);
    return;
  }
  const limits = seen["limits"] as { memoryMiB?: number; cpus?: number } | null;
  if (seen["placement"] !== "cell" || seen["status"] !== 200 || seen["body"] !== "bakery says hi") throw new Error(failedWith("the Small child's app did not answer through the cell", { seen }, errText));
  if (limits?.memoryMiB !== 2048 || limits.cpus !== 1) throw new Error(`the child did not hold the Small limits: ${JSON.stringify(limits)}`);
  if ((seen["statuses"] as number[]).includes(429)) throw new Error(failedWith("a child was answered 429, the sentence for a full host", { seen }, errText));
  if (seen["after"] !== 200) throw new Error(failedWith("the stopped child did not come back", { seen }, errText));
}, 240_000);
		@@ -7,6 +7,12 @@
7	7	* app, then asks `<app>--<child>--<parent>` through that server until the
8	8	* app's own body comes back. Nothing is faked: not the cell, not the child's
9	9	* runtime, not the route. Printed: one JSON line of what was seen.
	10	*
	11	* With the argument `recover` (test/spawn-small-cell.test.ts, #11981) it also
	12	* reads the child's cell limits, then stops the child as the host does for a
	13	* memory breach (a recorded stop and its process tree killed) and polls every
	14	* 100 ms until the app answers again, noting every status seen from the
	15	* first request on, so a 429 at any moment is reported.
10	16	*/
11	17	import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
12	18	import { tmpdir } from "node:os";
		@@ -14,13 +20,14 @@ import { join } from "node:path";
14	20	import { createRepo } from "../../src/actions/repos.ts";
15	21	import { spawnForge } from "../../src/actions/spawn.ts";
16	22	import { createApp } from "../../src/app.ts";
17		import { cellsAvailable } from "../../src/apps/contain.ts";
	23	import { cellsAvailable, killTree } from "../../src/apps/contain.ts";
18	24	import { createUser, type User } from "../../src/core/users.ts";
19	25	import { repoDir } from "../../src/git/repos.ts";
20	26	import { commitFiles } from "../../src/git/write.ts";
21	27	import { serve } from "../../src/http/server.ts";
22		import { cellPlacement, stopCellChildren } from "../../src/spawn/cell.ts";
	28	import { cellChildren, cellPlacement, stopCellChildren } from "../../src/spawn/cell.ts";
23	29	import { bootHostedChild } from "../../src/spawn/host.ts";
	30	import { noteStop } from "../../src/spawn/stopped.ts";
24	31	import { memoryBodies } from "../../src/spawn/placement.ts";
25	32	import "./roomy-disk.ts";
26	33
		@@ -29,7 +36,9 @@ import "./roomy-disk.ts";
29	36	// and a run that ran out says which phase did. The test's timeout is their sum
30	37	// plus slack (see test/spawn-cell-e2e.test.ts).
31	38	const SPAWN_MS = 120_000;
32		const POLL_MS = 150_000;
	39	const POLL_MS = 60_000;
	40	const RECOVER_MS = 60_000;
	41	const recover = process.argv[2] === "recover";
33	42	// A killed run (the test's timeout sends SIGTERM) still stops its cell children.
34	43	let cleanup: () => Promise<void> = async () => {};
35	44	// The spawn and boot work, kept so a teardown after its budget ran out waits
		@@ -105,21 +114,36 @@ try {
105	114	const spawning = spawnAndBoot(alice.value);
106	115	pending = spawning;
107	116	const made = await within(SPAWN_MS, "spawn and boot", spawning);
108		let status = 0;
109		let body = "";
110		let app: string \| null = null;
111		const until = Date.now() + POLL_MS;
112		while (Date.now() < until) {
113		const res = await fetch(`${servers.url}/`, { headers: { host: "bakery--kid--forge.test" }, redirect: "manual" }).catch(() => undefined);
114		if (res !== undefined) {
115		status = res.status;
116		body = await res.text();
117		app = res.headers.get("x-forge-app");
118		if (status === 200 && app === "bakery") break;
	117	const seen = new Set<number>();
	118	// Asks the app's address until it answers 200 from the app, every `everyMs`; every status is
	119	// noted.
	120	async function pollApp(budgetMs: number, everyMs: number) {
	121	let last = { status: 0, body: "", app: null as string \| null };
	122	const until = Date.now() + budgetMs;
	123	while (Date.now() < until) {
	124	const res = await fetch(`${servers.url}/`, { headers: { host: "bakery--kid--forge.test" }, redirect: "manual" }).catch(() => undefined);
	125	if (res !== undefined) {
	126	last = { status: res.status, body: await res.text(), app: res.headers.get("x-forge-app") };
	127	seen.add(last.status);
	128	if (last.status === 200 && last.app === "bakery") break;
	129	}
	130	await Bun.sleep(everyMs);
119	131	}
120		await Bun.sleep(1000);
	132	return last;
121	133	}
122		console.log(JSON.stringify({ placement: made.value.placement, status, body: body.slice(0, 200), app }));
	134	const first = await pollApp(POLL_MS, recover ? 100 : 1000);
	135	let limits = null;
	136	let after = null;
	137	if (recover) {
	138	const held = cellChildren(parent).list()[0];
	139	limits = held?.limits ?? null;
	140	if (held !== undefined) {
	141	noteStop(parent, held.bodyId, "kid", "used 2374 MiB of memory, over its 2048 MiB limit", Date.now(), 1_000);
	142	killTree(held.pid);
	143	after = (await pollApp(RECOVER_MS, 100)).status;
	144	}
	145	}
	146	console.log(JSON.stringify({ placement: made.value.placement, status: first.status, body: first.body.slice(0, 200), app: first.app, ...(recover ? { limits, after, statuses: [...seen] } : {}) }));
123	147	} finally {
124	148	await cleanup();
125	149	parent.close();
		@@ -0,0 +1,44 @@
	1	/**
	2	* Recovery of a Small child in a real cell (#11981, part of #11975). It proves
	3	* three things and no more: the child's cell holds the Small limits (2048 MiB,
	4	* 1 core), its app is served through the parent, and a child stopped as the
	5	* host stops one over its memory limit comes back by itself with no request,
	6	* from boot to recovery, answered 429 "the forge is full". The stop is
	7	* simulated (a recorded stop and the process tree killed), not a real memory
	8	* breach, and the app is a committed one-file app: building a first app,
	9	* passing its tests and landing a second change are not covered here.
	10	*
	11	* Runs where bwrap and socat are, else skips. Expected runtime: about a
	12	* minute (spawn and boot, a restart after a 1 s backoff); the 3 minute
	13	* ceiling is the worst case.
	14	*/
	15	import { test } from "bun:test";
	16	import { join } from "node:path";
	17	import { portInsteadOfSocket } from "../src/git/hook-address.ts";
	18	import { missing } from "./helpers/tools.ts";
	19	import { failedWith } from "./helpers/machine-note.ts";
	20	import { resultLine } from "./helpers/result-line.ts";
	21
	22	const cellTools = process.platform === "linux" && !portInsteadOfSocket() && !missing("bwrap", "socat");
	23
	24	test.skipIf(!cellTools)("a Small child in a real cell holds its limits and, once stopped, comes back with no 429", async () => {
	25	const env: Record<string, string> = {};
	26	for (const [k, v] of Object.entries(process.env)) if (v !== undefined && k !== "FORGE_CELLS") env[k] = v;
	27	const proc = Bun.spawn([process.execPath, join(import.meta.dir, "helpers", "real-spawn-e2e.ts"), "recover"], { env, stdout: "pipe", stderr: "pipe", timeout: 230_000 });
	28	const [out, errText] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited]);
	29	let seen: Record<string, unknown>;
	30	try {
	31	seen = JSON.parse(resultLine(out)) as Record<string, unknown>;
	32	} catch {
	33	throw new Error(failedWith("the recovery run printed no result", out.slice(-800), errText));
	34	}
	35	if (typeof seen["skipped"] === "string") {
	36	console.log(`SKIPPING: this host refuses a cell inside the test (${seen["skipped"]})`);
	37	return;
	38	}
	39	const limits = seen["limits"] as { memoryMiB?: number; cpus?: number } \| null;
	40	if (seen["placement"] !== "cell" \|\| seen["status"] !== 200 \|\| seen["body"] !== "bakery says hi") throw new Error(failedWith("the Small child's app did not answer through the cell", { seen }, errText));
	41	if (limits?.memoryMiB !== 2048 \|\| limits.cpus !== 1) throw new Error(`the child did not hold the Small limits: ${JSON.stringify(limits)}`);
	42	if ((seen["statuses"] as number[]).includes(429)) throw new Error(failedWith("a child was answered 429, the sentence for a full host", { seen }, errText));
	43	if (seen["after"] !== 200) throw new Error(failedWith("the stopped child did not come back", { seen }, errText));
	44	}, 240_000);