Public: anyone can read this; the crew can change it.

Full suite red: test/spawn-move-cell.test.ts #12854

Merged opened this in answer to

Check · Review · Land · Live done — Landed and live in production.

Merged

torque/forge:torque/12835-full-suite-red-test-spawn-move-cell-test → main 1 commit fork

  • Merged. torque/12835-full-suite-red-test-spawn-move-cell-test was merged into main as 57bc463.

Review

Sign in to review this change.
3 files changed +399 −322
docs/THREATS.md +1 −1
@@ -343,7 +343,7 @@ What it can reach: everything a child forge can reach, no more: its own data dir
343343 put there by an older release) is moved into a cell of its own, by an
344344 admin's button or by the forge's own pass at boot and when one sleeps; a
345345 move that does not prove itself is put back.
346 Enforced by: `test/spawn-move-cell.test.ts`
346 Enforced by: `test/spawn-move-cell.test.ts`, `test/spawn-move-cell-sizes.test.ts`
347347 - Before such a forge leaves its parent's process its repositories are read,
348348 running nothing, for what that process may have executed while it had no
349349 wall: hooks beyond the forge's own stubs and config keys that make git run
test/spawn-move-cell-sizes.test.ts added +398
@@ -0,0 +1,398 @@
1 /**
2 * The rest of `spawn-move-cell.test.ts`, split from it to keep each file under the file budget:
3 * a forge too big for its size is given the next; one whose repositories hold a hook is left for
4 * the admin; the automatic pass; the admin's action. The cell is the same stand-in
5 * (`helpers/moved-forge.ts`).
6 */
7 import {
8 afterAll,
9 afterEach,
10 beforeAll,
11 describe,
12 expect,
13 test,
14 } from "bun:test";
15 import { writeFileSync } from "node:fs";
16 import { join } from "node:path";
17 import { servedWindow, clientRows } from "../src/spawn/clients.ts";
18 import { provideCellsForTest, sleepIdleChildren } from "../src/spawn/host.ts";
19 import { moveDisk } from "../src/spawn/move_check.ts";
20 import {
21 healthWait,
22 keepSandboxing,
23 moveToCell,
24 sandboxPass,
25 } from "../src/spawn/move_cell.ts";
26 import { IN_PROCESS_WORDS } from "../src/spawn/where.ts";
27 import { quietLogs } from "./helpers/globals.ts";
28 import { mcpTool } from "./helpers/mcp.ts";
29 import {
30 inProcess,
31 makeFamily,
32 PARENT_HOST,
33 type Family,
34 } from "./helpers/moved-forge.ts";
35
36 quietLogs();
37
38 const window = servedWindow.ms;
39 const wait = { ...healthWait };
40 const realBytes = moveDisk.bytes;
41 beforeAll(() => {
42 servedWindow.ms = 0;
43 // A stand-in that does not come up is known at once: no test waits out a real cell's twenty
44 // seconds.
45 healthWait.ms = 3_000;
46 healthWait.everyMs = 20;
47 provideCellsForTest(() => Promise.resolve({ ok: true }));
48 });
49 afterAll(() => {
50 servedWindow.ms = window;
51 Object.assign(healthWait, wait);
52 provideCellsForTest();
53 });
54
55 const families: Family[] = [];
56 async function family(name?: string): Promise<Family> {
57 const f = await makeFamily(name);
58 families.push(f);
59 return f;
60 }
61 afterEach(async () => {
62 moveDisk.bytes = realBytes;
63 for (const f of families.splice(0)) await f.close();
64 });
65
66 const events = (
67 f: { parent: Family["parent"] },
68 kind: string,
69 ): { payload: string }[] =>
70 f.parent.db
71 .query<
72 { payload: string },
73 [string]
74 >("SELECT payload FROM event WHERE kind = ?1 ORDER BY rowid")
75 .all(kind);
76 const rowOf = async (f: Family) =>
77 (await clientRows(f.parent, { probe: false })).find(
78 (r) => r.name === f.name,
79 )!;
80
81 describe("a forge too big for its size", () => {
82 const MEMORY =
83 "its cell did not come up: it ran out of memory before it answered";
84 const KEY = "FORGE_CHILD_MEMORY_MIB";
85 const was = process.env[KEY];
86 afterEach(() => {
87 if (was === undefined) delete process.env[KEY];
88 else process.env[KEY] = was;
89 });
90
91 test("is given the next size up when its cell runs out of memory, and its row says so", async () => {
92 const f = await family("gamer");
93 const before = await f.facts();
94 f.cells.fail = (size) => (size === "small" ? MEMORY : undefined);
95 const moved = await moveToCell(f.parent, f.name, { by: "admin" });
96 expect(moved).toEqual({
97 ok: true,
98 name: "gamer",
99 says: "gamer now runs in its own sandbox, with a medium forge's room: a small one did not hold it. Nothing else changed.",
100 size: "medium",
101 sizeWas: "small",
102 });
103 expect(f.cells.starts).toEqual(["small", "medium"]);
104 expect(f.ledger()).toMatchObject({ placement: "cell", size: "medium" });
105 expect(f.ledger()["moved"]).toMatchObject({ sizeWas: "small" });
106 const row = await rowOf(f);
107 expect(row.size).toBe("medium");
108 expect(row.where.says).toEndWith(
109 "; given a medium forge's room, a small one did not hold it",
110 );
111 expect(events(f, "forge.moved").map((e) => JSON.parse(e.payload))).toEqual([
112 { name: "gamer", size: "medium", sizeWas: "small" },
113 ]);
114 expect(await f.facts()).toEqual(before);
115 }, 60_000);
116
117 test("is put back at its own size, with the limit named, when the host has no room for the next", async () => {
118 const f = await family("gamer");
119 f.cells.fail = (size) => (size === "small" ? MEMORY : undefined);
120 process.env[KEY] = "3000"; // a small forge fits; a medium one in its place does not
121 const failed = await moveToCell(f.parent, f.name, { by: "admin" });
122 expect(failed).toMatchObject({ ok: false, kind: "failed" });
123 expect((failed as { says: string }).says).toBe(
124 `gamer was not moved: ${MEMORY}; it needs a medium forge's room, and a medium forge in place of a small one would pass this host's child memory limit (${KEY}=3000 MiB; 2048 MiB already held). It runs inside this forge's own process as it did before.`,
125 );
126 expect(f.cells.starts).toEqual(["small"]);
127 expect(f.ledger()).toMatchObject({ placement: "hosted", size: "small" });
128 expect((await rowOf(f)).where.says).toContain(
129 "it needs a medium forge's room",
130 );
131 }, 60_000);
132
133 test("is never left stopped: a large forge that still runs out of memory is put back and says so", async () => {
134 const f = await family("gamer");
135 f.cells.fail = () => MEMORY;
136 const failed = await moveToCell(f.parent, f.name, { by: "admin" });
137 expect(failed).toMatchObject({ ok: false, kind: "failed" });
138 expect(f.cells.starts).toEqual(["small", "medium", "large"]);
139 expect(f.ledger()).toMatchObject({ placement: "hosted", size: "small" });
140 expect((await f.ask("/healthz")).status).toBe(200);
141 }, 60_000);
142
143 test("is given the size its disk needs before its cell is started at all", async () => {
144 const f = await family("gamer");
145 moveDisk.bytes = () => Promise.resolve(3000 * 1024 * 1024);
146 const moved = await moveToCell(f.parent, f.name, { by: "admin" });
147 expect(moved).toMatchObject({ ok: true, size: "medium", sizeWas: "small" });
148 expect(f.cells.starts).toEqual(["medium"]);
149 }, 60_000);
150 });
151
152 describe("a forge whose repositories hold a hook", () => {
153 test("is not moved by itself: the pass flags it for the admin by name, once, and it keeps running where it was", async () => {
154 const f = await family();
155 writeFileSync(
156 join(f.dir, "repos", "bob", "notes.git", "hooks", "post-update"),
157 "#!/bin/sh\necho never-print-this\n",
158 );
159 await sleepIdleChildren(f.parent, 0);
160 const WHY =
161 "its repositories hold what this forge's own process may have run (bob/notes: hooks post-update)";
162 const [held] = await sandboxPass(f.parent);
163 expect(held).toEqual({
164 ok: false,
165 name: f.name,
166 kind: "held",
167 says: `${f.name} is not moved by itself: ${WHY}. Look at them, then move it.`,
168 });
169 expect(f.ledger()).toMatchObject({ placement: "hosted" });
170 expect((f.ledger()["moveHeld"] as { why: string }).why).toBe(WHY);
171 expect(f.cells.starts).toEqual([]);
172 const row = await rowOf(f);
173 expect(row.where).toMatchObject({
174 inProcess: true,
175 movable: true,
176 held: true,
177 says: `${IN_PROCESS_WORDS}; not moved by itself: ${WHY}`,
178 });
179 // Said once, however often the pass looks; and never what the hook contains.
180 await sandboxPass(f.parent);
181 expect(
182 events(f, "forge.move_held").map((e) => JSON.parse(e.payload)),
183 ).toEqual([{ name: f.name, why: WHY }]);
184 expect(JSON.stringify(f.ledger()) + JSON.stringify(row)).not.toContain(
185 "never-print-this",
186 );
187 expect((await f.ask("/healthz")).status).toBe(200);
188 // The admin's move without saying they looked is held too; saying so moves it.
189 expect(await moveToCell(f.parent, f.name, { by: "admin" })).toMatchObject({
190 ok: false,
191 kind: "held",
192 });
193 const moved = await moveToCell(f.parent, f.name, {
194 by: "admin",
195 acknowledge: true,
196 });
197 expect(moved.ok).toBe(true);
198 expect(f.ledger()).toMatchObject({ placement: "cell" });
199 expect(f.ledger()["moveHeld"]).toBeUndefined();
200 }, 60_000);
201 });
202
203 describe("the automatic pass", () => {
204 test("moves a forge only while it is asleep, and takes it when it next sleeps", async () => {
205 const f = await family();
206 expect(inProcess(f.parent)).toEqual([f.name]);
207 await keepSandboxing(f.parent);
208 // Awake in the parent's process: left alone, nothing written.
209 expect(f.ledger()).toMatchObject({ placement: "hosted" });
210 expect(f.ledger()["move"]).toBeUndefined();
211 expect(f.cells.starts).toEqual([]);
212 // It sleeps; the pass that was waiting takes it.
213 await sleepIdleChildren(f.parent, 0);
214 for (let i = 0; i < 200 && f.ledger()["moved"] === undefined; i++)
215 await Bun.sleep(25); // real-timer: the pass runs behind the idle sweep, not under this test's hand; this waits for it to end
216 expect(f.ledger()).toMatchObject({ placement: "cell" });
217 expect(inProcess(f.parent)).toEqual([]);
218 expect((await f.ask("/healthz")).status).toBe(200);
219 expect(await sandboxPass(f.parent)).toEqual([]);
220 }, 60_000);
221
222 test("stops at the first forge it fails to move, and does not try that one again for a day", async () => {
223 const f = await family();
224 await sleepIdleChildren(f.parent, 0);
225 f.cells.fail = () => "its cell did not come up: it did not answer in time";
226 expect(await sandboxPass(f.parent)).toMatchObject([
227 { ok: false, kind: "failed" },
228 ]);
229 expect(f.cells.starts.length).toBe(1);
230 await sleepIdleChildren(f.parent, 0);
231 expect(await sandboxPass(f.parent)).toEqual([]);
232 expect(f.cells.starts.length).toBe(1);
233 // An admin's move is never held back by that.
234 f.cells.fail = () => undefined;
235 await sleepIdleChildren(f.parent, 0);
236 expect((await moveToCell(f.parent, f.name, { by: "admin" })).ok).toBe(true);
237 }, 60_000);
238
239 test("does nothing on a host that cannot make a cell, and drops an intent it finds there", async () => {
240 const f = await family();
241 await sleepIdleChildren(f.parent, 0);
242 provideCellsForTest(() =>
243 Promise.resolve({ ok: false, reason: "no cells on darwin" }),
244 );
245 try {
246 expect(await sandboxPass(f.parent)).toEqual([]);
247 expect(await moveToCell(f.parent, f.name, { by: "admin" })).toMatchObject(
248 {
249 ok: false,
250 kind: "blocked",
251 says: `${f.name} cannot be moved now: this host cannot make a sandbox (no cells on darwin).`,
252 },
253 );
254 writeFileSync(
255 f.ledgerPath,
256 JSON.stringify({
257 ...f.ledger(),
258 move: {
259 at: Date.now(),
260 step: "intent",
261 by: "auto",
262 sizeWas: "small",
263 },
264 }),
265 );
266 expect((await f.ask("/healthz")).status).toBe(503);
267 expect(await sandboxPass(f.parent)).toMatchObject([
268 { ok: false, kind: "blocked" },
269 ]);
270 expect(f.ledger()["move"]).toBeUndefined();
271 expect((await f.ask("/healthz")).status).toBe(200);
272 expect((await rowOf(f)).where).toMatchObject({
273 inProcess: true,
274 movable: false,
275 });
276 } finally {
277 provideCellsForTest(() => Promise.resolve({ ok: true }));
278 }
279 }, 60_000);
280 });
281
282 describe("the admin's action", () => {
283 test("is a button on the forge's row, an API route and a tool; nobody else has it", async () => {
284 const f = await family();
285 const page = await (
286 await fetch(`${f.servers.url}/forges`, {
287 headers: { cookie: f.cookie, host: PARENT_HOST },
288 })
289 ).text();
290 expect(page).toContain(`action="/forges/${f.name}/sandbox"`);
291 expect(page).toContain("Move into its own sandbox");
292 // The API plans unless told not to; the tool too.
293 const post = (body: unknown, token = f.adminToken) =>
294 fetch(`${f.servers.url}/api/v1/forges/${f.name}/sandbox`, {
295 method: "POST",
296 headers: {
297 authorization: `Bearer ${token}`,
298 host: PARENT_HOST,
299 "content-type": "application/json",
300 },
301 body: JSON.stringify(body),
302 });
303 const planned = await post({});
304 expect(planned.status).toBe(200);
305 expect(await planned.json()).toMatchObject({
306 dryRun: true,
307 name: f.name,
308 blocked: null,
309 held: null,
310 });
311 expect(f.ledger()).toMatchObject({ placement: "hosted" });
312 const tool = await mcpTool(f.servers.url, f.adminToken, "forges", {
313 forge: f.name,
314 sandbox: true,
315 });
316 expect(tool.value).toMatchObject({
317 name: f.name,
318 blocked: null,
319 plan: "This is the check; nothing moved. Send dryRun: false to move it.",
320 });
321 expect(
322 (
323 await fetch(`${f.servers.url}/api/v1/forges/${f.name}/sandbox`, {
324 method: "POST",
325 headers: { host: PARENT_HOST, "content-type": "application/json" },
326 body: "{}",
327 })
328 ).status,
329 ).toBeGreaterThanOrEqual(401);
330 // The button.
331 const pressed = await fetch(`${f.servers.url}/forges/${f.name}/sandbox`, {
332 method: "POST",
333 headers: {
334 cookie: f.cookie,
335 host: PARENT_HOST,
336 origin: `https://${PARENT_HOST}`,
337 "content-type": "application/x-www-form-urlencoded",
338 },
339 body: "",
340 });
341 expect(pressed.status).toBe(200);
342 expect(await pressed.text()).toContain(
343 `${f.name} now runs in its own sandbox. Nothing else changed.`,
344 );
345 expect(f.ledger()).toMatchObject({ placement: "cell" });
346 const after = await (
347 await fetch(`${f.servers.url}/forges`, {
348 headers: { cookie: f.cookie, host: PARENT_HOST },
349 })
350 ).text();
351 expect(after).not.toContain(`action="/forges/${f.name}/sandbox"`);
352 expect(after).toContain('data-placement="cell"');
353 // Asked again over the API: already there, and said so.
354 const again = await post({ dryRun: false });
355 expect(again.status).toBe(409);
356 expect(await again.json()).toMatchObject({
357 moved: false,
358 error: "blocked",
359 });
360 const viaTool = await mcpTool(f.servers.url, f.adminToken, "forges", {
361 forge: f.name,
362 sandbox: true,
363 dryRun: false,
364 });
365 expect(viaTool.error).toContain("already runs in its own sandbox");
366 }, 60_000);
367
368 test("moves it over the API and by the tool", async () => {
369 const f = await family();
370 const res = await fetch(
371 `${f.servers.url}/api/v1/forges/${f.name}/sandbox`,
372 {
373 method: "POST",
374 headers: {
375 authorization: `Bearer ${f.adminToken}`,
376 host: PARENT_HOST,
377 "content-type": "application/json",
378 },
379 body: JSON.stringify({ dryRun: false }),
380 },
381 );
382 expect(res.status).toBe(200);
383 expect(await res.json()).toMatchObject({
384 dryRun: false,
385 moved: true,
386 name: f.name,
387 size: "small",
388 });
389 const g = await family("mark");
390 const tool = await mcpTool(g.servers.url, g.adminToken, "forges", {
391 forge: "mark",
392 sandbox: true,
393 dryRun: false,
394 });
395 expect(tool.value).toMatchObject({ ok: true, name: "mark" });
396 expect(g.ledger()).toMatchObject({ placement: "cell" });
397 }, 60_000);
398 });
test/spawn-move-cell.test.ts −321
@@ -25,18 +25,15 @@ import { provideCellsForTest, sleepIdleChildren } from "../src/spawn/host.ts";
2525 import { moveDisk } from "../src/spawn/move_check.ts";
2626 import {
2727 healthWait,
28 keepSandboxing,
2928 moveToCell,
3029 sandboxPass,
3130 type MoveStep,
3231 } from "../src/spawn/move_cell.ts";
3332 import { IN_PROCESS_WORDS, SANDBOXED_WORDS } from "../src/spawn/where.ts";
3433 import { quietLogs } from "./helpers/globals.ts";
35 import { mcpTool } from "./helpers/mcp.ts";
3634 import {
3735 inProcess,
3836 makeFamily,
39 PARENT_HOST,
4037 type Family,
4138 } from "./helpers/moved-forge.ts";
4239
@@ -369,321 +366,3 @@ describe("a move that fails is put back exactly as it was, and a repeat complete
369366 }, 60_000);
370367 });
371368
372 describe("a forge too big for its size", () => {
373 const MEMORY =
374 "its cell did not come up: it ran out of memory before it answered";
375 const KEY = "FORGE_CHILD_MEMORY_MIB";
376 const was = process.env[KEY];
377 afterEach(() => {
378 if (was === undefined) delete process.env[KEY];
379 else process.env[KEY] = was;
380 });
381
382 test("is given the next size up when its cell runs out of memory, and its row says so", async () => {
383 const f = await family("gamer");
384 const before = await f.facts();
385 f.cells.fail = (size) => (size === "small" ? MEMORY : undefined);
386 const moved = await moveToCell(f.parent, f.name, { by: "admin" });
387 expect(moved).toEqual({
388 ok: true,
389 name: "gamer",
390 says: "gamer now runs in its own sandbox, with a medium forge's room: a small one did not hold it. Nothing else changed.",
391 size: "medium",
392 sizeWas: "small",
393 });
394 expect(f.cells.starts).toEqual(["small", "medium"]);
395 expect(f.ledger()).toMatchObject({ placement: "cell", size: "medium" });
396 expect(f.ledger()["moved"]).toMatchObject({ sizeWas: "small" });
397 const row = await rowOf(f);
398 expect(row.size).toBe("medium");
399 expect(row.where.says).toEndWith(
400 "; given a medium forge's room, a small one did not hold it",
401 );
402 expect(events(f, "forge.moved").map((e) => JSON.parse(e.payload))).toEqual([
403 { name: "gamer", size: "medium", sizeWas: "small" },
404 ]);
405 expect(await f.facts()).toEqual(before);
406 }, 60_000);
407
408 test("is put back at its own size, with the limit named, when the host has no room for the next", async () => {
409 const f = await family("gamer");
410 f.cells.fail = (size) => (size === "small" ? MEMORY : undefined);
411 process.env[KEY] = "3000"; // a small forge fits; a medium one in its place does not
412 const failed = await moveToCell(f.parent, f.name, { by: "admin" });
413 expect(failed).toMatchObject({ ok: false, kind: "failed" });
414 expect((failed as { says: string }).says).toBe(
415 `gamer was not moved: ${MEMORY}; it needs a medium forge's room, and a medium forge in place of a small one would pass this host's child memory limit (${KEY}=3000 MiB; 2048 MiB already held). It runs inside this forge's own process as it did before.`,
416 );
417 expect(f.cells.starts).toEqual(["small"]);
418 expect(f.ledger()).toMatchObject({ placement: "hosted", size: "small" });
419 expect((await rowOf(f)).where.says).toContain(
420 "it needs a medium forge's room",
421 );
422 }, 60_000);
423
424 test("is never left stopped: a large forge that still runs out of memory is put back and says so", async () => {
425 const f = await family("gamer");
426 f.cells.fail = () => MEMORY;
427 const failed = await moveToCell(f.parent, f.name, { by: "admin" });
428 expect(failed).toMatchObject({ ok: false, kind: "failed" });
429 expect(f.cells.starts).toEqual(["small", "medium", "large"]);
430 expect(f.ledger()).toMatchObject({ placement: "hosted", size: "small" });
431 expect((await f.ask("/healthz")).status).toBe(200);
432 }, 60_000);
433
434 test("is given the size its disk needs before its cell is started at all", async () => {
435 const f = await family("gamer");
436 moveDisk.bytes = () => Promise.resolve(3000 * 1024 * 1024);
437 const moved = await moveToCell(f.parent, f.name, { by: "admin" });
438 expect(moved).toMatchObject({ ok: true, size: "medium", sizeWas: "small" });
439 expect(f.cells.starts).toEqual(["medium"]);
440 }, 60_000);
441 });
442
443 describe("a forge whose repositories hold a hook", () => {
444 test("is not moved by itself: the pass flags it for the admin by name, once, and it keeps running where it was", async () => {
445 const f = await family();
446 writeFileSync(
447 join(f.dir, "repos", "bob", "notes.git", "hooks", "post-update"),
448 "#!/bin/sh\necho never-print-this\n",
449 );
450 await sleepIdleChildren(f.parent, 0);
451 const WHY =
452 "its repositories hold what this forge's own process may have run (bob/notes: hooks post-update)";
453 const [held] = await sandboxPass(f.parent);
454 expect(held).toEqual({
455 ok: false,
456 name: f.name,
457 kind: "held",
458 says: `${f.name} is not moved by itself: ${WHY}. Look at them, then move it.`,
459 });
460 expect(f.ledger()).toMatchObject({ placement: "hosted" });
461 expect((f.ledger()["moveHeld"] as { why: string }).why).toBe(WHY);
462 expect(f.cells.starts).toEqual([]);
463 const row = await rowOf(f);
464 expect(row.where).toMatchObject({
465 inProcess: true,
466 movable: true,
467 held: true,
468 says: `${IN_PROCESS_WORDS}; not moved by itself: ${WHY}`,
469 });
470 // Said once, however often the pass looks; and never what the hook contains.
471 await sandboxPass(f.parent);
472 expect(
473 events(f, "forge.move_held").map((e) => JSON.parse(e.payload)),
474 ).toEqual([{ name: f.name, why: WHY }]);
475 expect(JSON.stringify(f.ledger()) + JSON.stringify(row)).not.toContain(
476 "never-print-this",
477 );
478 expect((await f.ask("/healthz")).status).toBe(200);
479 // The admin's move without saying they looked is held too; saying so moves it.
480 expect(await moveToCell(f.parent, f.name, { by: "admin" })).toMatchObject({
481 ok: false,
482 kind: "held",
483 });
484 const moved = await moveToCell(f.parent, f.name, {
485 by: "admin",
486 acknowledge: true,
487 });
488 expect(moved.ok).toBe(true);
489 expect(f.ledger()).toMatchObject({ placement: "cell" });
490 expect(f.ledger()["moveHeld"]).toBeUndefined();
491 }, 60_000);
492 });
493
494 describe("the automatic pass", () => {
495 test("moves a forge only while it is asleep, and takes it when it next sleeps", async () => {
496 const f = await family();
497 expect(inProcess(f.parent)).toEqual([f.name]);
498 await keepSandboxing(f.parent);
499 // Awake in the parent's process: left alone, nothing written.
500 expect(f.ledger()).toMatchObject({ placement: "hosted" });
501 expect(f.ledger()["move"]).toBeUndefined();
502 expect(f.cells.starts).toEqual([]);
503 // It sleeps; the pass that was waiting takes it.
504 await sleepIdleChildren(f.parent, 0);
505 for (let i = 0; i < 200 && f.ledger()["moved"] === undefined; i++)
506 await Bun.sleep(25); // real-timer: the pass runs behind the idle sweep, not under this test's hand; this waits for it to end
507 expect(f.ledger()).toMatchObject({ placement: "cell" });
508 expect(inProcess(f.parent)).toEqual([]);
509 expect((await f.ask("/healthz")).status).toBe(200);
510 expect(await sandboxPass(f.parent)).toEqual([]);
511 }, 60_000);
512
513 test("stops at the first forge it fails to move, and does not try that one again for a day", async () => {
514 const f = await family();
515 await sleepIdleChildren(f.parent, 0);
516 f.cells.fail = () => "its cell did not come up: it did not answer in time";
517 expect(await sandboxPass(f.parent)).toMatchObject([
518 { ok: false, kind: "failed" },
519 ]);
520 expect(f.cells.starts.length).toBe(1);
521 await sleepIdleChildren(f.parent, 0);
522 expect(await sandboxPass(f.parent)).toEqual([]);
523 expect(f.cells.starts.length).toBe(1);
524 // An admin's move is never held back by that.
525 f.cells.fail = () => undefined;
526 await sleepIdleChildren(f.parent, 0);
527 expect((await moveToCell(f.parent, f.name, { by: "admin" })).ok).toBe(true);
528 }, 60_000);
529
530 test("does nothing on a host that cannot make a cell, and drops an intent it finds there", async () => {
531 const f = await family();
532 await sleepIdleChildren(f.parent, 0);
533 provideCellsForTest(() =>
534 Promise.resolve({ ok: false, reason: "no cells on darwin" }),
535 );
536 try {
537 expect(await sandboxPass(f.parent)).toEqual([]);
538 expect(await moveToCell(f.parent, f.name, { by: "admin" })).toMatchObject(
539 {
540 ok: false,
541 kind: "blocked",
542 says: `${f.name} cannot be moved now: this host cannot make a sandbox (no cells on darwin).`,
543 },
544 );
545 writeFileSync(
546 f.ledgerPath,
547 JSON.stringify({
548 ...f.ledger(),
549 move: {
550 at: Date.now(),
551 step: "intent",
552 by: "auto",
553 sizeWas: "small",
554 },
555 }),
556 );
557 expect((await f.ask("/healthz")).status).toBe(503);
558 expect(await sandboxPass(f.parent)).toMatchObject([
559 { ok: false, kind: "blocked" },
560 ]);
561 expect(f.ledger()["move"]).toBeUndefined();
562 expect((await f.ask("/healthz")).status).toBe(200);
563 expect((await rowOf(f)).where).toMatchObject({
564 inProcess: true,
565 movable: false,
566 });
567 } finally {
568 provideCellsForTest(() => Promise.resolve({ ok: true }));
569 }
570 }, 60_000);
571 });
572
573 describe("the admin's action", () => {
574 test("is a button on the forge's row, an API route and a tool; nobody else has it", async () => {
575 const f = await family();
576 const page = await (
577 await fetch(`${f.servers.url}/forges`, {
578 headers: { cookie: f.cookie, host: PARENT_HOST },
579 })
580 ).text();
581 expect(page).toContain(`action="/forges/${f.name}/sandbox"`);
582 expect(page).toContain("Move into its own sandbox");
583 // The API plans unless told not to; the tool too.
584 const post = (body: unknown, token = f.adminToken) =>
585 fetch(`${f.servers.url}/api/v1/forges/${f.name}/sandbox`, {
586 method: "POST",
587 headers: {
588 authorization: `Bearer ${token}`,
589 host: PARENT_HOST,
590 "content-type": "application/json",
591 },
592 body: JSON.stringify(body),
593 });
594 const planned = await post({});
595 expect(planned.status).toBe(200);
596 expect(await planned.json()).toMatchObject({
597 dryRun: true,
598 name: f.name,
599 blocked: null,
600 held: null,
601 });
602 expect(f.ledger()).toMatchObject({ placement: "hosted" });
603 const tool = await mcpTool(f.servers.url, f.adminToken, "forges", {
604 forge: f.name,
605 sandbox: true,
606 });
607 expect(tool.value).toMatchObject({
608 name: f.name,
609 blocked: null,
610 plan: "This is the check; nothing moved. Send dryRun: false to move it.",
611 });
612 expect(
613 (
614 await fetch(`${f.servers.url}/api/v1/forges/${f.name}/sandbox`, {
615 method: "POST",
616 headers: { host: PARENT_HOST, "content-type": "application/json" },
617 body: "{}",
618 })
619 ).status,
620 ).toBeGreaterThanOrEqual(401);
621 // The button.
622 const pressed = await fetch(`${f.servers.url}/forges/${f.name}/sandbox`, {
623 method: "POST",
624 headers: {
625 cookie: f.cookie,
626 host: PARENT_HOST,
627 origin: `https://${PARENT_HOST}`,
628 "content-type": "application/x-www-form-urlencoded",
629 },
630 body: "",
631 });
632 expect(pressed.status).toBe(200);
633 expect(await pressed.text()).toContain(
634 `${f.name} now runs in its own sandbox. Nothing else changed.`,
635 );
636 expect(f.ledger()).toMatchObject({ placement: "cell" });
637 const after = await (
638 await fetch(`${f.servers.url}/forges`, {
639 headers: { cookie: f.cookie, host: PARENT_HOST },
640 })
641 ).text();
642 expect(after).not.toContain(`action="/forges/${f.name}/sandbox"`);
643 expect(after).toContain('data-placement="cell"');
644 // Asked again over the API: already there, and said so.
645 const again = await post({ dryRun: false });
646 expect(again.status).toBe(409);
647 expect(await again.json()).toMatchObject({
648 moved: false,
649 error: "blocked",
650 });
651 const viaTool = await mcpTool(f.servers.url, f.adminToken, "forges", {
652 forge: f.name,
653 sandbox: true,
654 dryRun: false,
655 });
656 expect(viaTool.error).toContain("already runs in its own sandbox");
657 }, 60_000);
658
659 test("moves it over the API and by the tool", async () => {
660 const f = await family();
661 const res = await fetch(
662 `${f.servers.url}/api/v1/forges/${f.name}/sandbox`,
663 {
664 method: "POST",
665 headers: {
666 authorization: `Bearer ${f.adminToken}`,
667 host: PARENT_HOST,
668 "content-type": "application/json",
669 },
670 body: JSON.stringify({ dryRun: false }),
671 },
672 );
673 expect(res.status).toBe(200);
674 expect(await res.json()).toMatchObject({
675 dryRun: false,
676 moved: true,
677 name: f.name,
678 size: "small",
679 });
680 const g = await family("mark");
681 const tool = await mcpTool(g.servers.url, g.adminToken, "forges", {
682 forge: "mark",
683 sandbox: true,
684 dryRun: false,
685 });
686 expect(tool.value).toMatchObject({ ok: true, name: "mark" });
687 expect(g.ledger()).toMatchObject({ placement: "cell" });
688 }, 60_000);
689 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs