Public: anyone can read this; the crew can change it.
Full suite red: test/spawn-move-cell.test.ts #12854
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/12835-full-suite-red-test-spawn-move-cell-test → main 1 commit fork
3 files
changed +399
−322
docs/THREATS.md +1 −1
| @@ -343,7 +343,7 @@ What it can reach: everything a child forge can reach, no more: its own data dir | ||
| 343 | 343 | put there by an older release) is moved into a cell of its own, by an |
| 344 | 344 | admin's button or by the forge's own pass at boot and when one sleeps; a |
| 345 | 345 | move that does not prove itself is put back. |
| 346 | Enforced by: `test/spawn-move-cell.test.ts` | |
| 346 | Enforced by: `test/spawn-move-cell.test.ts`, `test/spawn-move-cell-sizes.test.ts` | |
| 347 | 347 | - Before such a forge leaves its parent's process its repositories are read, |
| 348 | 348 | running nothing, for what that process may have executed while it had no |
| 349 | 349 | wall: hooks beyond the forge's own stubs and config keys that make git run |
test/spawn-move-cell-sizes.test.ts added +398
| @@ -0,0 +1,398 @@ | ||
| 1 | /** | |
| 2 | * The rest of `spawn-move-cell.test.ts`, split from it to keep each file under the file budget: | |
| 3 | * a forge too big for its size is given the next; one whose repositories hold a hook is left for | |
| 4 | * the admin; the automatic pass; the admin's action. The cell is the same stand-in | |
| 5 | * (`helpers/moved-forge.ts`). | |
| 6 | */ | |
| 7 | import { | |
| 8 | afterAll, | |
| 9 | afterEach, | |
| 10 | beforeAll, | |
| 11 | describe, | |
| 12 | expect, | |
| 13 | test, | |
| 14 | } from "bun:test"; | |
| 15 | import { writeFileSync } from "node:fs"; | |
| 16 | import { join } from "node:path"; | |
| 17 | import { servedWindow, clientRows } from "../src/spawn/clients.ts"; | |
| 18 | import { provideCellsForTest, sleepIdleChildren } from "../src/spawn/host.ts"; | |
| 19 | import { moveDisk } from "../src/spawn/move_check.ts"; | |
| 20 | import { | |
| 21 | healthWait, | |
| 22 | keepSandboxing, | |
| 23 | moveToCell, | |
| 24 | sandboxPass, | |
| 25 | } from "../src/spawn/move_cell.ts"; | |
| 26 | import { IN_PROCESS_WORDS } from "../src/spawn/where.ts"; | |
| 27 | import { quietLogs } from "./helpers/globals.ts"; | |
| 28 | import { mcpTool } from "./helpers/mcp.ts"; | |
| 29 | import { | |
| 30 | inProcess, | |
| 31 | makeFamily, | |
| 32 | PARENT_HOST, | |
| 33 | type Family, | |
| 34 | } from "./helpers/moved-forge.ts"; | |
| 35 | ||
| 36 | quietLogs(); | |
| 37 | ||
| 38 | const window = servedWindow.ms; | |
| 39 | const wait = { ...healthWait }; | |
| 40 | const realBytes = moveDisk.bytes; | |
| 41 | beforeAll(() => { | |
| 42 | servedWindow.ms = 0; | |
| 43 | // A stand-in that does not come up is known at once: no test waits out a real cell's twenty | |
| 44 | // seconds. | |
| 45 | healthWait.ms = 3_000; | |
| 46 | healthWait.everyMs = 20; | |
| 47 | provideCellsForTest(() => Promise.resolve({ ok: true })); | |
| 48 | }); | |
| 49 | afterAll(() => { | |
| 50 | servedWindow.ms = window; | |
| 51 | Object.assign(healthWait, wait); | |
| 52 | provideCellsForTest(); | |
| 53 | }); | |
| 54 | ||
| 55 | const families: Family[] = []; | |
| 56 | async function family(name?: string): Promise<Family> { | |
| 57 | const f = await makeFamily(name); | |
| 58 | families.push(f); | |
| 59 | return f; | |
| 60 | } | |
| 61 | afterEach(async () => { | |
| 62 | moveDisk.bytes = realBytes; | |
| 63 | for (const f of families.splice(0)) await f.close(); | |
| 64 | }); | |
| 65 | ||
| 66 | const events = ( | |
| 67 | f: { parent: Family["parent"] }, | |
| 68 | kind: string, | |
| 69 | ): { payload: string }[] => | |
| 70 | f.parent.db | |
| 71 | .query< | |
| 72 | { payload: string }, | |
| 73 | [string] | |
| 74 | >("SELECT payload FROM event WHERE kind = ?1 ORDER BY rowid") | |
| 75 | .all(kind); | |
| 76 | const rowOf = async (f: Family) => | |
| 77 | (await clientRows(f.parent, { probe: false })).find( | |
| 78 | (r) => r.name === f.name, | |
| 79 | )!; | |
| 80 | ||
| 81 | describe("a forge too big for its size", () => { | |
| 82 | const MEMORY = | |
| 83 | "its cell did not come up: it ran out of memory before it answered"; | |
| 84 | const KEY = "FORGE_CHILD_MEMORY_MIB"; | |
| 85 | const was = process.env[KEY]; | |
| 86 | afterEach(() => { | |
| 87 | if (was === undefined) delete process.env[KEY]; | |
| 88 | else process.env[KEY] = was; | |
| 89 | }); | |
| 90 | ||
| 91 | test("is given the next size up when its cell runs out of memory, and its row says so", async () => { | |
| 92 | const f = await family("gamer"); | |
| 93 | const before = await f.facts(); | |
| 94 | f.cells.fail = (size) => (size === "small" ? MEMORY : undefined); | |
| 95 | const moved = await moveToCell(f.parent, f.name, { by: "admin" }); | |
| 96 | expect(moved).toEqual({ | |
| 97 | ok: true, | |
| 98 | name: "gamer", | |
| 99 | says: "gamer now runs in its own sandbox, with a medium forge's room: a small one did not hold it. Nothing else changed.", | |
| 100 | size: "medium", | |
| 101 | sizeWas: "small", | |
| 102 | }); | |
| 103 | expect(f.cells.starts).toEqual(["small", "medium"]); | |
| 104 | expect(f.ledger()).toMatchObject({ placement: "cell", size: "medium" }); | |
| 105 | expect(f.ledger()["moved"]).toMatchObject({ sizeWas: "small" }); | |
| 106 | const row = await rowOf(f); | |
| 107 | expect(row.size).toBe("medium"); | |
| 108 | expect(row.where.says).toEndWith( | |
| 109 | "; given a medium forge's room, a small one did not hold it", | |
| 110 | ); | |
| 111 | expect(events(f, "forge.moved").map((e) => JSON.parse(e.payload))).toEqual([ | |
| 112 | { name: "gamer", size: "medium", sizeWas: "small" }, | |
| 113 | ]); | |
| 114 | expect(await f.facts()).toEqual(before); | |
| 115 | }, 60_000); | |
| 116 | ||
| 117 | test("is put back at its own size, with the limit named, when the host has no room for the next", async () => { | |
| 118 | const f = await family("gamer"); | |
| 119 | f.cells.fail = (size) => (size === "small" ? MEMORY : undefined); | |
| 120 | process.env[KEY] = "3000"; // a small forge fits; a medium one in its place does not | |
| 121 | const failed = await moveToCell(f.parent, f.name, { by: "admin" }); | |
| 122 | expect(failed).toMatchObject({ ok: false, kind: "failed" }); | |
| 123 | expect((failed as { says: string }).says).toBe( | |
| 124 | `gamer was not moved: ${MEMORY}; it needs a medium forge's room, and a medium forge in place of a small one would pass this host's child memory limit (${KEY}=3000 MiB; 2048 MiB already held). It runs inside this forge's own process as it did before.`, | |
| 125 | ); | |
| 126 | expect(f.cells.starts).toEqual(["small"]); | |
| 127 | expect(f.ledger()).toMatchObject({ placement: "hosted", size: "small" }); | |
| 128 | expect((await rowOf(f)).where.says).toContain( | |
| 129 | "it needs a medium forge's room", | |
| 130 | ); | |
| 131 | }, 60_000); | |
| 132 | ||
| 133 | test("is never left stopped: a large forge that still runs out of memory is put back and says so", async () => { | |
| 134 | const f = await family("gamer"); | |
| 135 | f.cells.fail = () => MEMORY; | |
| 136 | const failed = await moveToCell(f.parent, f.name, { by: "admin" }); | |
| 137 | expect(failed).toMatchObject({ ok: false, kind: "failed" }); | |
| 138 | expect(f.cells.starts).toEqual(["small", "medium", "large"]); | |
| 139 | expect(f.ledger()).toMatchObject({ placement: "hosted", size: "small" }); | |
| 140 | expect((await f.ask("/healthz")).status).toBe(200); | |
| 141 | }, 60_000); | |
| 142 | ||
| 143 | test("is given the size its disk needs before its cell is started at all", async () => { | |
| 144 | const f = await family("gamer"); | |
| 145 | moveDisk.bytes = () => Promise.resolve(3000 * 1024 * 1024); | |
| 146 | const moved = await moveToCell(f.parent, f.name, { by: "admin" }); | |
| 147 | expect(moved).toMatchObject({ ok: true, size: "medium", sizeWas: "small" }); | |
| 148 | expect(f.cells.starts).toEqual(["medium"]); | |
| 149 | }, 60_000); | |
| 150 | }); | |
| 151 | ||
| 152 | describe("a forge whose repositories hold a hook", () => { | |
| 153 | test("is not moved by itself: the pass flags it for the admin by name, once, and it keeps running where it was", async () => { | |
| 154 | const f = await family(); | |
| 155 | writeFileSync( | |
| 156 | join(f.dir, "repos", "bob", "notes.git", "hooks", "post-update"), | |
| 157 | "#!/bin/sh\necho never-print-this\n", | |
| 158 | ); | |
| 159 | await sleepIdleChildren(f.parent, 0); | |
| 160 | const WHY = | |
| 161 | "its repositories hold what this forge's own process may have run (bob/notes: hooks post-update)"; | |
| 162 | const [held] = await sandboxPass(f.parent); | |
| 163 | expect(held).toEqual({ | |
| 164 | ok: false, | |
| 165 | name: f.name, | |
| 166 | kind: "held", | |
| 167 | says: `${f.name} is not moved by itself: ${WHY}. Look at them, then move it.`, | |
| 168 | }); | |
| 169 | expect(f.ledger()).toMatchObject({ placement: "hosted" }); | |
| 170 | expect((f.ledger()["moveHeld"] as { why: string }).why).toBe(WHY); | |
| 171 | expect(f.cells.starts).toEqual([]); | |
| 172 | const row = await rowOf(f); | |
| 173 | expect(row.where).toMatchObject({ | |
| 174 | inProcess: true, | |
| 175 | movable: true, | |
| 176 | held: true, | |
| 177 | says: `${IN_PROCESS_WORDS}; not moved by itself: ${WHY}`, | |
| 178 | }); | |
| 179 | // Said once, however often the pass looks; and never what the hook contains. | |
| 180 | await sandboxPass(f.parent); | |
| 181 | expect( | |
| 182 | events(f, "forge.move_held").map((e) => JSON.parse(e.payload)), | |
| 183 | ).toEqual([{ name: f.name, why: WHY }]); | |
| 184 | expect(JSON.stringify(f.ledger()) + JSON.stringify(row)).not.toContain( | |
| 185 | "never-print-this", | |
| 186 | ); | |
| 187 | expect((await f.ask("/healthz")).status).toBe(200); | |
| 188 | // The admin's move without saying they looked is held too; saying so moves it. | |
| 189 | expect(await moveToCell(f.parent, f.name, { by: "admin" })).toMatchObject({ | |
| 190 | ok: false, | |
| 191 | kind: "held", | |
| 192 | }); | |
| 193 | const moved = await moveToCell(f.parent, f.name, { | |
| 194 | by: "admin", | |
| 195 | acknowledge: true, | |
| 196 | }); | |
| 197 | expect(moved.ok).toBe(true); | |
| 198 | expect(f.ledger()).toMatchObject({ placement: "cell" }); | |
| 199 | expect(f.ledger()["moveHeld"]).toBeUndefined(); | |
| 200 | }, 60_000); | |
| 201 | }); | |
| 202 | ||
| 203 | describe("the automatic pass", () => { | |
| 204 | test("moves a forge only while it is asleep, and takes it when it next sleeps", async () => { | |
| 205 | const f = await family(); | |
| 206 | expect(inProcess(f.parent)).toEqual([f.name]); | |
| 207 | await keepSandboxing(f.parent); | |
| 208 | // Awake in the parent's process: left alone, nothing written. | |
| 209 | expect(f.ledger()).toMatchObject({ placement: "hosted" }); | |
| 210 | expect(f.ledger()["move"]).toBeUndefined(); | |
| 211 | expect(f.cells.starts).toEqual([]); | |
| 212 | // It sleeps; the pass that was waiting takes it. | |
| 213 | await sleepIdleChildren(f.parent, 0); | |
| 214 | for (let i = 0; i < 200 && f.ledger()["moved"] === undefined; i++) | |
| 215 | await Bun.sleep(25); // real-timer: the pass runs behind the idle sweep, not under this test's hand; this waits for it to end | |
| 216 | expect(f.ledger()).toMatchObject({ placement: "cell" }); | |
| 217 | expect(inProcess(f.parent)).toEqual([]); | |
| 218 | expect((await f.ask("/healthz")).status).toBe(200); | |
| 219 | expect(await sandboxPass(f.parent)).toEqual([]); | |
| 220 | }, 60_000); | |
| 221 | ||
| 222 | test("stops at the first forge it fails to move, and does not try that one again for a day", async () => { | |
| 223 | const f = await family(); | |
| 224 | await sleepIdleChildren(f.parent, 0); | |
| 225 | f.cells.fail = () => "its cell did not come up: it did not answer in time"; | |
| 226 | expect(await sandboxPass(f.parent)).toMatchObject([ | |
| 227 | { ok: false, kind: "failed" }, | |
| 228 | ]); | |
| 229 | expect(f.cells.starts.length).toBe(1); | |
| 230 | await sleepIdleChildren(f.parent, 0); | |
| 231 | expect(await sandboxPass(f.parent)).toEqual([]); | |
| 232 | expect(f.cells.starts.length).toBe(1); | |
| 233 | // An admin's move is never held back by that. | |
| 234 | f.cells.fail = () => undefined; | |
| 235 | await sleepIdleChildren(f.parent, 0); | |
| 236 | expect((await moveToCell(f.parent, f.name, { by: "admin" })).ok).toBe(true); | |
| 237 | }, 60_000); | |
| 238 | ||
| 239 | test("does nothing on a host that cannot make a cell, and drops an intent it finds there", async () => { | |
| 240 | const f = await family(); | |
| 241 | await sleepIdleChildren(f.parent, 0); | |
| 242 | provideCellsForTest(() => | |
| 243 | Promise.resolve({ ok: false, reason: "no cells on darwin" }), | |
| 244 | ); | |
| 245 | try { | |
| 246 | expect(await sandboxPass(f.parent)).toEqual([]); | |
| 247 | expect(await moveToCell(f.parent, f.name, { by: "admin" })).toMatchObject( | |
| 248 | { | |
| 249 | ok: false, | |
| 250 | kind: "blocked", | |
| 251 | says: `${f.name} cannot be moved now: this host cannot make a sandbox (no cells on darwin).`, | |
| 252 | }, | |
| 253 | ); | |
| 254 | writeFileSync( | |
| 255 | f.ledgerPath, | |
| 256 | JSON.stringify({ | |
| 257 | ...f.ledger(), | |
| 258 | move: { | |
| 259 | at: Date.now(), | |
| 260 | step: "intent", | |
| 261 | by: "auto", | |
| 262 | sizeWas: "small", | |
| 263 | }, | |
| 264 | }), | |
| 265 | ); | |
| 266 | expect((await f.ask("/healthz")).status).toBe(503); | |
| 267 | expect(await sandboxPass(f.parent)).toMatchObject([ | |
| 268 | { ok: false, kind: "blocked" }, | |
| 269 | ]); | |
| 270 | expect(f.ledger()["move"]).toBeUndefined(); | |
| 271 | expect((await f.ask("/healthz")).status).toBe(200); | |
| 272 | expect((await rowOf(f)).where).toMatchObject({ | |
| 273 | inProcess: true, | |
| 274 | movable: false, | |
| 275 | }); | |
| 276 | } finally { | |
| 277 | provideCellsForTest(() => Promise.resolve({ ok: true })); | |
| 278 | } | |
| 279 | }, 60_000); | |
| 280 | }); | |
| 281 | ||
| 282 | describe("the admin's action", () => { | |
| 283 | test("is a button on the forge's row, an API route and a tool; nobody else has it", async () => { | |
| 284 | const f = await family(); | |
| 285 | const page = await ( | |
| 286 | await fetch(`${f.servers.url}/forges`, { | |
| 287 | headers: { cookie: f.cookie, host: PARENT_HOST }, | |
| 288 | }) | |
| 289 | ).text(); | |
| 290 | expect(page).toContain(`action="/forges/${f.name}/sandbox"`); | |
| 291 | expect(page).toContain("Move into its own sandbox"); | |
| 292 | // The API plans unless told not to; the tool too. | |
| 293 | const post = (body: unknown, token = f.adminToken) => | |
| 294 | fetch(`${f.servers.url}/api/v1/forges/${f.name}/sandbox`, { | |
| 295 | method: "POST", | |
| 296 | headers: { | |
| 297 | authorization: `Bearer ${token}`, | |
| 298 | host: PARENT_HOST, | |
| 299 | "content-type": "application/json", | |
| 300 | }, | |
| 301 | body: JSON.stringify(body), | |
| 302 | }); | |
| 303 | const planned = await post({}); | |
| 304 | expect(planned.status).toBe(200); | |
| 305 | expect(await planned.json()).toMatchObject({ | |
| 306 | dryRun: true, | |
| 307 | name: f.name, | |
| 308 | blocked: null, | |
| 309 | held: null, | |
| 310 | }); | |
| 311 | expect(f.ledger()).toMatchObject({ placement: "hosted" }); | |
| 312 | const tool = await mcpTool(f.servers.url, f.adminToken, "forges", { | |
| 313 | forge: f.name, | |
| 314 | sandbox: true, | |
| 315 | }); | |
| 316 | expect(tool.value).toMatchObject({ | |
| 317 | name: f.name, | |
| 318 | blocked: null, | |
| 319 | plan: "This is the check; nothing moved. Send dryRun: false to move it.", | |
| 320 | }); | |
| 321 | expect( | |
| 322 | ( | |
| 323 | await fetch(`${f.servers.url}/api/v1/forges/${f.name}/sandbox`, { | |
| 324 | method: "POST", | |
| 325 | headers: { host: PARENT_HOST, "content-type": "application/json" }, | |
| 326 | body: "{}", | |
| 327 | }) | |
| 328 | ).status, | |
| 329 | ).toBeGreaterThanOrEqual(401); | |
| 330 | // The button. | |
| 331 | const pressed = await fetch(`${f.servers.url}/forges/${f.name}/sandbox`, { | |
| 332 | method: "POST", | |
| 333 | headers: { | |
| 334 | cookie: f.cookie, | |
| 335 | host: PARENT_HOST, | |
| 336 | origin: `https://${PARENT_HOST}`, | |
| 337 | "content-type": "application/x-www-form-urlencoded", | |
| 338 | }, | |
| 339 | body: "", | |
| 340 | }); | |
| 341 | expect(pressed.status).toBe(200); | |
| 342 | expect(await pressed.text()).toContain( | |
| 343 | `${f.name} now runs in its own sandbox. Nothing else changed.`, | |
| 344 | ); | |
| 345 | expect(f.ledger()).toMatchObject({ placement: "cell" }); | |
| 346 | const after = await ( | |
| 347 | await fetch(`${f.servers.url}/forges`, { | |
| 348 | headers: { cookie: f.cookie, host: PARENT_HOST }, | |
| 349 | }) | |
| 350 | ).text(); | |
| 351 | expect(after).not.toContain(`action="/forges/${f.name}/sandbox"`); | |
| 352 | expect(after).toContain('data-placement="cell"'); | |
| 353 | // Asked again over the API: already there, and said so. | |
| 354 | const again = await post({ dryRun: false }); | |
| 355 | expect(again.status).toBe(409); | |
| 356 | expect(await again.json()).toMatchObject({ | |
| 357 | moved: false, | |
| 358 | error: "blocked", | |
| 359 | }); | |
| 360 | const viaTool = await mcpTool(f.servers.url, f.adminToken, "forges", { | |
| 361 | forge: f.name, | |
| 362 | sandbox: true, | |
| 363 | dryRun: false, | |
| 364 | }); | |
| 365 | expect(viaTool.error).toContain("already runs in its own sandbox"); | |
| 366 | }, 60_000); | |
| 367 | ||
| 368 | test("moves it over the API and by the tool", async () => { | |
| 369 | const f = await family(); | |
| 370 | const res = await fetch( | |
| 371 | `${f.servers.url}/api/v1/forges/${f.name}/sandbox`, | |
| 372 | { | |
| 373 | method: "POST", | |
| 374 | headers: { | |
| 375 | authorization: `Bearer ${f.adminToken}`, | |
| 376 | host: PARENT_HOST, | |
| 377 | "content-type": "application/json", | |
| 378 | }, | |
| 379 | body: JSON.stringify({ dryRun: false }), | |
| 380 | }, | |
| 381 | ); | |
| 382 | expect(res.status).toBe(200); | |
| 383 | expect(await res.json()).toMatchObject({ | |
| 384 | dryRun: false, | |
| 385 | moved: true, | |
| 386 | name: f.name, | |
| 387 | size: "small", | |
| 388 | }); | |
| 389 | const g = await family("mark"); | |
| 390 | const tool = await mcpTool(g.servers.url, g.adminToken, "forges", { | |
| 391 | forge: "mark", | |
| 392 | sandbox: true, | |
| 393 | dryRun: false, | |
| 394 | }); | |
| 395 | expect(tool.value).toMatchObject({ ok: true, name: "mark" }); | |
| 396 | expect(g.ledger()).toMatchObject({ placement: "cell" }); | |
| 397 | }, 60_000); | |
| 398 | }); |
test/spawn-move-cell.test.ts −321
| @@ -25,18 +25,15 @@ import { provideCellsForTest, sleepIdleChildren } from "../src/spawn/host.ts"; | ||
| 25 | 25 | import { moveDisk } from "../src/spawn/move_check.ts"; |
| 26 | 26 | import { |
| 27 | 27 | healthWait, |
| 28 | keepSandboxing, | |
| 29 | 28 | moveToCell, |
| 30 | 29 | sandboxPass, |
| 31 | 30 | type MoveStep, |
| 32 | 31 | } from "../src/spawn/move_cell.ts"; |
| 33 | 32 | import { IN_PROCESS_WORDS, SANDBOXED_WORDS } from "../src/spawn/where.ts"; |
| 34 | 33 | import { quietLogs } from "./helpers/globals.ts"; |
| 35 | import { mcpTool } from "./helpers/mcp.ts"; | |
| 36 | 34 | import { |
| 37 | 35 | inProcess, |
| 38 | 36 | makeFamily, |
| 39 | PARENT_HOST, | |
| 40 | 37 | type Family, |
| 41 | 38 | } from "./helpers/moved-forge.ts"; |
| 42 | 39 | |
| @@ -369,321 +366,3 @@ describe("a move that fails is put back exactly as it was, and a repeat complete | ||
| 369 | 366 | }, 60_000); |
| 370 | 367 | }); |
| 371 | 368 | |
| 372 | describe("a forge too big for its size", () => { | |
| 373 | const MEMORY = | |
| 374 | "its cell did not come up: it ran out of memory before it answered"; | |
| 375 | const KEY = "FORGE_CHILD_MEMORY_MIB"; | |
| 376 | const was = process.env[KEY]; | |
| 377 | afterEach(() => { | |
| 378 | if (was === undefined) delete process.env[KEY]; | |
| 379 | else process.env[KEY] = was; | |
| 380 | }); | |
| 381 | ||
| 382 | test("is given the next size up when its cell runs out of memory, and its row says so", async () => { | |
| 383 | const f = await family("gamer"); | |
| 384 | const before = await f.facts(); | |
| 385 | f.cells.fail = (size) => (size === "small" ? MEMORY : undefined); | |
| 386 | const moved = await moveToCell(f.parent, f.name, { by: "admin" }); | |
| 387 | expect(moved).toEqual({ | |
| 388 | ok: true, | |
| 389 | name: "gamer", | |
| 390 | says: "gamer now runs in its own sandbox, with a medium forge's room: a small one did not hold it. Nothing else changed.", | |
| 391 | size: "medium", | |
| 392 | sizeWas: "small", | |
| 393 | }); | |
| 394 | expect(f.cells.starts).toEqual(["small", "medium"]); | |
| 395 | expect(f.ledger()).toMatchObject({ placement: "cell", size: "medium" }); | |
| 396 | expect(f.ledger()["moved"]).toMatchObject({ sizeWas: "small" }); | |
| 397 | const row = await rowOf(f); | |
| 398 | expect(row.size).toBe("medium"); | |
| 399 | expect(row.where.says).toEndWith( | |
| 400 | "; given a medium forge's room, a small one did not hold it", | |
| 401 | ); | |
| 402 | expect(events(f, "forge.moved").map((e) => JSON.parse(e.payload))).toEqual([ | |
| 403 | { name: "gamer", size: "medium", sizeWas: "small" }, | |
| 404 | ]); | |
| 405 | expect(await f.facts()).toEqual(before); | |
| 406 | }, 60_000); | |
| 407 | ||
| 408 | test("is put back at its own size, with the limit named, when the host has no room for the next", async () => { | |
| 409 | const f = await family("gamer"); | |
| 410 | f.cells.fail = (size) => (size === "small" ? MEMORY : undefined); | |
| 411 | process.env[KEY] = "3000"; // a small forge fits; a medium one in its place does not | |
| 412 | const failed = await moveToCell(f.parent, f.name, { by: "admin" }); | |
| 413 | expect(failed).toMatchObject({ ok: false, kind: "failed" }); | |
| 414 | expect((failed as { says: string }).says).toBe( | |
| 415 | `gamer was not moved: ${MEMORY}; it needs a medium forge's room, and a medium forge in place of a small one would pass this host's child memory limit (${KEY}=3000 MiB; 2048 MiB already held). It runs inside this forge's own process as it did before.`, | |
| 416 | ); | |
| 417 | expect(f.cells.starts).toEqual(["small"]); | |
| 418 | expect(f.ledger()).toMatchObject({ placement: "hosted", size: "small" }); | |
| 419 | expect((await rowOf(f)).where.says).toContain( | |
| 420 | "it needs a medium forge's room", | |
| 421 | ); | |
| 422 | }, 60_000); | |
| 423 | ||
| 424 | test("is never left stopped: a large forge that still runs out of memory is put back and says so", async () => { | |
| 425 | const f = await family("gamer"); | |
| 426 | f.cells.fail = () => MEMORY; | |
| 427 | const failed = await moveToCell(f.parent, f.name, { by: "admin" }); | |
| 428 | expect(failed).toMatchObject({ ok: false, kind: "failed" }); | |
| 429 | expect(f.cells.starts).toEqual(["small", "medium", "large"]); | |
| 430 | expect(f.ledger()).toMatchObject({ placement: "hosted", size: "small" }); | |
| 431 | expect((await f.ask("/healthz")).status).toBe(200); | |
| 432 | }, 60_000); | |
| 433 | ||
| 434 | test("is given the size its disk needs before its cell is started at all", async () => { | |
| 435 | const f = await family("gamer"); | |
| 436 | moveDisk.bytes = () => Promise.resolve(3000 * 1024 * 1024); | |
| 437 | const moved = await moveToCell(f.parent, f.name, { by: "admin" }); | |
| 438 | expect(moved).toMatchObject({ ok: true, size: "medium", sizeWas: "small" }); | |
| 439 | expect(f.cells.starts).toEqual(["medium"]); | |
| 440 | }, 60_000); | |
| 441 | }); | |
| 442 | ||
| 443 | describe("a forge whose repositories hold a hook", () => { | |
| 444 | test("is not moved by itself: the pass flags it for the admin by name, once, and it keeps running where it was", async () => { | |
| 445 | const f = await family(); | |
| 446 | writeFileSync( | |
| 447 | join(f.dir, "repos", "bob", "notes.git", "hooks", "post-update"), | |
| 448 | "#!/bin/sh\necho never-print-this\n", | |
| 449 | ); | |
| 450 | await sleepIdleChildren(f.parent, 0); | |
| 451 | const WHY = | |
| 452 | "its repositories hold what this forge's own process may have run (bob/notes: hooks post-update)"; | |
| 453 | const [held] = await sandboxPass(f.parent); | |
| 454 | expect(held).toEqual({ | |
| 455 | ok: false, | |
| 456 | name: f.name, | |
| 457 | kind: "held", | |
| 458 | says: `${f.name} is not moved by itself: ${WHY}. Look at them, then move it.`, | |
| 459 | }); | |
| 460 | expect(f.ledger()).toMatchObject({ placement: "hosted" }); | |
| 461 | expect((f.ledger()["moveHeld"] as { why: string }).why).toBe(WHY); | |
| 462 | expect(f.cells.starts).toEqual([]); | |
| 463 | const row = await rowOf(f); | |
| 464 | expect(row.where).toMatchObject({ | |
| 465 | inProcess: true, | |
| 466 | movable: true, | |
| 467 | held: true, | |
| 468 | says: `${IN_PROCESS_WORDS}; not moved by itself: ${WHY}`, | |
| 469 | }); | |
| 470 | // Said once, however often the pass looks; and never what the hook contains. | |
| 471 | await sandboxPass(f.parent); | |
| 472 | expect( | |
| 473 | events(f, "forge.move_held").map((e) => JSON.parse(e.payload)), | |
| 474 | ).toEqual([{ name: f.name, why: WHY }]); | |
| 475 | expect(JSON.stringify(f.ledger()) + JSON.stringify(row)).not.toContain( | |
| 476 | "never-print-this", | |
| 477 | ); | |
| 478 | expect((await f.ask("/healthz")).status).toBe(200); | |
| 479 | // The admin's move without saying they looked is held too; saying so moves it. | |
| 480 | expect(await moveToCell(f.parent, f.name, { by: "admin" })).toMatchObject({ | |
| 481 | ok: false, | |
| 482 | kind: "held", | |
| 483 | }); | |
| 484 | const moved = await moveToCell(f.parent, f.name, { | |
| 485 | by: "admin", | |
| 486 | acknowledge: true, | |
| 487 | }); | |
| 488 | expect(moved.ok).toBe(true); | |
| 489 | expect(f.ledger()).toMatchObject({ placement: "cell" }); | |
| 490 | expect(f.ledger()["moveHeld"]).toBeUndefined(); | |
| 491 | }, 60_000); | |
| 492 | }); | |
| 493 | ||
| 494 | describe("the automatic pass", () => { | |
| 495 | test("moves a forge only while it is asleep, and takes it when it next sleeps", async () => { | |
| 496 | const f = await family(); | |
| 497 | expect(inProcess(f.parent)).toEqual([f.name]); | |
| 498 | await keepSandboxing(f.parent); | |
| 499 | // Awake in the parent's process: left alone, nothing written. | |
| 500 | expect(f.ledger()).toMatchObject({ placement: "hosted" }); | |
| 501 | expect(f.ledger()["move"]).toBeUndefined(); | |
| 502 | expect(f.cells.starts).toEqual([]); | |
| 503 | // It sleeps; the pass that was waiting takes it. | |
| 504 | await sleepIdleChildren(f.parent, 0); | |
| 505 | for (let i = 0; i < 200 && f.ledger()["moved"] === undefined; i++) | |
| 506 | await Bun.sleep(25); // real-timer: the pass runs behind the idle sweep, not under this test's hand; this waits for it to end | |
| 507 | expect(f.ledger()).toMatchObject({ placement: "cell" }); | |
| 508 | expect(inProcess(f.parent)).toEqual([]); | |
| 509 | expect((await f.ask("/healthz")).status).toBe(200); | |
| 510 | expect(await sandboxPass(f.parent)).toEqual([]); | |
| 511 | }, 60_000); | |
| 512 | ||
| 513 | test("stops at the first forge it fails to move, and does not try that one again for a day", async () => { | |
| 514 | const f = await family(); | |
| 515 | await sleepIdleChildren(f.parent, 0); | |
| 516 | f.cells.fail = () => "its cell did not come up: it did not answer in time"; | |
| 517 | expect(await sandboxPass(f.parent)).toMatchObject([ | |
| 518 | { ok: false, kind: "failed" }, | |
| 519 | ]); | |
| 520 | expect(f.cells.starts.length).toBe(1); | |
| 521 | await sleepIdleChildren(f.parent, 0); | |
| 522 | expect(await sandboxPass(f.parent)).toEqual([]); | |
| 523 | expect(f.cells.starts.length).toBe(1); | |
| 524 | // An admin's move is never held back by that. | |
| 525 | f.cells.fail = () => undefined; | |
| 526 | await sleepIdleChildren(f.parent, 0); | |
| 527 | expect((await moveToCell(f.parent, f.name, { by: "admin" })).ok).toBe(true); | |
| 528 | }, 60_000); | |
| 529 | ||
| 530 | test("does nothing on a host that cannot make a cell, and drops an intent it finds there", async () => { | |
| 531 | const f = await family(); | |
| 532 | await sleepIdleChildren(f.parent, 0); | |
| 533 | provideCellsForTest(() => | |
| 534 | Promise.resolve({ ok: false, reason: "no cells on darwin" }), | |
| 535 | ); | |
| 536 | try { | |
| 537 | expect(await sandboxPass(f.parent)).toEqual([]); | |
| 538 | expect(await moveToCell(f.parent, f.name, { by: "admin" })).toMatchObject( | |
| 539 | { | |
| 540 | ok: false, | |
| 541 | kind: "blocked", | |
| 542 | says: `${f.name} cannot be moved now: this host cannot make a sandbox (no cells on darwin).`, | |
| 543 | }, | |
| 544 | ); | |
| 545 | writeFileSync( | |
| 546 | f.ledgerPath, | |
| 547 | JSON.stringify({ | |
| 548 | ...f.ledger(), | |
| 549 | move: { | |
| 550 | at: Date.now(), | |
| 551 | step: "intent", | |
| 552 | by: "auto", | |
| 553 | sizeWas: "small", | |
| 554 | }, | |
| 555 | }), | |
| 556 | ); | |
| 557 | expect((await f.ask("/healthz")).status).toBe(503); | |
| 558 | expect(await sandboxPass(f.parent)).toMatchObject([ | |
| 559 | { ok: false, kind: "blocked" }, | |
| 560 | ]); | |
| 561 | expect(f.ledger()["move"]).toBeUndefined(); | |
| 562 | expect((await f.ask("/healthz")).status).toBe(200); | |
| 563 | expect((await rowOf(f)).where).toMatchObject({ | |
| 564 | inProcess: true, | |
| 565 | movable: false, | |
| 566 | }); | |
| 567 | } finally { | |
| 568 | provideCellsForTest(() => Promise.resolve({ ok: true })); | |
| 569 | } | |
| 570 | }, 60_000); | |
| 571 | }); | |
| 572 | ||
| 573 | describe("the admin's action", () => { | |
| 574 | test("is a button on the forge's row, an API route and a tool; nobody else has it", async () => { | |
| 575 | const f = await family(); | |
| 576 | const page = await ( | |
| 577 | await fetch(`${f.servers.url}/forges`, { | |
| 578 | headers: { cookie: f.cookie, host: PARENT_HOST }, | |
| 579 | }) | |
| 580 | ).text(); | |
| 581 | expect(page).toContain(`action="/forges/${f.name}/sandbox"`); | |
| 582 | expect(page).toContain("Move into its own sandbox"); | |
| 583 | // The API plans unless told not to; the tool too. | |
| 584 | const post = (body: unknown, token = f.adminToken) => | |
| 585 | fetch(`${f.servers.url}/api/v1/forges/${f.name}/sandbox`, { | |
| 586 | method: "POST", | |
| 587 | headers: { | |
| 588 | authorization: `Bearer ${token}`, | |
| 589 | host: PARENT_HOST, | |
| 590 | "content-type": "application/json", | |
| 591 | }, | |
| 592 | body: JSON.stringify(body), | |
| 593 | }); | |
| 594 | const planned = await post({}); | |
| 595 | expect(planned.status).toBe(200); | |
| 596 | expect(await planned.json()).toMatchObject({ | |
| 597 | dryRun: true, | |
| 598 | name: f.name, | |
| 599 | blocked: null, | |
| 600 | held: null, | |
| 601 | }); | |
| 602 | expect(f.ledger()).toMatchObject({ placement: "hosted" }); | |
| 603 | const tool = await mcpTool(f.servers.url, f.adminToken, "forges", { | |
| 604 | forge: f.name, | |
| 605 | sandbox: true, | |
| 606 | }); | |
| 607 | expect(tool.value).toMatchObject({ | |
| 608 | name: f.name, | |
| 609 | blocked: null, | |
| 610 | plan: "This is the check; nothing moved. Send dryRun: false to move it.", | |
| 611 | }); | |
| 612 | expect( | |
| 613 | ( | |
| 614 | await fetch(`${f.servers.url}/api/v1/forges/${f.name}/sandbox`, { | |
| 615 | method: "POST", | |
| 616 | headers: { host: PARENT_HOST, "content-type": "application/json" }, | |
| 617 | body: "{}", | |
| 618 | }) | |
| 619 | ).status, | |
| 620 | ).toBeGreaterThanOrEqual(401); | |
| 621 | // The button. | |
| 622 | const pressed = await fetch(`${f.servers.url}/forges/${f.name}/sandbox`, { | |
| 623 | method: "POST", | |
| 624 | headers: { | |
| 625 | cookie: f.cookie, | |
| 626 | host: PARENT_HOST, | |
| 627 | origin: `https://${PARENT_HOST}`, | |
| 628 | "content-type": "application/x-www-form-urlencoded", | |
| 629 | }, | |
| 630 | body: "", | |
| 631 | }); | |
| 632 | expect(pressed.status).toBe(200); | |
| 633 | expect(await pressed.text()).toContain( | |
| 634 | `${f.name} now runs in its own sandbox. Nothing else changed.`, | |
| 635 | ); | |
| 636 | expect(f.ledger()).toMatchObject({ placement: "cell" }); | |
| 637 | const after = await ( | |
| 638 | await fetch(`${f.servers.url}/forges`, { | |
| 639 | headers: { cookie: f.cookie, host: PARENT_HOST }, | |
| 640 | }) | |
| 641 | ).text(); | |
| 642 | expect(after).not.toContain(`action="/forges/${f.name}/sandbox"`); | |
| 643 | expect(after).toContain('data-placement="cell"'); | |
| 644 | // Asked again over the API: already there, and said so. | |
| 645 | const again = await post({ dryRun: false }); | |
| 646 | expect(again.status).toBe(409); | |
| 647 | expect(await again.json()).toMatchObject({ | |
| 648 | moved: false, | |
| 649 | error: "blocked", | |
| 650 | }); | |
| 651 | const viaTool = await mcpTool(f.servers.url, f.adminToken, "forges", { | |
| 652 | forge: f.name, | |
| 653 | sandbox: true, | |
| 654 | dryRun: false, | |
| 655 | }); | |
| 656 | expect(viaTool.error).toContain("already runs in its own sandbox"); | |
| 657 | }, 60_000); | |
| 658 | ||
| 659 | test("moves it over the API and by the tool", async () => { | |
| 660 | const f = await family(); | |
| 661 | const res = await fetch( | |
| 662 | `${f.servers.url}/api/v1/forges/${f.name}/sandbox`, | |
| 663 | { | |
| 664 | method: "POST", | |
| 665 | headers: { | |
| 666 | authorization: `Bearer ${f.adminToken}`, | |
| 667 | host: PARENT_HOST, | |
| 668 | "content-type": "application/json", | |
| 669 | }, | |
| 670 | body: JSON.stringify({ dryRun: false }), | |
| 671 | }, | |
| 672 | ); | |
| 673 | expect(res.status).toBe(200); | |
| 674 | expect(await res.json()).toMatchObject({ | |
| 675 | dryRun: false, | |
| 676 | moved: true, | |
| 677 | name: f.name, | |
| 678 | size: "small", | |
| 679 | }); | |
| 680 | const g = await family("mark"); | |
| 681 | const tool = await mcpTool(g.servers.url, g.adminToken, "forges", { | |
| 682 | forge: "mark", | |
| 683 | sandbox: true, | |
| 684 | dryRun: false, | |
| 685 | }); | |
| 686 | expect(tool.value).toMatchObject({ ok: true, name: "mark" }); | |
| 687 | expect(g.ledger()).toMatchObject({ placement: "cell" }); | |
| 688 | }, 60_000); | |
| 689 | }); |
new above
offline