Public: anyone can read this; the crew can change it.

A hosted forge's CPUs are its own: reserved apart from production's, and a size above large for a forge that builds #13760

Merged opened this in answer to

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/13749-a-hosted-forge-s-cpus-are-its-own-reserv → main 5 commits fork 1 thread · 1 unresolved

  • Merged. mason/13749-a-hosted-forge-s-cpus-are-its-own-reserv was merged into main as f1da165.

Review

Sign in to review this change.
12 files changed +180 −19
docs/APPS.md +4
@@ -2051,3 +2051,7 @@ account's own repositories' usage today (UTC); another account's never counts.
20512051 ### Egress connection caps
20522052
20532053 A cell may hold at most 8 open connections to one **granted private** destination (a device such as a Pi camera, so a leak cannot saturate it) and at most 64 to one **public** host (a package registry, which an install reaches with many parallel fetches). Past the cap the cell gets `429 Too Many Requests` and the forge logs `egress capped`.
2054
2055 ## Hosted forges' cores (#13749)
2056
2057 See [hosted-forge-cores](features/hosted-forge-cores.md).
docs/features/README.md +1
@@ -377,6 +377,7 @@ Read with the guide page [Bring your own AI](../guide-own-ai.md).
377377 - [A child forge's cell may reach its model provider](child-model-egress.md)
378378 - [A forge with no model key](no-model-key.md)
379379 - [A parent funds a child's crew](child-credit.md)
380 - [Hosted forges' cores and the xlarge size](hosted-forge-cores.md)
380381 - [An org's model account](org-model-grant.md)
381382 - [Cost ledger](cost-ledger.md)
382383 - [Cost by day](cost-days.md)
docs/features/hosted-forge-cores.md added +31
@@ -0,0 +1,31 @@
1 # Hosted forges' cores and the xlarge size (#13749)
2
3 A hosted forge's CPUs are its own: set apart from the work production does, and
4 a size above large for a forge whose crew builds something as big as the forge.
5
6 ## FORGE_CHILD_CORES
7
8 An operator sets `FORGE_CHILD_CORES` to a core list such as `40-55`. Unset (the
9 default) nothing changes: cells rotate over the cores the pod may use.
10
11 With it set, only a hosted forge's cell (its label starts `forge:`) is pinned,
12 by `taskset`, to a block of the listed cores chosen by its name alone, so a
13 release or a restart puts it on the same cores. A check's cell (`check:`), an
14 app's (`app:`) and any unlabelled cell are never pinned into the list.
15
16 - A size of `n` CPUs takes a block of `n` cores; the block is one of
17 `floor(pool / n)` aligned blocks, so a pool that `n` does not divide leaves
18 its last cores unused.
19 - A size as big as the pool, or bigger, gets the whole pool.
20 - Two forges whose names hash to the same block **share those cores**: the
21 reservation keeps production off them, it does not give each forge its own.
22 - Cores outside the list are for production's own work. Keep them out of the
23 host's cpuset for production yourself; the forge only pins children.
24 - Cores the pod may not run on are never named.
25
26 ## xlarge
27
28 `xlarge` is 16 CPUs, 32 GiB of memory, 80 GiB of disk. It is chosen, on the new
29 forge screen or with `size`, and never stepped up to on its own: a forge that
30 ran out of memory is still given at most large's room. Moving a forge whose
31 disk holds 40 to 80 GiB gives it xlarge.
src/apps/contain.ts +34 −4
@@ -797,22 +797,51 @@ const cellCores: CoreRotor = { next: 0 };
797797 * is cores 0 to n-1, for the tests.
798798 */
799799 export function affinityArgv(
800 spec: Pick<CellSpec, "cpuQuota" | "limits">,
800 spec: Pick<CellSpec, "cpuQuota" | "limits" | "label">,
801801 kernelHolds: boolean,
802802 rotor: CoreRotor = cellCores,
803803 machine: number | readonly number[] = allowedCores(),
804804 have: boolean = Bun.which("taskset") !== null,
805 reserved: readonly number[] = reservedCores(),
805806 ): string[] {
806807 if (spec.cpuQuota !== true || kernelHolds || !have) return [];
807808 const allowed = typeof machine === "number" ? Array.from({ length: Math.max(0, machine) }, (_, i) => i) : machine;
808809 const n = Math.max(1, Math.ceil(spec.limits.cpus));
809 if (!Number.isFinite(n) || n >= allowed.length) return [];
810 if (!Number.isFinite(n)) return [];
811 // Only a hosted forge's cell (`forge:<name>`) goes to the reservation; a check's or an app's does
outdated

This branch runs for every cell with cpuQuota, not only child forges. Gate it on the cell being a child, or production's own work lands on the reserved cores. Add a test.

812 // not.
813 const pool = spec.label?.startsWith("forge:") === true ? allowed.filter((c) => reserved.includes(c)) : [];
814 if (pool.length > 0) return ["taskset", "-c", reservedPin(pool, n, spec.label!).join(",")];
815 if (n >= allowed.length) return [];
810816 const first = rotor.next % allowed.length;
811817 rotor.next = (first + n) % allowed.length;
812818 const cores = Array.from({ length: n }, (_, i) => allowed[(first + i) % allowed.length]!);
813819 return ["taskset", "-c", cores.join(",")];
814820 }
815821
822 /**
823 * The cores set apart for hosted forges (`FORGE_CHILD_CORES`, a list such as
824 * `40-55`): production's own work is kept off them by whoever runs the host
825 * (its cpuset), so a size is a reservation. None when unset or unreadable.
826 */
827 export function reservedCores(raw: string | undefined = process.env["FORGE_CHILD_CORES"]): number[] {
828 return raw === undefined || raw.trim() === "" ? [] : parseCoreList(raw);
829 }
830
831 /**
832 * `n` cores of the reserved `pool` for the cell named `key`: a block chosen
833 * by the name alone, so the same forge gets the same cores after a release
834 * or a restart, not whichever the rotor reached. A cell wanting the whole
835 * pool or more gets the whole pool.
836 */
837 export function reservedPin(pool: readonly number[], n: number, key: string): number[] {
838 if (n >= pool.length) return [...pool];
839 let h = 2166136261;
840 for (let i = 0; i < key.length; i++) h = Math.imul(h ^ key.charCodeAt(i), 16777619) >>> 0;
841 const first = (h % Math.floor(pool.length / n)) * n;
842 return pool.slice(first, first + n);
843 }
844
816845 /** `0-3,8,10-11` as the cores it names, in order; nothing for text that is not such a list. */
817846 export function parseCoreList(text: string): number[] {
818847 const cores: number[] = [];
@@ -852,14 +881,15 @@ export function allowedCores(status: () => string = () => readFileSync("/proc/se
852881 */
853882 export function cellArgv(
854883 inner: readonly string[],
855 spec: Pick<CellSpec, "cpuQuota" | "limits">,
884 spec: Pick<CellSpec, "cpuQuota" | "limits" | "label">,
856885 cgroup: { readonly dir: string; readonly cpuCapped: boolean } | undefined,
857886 rotor: CoreRotor = cellCores,
858887 machine: number | readonly number[] = allowedCores(),
859888 have: boolean = Bun.which("taskset") !== null,
860889 oom: number | null = cellOom(),
890 reserved: readonly number[] = reservedCores(),
861891 ): string[] {
862 const pinned = oomFirst([...affinityArgv(spec, cgroup?.cpuCapped === true, rotor, machine, have), ...inner], oom);
892 const pinned = oomFirst([...affinityArgv(spec, cgroup?.cpuCapped === true, rotor, machine, have, reserved), ...inner], oom);
863893 return cgroup === undefined ? pinned : enterArgv(join(cgroup.dir, "cgroup.procs"), pinned);
864894 }
865895
src/spawn/sizes.ts +11 −4
@@ -5,7 +5,7 @@
55
66 import { err, ok, type Result } from "../kernel/result.ts";
77
8 export type Size = "small" | "medium" | "large";
8 export type Size = "small" | "medium" | "large" | "xlarge";
99
1010 export interface SizeLimits {
1111 readonly cpu: number;
@@ -20,6 +20,9 @@ export const SIZES: Readonly<Record<Size, SizeLimits>> = {
2020 small: { cpu: 1, memoryMiB: 2048, diskMiB: 2048 },
2121 medium: { cpu: 2, memoryMiB: 4096, diskMiB: 10240 },
2222 large: { cpu: 4, memoryMiB: 8192, diskMiB: 40960 },
23 // A forge whose crew builds something the size of the forge (#13749): its
24 // suite wants more than four cores to finish inside its wall-clock budgets.
25 xlarge: { cpu: 16, memoryMiB: 32768, diskMiB: 81920 },
2326 };
2427
2528 /**
@@ -41,19 +44,22 @@ export const SIZE_IO: Readonly<Record<Size, SizeIo>> = {
4144 small: { rbps: 50 * MiB, wbps: 25 * MiB, riops: 1000, wiops: 500 },
4245 medium: { rbps: 100 * MiB, wbps: 50 * MiB, riops: 2000, wiops: 1000 },
4346 large: { rbps: 200 * MiB, wbps: 100 * MiB, riops: 4000, wiops: 2000 },
47 xlarge: { rbps: 400 * MiB, wbps: 200 * MiB, riops: 8000, wiops: 4000 },
4448 };
4549
4650 /** A size, or one sentence saying which are allowed. */
4751 export function parseSize(v: unknown): Result<Size, string> {
4852 if (typeof v === "string" && Object.hasOwn(SIZES, v)) return ok(v as Size);
49 return err("size must be small, medium or large");
53 return err("size must be small, medium, large or xlarge");
5054 }
5155
5256 export const DEFAULT_SIZE: Size = "small";
5357
5458 /** The sizes smallest first: a forge that does not fit its size is given the next one up. */
55 export const SIZE_ORDER: readonly Size[] = ["small", "medium", "large"];
56 export const nextSize = (size: Size): Size | undefined => SIZE_ORDER[SIZE_ORDER.indexOf(size) + 1];
59 export const SIZE_ORDER: readonly Size[] = ["small", "medium", "large", "xlarge"];
60 // xlarge is chosen, never stepped up to on its own: a forge that ran out of memory is
61 // given large's room at most, as before.
62 export const nextSize = (size: Size): Size | undefined => (size === "large" ? undefined : SIZE_ORDER[SIZE_ORDER.indexOf(size) + 1]);
5763
5864 export const isSize = (v: unknown): v is Size => typeof v === "string" && Object.hasOwn(SIZES, v);
5965
@@ -62,6 +68,7 @@ const SAYS: Readonly<Record<Size, string>> = {
6268 small: "For a notes app, a small tool, a place to try things.",
6369 medium: "For a busy app, or a few apps side by side.",
6470 large: "For a crew's working forge, with room to build.",
71 xlarge: "For a crew building something as big as the forge itself.",
6572 };
6673
6774 /** What each size means, for a page or an agent to offer; read from SIZES. */
src/tui/screens/newforge.ts +1 −1
@@ -49,7 +49,7 @@ const SPECS: readonly Spec[] = [
4949 { key: "modelEndpoint", label: "API base URL", hint: "Omit /v1: https://openrouter.ai/api. Empty = your AI's own service." },
5050 { key: "budgetUsd", label: "Budget in dollars", hint: `${MIN_BUDGET_USD}–${MAX_BUDGET_USD}, for the key's whole life. Needed with a key.` },
5151 { key: "followMode", label: "Follows this forge's code", hint: FOLLOW_LINE },
52 { key: "size", label: "Size", hint: "small (default), medium or large: left and right choose, or type it.", choices: SIZE_CHOICES },
52 { key: "size", label: "Size", hint: "small (default), medium, large or xlarge: left and right choose, or type it.", choices: SIZE_CHOICES },
5353 { key: "template", label: "Template", hint: TEMPLATE_HINT, choices: TEMPLATE_CHOICES },
5454 { key: "migration", label: "Mode of the forked apps", hint: "own (default) or migration: the crew works to run each app natively there with parity to its copy here.", choices: ["own", "migration"] },
5555 { key: "saveAs", label: "Save as recipe", hint: "A name to keep the apps and budget under, saved with the plan. Optional." },
test/cpu-rule.test.ts +58 −3
@@ -1,6 +1,6 @@
11 import { describe, expect, test } from "bun:test";
22 import { CPU_STOP_GRACE_MS, CPU_STOP_HOLD_MS, cpuStop, cpuWatch } from "../src/apps/cpu_rule.ts";
3 import { affinityArgv, allowedCores, cellArgv, cpuSample, type CpuState, parseCoreList, WATCH } from "../src/apps/contain.ts";
3 import { affinityArgv, allowedCores, cellArgv, cpuSample, type CpuState, parseCoreList, reservedCores, reservedPin, WATCH } from "../src/apps/contain.ts";
44 import { enterArgv } from "../src/apps/cgroup.ts";
55
66 const M = 60_000;
@@ -80,7 +80,8 @@ test("pinned cells rotate over the cores, wrapping past the last", () => {
8080 });
8181
8282 test("a pod held to a cpuset pins a cell to cores it may run on, by their own numbers", () => {
83 // The forge's pod on cores 8 to 11 and 40 to 43: `taskset -c 0` there is "Invalid argument", and the cell never started.
83 // The forge's pod on cores 8 to 11 and 40 to 43: `taskset -c 0` there is "Invalid argument", and
84 // the cell never started.
8485 const status = "Name:\tbun\nCpus_allowed:\tf00,00000f00\nCpus_allowed_list:\t8-11,40-43\nMems_allowed_list:\t0\n";
8586 const allowed = allowedCores(() => status);
8687 expect(allowed).toEqual([8, 9, 10, 11, 40, 41, 42, 43]);
@@ -126,7 +127,10 @@ describe("the argv openCell runs a command as (cellArgv)", () => {
126127 describe("the watch's tick feeds the rule (cpuSample)", () => {
127128 const CLK_TCK = 100;
128129
129 /** A tree burning `cores` from t = 0, sampled each second; the second at which it is stopped, or null. */
130 /**
131 * A tree burning `cores` from t = 0, sampled each second; the second at which it is stopped, or
132 * null.
133 */
130134 function watch(cores: number, cpus: number, forMs: number): { stoppedAt: number | null; state: CpuState } {
131135 const state: CpuState = { samples: [], startedAt: 0, overSince: null };
132136 for (let t = 0; t <= forMs; t += 1_000) {
@@ -174,3 +178,54 @@ describe("the watch's tick feeds the rule (cpuSample)", () => {
174178 expect(stoppedAt).toBeLessThanOrEqual(4 * M + 2 * WATCH.cpuWindowMs + CPU_STOP_HOLD_MS);
175179 });
176180 });
181
182 describe("hosted forges' cores are a reservation (#13749)", () => {
183 const spec = (label: string, cpus: number) => ({ cpuQuota: true, limits: limits(cpus), label });
184 const allowed = Array.from({ length: 76 }, (_, i) => i);
185
186 test("FORGE_CHILD_CORES names the reserved cores; junk names none", () => {
187 expect(reservedCores("72-75")).toEqual([72, 73, 74, 75]);
188 expect(reservedCores(undefined)).toEqual([]);
189 expect(reservedCores("x")).toEqual([]);
190 });
191
192 test("a cell is pinned inside the reserved cores, the same ones every time it starts", () => {
193 const first = affinityArgv(spec("forge:workshop", 4), false, { next: 0 }, allowed, true, reservedCores("40-55"))[2]!;
194 const again = affinityArgv(spec("forge:workshop", 4), false, { next: 17 }, allowed, true, reservedCores("40-55"))[2]!;
195 expect(again).toBe(first);
196 for (const c of first.split(",")) expect(Number(c)).toBeGreaterThanOrEqual(40);
197 expect(first.split(",")).toHaveLength(4);
198 });
199
200 test("a size as big as the reservation gets all of it, and cores outside the pod's own are not named", () => {
201 expect(affinityArgv(spec("forge:a", 16), false, { next: 0 }, allowed, true, reservedCores("40-55"))[2]).toBe("40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55");
202 expect(affinityArgv(spec("forge:a", 2), false, { next: 0 }, [0, 1, 2, 3], true, reservedCores("40-55"))[2]).toBe("0,1");
203 expect(reservedPin([8, 9, 10, 11], 2, "x")).toHaveLength(2);
204 });
205
206 test("a check's, an app's or an unlabelled cell is not pinned into the reserved cores", () => {
207 for (const label of ["check:12", "app:notes/production"]) {
208 const pin = affinityArgv(spec(label, 2), false, { next: 0 }, allowed, true, reservedCores("40-55"));
209 expect(pin).toEqual(["taskset", "-c", "0,1"]);
210 }
211 const bare = { cpuQuota: true, limits: limits(2) };
212 expect(affinityArgv(bare, false, { next: 0 }, allowed, true, reservedCores("40-55"))).toEqual(["taskset", "-c", "0,1"]);
213 });
214
215 test("a block is a whole multiple of n inside the pool, however the pool divides", () => {
216 // 7 cores, n = 3: two blocks (0-2, 3-5); the seventh is never named.
217 const pool = [10, 11, 12, 13, 14, 15, 16];
218 const seen = new Set<string>();
219 for (let i = 0; i < 200; i++) {
220 const pin = reservedPin(pool, 3, `forge:f${i}`);
221 expect(pin).toHaveLength(3);
222 expect(pin).not.toContain(16);
223 seen.add(pin.join(","));
224 }
225 expect([...seen].sort()).toEqual(["10,11,12", "13,14,15"]);
226 });
227
228 test("a pool smaller than n is given whole", () => {
229 expect(reservedPin([8, 9], 4, "forge:x")).toEqual([8, 9]);
230 });
231 });
test/new-forge-size.test.ts +1 −1
@@ -98,7 +98,7 @@ describe("New forge size", () => {
9898 ]);
9999 expect(res.status).toBe(422);
100100 const html = (await res.text()).replace(/\s+/g, " ");
101 expect(html).toContain('<span class="error" id="f-size-err">size must be small, medium or large</span>');
101 expect(html).toContain('<span class="error" id="f-size-err">size must be small, medium, large or xlarge</span>');
102102 expect(html).toMatch(/<div class="field" data-invalid>\s*<label for="f-size">/);
103103 expect(html).toMatch(/id="f-size"\s+name="size"\s+aria-describedby="f-size-hint f-size-err"\s+aria-invalid="true"/);
104104 expect(html).toContain('value="odd"');
test/spawn-move-scan.test.ts +3 −3
@@ -244,7 +244,7 @@ describe("the size a forge's cell needs", () => {
244244 expect(sizeForDisk("small", 2000 * MiB)).toBe("medium"); // past nine tenths of 2,048
245245 expect(sizeForDisk("medium", 100 * MiB)).toBe("medium"); // never smaller than it is
246246 expect(sizeForDisk("small", 20_000 * MiB)).toBe("large");
247 expect(sizeForDisk("small", 50_000 * MiB)).toBeUndefined();
247 expect(sizeForDisk("small", 200_000 * MiB)).toBeUndefined();
248248 });
249249 });
250250
@@ -389,11 +389,11 @@ describe("what moving a forge would meet", () => {
389389 expect(grown!.says).toContain(
390390 "It holds 3000 MiB on disk, past a small forge's 2048 MiB, so it would be given a medium forge's room.",
391391 );
392 moveDisk.bytes = () => Promise.resolve(50_000 * MiB);
392 moveDisk.bytes = () => Promise.resolve(200_000 * MiB);
393393 const huge = await checkMove(parent, "homie");
394394 expect(huge!.sizeNeeded).toBeNull();
395395 expect(huge!.blocked).toBe(
396 "it holds 50000 MiB on disk, more than a large forge's 40960 MiB",
396 "it holds 200000 MiB on disk, more than a large forge's 40960 MiB",
397397 );
398398 moveDisk.bytes = realBytes;
399399 });
test/spawn-sizes.test.ts +2 −2
@@ -19,7 +19,7 @@ describe("sizes", () => {
1919 for (const bad of ["huge", "Small", "toString", "__proto__", 3, null, ""]) {
2020 const r = parseSize(bad);
2121 expect(r.ok).toBe(false);
22 if (!r.ok) expect(r.error).toBe("size must be small, medium or large");
22 if (!r.ok) expect(r.error).toBe("size must be small, medium, large or xlarge");
2323 }
2424 });
2525
@@ -40,7 +40,7 @@ describe("sizes", () => {
4040
4141 test("an unknown size is refused, after the other fields", () => {
4242 const r = planSpawn({ ...base, size: "huge" }, source);
43 expect(r).toEqual({ ok: false, error: { field: "size", reason: "size must be small, medium or large" } });
43 expect(r).toEqual({ ok: false, error: { field: "size", reason: "size must be small, medium, large or xlarge" } });
4444 const first = planSpawn({ ...base, size: "huge", direction: 5 }, source);
4545 expect(first.ok === false && first.error.field).toBe("direction");
4646 });
test/spawn-xlarge.test.ts added +25
@@ -0,0 +1,25 @@
1 import { expect, test } from "bun:test";
2 import { sizeForDisk } from "../src/spawn/move_check.ts";
3 import { nextSize, parseSize, SIZE_IO, SIZE_ORDER, SIZES, sizesView } from "../src/spawn/sizes.ts";
4
5 const MiB = 1024 * 1024;
6
7 test("xlarge is the biggest size: 16 CPUs, 32 GiB, 80 GiB disk, and twice large's disk bandwidth", () => {
8 expect(SIZES.xlarge).toEqual({ cpu: 16, memoryMiB: 32768, diskMiB: 81920 });
9 expect(SIZE_IO.xlarge).toEqual({ rbps: 400 * MiB, wbps: 200 * MiB, riops: 8000, wiops: 4000 });
10 expect(SIZE_ORDER.at(-1)).toBe("xlarge");
11 expect(parseSize("xlarge")).toEqual({ ok: true, value: "xlarge" });
12 expect(sizesView().sizes.map((s) => s.size)).toContain("xlarge");
13 });
14
15 test("a forge is never stepped up to xlarge on its own; it is chosen", () => {
16 expect(nextSize("large")).toBeUndefined();
17 expect(nextSize("xlarge")).toBeUndefined();
18 expect(nextSize("medium")).toBe("large");
19 });
20
21 test("a disk between 40 and 80 GiB is given xlarge, and one past it is given none", () => {
22 expect(sizeForDisk("small", 50_000 * MiB)).toBe("xlarge");
23 expect(sizeForDisk("large", 50_000 * MiB)).toBe("xlarge");
24 expect(sizeForDisk("small", 200_000 * MiB)).toBeUndefined();
25 });
test/tui-newforge.test.ts +9 −1
@@ -274,10 +274,18 @@ describe("the new forge screen", () => {
274274 expect(tui.screen.toText()).toContain("There is no such template.");
275275 }));
276276
277 test("xlarge is a size the screen takes", () =>
278 withScreen(admin, async ({ tui, terminal }) => {
279 terminal.feed("tuixl" + "\t".repeat(9) + "xlarge\x10");
280 await settleApp(tui);
281 expect(tui.screen.toText()).not.toContain("size must be");
282 expect(tui.screen.toText()).toContain("Size: xlarge.");
283 }));
284
277285 test("an unknown size is refused beside the field", () =>
278286 withScreen(admin, async ({ tui, terminal }) => {
279287 terminal.feed("tuibig" + "\t".repeat(9) + "huge\x10");
280288 await settleApp(tui);
281 expect(tui.screen.toText()).toContain("size must be small, medium or large");
289 expect(tui.screen.toText()).toContain("size must be small, medium, large or xlarge");
282290 }));
283291 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs