Public: anyone can read this; the crew can change it.
Hosted forges' cells run on CPUs apart from the forge's own serving cores, set on a page with its reason #13673
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/13642-hosted-forges-cells-run-on-cpus-apart-fr → main 15 commits fork 3 threads · 3 unresolved
10 files
changed +413
−40
docs/MCP.md +1 −1
| @@ -88,7 +88,7 @@ bytes in `tools/list`. After changing a tool run `bun scripts/mcp-table.ts`; | ||
| 88 | 88 | | `boards` | — | The forge's boards for person admins: slug, title, range, refresh and panels | 518 | |
| 89 | 89 | | `branches` | `create`, `delete`, `branches`, `tags` | A repo's branches; `name`, `action` create (from `from`) or delete one, not the default | 407 | |
| 90 | 90 | | `builds` | — | A repo's build pins (`refs/builds/`): label, commit, pinned by, note | 231 | |
| 91 | | `capacity` | — | Forge capacity and guards, live | 514 | | |
| 91 | | `capacity` | — | Forge capacity and guards, live | 535 | | |
| 92 | 92 | | `claim_link` | `revoke`, `undo` | A child's claim link: `reveal` a new one, or `action` revoke/undo | 304 | |
| 93 | 93 | | `close` | — | Close or reopen (`state` open) an issue or pull | 463 | |
| 94 | 94 | | `cloud_account` | — | Change a cloud account by `verb` (admin) | 549 | |
docs/features/README.md +1
| @@ -340,6 +340,7 @@ Read with the guide page [Hosting forges for others, and getting paid](../guide- | ||
| 340 | 340 | - [](billing.md#what-a-paid-forge-is-at-terms-13207) |
| 341 | 341 | - [](billing-links.md) |
| 342 | 342 | - [](capacity.md) |
| 343 | - [](serving-cores.md) | |
| 343 | 344 | - [](cloud.md) |
| 344 | 345 | - [](machines-disks.md) |
| 345 | 346 | - [](forge-ready.md) |
docs/features/serving-cores.md added +11
| @@ -0,0 +1,11 @@ | ||
| 1 | # CPUs kept for the forge's own process (#13642) | |
| 2 | ||
| 3 | On a busy machine, cells and checks running on every core can stall the forge's own serving thread, so pages wait. The capacity setting `cells.servingCores` (**Settings → Capacity**, "CPUs kept for the forge itself"; env `FORGE_SERVING_CORES`) keeps that many CPUs for the forge alone. | |
| 4 | ||
| 5 | - **What it does:** the forge keeps the first N cores of the ones its process may run on (in a pod held to a cpuset, those are the cpuset's own numbers, not 0 to N-1). Every cell is started under `taskset -c` on the other cores: apps, terminals, hosted child forges and check steps. A check's shards are forked inside its step's cell, so they keep off those cores too. A cell with a CPU quota of its own still rotates over its cores, but only among the cores left over. | |
| 6 | - **Default, by the CPUs the forge has:** none below 8 CPUs, so a small forge runs as it always has. From 8 CPUs up it keeps one per 16, at least 1 and at most 4: 8–31 CPUs keep 1, 32–47 keep 2, 48–63 keep 3, 64 or more keep 4. | |
| 7 | - **The cap:** cells always keep at least half the cores. A setting above half of them is held to half. | |
| 8 | - **No `taskset`:** where util-linux's `taskset` is missing, nothing is pinned and cells run on every core as before. The forge's log says so once: "cpu: taskset is absent, so the N cores kept for this forge are not held; cells run on every core". | |
| 9 | - **When it applies:** the setting is read every time a cell opens, so a change reaches the next cell started. Cells already running keep the cores they had. | |
| 10 | ||
| 11 | Set it like any capacity row, with a reason. **Put back** returns it to the default. |
src/actions/capacity.ts +61 −21
| @@ -16,7 +16,6 @@ import { | ||
| 16 | 16 | bootDefault, |
| 17 | 17 | isCapacityKey, |
| 18 | 18 | setCapacity, |
| 19 | storedCapacity, | |
| 20 | 19 | validCapacity, |
| 21 | 20 | type CapacityChange, |
| 22 | 21 | type CapacityKey, |
| @@ -48,6 +47,7 @@ export const SETTABLE_CAPACITY = [ | ||
| 48 | 47 | "checks.concurrency", |
| 49 | 48 | "checks.shards", |
| 50 | 49 | "cells.egress", |
| 50 | "cells.servingCores", | |
| 51 | 51 | "guard_new_accounts", |
| 52 | 52 | "signin.edge", |
| 53 | 53 | "children.ownFilesystem", |
| @@ -101,16 +101,57 @@ function admin(actor: User | undefined): Result<User, "not_found"> { | ||
| 101 | 101 | : err("not_found"); |
| 102 | 102 | } |
| 103 | 103 | |
| 104 | /** Who set `key` and when, only while its stored value is not the default. */ | |
| 105 | export function changedOf(db: Database, key: CapacityKey): { changed?: { at: string; by: string } } { | |
| 106 | const value = storedCapacity(db, key); | |
| 107 | if (value === undefined || value === bootDefault(db, key)) return {}; | |
| 108 | const row = db | |
| 109 | .query<{ updated_at: string; username: string | null }, [string]>( | |
| 110 | "SELECT s.updated_at, u.username FROM forge_setting s LEFT JOIN user u ON u.id = s.updated_by WHERE s.key = ?1", | |
| 111 | ) | |
| 112 | .get(key); | |
| 113 | return row === null ? {} : { changed: { at: row.updated_at, by: row.username ?? "someone" } }; | |
| 104 | interface StoredRow { | |
| 105 | readonly key: string; | |
| 106 | readonly value: string; | |
| 107 | readonly updated_at: string; | |
| 108 | readonly username: string | null; | |
| 109 | } | |
| 110 | ||
| 111 | function storedRow(db: Database, key: CapacityKey): StoredRow | undefined { | |
| 112 | return ( | |
| 113 | db | |
| 114 | .query<StoredRow, [string]>( | |
| 115 | "SELECT s.key, s.value, s.updated_at, u.username FROM forge_setting s LEFT JOIN user u ON u.id = s.updated_by WHERE s.key = ?1", | |
| 116 | ) | |
| 117 | .get(key) ?? undefined | |
| 118 | ); | |
| 119 | } | |
| 120 | ||
| 121 | /** | |
| 122 | * Who set `key` and when, only while its stored value is not the default. | |
| 123 | * `row` is the stored row when the caller has read it already (the list reads | |
| 124 | * every row in one statement); left out, it is read here. | |
| 125 | */ | |
| 126 | export function changedOf( | |
| 127 | db: Database, | |
| 128 | key: CapacityKey, | |
| 129 | row: StoredRow | undefined = storedRow(db, key), | |
| 130 | ): { changed?: { at: string; by: string } } { | |
| 131 | const value = row !== undefined && validCapacity(key, row.value) ? row.value : undefined; | |
| 132 | if (row === undefined || value === undefined || value === bootDefault(db, key)) return {}; | |
| 133 | return { changed: { at: row.updated_at, by: row.username ?? "someone" } }; | |
| 134 | } | |
| 135 | ||
| 136 | /** | |
| 137 | * One capacity row from what is stored, read once for the whole list: the | |
| 138 | * same answer `storedCapacity`, `capacityValue` and `changedOf` give a key, | |
| 139 | * without a statement per row (the page's budget, `pages-budget.test.ts`). | |
| 140 | */ | |
| 141 | function capacityRow(db: Database, key: CapacityKey, row: StoredRow | undefined): CapacityRow { | |
| 142 | const stored = row !== undefined && validCapacity(key, row.value) ? row.value : undefined; | |
| 143 | const boot = bootDefault(db, key); | |
| 144 | const derived = followsSpeed(key); | |
| 145 | return { | |
| 146 | key, | |
| 147 | label: CAPACITY[key].label, | |
| 148 | help: CAPACITY[key].help, | |
| 149 | value: derived ? effectiveCapacity(db, key) : (stored ?? boot), | |
| 150 | source: derived ? "derived" : stored === undefined ? "default" : "setting", | |
| 151 | default: boot, | |
| 152 | env: CAPACITY[key].env, | |
| 153 | ...changedOf(db, key, row), | |
| 154 | }; | |
| 114 | 155 | } |
| 115 | 156 | |
| 116 | 157 | export function listCapacity( |
| @@ -122,17 +163,16 @@ export function listCapacity( | ||
| 122 | 163 | > { |
| 123 | 164 | const who = admin(actor); |
| 124 | 165 | if (!who.ok) return who; |
| 166 | const stored = new Map( | |
| 167 | app.db | |
| 168 | .query<StoredRow, []>( | |
| 169 | "SELECT s.key, s.value, s.updated_at, u.username FROM forge_setting s LEFT JOIN user u ON u.id = s.updated_by", | |
| 170 | ) | |
| 171 | .all() | |
| 172 | .map((r) => [r.key, r]), | |
| 173 | ); | |
| 125 | 174 | return ok({ |
| 126 | settings: CAPACITY_KEYS.map((key) => ({ | |
| 127 | key, | |
| 128 | label: CAPACITY[key].label, | |
| 129 | help: CAPACITY[key].help, | |
| 130 | value: effectiveCapacity(app.db, key), | |
| 131 | source: followsSpeed(key) ? "derived" : storedCapacity(app.db, key) === undefined ? "default" : "setting", | |
| 132 | default: bootDefault(app.db, key), | |
| 133 | env: CAPACITY[key].env, | |
| 134 | ...changedOf(app.db, key), | |
| 135 | })), | |
| 175 | settings: CAPACITY_KEYS.map((key) => capacityRow(app.db, key, stored.get(key))), | |
| 136 | 176 | changes: capacityChanges(app.db), |
| 137 | 177 | egress: egressHour(app), |
| 138 | 178 | selfSwap: selfSwapState(app.config.selfSwap === true), |
src/apps/contain.ts +93 −18
| @@ -33,6 +33,7 @@ import { | ||
| 33 | 33 | rmSync, |
| 34 | 34 | statSync, |
| 35 | 35 | } from "node:fs"; |
| 36 | import type { Database } from "bun:sqlite"; | |
| 36 | 37 | import { availableParallelism, tmpdir } from "node:os"; |
| 37 | 38 | import { dirname, join } from "node:path"; |
| 38 | 39 | import { portInsteadOfSocket } from "../git/hook-address.ts"; |
| @@ -53,7 +54,7 @@ import { during } from "../perf/jobs.ts"; | ||
| 53 | 54 | import { diskBytes, diskWalk, type DiskWalk } from "./disk_walk.ts"; |
| 54 | 55 | import { below, readTable, readTrees, treeOfTable, type ProcessTable, type TreeAsk, type TreeReading } from "./tree_read.ts"; |
| 55 | 56 | import { isSize, SIZES, type Size } from "../spawn/sizes.ts"; |
| 56 | import { CELL_MEMORY_FLOOR_MIB, CELL_MEMORY_RESERVE_MIB, cellMemoryCeilingMiB } from "../core/capacity.ts"; | |
| 57 | import { CELL_MEMORY_FLOOR_MIB, CELL_MEMORY_RESERVE_MIB, cellMemoryCeilingMiB, servingCores } from "../core/capacity.ts"; | |
| 57 | 58 | import { cpuWatch } from "./cpu_rule.ts"; |
| 58 | 59 | import { dutyTick, newDuty, resumeNow, type Duty, type DutyLater, type Signaller } from "./cpu_duty.ts"; |
| 59 | 60 | |
| @@ -774,21 +775,88 @@ export function noteCpuHolder(kernel: boolean, log: (line: string) => void, forg | ||
| 774 | 775 | return who; |
| 775 | 776 | } |
| 776 | 777 | |
| 778 | const servingSaid: { said: string | undefined } = { said: undefined }; | |
| 779 | ||
| 780 | /** | |
| 781 | * What the reserve does here, said once per process and again only if it | |
| 782 | * changes: where `taskset` is absent the reserve cannot be held, and cells | |
| 783 | * run on every core as they always did. `forget` starts over (tests). | |
| 784 | */ | |
| 785 | export function noteServingCores(reserved: number, have: boolean, log: (line: string) => void, forget = false): string | undefined { | |
| 786 | if (forget) servingSaid.said = undefined; | |
| 787 | const line = | |
| 788 | reserved <= 0 | |
| 789 | ? undefined | |
| 790 | : have | |
| 791 | ? `cpu: ${reserved} of this forge's cores are kept from its cells` | |
| 792 | : `cpu: taskset is absent, so the ${reserved} cores kept for this forge are not held; cells run on every core`; | |
| 793 | if (line !== undefined && servingSaid.said !== line) log(line); | |
| 794 | servingSaid.said = line; | |
| 795 | return line; | |
| 796 | } | |
| 797 | ||
| 777 | 798 | /** Where the next pinned cell's cores begin; `affinityArgv` advances it. */ |
| 778 | 799 | export interface CoreRotor { |
| 779 | 800 | next: number; |
| 780 | 801 | } |
| 781 | 802 | |
| 782 | 803 | /** The process's one rotor: every cell opened here draws its cores from it. */ |
| 783 | const cellCores: CoreRotor = { next: 0 }; | |
| 804 | const cellRotor: CoreRotor = { next: 0 }; | |
| 805 | ||
| 806 | /** | |
| 807 | * The CPUs kept for the forge's own process (`cells.servingCores`, #13642), | |
| 808 | * asked on every cell opened; 0 until the forge says (`provideServingCores`). | |
| 809 | */ | |
| 810 | let servingReserve: () => number = () => 0; | |
| 811 | ||
| 812 | /** Where the reserve is read from: the forge registers its setting at boot. */ | |
| 813 | export function provideServingCores(read: () => number): void { | |
| 814 | servingReserve = read; | |
| 815 | } | |
| 816 | ||
| 817 | /** | |
| 818 | * The forge's wiring (`main.ts`): every cell opened from here on asks `db`'s | |
| 819 | * `cells.servingCores` as it opens, so an admin's change reaches the next one. | |
| 820 | */ | |
| 821 | export function keepServingCoresFrom(db: Database): void { | |
| 822 | provideServingCores(() => servingCores(db)); | |
| 823 | } | |
| 824 | ||
| 825 | /** The reserve in force now: a whole number, 0 when unset or unreadable. */ | |
| 826 | export function servingCoresNow(): number { | |
| 827 | try { | |
| 828 | const n = servingReserve(); | |
| 829 | return Number.isInteger(n) && n > 0 ? n : 0; | |
| 830 | } catch { | |
| 831 | return 0; | |
| 832 | } | |
| 833 | } | |
| 834 | ||
| 835 | /** | |
| 836 | * The cores a cell may run on: `allowed` less the first `reserved`, which | |
| 837 | * are the forge's own, never more than half of them so cells keep the rest. | |
| 838 | */ | |
| 839 | export function cellCores(allowed: readonly number[], reserved: number): number[] { | |
| 840 | const r = Math.min(Math.max(0, Math.floor(reserved)), Math.floor(allowed.length / 2)); | |
| 841 | return allowed.slice(r); | |
| 842 | } | |
| 784 | 843 | |
| 785 | 844 | /** |
| 786 | 845 | * Where the kernel's `cpu.max` is not ours to write, a cell that declares a |
| 787 | 846 | * quota is held to its cores by CPU affinity instead (`taskset -c`, util-linux; |
| 788 | 847 | * sched_setaffinity(2)): a 1-CPU forge builds slowly on one core instead of |
| 789 | 848 | * being stopped. The cores rotate so forges do not all pile on core 0. |
| 790 | * Nothing is added when the kernel holds the cores, `taskset` is absent, or | |
| 791 | * the cell may have every core the machine has (#11657). | |
| 849 | * | |
| 850 | * Every cell, quota or not, is also kept off the cores reserved for the | |
| 851 | * forge's own process (`reserved`, #13642): a cell the kernel holds, or one | |
| 852 | * with no quota, is pinned to all the rest. Nothing is added when `taskset` | |
| 853 | * is absent (today's behaviour, said once by `openCell`), or when nothing is | |
| 854 | * reserved and the kernel holds the cores or the cell may have every core | |
| 855 | * the machine has (#11657). | |
| 856 | * | |
| 857 | * A hosted forge's cell (`forge:<name>`) that declares a quota goes to the | |
| 858 | * cores set apart for hosted forges (`forgeCores`, #13760) when any of them | |
| 859 | * are left after the serving reserve, by `reservedPin`. | |
| 792 | 860 | * |
| 793 | 861 | * The cores are drawn from the ones this process may run on (`allowedCores`), |
| 794 | 862 | * by their own numbers: in a pod held to a cpuset those are not 0 to n-1, and |
| @@ -799,24 +867,29 @@ const cellCores: CoreRotor = { next: 0 }; | ||
| 799 | 867 | export function affinityArgv( |
| 800 | 868 | spec: Pick<CellSpec, "cpuQuota" | "limits" | "label">, |
| 801 | 869 | kernelHolds: boolean, |
| 802 | rotor: CoreRotor = cellCores, | |
| 870 | rotor: CoreRotor = cellRotor, | |
| 803 | 871 | machine: number | readonly number[] = allowedCores(), |
| 804 | 872 | have: boolean = Bun.which("taskset") !== null, |
| 805 | reserved: readonly number[] = reservedCores(), | |
| 873 | forgeCores: readonly number[] = reservedCores(), | |
| 874 | reserved: number = servingCoresNow(), | |
| 806 | 875 | ): string[] { |
| 807 | if (spec.cpuQuota !== true || kernelHolds || !have) return []; | |
| 808 | const allowed = typeof machine === "number" ? Array.from({ length: Math.max(0, machine) }, (_, i) => i) : machine; | |
| 876 | if (!have) return []; | |
| 877 | const all = typeof machine === "number" ? Array.from({ length: Math.max(0, machine) }, (_, i) => i) : machine; | |
| 878 | const allowed = cellCores(all, reserved); | |
| 879 | const kept = allowed.length < all.length; | |
| 809 | 880 | const n = Math.max(1, Math.ceil(spec.limits.cpus)); |
| 810 | if (!Number.isFinite(n)) return []; | |
| 881 | const held = spec.cpuQuota === true && !kernelHolds && Number.isFinite(n); | |
| 811 | 882 | // Only a hosted forge's cell (`forge:<name>`) goes to the reservation; a check's or an app's does |
| 812 | 883 | // not. |
| 813 | const pool = spec.label?.startsWith("forge:") === true ? allowed.filter((c) => reserved.includes(c)) : []; | |
| 884 | const pool = held && spec.label?.startsWith("forge:") === true ? allowed.filter((c) => forgeCores.includes(c)) : []; | |
| 814 | 885 | if (pool.length > 0) return ["taskset", "-c", reservedPin(pool, n, spec.label!).join(",")]; |
| 815 | if (n >= allowed.length) return []; | |
| 816 | const first = rotor.next % allowed.length; | |
| 817 | rotor.next = (first + n) % allowed.length; | |
| 818 | const cores = Array.from({ length: n }, (_, i) => allowed[(first + i) % allowed.length]!); | |
| 819 | return ["taskset", "-c", cores.join(",")]; | |
| 886 | if (held && n < allowed.length) { | |
| 887 | const first = rotor.next % allowed.length; | |
| 888 | rotor.next = (first + n) % allowed.length; | |
| 889 | const cores = Array.from({ length: n }, (_, i) => allowed[(first + i) % allowed.length]!); | |
| 890 | return ["taskset", "-c", cores.join(",")]; | |
| 891 | } | |
| 892 | return kept ? ["taskset", "-c", allowed.join(",")] : []; | |
| 820 | 893 | } |
| 821 | 894 | |
| 822 | 895 | /** |
| @@ -883,13 +956,14 @@ export function cellArgv( | ||
| 883 | 956 | inner: readonly string[], |
| 884 | 957 | spec: Pick<CellSpec, "cpuQuota" | "limits" | "label">, |
| 885 | 958 | cgroup: { readonly dir: string; readonly cpuCapped: boolean } | undefined, |
| 886 | rotor: CoreRotor = cellCores, | |
| 959 | rotor: CoreRotor = cellRotor, | |
| 887 | 960 | machine: number | readonly number[] = allowedCores(), |
| 888 | 961 | have: boolean = Bun.which("taskset") !== null, |
| 889 | 962 | oom: number | null = cellOom(), |
| 890 | reserved: readonly number[] = reservedCores(), | |
| 963 | forgeCores: readonly number[] = reservedCores(), | |
| 964 | reserved: number = servingCoresNow(), | |
| 891 | 965 | ): string[] { |
| 892 | const pinned = oomFirst([...affinityArgv(spec, cgroup?.cpuCapped === true, rotor, machine, have, reserved), ...inner], oom); | |
| 966 | const pinned = oomFirst([...affinityArgv(spec, cgroup?.cpuCapped === true, rotor, machine, have, forgeCores, reserved), ...inner], oom); | |
| 893 | 967 | return cgroup === undefined ? pinned : enterArgv(join(cgroup.dir, "cgroup.procs"), pinned); |
| 894 | 968 | } |
| 895 | 969 | |
| @@ -955,6 +1029,7 @@ export function openCell( | ||
| 955 | 1029 | |
| 956 | 1030 | const cgroup = cellCgroup(spec); |
| 957 | 1031 | if (spec.cpuQuota === true) noteCpuHolder(cgroup?.cpuCapped === true, log); |
| 1032 | noteServingCores(servingCoresNow(), Bun.which("taskset") !== null, log); | |
| 958 | 1033 | const argv = (command: string): string[] => cellArgv(bwrapArgv(command), spec, cgroup); |
| 959 | 1034 | const rlimits = cellRlimits(spec.limits); |
| 960 | 1035 | const bwrapArgv = (command: string): string[] => [ |
src/core/capacity.ts +26
| @@ -27,6 +27,7 @@ export type CapacityKey = | ||
| 27 | 27 | | "week.use" |
| 28 | 28 | | "children.max" |
| 29 | 29 | | "cells.egress" |
| 30 | | "cells.servingCores" | |
| 30 | 31 | | "children.modelEgress" |
| 31 | 32 | | "children.registryEgress" |
| 32 | 33 | | "children.paymentEgress" |
| @@ -113,6 +114,14 @@ export const CAPACITY: Record<CapacityKey, Definition> = { | ||
| 113 | 114 | fallback: "enforce", |
| 114 | 115 | help: "enforce refuses a host an app or check did not declare; report lets it through on 80 and 443 and lists it on the app's page. Enforce unless an admin sets report here, with a reason. Takes effect on the next cell started.", |
| 115 | 116 | }, |
| 117 | "cells.servingCores": { | |
| 118 | label: "CPUs kept for the forge itself", | |
| 119 | env: "FORGE_SERVING_CORES", | |
| 120 | kind: { type: "count", min: 0, max: 16 }, | |
| 121 | fallback: "0", | |
| 122 | sized: (cpus) => servingCoresForCpus(cpus), | |
| 123 | help: "Why: cells and checks busy on every core stall the forge's own serving thread, so pages wait. This many CPUs (the first ones this process may run on) are left to the forge alone; every cell and check is pinned to the rest (taskset), never more than half. 0 keeps none. Empty: none below 8 CPUs, else one per 16, 1 to 4. Without taskset nothing is pinned, and the log says so. Takes effect on the next cell started.", | |
| 124 | }, | |
| 116 | 125 | "children.modelEgress": { |
| 117 | 126 | label: "Model provider a child forge cell may reach", |
| 118 | 127 | env: "FORGE_CHILDREN_MODEL_EGRESS", |
| @@ -235,6 +244,23 @@ export function checksForCpus(cpus: number, shards = shardsForCpus(cpus)): numbe | ||
| 235 | 244 | return Number.isFinite(n) ? Math.max(1, Math.min(16, n)) : 1; |
| 236 | 245 | } |
| 237 | 246 | |
| 247 | /** | |
| 248 | * The CPUs kept for the forge's own process by default (#13642): none on a | |
| 249 | * machine under 8, so a small forge runs as it always has; one per 16 above, | |
| 250 | * 1 to 4 — 80 CPUs keep 4. | |
| 251 | */ | |
| 252 | export function servingCoresForCpus(cpus: number): number { | |
| 253 | const n = Math.floor(cpus); | |
| 254 | if (!Number.isFinite(n) || n < 8) return 0; | |
| 255 | return Math.max(1, Math.min(4, Math.floor(n / 16))); | |
| 256 | } | |
| 257 | ||
| 258 | /** The CPUs no cell may run on (`cells.servingCores`), read on every cell started. */ | |
| 259 | export function servingCores(db: Database): number { | |
| 260 | const n = Number(capacityValue(db, "cells.servingCores")); | |
| 261 | return Number.isInteger(n) && n > 0 ? n : 0; | |
outdated
|
||
| 262 | } | |
| 263 | ||
| 238 | 264 | let cpusForTest: number | undefined; |
| 239 | 265 | |
| 240 | 266 | /** |
src/main.ts +4
| @@ -125,6 +125,7 @@ import { forgesMemory, memoryOverLine } from "./spawn/memory_fit.ts"; | ||
| 125 | 125 | import { keepFloor } from "./apps/cgroup.ts"; |
| 126 | 126 | import { limitValue } from "./settings/limits.ts"; |
| 127 | 127 | import { capacityDefault, egressEnvIgnored } from "./core/capacity.ts"; |
| 128 | import { keepServingCoresFrom } from "./apps/contain.ts"; | |
| 128 | 129 | import { bootCounts, recordBoot, recordCleanExit } from "./kernel/boots.ts"; |
| 129 | 130 | import { census, censusWords, processRows } from "./kernel/census.ts"; |
| 130 | 131 | import { reapLeftoverBridges } from "./crew/leftovers.ts"; |
| @@ -501,6 +502,9 @@ if (appDomain?.ok === true && appDomain.note !== undefined) | ||
| 501 | 502 | // An environment asking for egress report is ignored, and said so once (#10778). |
| 502 | 503 | const egressIgnored = egressEnvIgnored(); |
| 503 | 504 | if (egressIgnored !== null) app.log.info(egressIgnored); |
| 505 | // Cells keep off the cores reserved for the forge's own process, asked on | |
| 506 | // every cell opened, so an admin's change reaches the next one (#13642). | |
| 507 | keepServingCoresFrom(app.db); | |
outdated
The wiring has no test. Show that a change to the setting reaches the next cell, and that checks use the same path. |
||
| 504 | 508 | // The memory the forge keeps for itself goes to its own cgroup where one is |
| 505 | 509 | // delegated (`apps/cgroup.ts`), and who holds it is said once. |
| 506 | 510 | keepFloor(limitValue(app.db, "forge.memory_floor_mb"), (line) => app.log.info(line)); |
test/serving-cores-wiring.test.ts added +126
| @@ -0,0 +1,126 @@ | ||
| 1 | /** | |
| 2 | * #13642, the stored setting and the path a cell and a check take: what | |
| 3 | * `cells.servingCores` holds reaches the next cell's argv, a check's step | |
| 4 | * cell included, and the capacity list still answers every row as it did | |
| 5 | * when it read each one with its own statement. | |
| 6 | */ | |
| 7 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 8 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 9 | import { readFileSync } from "node:fs"; | |
| 10 | import { tmpdir } from "node:os"; | |
| 11 | import { join } from "node:path"; | |
| 12 | import { createApp, type App } from "../src/app.ts"; | |
| 13 | import { changedOf, listCapacity } from "../src/actions/capacity.ts"; | |
| 14 | import { | |
| 15 | CAPACITY_KEYS, | |
| 16 | bootDefault, | |
| 17 | effectiveCapacity, | |
| 18 | followsSpeed, | |
| 19 | provideCpusForTest, | |
| 20 | servingCores, | |
| 21 | setCapacity, | |
| 22 | storedCapacity, | |
| 23 | } from "../src/core/capacity.ts"; | |
| 24 | import { cellArgv, cellsAvailable, keepServingCoresFrom, provideServingCores, resolveLimits } from "../src/apps/contain.ts"; | |
| 25 | import { runStep } from "../src/apps/process.ts"; | |
| 26 | import type { User } from "../src/core/users.ts"; | |
| 27 | import { seedAdmin } from "./helpers/admin.ts"; | |
| 28 | ||
| 29 | let dataDir = ""; | |
| 30 | let app: App, alice: User; | |
| 31 | ||
| 32 | beforeAll(async () => { | |
| 33 | provideCpusForTest(4); | |
| 34 | dataDir = await mkdtemp(join(tmpdir(), "forge-serving-")); | |
| 35 | app = createApp({ dataDir, port: 0 }); | |
| 36 | const admin = await seedAdmin(app, { username: "alice", email: "[email protected]", password: "correct-horse-battery" }); | |
| 37 | if (!admin.ok) throw new Error(admin.error); | |
| 38 | alice = admin.value; | |
| 39 | }); | |
| 40 | ||
| 41 | afterAll(async () => { | |
| 42 | provideServingCores(() => 0); | |
| 43 | provideCpusForTest(undefined); | |
| 44 | app.close(); | |
| 45 | await rm(dataDir, { recursive: true, force: true }); | |
| 46 | }); | |
| 47 | ||
| 48 | const store = (value: string): void => { | |
| 49 | app.db.query("UPDATE forge_setting SET value = ?1 WHERE key = 'cells.servingCores'").run(value); | |
| 50 | }; | |
| 51 | ||
| 52 | test("servingCores reads the stored count; unset is the default, a stored value that is not a count is too", () => { | |
| 53 | setCapacity(app.db, "cells.servingCores", null, alice.id); | |
| 54 | expect(servingCores(app.db)).toBe(0); // 4 CPUs keep none | |
| 55 | setCapacity(app.db, "cells.servingCores", "3", alice.id); | |
| 56 | expect(servingCores(app.db)).toBe(3); | |
| 57 | for (const bad of ["two", "-1", "1.5", "99", ""]) { | |
| 58 | store(bad); | |
| 59 | expect(servingCores(app.db)).toBe(0); | |
| 60 | } | |
| 61 | setCapacity(app.db, "cells.servingCores", null, alice.id); | |
| 62 | }); | |
| 63 | ||
| 64 | // What a check's step is opened as (`checks/service.ts`): step limits, no | |
| 65 | // quota of its own; `runStep` hands it to `openCell`, whose argv is this. | |
| 66 | const checkSpec = { limits: resolveLimits("step", undefined) }; | |
| 67 | const allowed = (argv: string[]): number[] => (argv[0] === "taskset" ? argv[2]!.split(",").map(Number) : []); | |
| 68 | ||
| 69 | test("the forge's wiring: a change to the setting reaches the next cell and the next check", () => { | |
| 70 | keepServingCoresFrom(app.db); | |
| 71 | const open = () => cellArgv(["bwrap", "sh"], checkSpec, undefined, { next: 0 }, 16, true, null); | |
| 72 | expect(open()).toEqual(["bwrap", "sh"]); | |
| 73 | setCapacity(app.db, "cells.servingCores", "2", alice.id); | |
| 74 | expect(allowed(open())).toEqual([2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); | |
| 75 | setCapacity(app.db, "cells.servingCores", "1", alice.id); | |
| 76 | expect(allowed(open())[0]).toBe(1); | |
| 77 | // a cell with a quota of its own draws its cores from the rest too | |
| 78 | expect(allowed(cellArgv(["bwrap"], { cpuQuota: true, limits: resolveLimits("step", undefined) }, undefined, { next: 0 }, 16, true, null))).not.toContain(0); | |
| 79 | setCapacity(app.db, "cells.servingCores", null, alice.id); | |
| 80 | // and the forge wires it at boot | |
| 81 | expect(readFileSync(join(import.meta.dir, "../src/main.ts"), "utf8")).toContain("keepServingCoresFrom(app.db);"); | |
| 82 | }); | |
| 83 | ||
| 84 | const cells = await cellsAvailable(); | |
| 85 | ||
| 86 | test.skipIf(!cells.ok)("a check's step, opened as the check service opens it, runs off the reserved cores", async () => { | |
| 87 | provideServingCores(() => 1); | |
| 88 | const dir = await mkdtemp(join(tmpdir(), "forge-serving-step-")); | |
| 89 | const lines: string[] = []; | |
| 90 | const ran = await runStep("check", "grep Cpus_allowed_list /proc/self/status", { | |
| 91 | cwd: dir, | |
| 92 | env: { PATH: process.env["PATH"] ?? "/usr/bin:/bin" }, | |
| 93 | log: (line) => { | |
| 94 | lines.push(line); | |
| 95 | }, | |
| 96 | signal: new AbortController().signal, | |
| 97 | cell: { dir, writable: [], home: `${dir}.home`, hide: [], egress: [], reach: [], limits: resolveLimits("step", undefined), label: "check:serving" }, | |
| 98 | }); | |
| 99 | provideServingCores(() => 0); | |
| 100 | await rm(dir, { recursive: true, force: true }); | |
| 101 | await rm(`${dir}.home`, { recursive: true, force: true }); | |
| 102 | expect(ran.ok).toBe(true); | |
| 103 | const list = lines.find((l) => /^Cpus_allowed_list:/.test(l)); | |
| 104 | expect(list).toBeDefined(); | |
| 105 | if (Bun.which("taskset") !== null && (navigator.hardwareConcurrency ?? 1) > 1) expect(list).not.toMatch(/:\s*0(?:[-,]|$)/); | |
| 106 | }); | |
| 107 | ||
| 108 | test("the list answers every row as reading each key on its own did", () => { | |
| 109 | setCapacity(app.db, "crew.concurrency", "3", alice.id); | |
| 110 | setCapacity(app.db, "cells.servingCores", "2", alice.id); | |
| 111 | setCapacity(app.db, "checks.shards", bootDefault(app.db, "checks.shards"), alice.id); | |
| 112 | setCapacity(app.db, "week.use", "50", alice.id); | |
| 113 | app.db.query("UPDATE forge_setting SET value = 'nonsense' WHERE key = 'week.use'").run(); | |
| 114 | const listed = listCapacity(app, alice); | |
| 115 | if (!listed.ok) throw new Error(listed.error); | |
| 116 | const old = CAPACITY_KEYS.map((key) => ({ | |
| 117 | key, | |
| 118 | value: effectiveCapacity(app.db, key), | |
| 119 | source: (followsSpeed(key) ? "derived" : storedCapacity(app.db, key) === undefined ? "default" : "setting") as "derived" | "default" | "setting", | |
| 120 | default: bootDefault(app.db, key), | |
| 121 | ...changedOf(app.db, key), | |
| 122 | })); | |
| 123 | expect(listed.value.settings.map(({ key, value, source, default: d, changed }) => ({ key, value, source, default: d, ...(changed === undefined ? {} : { changed }) }))).toEqual(old); | |
| 124 | expect(listed.value.settings.find((s) => s.key === "cells.servingCores")?.changed?.by).toBe("alice"); | |
| 125 | expect(listed.value.settings.find((s) => s.key === "week.use")?.source).toBe("default"); | |
| 126 | }); |
test/serving-cores.test.ts added +70
| @@ -0,0 +1,70 @@ | ||
| 1 | import { expect, test } from "bun:test"; | |
| 2 | import { affinityArgv, cellArgv, cellCores, noteServingCores, provideServingCores, servingCoresNow } from "../src/apps/contain.ts"; | |
| 3 | import { CAPACITY, servingCoresForCpus } from "../src/core/capacity.ts"; | |
| 4 | import type { Limits } from "../src/apps/contain.ts"; | |
| 5 | ||
| 6 | // #13642: cells and checks keep off the cores reserved for the forge's own | |
| 7 | // serving process, so its thread is not stalled by builds on every core. | |
| 8 | ||
| 9 | const limits = (cpus: number): Limits => ({ cpus, memoryMiB: 512 }) as Limits; | |
| 10 | const cores = (argv: string[]): number[] => (argv[2] ?? "").split(",").map(Number); | |
| 11 | ||
| 12 | test("a cell's affinity excludes the reserved cores, quota or not, kernel-held or not", () => { | |
| 13 | const all = Array.from({ length: 16 }, (_, i) => i); | |
| 14 | for (const spec of [ | |
| 15 | { cpuQuota: true, limits: limits(2) }, | |
| 16 | { cpuQuota: false, limits: limits(2) }, | |
| 17 | { cpuQuota: true, limits: limits(64) }, | |
| 18 | ]) | |
| 19 | for (const kernelHolds of [false, true]) | |
| 20 | for (let i = 0; i < 20; i++) { | |
| 21 | const argv = affinityArgv(spec, kernelHolds, { next: i }, all, true, [], 2); | |
| 22 | expect(argv.slice(0, 2)).toEqual(["taskset", "-c"]); | |
| 23 | expect(cores(argv)).not.toContain(0); | |
| 24 | expect(cores(argv)).not.toContain(1); | |
| 25 | } | |
| 26 | // a pod held to a cpuset reserves its own first cores, by their numbers | |
| 27 | const pod = [8, 9, 10, 11, 12, 13]; | |
| 28 | expect(affinityArgv({ cpuQuota: false, limits: limits(1) }, false, { next: 0 }, pod, true, [], 1)).toEqual(["taskset", "-c", "9,10,11,12,13"]); | |
| 29 | // and the whole argv a cell runs as carries it | |
| 30 | expect(cellArgv(["bwrap", "x"], { cpuQuota: false, limits: limits(1) }, undefined, { next: 0 }, 4, true, null, [], 1)).toEqual([ | |
| 31 | "taskset", | |
| 32 | "-c", | |
| 33 | "1,2,3", | |
| 34 | "bwrap", | |
| 35 | "x", | |
| 36 | ]); | |
| 37 | }); | |
| 38 | ||
| 39 | test("nothing reserved, or no taskset, is today's behaviour", () => { | |
| 40 | const spec = { cpuQuota: false, limits: limits(1) }; | |
| 41 | expect(affinityArgv(spec, false, { next: 0 }, 16, true, [], 0)).toEqual([]); | |
| 42 | expect(affinityArgv(spec, false, { next: 0 }, 16, false, [], 4)).toEqual([]); | |
| 43 | expect(affinityArgv({ cpuQuota: true, limits: limits(1) }, false, { next: 0 }, 16, false, [], 4)).toEqual([]); | |
| 44 | const said: string[] = []; | |
| 45 | expect(noteServingCores(4, false, (l) => said.push(l), true)).toContain("taskset is absent"); | |
| 46 | noteServingCores(4, false, (l) => said.push(l)); | |
| 47 | expect(said.length).toBe(1); | |
| 48 | expect(noteServingCores(0, false, (l) => said.push(l), true)).toBeUndefined(); | |
| 49 | }); | |
| 50 | ||
| 51 | test("cells always keep at least half the cores", () => { | |
| 52 | expect(cellCores([0, 1, 2, 3], 16)).toEqual([2, 3]); | |
| 53 | expect(cellCores([0], 4)).toEqual([0]); | |
| 54 | expect(cellCores([0, 1, 2, 3], -1)).toEqual([0, 1, 2, 3]); | |
| 55 | }); | |
| 56 | ||
| 57 | test("the default keeps none below 8 CPUs, one per 16 above, at most 4", () => { | |
| 58 | expect([1, 4, 7, 8, 16, 32, 64, 80, 256].map(servingCoresForCpus)).toEqual([0, 0, 0, 1, 1, 2, 4, 4, 4]); | |
| 59 | expect(CAPACITY["cells.servingCores"].help).toStartWith("Why:"); | |
| 60 | }); | |
| 61 | ||
| 62 | test("the reserve is read from the forge's setting on every cell, 0 when unreadable", () => { | |
| 63 | provideServingCores(() => 3); | |
| 64 | expect(servingCoresNow()).toBe(3); | |
| 65 | provideServingCores(() => { | |
| 66 | throw new Error("closed"); | |
| 67 | }); | |
| 68 | expect(servingCoresNow()).toBe(0); | |
| 69 | provideServingCores(() => 0); | |
| 70 | }); |
test/tui-capacity.test.ts +20
| @@ -137,6 +137,26 @@ describe("tui capacity", () => { | ||
| 137 | 137 | } |
| 138 | 138 | }); |
| 139 | 139 | |
| 140 | test("cells.servingCores is listed with its bounds and set from the terminal (#13642)", async () => { | |
| 141 | const serving = () => capacityValue(app.db, "cells.servingCores"); | |
| 142 | const h = boot(carol, 80); | |
| 143 | try { | |
| 144 | await h.press(); | |
| 145 | expect(h.text()).toContain("CPUs kept for the forge itself (cells.servingCores)"); | |
| 146 | expect(h.text()).toContain("takes 0–16"); | |
| 147 | for (let i = 0; i < CAPACITY_KEYS.indexOf("cells.servingCores"); i++) await h.press("j"); | |
| 148 | await h.press("e"); | |
| 149 | for (let i = 0; i < 4; i++) await h.press("backspace"); | |
| 150 | await h.press("2", "enter"); | |
| 151 | expect(h.text()).toContain("→ 2"); | |
| 152 | await h.press("left", "enter"); | |
| 153 | expect(serving()).toBe("2"); | |
| 154 | } finally { | |
| 155 | h.stop(); | |
| 156 | changeCapacity(app, carol, "cells.servingCores", null); | |
| 157 | } | |
| 158 | }); | |
| 159 | ||
| 140 | 160 | test("a crew concurrency below its ceiling is marked, and not at it", async () => { |
| 141 | 161 | const ceiling = capacityDefault("crew.concurrency"); |
| 142 | 162 | try { |
new above
offline
Hoist this IIFE into a named
const stored = new Map(...)and add a test that the list matches the old per-row answers (value, source, changed).