Public: anyone can read this; the crew can change it.
One process, one thread by default: the serve/work split switches on only when the machine has the cores #12531
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/12520-one-process-one-thread-by-default-the-se → main 3 commits fork 2 threads · 2 unresolved
5 files
changed +229
−12
docs/ARCHITECTURE.md +4 −1
| @@ -471,7 +471,10 @@ keep serving on the database the new one migrated. | ||
| 471 | 471 | |
| 472 | 472 | ## Two threads |
| 473 | 473 | |
| 474 | One thread serves and works, by default. `FORGE_ENGINE=worker` puts the | |
| 474 | One thread serves and works unless the machine has cores to spare: with | |
| 475 | `FORGE_ENGINE` unset, 4 CPUs and a 1024 MiB memory limit (`engine/mode.ts` | |
| 476 | `machineChoice`) give the split, and an explicit `FORGE_ENGINE=worker|inline` | |
| 477 | wins. The split puts the | |
| 475 | 478 | engine's duties (`engine/duties.ts`: delivery, retention, the keeper, the clog watch, the asks ledger) on a Worker of the same process with a |
| 476 | 479 | database handle of its own, and keeps one thread for a hosted child, a |
| 477 | 480 | read-only standby and a test boot whatever the variable says. The two threads |
docs/features/engine-shape.md +9 −3
| @@ -1,8 +1,14 @@ | ||
| 1 | 1 | # Engine shape |
| 2 | 2 | |
| 3 | The forge runs on one thread unless asked. `FORGE_ENGINE=worker` moves the | |
| 4 | engine's duties onto a Worker of the same process; `FORGE_ENGINE=inline` says | |
| 5 | the default out loud. | |
| 3 | With `FORGE_ENGINE` unset the machine decides at boot: at least 4 CPUs and a | |
| 4 | memory limit of at least 1024 MiB (or none known) move the engine's duties onto a Worker of the same process; | |
| 5 | anything less keeps one thread, since the Worker costs about 20 MiB and only | |
| 6 | pays with a core to spare. CPUs are what the scheduler offers held under the | |
| 7 | cgroup's quota (v2 `cpu.max`, else v1 `cpu.cfs_quota_us`/`cpu.cfs_period_us`), | |
| 8 | so a 1-CPU container on a big host counts as 1; memory is the smaller of the | |
| 9 | cgroup's cap (v2 `memory.max`, else v1 `memory.limit_in_bytes`) and the | |
| 10 | machine's. An explicit `FORGE_ENGINE=worker` or | |
| 11 | `FORGE_ENGINE=inline` wins over the machine. | |
| 6 | 12 | |
| 7 | 13 | A hosted child, a read-only standby and a test boot stay on one thread whatever |
| 8 | 14 | the variable says. |
src/engine/mode.ts +110 −5
| @@ -1,14 +1,23 @@ | ||
| 1 | 1 | /** |
| 2 | * Where the engine runs. `inline` (the default) is the forge as it always was: | |
| 3 | * one thread serves and works. `worker` (`FORGE_ENGINE=worker`) puts the | |
| 4 | * engine's duties (`duties.ts`) on a Worker of this process with its own | |
| 5 | * database handle, so what they cost never holds a page. | |
| 2 | * Where the engine runs. `inline` is the forge as it always was: one thread | |
| 3 | * serves and works. `worker` puts the engine's duties (`duties.ts`) on a Worker | |
| 4 | * of this process with its own database handle, so what they cost never holds | |
| 5 | * a page. | |
| 6 | * | |
| 7 | * The Worker costs about 20 MiB and only pays when a core is free to run it, so | |
| 8 | * with no `FORGE_ENGINE` the machine decides: at least `SPLIT_MIN_CPUS` CPUs | |
| 9 | * and a memory limit of at least `SPLIT_MIN_MEMORY` (or none known) give the | |
| 10 | * split, anything less one thread. An explicit `FORGE_ENGINE=worker|inline` | |
| 11 | * wins over the machine. | |
| 6 | 12 | * |
| 7 | 13 | * A hosted child (a cell of another forge: 1 CPU, one thread to share), a |
| 8 | 14 | * read-only standby and a test boot stay `inline` whatever the variable says; |
| 9 | 15 | * a test that wants the split asks for it itself (`optIn`). |
| 10 | 16 | */ |
| 11 | 17 | |
| 18 | import { availableParallelism, totalmem } from "node:os"; | |
| 19 | import { readFileSync } from "node:fs"; | |
| 20 | ||
| 12 | 21 | export type EngineMode = "inline" | "worker"; |
| 13 | 22 | |
| 14 | 23 | /** The shape that runs and the reason, in plain words, for `/system` and the terminal. */ |
| @@ -17,15 +26,111 @@ export interface EngineChoice { | ||
| 17 | 26 | why: string; |
| 18 | 27 | } |
| 19 | 28 | |
| 29 | /** CPUs the split asks for: one serves, one runs the engine, two spare for runs and checks. */ | |
| 30 | export const SPLIT_MIN_CPUS = 4; | |
| 31 | /** Memory limit the split asks for: the Worker's ~20 MiB is under 2% of it. */ | |
| 32 | export const SPLIT_MIN_MEMORY = 1024 * 1024 * 1024; | |
| 33 | ||
| 34 | /** What the machine gives this process: usable CPUs, and its memory limit in bytes (null: none). */ | |
| 35 | export interface Machine { | |
| 36 | cpus: number; | |
| 37 | memoryBytes: number | null; | |
| 38 | } | |
| 39 | ||
| 40 | const mib = (bytes: number): string => `${Math.round(bytes / (1024 * 1024))} MiB`; | |
| 41 | ||
| 42 | /** The default when `FORGE_ENGINE` is not set: a pure function of the machine. */ | |
| 43 | export function machineChoice(m: Machine): EngineChoice { | |
| 44 | const cpus = `${m.cpus} CPU${m.cpus === 1 ? "" : "s"}`; | |
| 45 | if (m.cpus < SPLIT_MIN_CPUS) { | |
| 46 | return { mode: "inline", why: `FORGE_ENGINE is not set and ${cpus} is under the ${SPLIT_MIN_CPUS} the split needs` }; | |
| 47 | } | |
| 48 | if (m.memoryBytes !== null && m.memoryBytes < SPLIT_MIN_MEMORY) { | |
| 49 | return { | |
| 50 | mode: "inline", | |
| 51 | why: `FORGE_ENGINE is not set and the memory limit, ${mib(m.memoryBytes)}, is under the ${mib(SPLIT_MIN_MEMORY)} the split needs`, | |
| 52 | }; | |
| 53 | } | |
| 54 | const memory = m.memoryBytes === null ? "no memory limit" : `a ${mib(m.memoryBytes)} memory limit`; | |
| 55 | return { mode: "worker", why: `FORGE_ENGINE is not set and ${cpus} with ${memory} meet the split's ${SPLIT_MIN_CPUS} CPUs and ${mib(SPLIT_MIN_MEMORY)}` }; | |
| 56 | } | |
| 57 | ||
| 58 | /** A cgroup memory cap in bytes; null for "max", empty, junk or v1's near-2^63 "unlimited". */ | |
| 59 | export function parseMemoryMax(text: string): number | null { | |
| 60 | const t = text.trim(); | |
| 61 | if (!/^[0-9]+$/.test(t)) return null; | |
| 62 | const n = Number(t); | |
| 63 | return Number.isSafeInteger(n) && n > 0 ? n : null; | |
| 64 | } | |
| 65 | ||
| 66 | /** | |
| 67 | * The memory limit from the cgroup's cap (its file's text, null when unreadable) and the | |
| 68 | * machine's total: the smaller of the two, the total without a cap, null with neither. | |
| 69 | */ | |
| 70 | export function memoryLimit(cgroupText: string | null, total: number): number | null { | |
| 71 | const cap = cgroupText === null ? null : parseMemoryMax(cgroupText); | |
| 72 | const known = total > 0 ? total : null; | |
| 73 | if (cap === null) return known; | |
| 74 | return known === null ? cap : Math.min(cap, known); | |
outdated
availableParallelism ignores a cgroup cpu.max quota, so a 1-CPU container on a big host reads as many CPUs and gets the split. Read cpu.max and take the min, with a test, or document the limitation. |
||
| 75 | } | |
| 76 | ||
| 77 | /** | |
| 78 | * CPUs a cgroup quota allows, whole ones and at least 1: v2 `cpu.max` ("quota period" or | |
| 79 | * "max period"), or v1's quota and period joined by a space. Null for no quota or junk. | |
| 80 | */ | |
| 81 | export function cpuQuota(text: string | null): number | null { | |
| 82 | if (text === null) return null; | |
| 83 | const m = /^\s*([0-9]+)\s+([0-9]+)\s*$/.exec(text); | |
| 84 | if (m === null) return null; | |
| 85 | const quota = Number(m[1]); | |
| 86 | const period = Number(m[2]); | |
| 87 | if (!(quota > 0 && period > 0)) return null; | |
| 88 | return Math.max(1, Math.floor(quota / period)); | |
| 89 | } | |
| 90 | ||
| 91 | /** Usable CPUs: what the scheduler offers, held under the cgroup's quota, at least 1. */ | |
| 92 | export const usableCpus = (parallelism: number, quota: number | null): number => | |
| 93 | Math.max(1, Math.min(parallelism, quota ?? parallelism)); | |
| 94 | ||
| 95 | /** A file's text, or null when it cannot be read. */ | |
| 96 | const readText = (path: string): string | null => { | |
| 97 | try { | |
| 98 | return readFileSync(path, "utf8"); | |
| 99 | } catch { | |
| 100 | return null; | |
| 101 | } | |
| 102 | }; | |
| 103 | ||
| 104 | const v1Quota = (read: (path: string) => string | null): string | null => { | |
| 105 | const quota = read("/sys/fs/cgroup/cpu/cpu.cfs_quota_us"); | |
| 106 | const period = read("/sys/fs/cgroup/cpu/cpu.cfs_period_us"); | |
| 107 | return quota === null || period === null ? null : `${quota.trim()} ${period.trim()}`; | |
| 108 | }; | |
| 109 | ||
| 110 | /** | |
| 111 | * This machine, read once at boot: CPUs under the cgroup's `cpu.max` quota, and memory under | |
| 112 | * its `memory.max` cap; cgroup v2 first, v1's files when v2's are missing. | |
| 113 | */ | |
| 114 | export function readMachine( | |
| 115 | read: (path: string) => string | null = readText, | |
| 116 | parallelism: number = availableParallelism(), | |
| 117 | total: number = totalmem(), | |
| 118 | ): Machine { | |
| 119 | const memory = read("/sys/fs/cgroup/memory.max") ?? read("/sys/fs/cgroup/memory/memory.limit_in_bytes"); | |
| 120 | const cpu = read("/sys/fs/cgroup/cpu.max") ?? v1Quota(read); | |
| 121 | return { cpus: usableCpus(parallelism, cpuQuota(cpu)), memoryBytes: memoryLimit(memory, total) }; | |
| 122 | } | |
| 123 | ||
| 20 | 124 | export function engineChoice( |
| 21 | 125 | env: Record<string, string | undefined>, |
| 22 | where: { hosted: boolean; readOnly: boolean; optIn?: boolean }, | |
| 126 | where: { hosted: boolean; readOnly: boolean; optIn?: boolean; machine?: Machine }, | |
| 23 | 127 | ): EngineChoice { |
| 24 | 128 | if (where.hosted) return { mode: "inline", why: "this forge is a hosted child" }; |
| 25 | 129 | if (where.readOnly) return { mode: "inline", why: "this forge is a read-only standby" }; |
| 26 | 130 | if (env["NODE_ENV"] === "test" && where.optIn !== true) return { mode: "inline", why: "this is a test boot" }; |
| 27 | 131 | if (env["FORGE_ENGINE"] === "worker") return { mode: "worker", why: "FORGE_ENGINE=worker is set" }; |
| 28 | 132 | if (env["FORGE_ENGINE"] === "inline") return { mode: "inline", why: "FORGE_ENGINE=inline is set" }; |
| 133 | if (where.machine !== undefined) return machineChoice(where.machine); | |
| 29 | 134 | return { mode: "inline", why: "FORGE_ENGINE is not set" }; |
| 30 | 135 | } |
| 31 | 136 |
src/main.ts +2 −2
| @@ -87,7 +87,7 @@ import { graphInstagram } from "./crew/instagram.ts"; | ||
| 87 | 87 | import { peopleOnCrews } from "./core/agents.ts"; |
| 88 | 88 | import { startNotifier } from "./crew/notify.ts"; |
| 89 | 89 | import { tailEvents } from "./core/event_tail.ts"; |
| 90 | import { engineChoice } from "./engine/mode.ts"; | |
| 90 | import { engineChoice, readMachine } from "./engine/mode.ts"; | |
| 91 | 91 | import { startEngine } from "./engine/host.ts"; |
| 92 | 92 | import { startDuties } from "./engine/duties.ts"; |
| 93 | 93 | import { startRetention } from "./system/retention.ts"; |
| @@ -277,7 +277,7 @@ watchStalls(loopLog, undefined, undefined, undefined, (late, at) => { | ||
| 277 | 277 | |
| 278 | 278 | const previewKey = previewIdentityKey(process.env); |
| 279 | 279 | |
| 280 | const engineShape = engineChoice(process.env, { hosted, readOnly }); | |
| 280 | const engineShape = engineChoice(process.env, { hosted, readOnly, machine: readMachine() }); | |
| 281 | 281 | |
| 282 | 282 | const app = createApp({ |
| 283 | 283 | dataDir, |
test/engine-shape.test.ts +104 −1
| @@ -6,7 +6,17 @@ import { join } from "node:path"; | ||
| 6 | 6 | import { createApp } from "../src/app.ts"; |
| 7 | 7 | import type { User } from "../src/core/users.ts"; |
| 8 | 8 | import type { Ctx } from "../src/http/context.ts"; |
| 9 | import { engineChoice } from "../src/engine/mode.ts"; | |
| 9 | import { | |
| 10 | cpuQuota, | |
| 11 | engineChoice, | |
| 12 | machineChoice, | |
| 13 | memoryLimit, | |
| 14 | parseMemoryMax, | |
| 15 | readMachine, | |
| 16 | SPLIT_MIN_CPUS, | |
| 17 | SPLIT_MIN_MEMORY, | |
| 18 | usableCpus, | |
| 19 | } from "../src/engine/mode.ts"; | |
| 10 | 20 | import { engineLines } from "../src/tui/screens/system.ts"; |
| 11 | 21 | import { systemPage } from "../src/web/pages/system.ts"; |
| 12 | 22 | |
| @@ -51,6 +61,93 @@ describe("the choice", () => { | ||
| 51 | 61 | }); |
| 52 | 62 | }); |
| 53 | 63 | |
| 64 | describe("the machine decides when FORGE_ENGINE is not set", () => { | |
| 65 | const gib = 1024 * 1024 * 1024; | |
| 66 | test("1 CPU gives one thread, 8 CPUs give the split", () => { | |
| 67 | const one = engineChoice({}, { ...where, machine: { cpus: 1, memoryBytes: 8 * gib } }); | |
| 68 | expect(one.mode).toBe("inline"); | |
| 69 | expect(one.why).toBe(`FORGE_ENGINE is not set and 1 CPU is under the ${SPLIT_MIN_CPUS} the split needs`); | |
| 70 | const eight = engineChoice({}, { ...where, machine: { cpus: 8, memoryBytes: 8 * gib } }); | |
| 71 | expect(eight.mode).toBe("worker"); | |
| 72 | expect(eight.why).toContain("8 CPUs with a 8192 MiB memory limit"); | |
| 73 | }); | |
| 74 | test("the thresholds are where they say, and a small memory limit keeps one thread", () => { | |
| 75 | expect(machineChoice({ cpus: SPLIT_MIN_CPUS - 1, memoryBytes: null }).mode).toBe("inline"); | |
| 76 | expect(machineChoice({ cpus: SPLIT_MIN_CPUS, memoryBytes: SPLIT_MIN_MEMORY }).mode).toBe("worker"); | |
| 77 | expect(machineChoice({ cpus: SPLIT_MIN_CPUS, memoryBytes: null }).why).toContain("no memory limit"); | |
| 78 | const tight = machineChoice({ cpus: 8, memoryBytes: 512 * 1024 * 1024 }); | |
| 79 | expect(tight).toEqual({ mode: "inline", why: "FORGE_ENGINE is not set and the memory limit, 512 MiB, is under the 1024 MiB the split needs" }); | |
| 80 | }); | |
| 81 | test("an explicit FORGE_ENGINE wins over the machine, and a hosted child stays one thread", () => { | |
| 82 | expect(engineChoice({ FORGE_ENGINE: "inline" }, { ...where, machine: { cpus: 8, memoryBytes: null } }).mode).toBe("inline"); | |
| 83 | expect(engineChoice({ FORGE_ENGINE: "worker" }, { ...where, machine: { cpus: 1, memoryBytes: null } }).mode).toBe("worker"); | |
| 84 | expect(engineChoice({}, { ...where, hosted: true, machine: { cpus: 8, memoryBytes: null } }).mode).toBe("inline"); | |
| 85 | expect(engineChoice({ NODE_ENV: "test" }, { ...where, machine: { cpus: 8, memoryBytes: null } }).mode).toBe("inline"); | |
| 86 | }); | |
| 87 | test("memory.max reads bytes, and max or junk as no limit", () => { | |
| 88 | expect(parseMemoryMax("1073741824\n")).toBe(1073741824); | |
| 89 | expect(parseMemoryMax("max\n")).toBeNull(); | |
| 90 | expect(parseMemoryMax("")).toBeNull(); | |
| 91 | expect(parseMemoryMax("-5")).toBeNull(); | |
| 92 | }); | |
| 93 | }); | |
| 94 | ||
| 95 | describe("reading the machine", () => { | |
| 96 | const gib = 1024 * 1024 * 1024; | |
| 97 | const files = (map: Record<string, string>) => (path: string) => map[path] ?? null; | |
| 98 | test("the memory limit is the smaller of the cgroup's cap and the total", () => { | |
| 99 | expect(memoryLimit(String(gib), 8 * gib)).toBe(gib); | |
| 100 | expect(memoryLimit(String(16 * gib), 8 * gib)).toBe(8 * gib); | |
| 101 | expect(memoryLimit("max\n", 8 * gib)).toBe(8 * gib); | |
| 102 | expect(memoryLimit(null, 8 * gib)).toBe(8 * gib); | |
| 103 | expect(memoryLimit(null, 0)).toBeNull(); | |
| 104 | expect(memoryLimit(String(gib), 0)).toBe(gib); | |
| 105 | // v1's "unlimited" is near 2^63: no cap, so the total. | |
| 106 | expect(memoryLimit("9223372036854771712\n", 8 * gib)).toBe(8 * gib); | |
| 107 | }); | |
| 108 | test("a cpu quota counts whole CPUs, at least 1, and none for max or junk", () => { | |
| 109 | expect(cpuQuota("100000 100000\n")).toBe(1); | |
| 110 | expect(cpuQuota("50000 100000")).toBe(1); | |
| 111 | expect(cpuQuota("250000 100000")).toBe(2); | |
| 112 | expect(cpuQuota("max 100000\n")).toBeNull(); | |
| 113 | expect(cpuQuota("-1 100000")).toBeNull(); | |
| 114 | expect(cpuQuota("0 100000")).toBeNull(); | |
| 115 | expect(cpuQuota(null)).toBeNull(); | |
| 116 | }); | |
| 117 | test("usable CPUs are held under the quota and never under 1", () => { | |
| 118 | expect(usableCpus(64, 1)).toBe(1); | |
| 119 | expect(usableCpus(2, 8)).toBe(2); | |
| 120 | expect(usableCpus(8, null)).toBe(8); | |
| 121 | expect(usableCpus(0, null)).toBe(1); | |
| 122 | }); | |
| 123 | test("a 1-CPU, 512 MiB container on a big host reads as small, on v2 and on v1", () => { | |
| 124 | const v2 = readMachine(files({ "/sys/fs/cgroup/cpu.max": "100000 100000\n", "/sys/fs/cgroup/memory.max": "536870912\n" }), 64, 256 * gib); | |
| 125 | expect(v2).toEqual({ cpus: 1, memoryBytes: 512 * 1024 * 1024 }); | |
| 126 | const v1 = readMachine( | |
| 127 | files({ | |
| 128 | "/sys/fs/cgroup/cpu/cpu.cfs_quota_us": "100000\n", | |
| 129 | "/sys/fs/cgroup/cpu/cpu.cfs_period_us": "100000\n", | |
| 130 | "/sys/fs/cgroup/memory/memory.limit_in_bytes": "536870912\n", | |
| 131 | }), | |
| 132 | 64, | |
| 133 | 256 * gib, | |
| 134 | ); | |
| 135 | expect(v1).toEqual(v2); | |
| 136 | expect(machineChoice(v1).mode).toBe("inline"); | |
| 137 | }); | |
| 138 | test("with no cgroup files the scheduler and the total decide", () => { | |
| 139 | expect(readMachine(files({}), 8, 16 * gib)).toEqual({ cpus: 8, memoryBytes: 16 * gib }); | |
| 140 | expect(readMachine(files({ "/sys/fs/cgroup/cpu/cpu.cfs_quota_us": "-1", "/sys/fs/cgroup/cpu/cpu.cfs_period_us": "100000" }), 8, 0)) | |
| 141 | .toEqual({ cpus: 8, memoryBytes: null }); | |
| 142 | // The real read answers a sane shape on whatever runs the test. | |
| 143 | const here = readMachine(); | |
| 144 | expect(here.cpus).toBeGreaterThanOrEqual(1); | |
| 145 | }); | |
| 146 | test("a caller that passes no machine still gets one thread", () => { | |
| 147 | expect(engineChoice({}, where)).toEqual({ mode: "inline", why: "FORGE_ENGINE is not set" }); | |
| 148 | }); | |
| 149 | }); | |
| 150 | ||
| 54 | 151 | describe("/system", () => { |
| 55 | 152 | const admin = person("a", true); |
| 56 | 153 | test("an admin who is a person reads the shape and why", async () => { |
| @@ -59,6 +156,12 @@ describe("/system", () => { | ||
| 59 | 156 | expect(body).toContain("on a Worker"); |
| 60 | 157 | expect(body).toContain("FORGE_ENGINE=worker is set"); |
| 61 | 158 | }); |
| 159 | test("a 1-CPU machine's reason is what /system shows", async () => { | |
| 160 | const choice = engineChoice({}, { ...where, machine: { cpus: 1, memoryBytes: null } }); | |
| 161 | const { body } = await page(fresh(choice), admin); | |
| 162 | expect(body).toContain("one thread serves and works"); | |
| 163 | expect(body).toContain("FORGE_ENGINE is not set and 1 CPU is under the 4 the split needs"); | |
| 164 | }); | |
| 62 | 165 | test("a forge that recorded none has no section", async () => { |
| 63 | 166 | expect((await page(fresh(), admin)).body).not.toContain("Engine shape"); |
| 64 | 167 | }); |
new above
offline
Untested and tangled with the file read. Pull the cap/total arithmetic into a pure function and test the cases (cap<total, cap>total, max, unreadable, total 0). Also read cgroup v1 or document that only v2 is read.