Public: anyone can read this; the crew can change it.
The forge starts, sizes and updates its own runners: nine were started by hand on production and outlive each release with the old image #14210
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/14201-the-forge-starts-sizes-and-updates-its-o → main 5 commits fork
5 files
changed +479
−4
docs/features/runners.md +26 −3
| @@ -11,6 +11,28 @@ runs on the forge's machine at once, as `process` does; when runners are | ||
| 11 | 11 | connected and none takes the job in 5 seconds it runs on the forge's machine |
| 12 | 12 | too. |
| 13 | 13 | |
| 14 | ## The forge's own runners | |
| 15 | ||
| 16 | Under `FORGE_CHECKS_PLACEMENT=runner` the forge starts runners of its own, as | |
| 17 | children of its process, so a release stops them and the next forge starts them | |
| 18 | from its own image: none outlives a release with old code. `FORGE_RUNNERS` is | |
| 19 | `auto` (the default), a count, or `0` for none. `auto` is what fits beside | |
| 20 | serving: the cores left after `FORGE_RUNNERS_SERVING` (default 2) divided by | |
| 21 | the cores the repository's own `forge.json` declares for its check | |
| 22 | (`check.limits.cpus`), and likewise the memory left after a fixed 4 GiB kept | |
| 23 | for serving divided by `check.limits.memory`, never fewer than one on a | |
| 24 | machine with room and never more than 64. They are enrolled as `local-1`…`local-N` with a token made | |
| 25 | at each start (a token from before is removed) and listed by | |
| 26 | `GET /api/v1/admin/runners` like any other; one that exits is started again | |
| 27 | after a pause that doubles up to a minute. Their environment holds only | |
| 28 | `FORGE_URL`, their token, their directory and `PATH`, `HOME`, `LANG`. Runners | |
| 29 | started by hand are in addition: an admin removes them with | |
| 30 | `DELETE /api/v1/admin/runners/<id>`. Every `local-*` row created by `forge` is | |
| 31 | deleted at each boot, so never name a hand-made runner `local-N`. | |
| 32 | ||
| 33 | Not yet: a page for them, `checks.concurrency` counting runner capacity, a job | |
| 34 | that survives a restart, caches by hash (#14201's follow-ups). | |
| 35 | ||
| 14 | 36 | ## Start one, in two lines |
| 15 | 37 | |
| 16 | 38 | An admin enrols it (admin scope; the token is shown this once): |
| @@ -105,6 +127,7 @@ check starts without them: correct, and slower than the forge's own. | ||
| 105 | 127 | A runner's requests are counted like a stranger's (600 a minute from one |
| 106 | 128 | address); one job is about 60 a minute. |
| 107 | 129 | |
| 108 | Code: `src/checks/runner_jobs.ts` (the desk), `src/checks/runner_step.ts` (the | |
| 109 | forge's side), `src/runner/main.ts` (the runner), `src/http/api/runners.ts`. | |
| 110 | Tests: `test/checks-runner-placement.test.ts`. Epic: #13652. | |
| 130 | Code: `src/checks/runner_pool.ts` (the forge's own), `src/checks/runner_jobs.ts` | |
| 131 | (the desk), `src/checks/runner_step.ts` (the forge's side), `src/runner/main.ts` | |
| 132 | (the runner), `src/http/api/runners.ts`. | |
| 133 | Tests: `test/checks-runner-placement.test.ts`, `test/checks-runner-pool.test.ts`. Epic: #13652. |
src/checks/runner_pool.ts added +218
| @@ -0,0 +1,218 @@ | ||
| 1 | /** | |
| 2 | * The forge as its own first runner pool (#14201): with `FORGE_CHECKS_PLACEMENT=runner` the forge | |
| 3 | * starts N runner processes of its own, sized from this machine, and they are its children: a | |
| 4 | * release stops them with the forge and the next forge starts them from its own image, so none | |
| 5 | * outlives a release with the old code. | |
| 6 | * | |
| 7 | * FORGE_RUNNERS `auto` (the default under the runner placement), a count, or `0` for none | |
| 8 | * FORGE_RUNNERS_SERVING cores kept for serving. @default 2 | |
| 9 | * | |
| 10 | * Each is enrolled as `local-<n>` with a token made at this boot (the rows of the last boot are | |
| 11 | * removed first, so an old token opens nothing) and is shown by `GET /api/v1/admin/runners` like | |
| 12 | * any other. A runner that exits is started again after a pause that doubles up to a minute. | |
| 13 | */ | |
| 14 | ||
| 15 | import { mkdirSync, readFileSync } from "node:fs"; | |
| 16 | import { join } from "node:path"; | |
| 17 | import type { App } from "../app.ts"; | |
| 18 | import { enrolRunner } from "./runner_jobs.ts"; | |
| 19 | ||
| 20 | /** The name prefix of the runners the forge starts itself: the boot replaces all of them. */ | |
| 21 | export const LOCAL_PREFIX = "local-"; | |
| 22 | /** Who the enrolment is recorded as made by. */ | |
| 23 | const SELF = "forge"; | |
| 24 | const MAX_RUNNERS = 64; | |
| 25 | const FIRST_BACKOFF_MS = 1_000; | |
| 26 | const MAX_BACKOFF_MS = 60_000; | |
| 27 | /** A runner that lived this long was healthy: its next pause starts over. */ | |
| 28 | const HEALTHY_MS = 60_000; | |
| 29 | /** Cores and memory kept for serving unless `FORGE_RUNNERS_SERVING` says otherwise (cores only). */ | |
| 30 | const SERVING_CPUS = 2; | |
| 31 | const SERVING_MEMORY_MIB = 4096; | |
| 32 | /** What a check is taken to need when the repository's `forge.json` declares none. */ | |
| 33 | const DEFAULT_CHECK_CPUS = 4; | |
| 34 | const DEFAULT_CHECK_MEMORY_MIB = 4096; | |
| 35 | ||
| 36 | export interface Machine { | |
| 37 | readonly cpus: number; | |
| 38 | readonly memoryMiB: number; | |
| 39 | } | |
| 40 | ||
| 41 | export interface PoolShape { | |
| 42 | /** Cores a check may use: the repository's declared `check.limits.cpus`. */ | |
| 43 | readonly perRunnerCpus: number; | |
| 44 | /** Memory a check may use, in MiB: `check.limits.memory`. */ | |
| 45 | readonly perRunnerMemoryMiB: number; | |
| 46 | /** Cores and memory kept for serving. */ | |
| 47 | readonly servingCpus: number; | |
| 48 | readonly servingMemoryMiB: number; | |
| 49 | } | |
| 50 | ||
| 51 | /** | |
| 52 | * How many runners fit beside serving: what is left of the cores, then of the memory, divided by | |
| 53 | * what one check is declared to use; at least one on a machine with any room at all, since a | |
| 54 | * check may use fewer cores than it declares. | |
| 55 | */ | |
| 56 | export function poolSize(machine: Machine, shape: PoolShape): number { | |
| 57 | const cores = machine.cpus - shape.servingCpus; | |
| 58 | const memory = machine.memoryMiB - shape.servingMemoryMiB; | |
| 59 | if (cores < 1 || memory < 1) return 0; | |
| 60 | const byCores = Math.floor(cores / Math.max(1, shape.perRunnerCpus)); | |
| 61 | const byMemory = Math.floor(memory / Math.max(1, shape.perRunnerMemoryMiB)); | |
| 62 | return Math.min(MAX_RUNNERS, Math.max(1, Math.min(byCores, byMemory))); | |
| 63 | } | |
| 64 | ||
| 65 | /** `FORGE_RUNNERS`: undefined is auto, a count is a count, anything else is none. */ | |
| 66 | export function runnersWanted(raw: string | undefined): number | "auto" { | |
| 67 | const value = (raw ?? "auto").trim().toLowerCase(); | |
| 68 | if (value === "" || value === "auto") return "auto"; | |
| 69 | return /^\d{1,2}$/.test(value) ? Math.min(Number(value), MAX_RUNNERS) : 0; | |
| 70 | } | |
| 71 | ||
| 72 | /** The check's declared limits from a `forge.json` text; the repository's own file, or none. */ | |
| 73 | export function declaredCheckLimits( | |
| 74 | text: string, | |
| 75 | ): { cpus: number; memoryMiB: number } | undefined { | |
| 76 | try { | |
| 77 | const limits = ( | |
| 78 | JSON.parse(text) as { check?: { limits?: Record<string, unknown> } } | |
| 79 | ).check?.limits; | |
| 80 | const cpus = Number(limits?.["cpus"]); | |
| 81 | const memoryMiB = Number(limits?.["memory"]); | |
| 82 | return Number.isFinite(cpus) && | |
| 83 | cpus >= 1 && | |
| 84 | Number.isFinite(memoryMiB) && | |
| 85 | memoryMiB >= 1 | |
| 86 | ? { cpus: Math.trunc(cpus), memoryMiB: Math.trunc(memoryMiB) } | |
| 87 | : undefined; | |
| 88 | } catch { | |
| 89 | return undefined; | |
| 90 | } | |
| 91 | } | |
| 92 | ||
| 93 | /** `FORGE_RUNNERS_SERVING`: a whole number of cores, or the default when unset or not one. */ | |
| 94 | export function servingCpusFrom(raw: string | undefined): number { | |
| 95 | const value = (raw ?? "").trim(); | |
| 96 | return /^\d{1,4}$/.test(value) ? Number(value) : SERVING_CPUS; | |
| 97 | } | |
| 98 | ||
| 99 | /** The shape this forge's own `forge.json` declares, with the defaults of a small check. */ | |
| 100 | export function shapeFromRepository( | |
| 101 | root: string, | |
| 102 | env: Record<string, string | undefined> = process.env, | |
| 103 | ): PoolShape { | |
| 104 | let declared: ReturnType<typeof declaredCheckLimits>; | |
| 105 | try { | |
| 106 | declared = declaredCheckLimits(readFileSync(join(root, "forge.json"), "utf8")); | |
| 107 | } catch { | |
| 108 | declared = undefined; | |
| 109 | } | |
| 110 | return { | |
| 111 | perRunnerCpus: declared?.cpus ?? DEFAULT_CHECK_CPUS, | |
| 112 | perRunnerMemoryMiB: declared?.memoryMiB ?? DEFAULT_CHECK_MEMORY_MIB, | |
| 113 | servingCpus: servingCpusFrom(env["FORGE_RUNNERS_SERVING"]), | |
| 114 | servingMemoryMiB: SERVING_MEMORY_MIB, | |
| 115 | }; | |
| 116 | } | |
| 117 | ||
| 118 | /** How many runners this boot starts: `FORGE_RUNNERS` as asked, `auto` sized from the machine. */ | |
| 119 | export function localPoolSize( | |
| 120 | env: Record<string, string | undefined>, | |
| 121 | machine: Machine, | |
| 122 | root: string, | |
| 123 | ): number { | |
| 124 | const wanted = runnersWanted(env["FORGE_RUNNERS"]); | |
| 125 | return wanted === "auto" ? poolSize(machine, shapeFromRepository(root, env)) : wanted; | |
| 126 | } | |
| 127 | ||
| 128 | export interface Child { | |
| 129 | readonly exited: Promise<number>; | |
| 130 | kill(): void; | |
| 131 | } | |
| 132 | ||
| 133 | export type Spawn = (argv: string[], env: Record<string, string>) => Child; | |
| 134 | ||
| 135 | const spawnChild: Spawn = (argv, env) => { | |
| 136 | const proc = Bun.spawn(argv, { | |
| 137 | env, | |
| 138 | stdin: "ignore", | |
| 139 | stdout: "inherit", | |
| 140 | stderr: "inherit", | |
| 141 | }); | |
| 142 | return { exited: proc.exited, kill: () => proc.kill("SIGTERM") }; | |
| 143 | }; | |
| 144 | ||
| 145 | export interface PoolOptions { | |
| 146 | readonly forgeUrl: string; | |
| 147 | readonly size: number; | |
| 148 | /** Where each runner keeps its scratch and store: a directory under the forge's data. */ | |
| 149 | readonly dir: string; | |
| 150 | /** The runner's entry. */ | |
| 151 | readonly entry: string; | |
| 152 | readonly spawn?: Spawn | undefined; | |
| 153 | readonly sleep?: ((ms: number) => Promise<void>) | undefined; | |
| 154 | /** The clock the healthy-run reset reads. @default Date.now */ | |
| 155 | readonly now?: (() => number) | undefined; | |
| 156 | } | |
| 157 | ||
| 158 | export interface Pool { | |
| 159 | readonly names: string[]; | |
| 160 | stop(): Promise<void>; | |
| 161 | } | |
| 162 | ||
| 163 | export function startLocalPool(app: App, options: PoolOptions): Pool { | |
| 164 | const spawn = options.spawn ?? spawnChild; | |
| 165 | const sleep = options.sleep ?? ((ms: number) => Bun.sleep(ms)); | |
| 166 | const now = options.now ?? Date.now; | |
| 167 | // Last boot's runners: their tokens open nothing from here on. | |
| 168 | app.db | |
| 169 | .query("DELETE FROM check_runner WHERE name LIKE ?1 AND created_by = ?2") | |
| 170 | .run(`${LOCAL_PREFIX}%`, SELF); | |
| 171 | const live = new Set<Child>(); | |
| 172 | let stopping = false; | |
| 173 | const names: string[] = []; | |
| 174 | ||
| 175 | async function keep(name: string): Promise<void> { | |
| 176 | const dir = join(options.dir, name); | |
| 177 | mkdirSync(dir, { recursive: true }); | |
| 178 | let pause = FIRST_BACKOFF_MS; | |
| 179 | while (!stopping) { | |
| 180 | const started = now(); | |
| 181 | // A fresh enrolment each start: a token an admin removed is not asked for again. | |
| 182 | app.db.query("DELETE FROM check_runner WHERE name = ?1 AND created_by = ?2").run(name, SELF); | |
| 183 | const { token } = enrolRunner(app.db, name, SELF); | |
| 184 | // Only what a runner needs: never the forge's own environment. | |
| 185 | const child = spawn(["bun", options.entry], { | |
| 186 | PATH: process.env["PATH"] ?? "/usr/local/bin:/usr/bin:/bin", | |
| 187 | HOME: process.env["HOME"] ?? "/", | |
| 188 | LANG: process.env["LANG"] ?? "C.UTF-8", | |
| 189 | FORGE_URL: options.forgeUrl, | |
| 190 | FORGE_RUNNER_TOKEN: token, | |
| 191 | FORGE_RUNNER_DIR: dir, | |
| 192 | }); | |
| 193 | live.add(child); | |
| 194 | await child.exited.catch(() => 1); | |
| 195 | live.delete(child); | |
| 196 | if (stopping) return; | |
| 197 | pause = now() - started >= HEALTHY_MS ? FIRST_BACKOFF_MS : pause; | |
| 198 | app.log.warn("a local runner exited; starting it again", { name, pauseMs: pause }); | |
| 199 | await sleep(pause); | |
| 200 | pause = Math.min(pause * 2, MAX_BACKOFF_MS); | |
| 201 | } | |
| 202 | } | |
| 203 | ||
| 204 | for (let n = 1; n <= options.size; n++) { | |
| 205 | const name = `${LOCAL_PREFIX}${n}`; | |
| 206 | names.push(name); | |
| 207 | void keep(name); | |
| 208 | } | |
| 209 | return { | |
| 210 | names, | |
| 211 | stop: async () => { | |
| 212 | stopping = true; | |
| 213 | const going = [...live]; | |
| 214 | for (const child of going) child.kill(); | |
| 215 | await Promise.all(going.map((child) => child.exited.catch(() => 1))); | |
| 216 | }, | |
| 217 | }; | |
| 218 | } |
src/main.ts +26 −1
| @@ -64,6 +64,10 @@ import { LAND_SETTLE_MS, requeueAbandonedChecks, startChecks } from "./checks/se | ||
| 64 | 64 | import { VERIFY_EVERY_MS } from "./checks/verify.ts"; |
| 65 | 65 | import { RED_HOLD_MS } from "./checks/red_main.ts"; |
| 66 | 66 | import { stopQueue } from "./checks/queue.ts"; |
| 67 | import { placementFromEnv } from "./checks/placement.ts"; | |
| 68 | import { localPoolSize, startLocalPool } from "./checks/runner_pool.ts"; | |
| 69 | import { memoryLimitBytes } from "./system/memory_limit.ts"; | |
| 70 | import { cpuLimit } from "./core/capacity.ts"; | |
| 67 | 71 | import { startLooks } from "./looks/service.ts"; |
| 68 | 72 | import { startMaintain } from "./maintain/runner.ts"; |
| 69 | 73 | import { startMirrorOut } from "./origins/mirror-out.ts"; |
| @@ -617,6 +621,24 @@ const checks = | ||
| 617 | 621 | // Main is verified after landing at most this often (#7786). |
| 618 | 622 | verifyEveryMs: VERIFY_EVERY_MS, |
| 619 | 623 | }); |
| 624 | // Under the runner placement the forge is its own first runner pool: N runner processes sized | |
| 625 | // to this machine beside serving, children of this process so a release replaces them (#14201). | |
| 626 | const localRunners = (() => { | |
| 627 | if (checks === undefined || placementFromEnv().mode !== "runner") return undefined; | |
| 628 | const size = localPoolSize( | |
| 629 | process.env, | |
| 630 | { cpus: cpuLimit(), memoryMiB: Math.floor(memoryLimitBytes() / 1_048_576) }, | |
| 631 | join(import.meta.dir, ".."), | |
| 632 | ); | |
| 633 | return size === 0 | |
| 634 | ? undefined | |
| 635 | : startLocalPool(app, { | |
| 636 | forgeUrl: servers.url, | |
| 637 | size, | |
| 638 | dir: join(app.config.dataDir, "runners"), | |
| 639 | entry: join(import.meta.dir, "runner", "main.ts"), | |
| 640 | }); | |
| 641 | })(); | |
| 620 | 642 | // Migrated repositories kept as a read-only mirror push to their source. |
| 621 | 643 | const mirrorOut = startMirrorOut(app); |
| 622 | 644 | // …and take the pulls opened on that mirror as pulls here (#9011). |
| @@ -1135,7 +1157,10 @@ function drain(signal: NodeJS.Signals): Promise<unknown> { | ||
| 1135 | 1157 | (linger ?? Promise.resolve()).then(() => stopCellChildren(app)), |
| 1136 | 1158 | // A step leads a process group of its own, so a terminal's Ctrl-C no |
| 1137 | 1159 | // longer ends it: a person's stop cuts checks off (queued again) at once. |
| 1138 | checks?.stop({ drain: signal === "SIGTERM" }), | |
| 1160 | // The runners go after the checks' drain: a check on one finishes first. | |
| 1161 | Promise.resolve(checks?.stop({ drain: signal === "SIGTERM" })).then(() => | |
| 1162 | localRunners?.stop(), | |
| 1163 | ), | |
| 1139 | 1164 | mirrorOut(), |
| 1140 | 1165 | mirrorPulls(), |
| 1141 | 1166 | looks?.stop(), |
test/checks-runner-pool.test.ts added +207
| @@ -0,0 +1,207 @@ | ||
| 1 | /** | |
| 2 | * The forge as its own first runner pool (#14201): sized from the machine, enrolled each start, | |
| 3 | * started again when one exits, stopped with the forge. | |
| 4 | */ | |
| 5 | import { describe, expect, test } from "bun:test"; | |
| 6 | import { | |
| 7 | declaredCheckLimits, | |
| 8 | localPoolSize, | |
| 9 | poolSize, | |
| 10 | runnersWanted, | |
| 11 | servingCpusFrom, | |
| 12 | shapeFromRepository, | |
| 13 | startLocalPool, | |
| 14 | type Child, | |
| 15 | } from "../src/checks/runner_pool.ts"; | |
| 16 | import { listRunners, runnerByToken } from "../src/checks/runner_jobs.ts"; | |
| 17 | import { createApp } from "../src/app.ts"; | |
| 18 | import { mkdtempSync, writeFileSync } from "node:fs"; | |
| 19 | import { tmpdir } from "node:os"; | |
| 20 | import { join } from "node:path"; | |
| 21 | ||
| 22 | const shape = { perRunnerCpus: 16, perRunnerMemoryMiB: 8192, servingCpus: 2, servingMemoryMiB: 4096 }; | |
| 23 | ||
| 24 | describe("poolSize", () => { | |
| 25 | test("what fits beside serving, by cores then by memory", () => { | |
| 26 | expect(poolSize({ cpus: 66, memoryMiB: 200_000 }, shape)).toBe(4); | |
| 27 | expect(poolSize({ cpus: 66, memoryMiB: 20_000 }, shape)).toBe(1); | |
| 28 | expect(poolSize({ cpus: 66, memoryMiB: 4096 + 8192 * 2 }, shape)).toBe(2); | |
| 29 | }); | |
| 30 | test("a small machine still gets one, a machine with no room none", () => { | |
| 31 | expect(poolSize({ cpus: 8, memoryMiB: 32_000 }, shape)).toBe(1); | |
| 32 | expect(poolSize({ cpus: 2, memoryMiB: 32_000 }, shape)).toBe(0); | |
| 33 | expect(poolSize({ cpus: 64, memoryMiB: 4096 }, shape)).toBe(0); | |
| 34 | }); | |
| 35 | test("never more than 64", () => { | |
| 36 | const tiny = { ...shape, perRunnerCpus: 1, perRunnerMemoryMiB: 1 }; | |
| 37 | expect(poolSize({ cpus: 4000, memoryMiB: 9_999_999 }, tiny)).toBe(64); | |
| 38 | }); | |
| 39 | test("a declared zero is taken as one, not a division by nothing", () => { | |
| 40 | expect(poolSize({ cpus: 10, memoryMiB: 200_000 }, { ...shape, perRunnerCpus: 0 })).toBe(8); | |
| 41 | expect(poolSize({ cpus: 66, memoryMiB: 20_000 }, { ...shape, perRunnerMemoryMiB: 0 })).toBe(4); | |
| 42 | }); | |
| 43 | }); | |
| 44 | ||
| 45 | describe("what is asked for", () => { | |
| 46 | test("FORGE_RUNNERS", () => { | |
| 47 | expect(runnersWanted(undefined)).toBe("auto"); | |
| 48 | expect(runnersWanted("auto")).toBe("auto"); | |
| 49 | expect(runnersWanted("3")).toBe(3); | |
| 50 | expect(runnersWanted("0")).toBe(0); | |
| 51 | expect(runnersWanted("many")).toBe(0); | |
| 52 | expect(runnersWanted("-1")).toBe(0); | |
| 53 | expect(runnersWanted("99")).toBe(64); | |
| 54 | }); | |
| 55 | test("FORGE_RUNNERS_SERVING", () => { | |
| 56 | expect(servingCpusFrom(undefined)).toBe(2); | |
| 57 | expect(servingCpusFrom("0")).toBe(0); | |
| 58 | expect(servingCpusFrom("3")).toBe(3); | |
| 59 | expect(servingCpusFrom("abc")).toBe(2); | |
| 60 | expect(servingCpusFrom("-1")).toBe(2); | |
| 61 | expect(servingCpusFrom("")).toBe(2); | |
| 62 | }); | |
| 63 | test("the shape reads the repository's forge.json, and falls back without one", () => { | |
| 64 | const dir = mkdtempSync(join(tmpdir(), "forge-shape-")); | |
| 65 | expect(shapeFromRepository(dir, {})).toEqual({ | |
| 66 | perRunnerCpus: 4, | |
| 67 | perRunnerMemoryMiB: 4096, | |
| 68 | servingCpus: 2, | |
| 69 | servingMemoryMiB: 4096, | |
| 70 | }); | |
| 71 | writeFileSync(join(dir, "forge.json"), '{"check":{"limits":{"cpus":16,"memory":8192}}}'); | |
| 72 | expect(shapeFromRepository(dir, { FORGE_RUNNERS_SERVING: "3" })).toEqual({ | |
| 73 | perRunnerCpus: 16, | |
| 74 | perRunnerMemoryMiB: 8192, | |
| 75 | servingCpus: 3, | |
| 76 | servingMemoryMiB: 4096, | |
| 77 | }); | |
| 78 | }); | |
| 79 | test("how many this boot starts: none, a count, or sized from the machine", () => { | |
| 80 | const dir = mkdtempSync(join(tmpdir(), "forge-size-")); | |
| 81 | writeFileSync(join(dir, "forge.json"), '{"check":{"limits":{"cpus":16,"memory":8192}}}'); | |
| 82 | const machine = { cpus: 66, memoryMiB: 200_000 }; | |
| 83 | expect(localPoolSize({ FORGE_RUNNERS: "0" }, machine, dir)).toBe(0); | |
| 84 | expect(localPoolSize({ FORGE_RUNNERS: "5" }, machine, dir)).toBe(5); | |
| 85 | expect(localPoolSize({}, machine, dir)).toBe(4); | |
| 86 | expect(localPoolSize({ FORGE_RUNNERS: "auto", FORGE_RUNNERS_SERVING: "34" }, machine, dir)).toBe(2); | |
| 87 | }); | |
| 88 | test("the check's declared limits are read, and nonsense is none", () => { | |
| 89 | const text = '{"check":{"limits":{"cpus":16,"memory":8192}}}'; | |
| 90 | expect(declaredCheckLimits(text)).toEqual({ cpus: 16, memoryMiB: 8192 }); | |
| 91 | expect(declaredCheckLimits('{"check":{}}')).toBeUndefined(); | |
| 92 | expect(declaredCheckLimits("not json")).toBeUndefined(); | |
| 93 | }); | |
| 94 | }); | |
| 95 | ||
| 96 | describe("startLocalPool", () => { | |
| 97 | test("enrols each runner, starts it again when it exits, and stops them with the forge", async () => { | |
| 98 | const dir = mkdtempSync(join(tmpdir(), "forge-pool-")); | |
| 99 | const app = createApp({ dataDir: dir }); | |
| 100 | const starts: Record<string, string>[] = []; | |
| 101 | const killed: number[] = []; | |
| 102 | let thirdStarted!: () => void; | |
| 103 | const third = new Promise<void>((resolve) => (thirdStarted = resolve)); | |
| 104 | const spawn = (_argv: string[], env: Record<string, string>): Child => { | |
| 105 | starts.push(env); | |
| 106 | const n = starts.length; | |
| 107 | if (n === 3) thirdStarted(); | |
| 108 | let done!: (code: number) => void; | |
| 109 | const exited = new Promise<number>((resolve) => (done = resolve)); | |
| 110 | // The first start crashes at once; every other waits for a kill. | |
| 111 | if (n === 1) done(1); | |
| 112 | return { exited, kill: () => (killed.push(n), done(0)) }; | |
| 113 | }; | |
| 114 | const pool = startLocalPool(app, { | |
| 115 | forgeUrl: "http://localhost:1", | |
| 116 | size: 2, | |
| 117 | dir: join(dir, "runners"), | |
| 118 | entry: "src/runner/main.ts", | |
| 119 | spawn, | |
| 120 | sleep: async () => {}, | |
| 121 | }); | |
| 122 | expect(pool.names).toEqual(["local-1", "local-2"]); | |
| 123 | await third; // the pool's restart loop starts the crashed runner again | |
| 124 | expect(starts).toHaveLength(3); | |
| 125 | // The environment is the runner's own: the forge's secrets are not in it. | |
| 126 | for (const env of starts) { | |
| 127 | expect(Object.keys(env).sort()).toEqual([ | |
| 128 | "FORGE_RUNNER_DIR", | |
| 129 | "FORGE_RUNNER_TOKEN", | |
| 130 | "FORGE_URL", | |
| 131 | "HOME", | |
| 132 | "LANG", | |
| 133 | "PATH", | |
| 134 | ]); | |
| 135 | expect(env["FORGE_URL"]).toBe("http://localhost:1"); | |
| 136 | } | |
| 137 | // The crashed one's old token is gone, the new one opens a runner; two rows, not three. | |
| 138 | expect(listRunners(app.db).map((r) => r.name).sort()).toEqual(["local-1", "local-2"]); | |
| 139 | expect(runnerByToken(app.db, starts[0]!["FORGE_RUNNER_TOKEN"]!, Date.now())).toBeUndefined(); | |
| 140 | expect(runnerByToken(app.db, starts[2]!["FORGE_RUNNER_TOKEN"]!, Date.now())?.name).toBe("local-1"); | |
| 141 | await pool.stop(); | |
| 142 | expect(killed.sort()).toEqual([2, 3]); | |
| 143 | }); | |
| 144 | ||
| 145 | test("a second boot removes the first boot's runners", async () => { | |
| 146 | const dir = mkdtempSync(join(tmpdir(), "forge-pool-")); | |
| 147 | const app = createApp({ dataDir: dir }); | |
| 148 | const spawn = (): Child => ({ exited: new Promise(() => {}), kill() {} }); | |
| 149 | const options = { | |
| 150 | forgeUrl: "http://localhost:1", | |
| 151 | size: 1, | |
| 152 | dir: join(dir, "r"), | |
| 153 | entry: "x", | |
| 154 | spawn, | |
| 155 | sleep: async () => {}, | |
| 156 | }; | |
| 157 | startLocalPool(app, options); | |
| 158 | const first = listRunners(app.db)[0]!.id; | |
| 159 | startLocalPool(app, options); | |
| 160 | const now = listRunners(app.db); | |
| 161 | expect(now).toHaveLength(1); | |
| 162 | expect(now[0]!.id).not.toBe(first); | |
| 163 | }); | |
| 164 | }); | |
| 165 | ||
| 166 | describe("the restart pause", () => { | |
| 167 | /** A pool whose runner exits at once, `lives` ms after it started, each time; pauses recorded. */ | |
| 168 | async function pauses(lives: number[]): Promise<number[]> { | |
| 169 | const dir = mkdtempSync(join(tmpdir(), "forge-pool-")); | |
| 170 | const app = createApp({ dataDir: dir }); | |
| 171 | const seen: number[] = []; | |
| 172 | let clock = 0; | |
| 173 | let starts = 0; | |
| 174 | let finished!: () => void; | |
| 175 | const done = new Promise<void>((resolve) => (finished = resolve)); | |
| 176 | const spawn = (): Child => { | |
| 177 | const life = lives[starts++]; | |
| 178 | if (life === undefined) { | |
| 179 | finished(); | |
| 180 | return { exited: new Promise(() => {}), kill() {} }; | |
| 181 | } | |
| 182 | clock += life; | |
| 183 | return { exited: Promise.resolve(1), kill() {} }; | |
| 184 | }; | |
| 185 | const pool = startLocalPool(app, { | |
| 186 | forgeUrl: "http://localhost:1", | |
| 187 | size: 1, | |
| 188 | dir: join(dir, "r"), | |
| 189 | entry: "x", | |
| 190 | spawn, | |
| 191 | now: () => clock, | |
| 192 | sleep: async (ms) => void seen.push(ms), | |
| 193 | }); | |
| 194 | await done; | |
| 195 | void pool.stop(); | |
| 196 | return seen; | |
| 197 | } | |
| 198 | ||
| 199 | test("doubles from a second up to a minute and stays there", async () => { | |
| 200 | const quick = Array(9).fill(0) as number[]; | |
| 201 | expect(await pauses(quick)).toEqual([1000, 2000, 4000, 8000, 16000, 32000, 60000, 60000, 60000]); | |
| 202 | }); | |
| 203 | ||
| 204 | test("a runner that lived a minute starts the pause over", async () => { | |
| 205 | expect(await pauses([0, 0, 0, 60_000, 0])).toEqual([1000, 2000, 4000, 1000, 2000]); | |
| 206 | }); | |
| 207 | }); |
test/spawn-sites.test.ts +2
| @@ -25,6 +25,8 @@ const SITES: Record<string, string> = { | ||
| 25 | 25 | "chaos harness: `git cat-file -e <oid>^{commit}` with an oid read from the forge's own pulls table", |
| 26 | 26 | "src/checks/install.ts": |
| 27 | 27 | "a fixed argv, `bun install --frozen-lockfile --ignore-scripts`, in the check's checkout", |
| 28 | "src/checks/runner_pool.ts": | |
| 29 | "starts `bun` on the forge's own runner entry, a path the forge builds, with an environment it builds (its URL, a token it made, a directory); no request text", | |
| 28 | 30 | "src/cli/commands/open.ts": |
| 29 | 31 | "the CLI hands a forge URL it built to open/xdg-open as one argv element, never through a shell", |
| 30 | 32 | "src/cli/config.ts": |
new above
offline