Public: anyone can read this; the crew can change it.
The engine's thread goes silent and is restarted two or three times in a few hours on production #14208
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/14202-the-engine-s-thread-goes-silent-and-is-r → main 4 commits fork 1 thread · 1 unresolved
6 files
changed +102
−11
docs/features/engine-shape.md +13
| @@ -36,6 +36,19 @@ error level, the serving thread takes the duties back, and the forge runs one | ||
| 36 | 36 | thread until its next start. `/system` then reads one thread with what |
| 37 | 37 | happened, and an admin's needs list carries one line linking there. |
| 38 | 38 | |
| 39 | ## How the serving thread knows the engine is alive | |
| 40 | ||
| 41 | The engine beats every second (`beat.ts`): a message of `h` and the jobs that held its loop | |
| 42 | longest in the last two seconds, at most three, as `name=12ms`. Any message from the engine | |
| 43 | counts as a word; one silent past `STALL_MS` is ended and started again, and the log line says | |
| 44 | `why=silent` with `quietMs` and `lastJobs`, the jobs of its last beat. `inMs` on that line is the | |
| 45 | wait before the restart, not the window. | |
| 46 | ||
| 47 | The watch runs on the serving thread, so a stall there delays the engine's beats behind it. A watch | |
| 48 | tick that comes more than two beats after the one before is the serving thread's own lateness: the | |
| 49 | gap is forgiven and the engine's window starts again. This is bounded: it applies to a late tick | |
| 50 | only, so an engine that is truly hung is ended on the next on-time tick. | |
| 51 | ||
| 39 | 52 | ## Which shape runs |
| 40 | 53 | |
| 41 | 54 | Open `/system` and read "Engine shape", or open the System command in the |
src/engine/beat.ts +34
| @@ -15,3 +15,37 @@ export const STALL_MS = 3 * BUSY_TIMEOUT_MS; | ||
| 15 | 15 | * thread counts it a death at once. |
| 16 | 16 | */ |
| 17 | 17 | export const FAILED = "failed:"; |
| 18 | ||
| 19 | /** | |
| 20 | * What a beat message begins with; the rest is the jobs that held the engine's loop since the last | |
| 21 | * (`worker.ts`). | |
| 22 | */ | |
| 23 | export const BEAT = "h"; | |
| 24 | ||
| 25 | /** | |
| 26 | * Is the engine silent? Silence is only the engine's when the watcher itself kept time: a serving | |
| 27 | * thread whose loop was held past a tick had the engine's beats queued behind the stall, so the | |
| 28 | * gap it measures is its own. Such a tick forgives the gap (the beat counts from now) instead of | |
| 29 | * ending an engine for the serving thread's lateness. The forgiveness is bounded: it applies only | |
| 30 | * to a late tick, so a truly hung engine is still ended on the next on-time tick, one `BEAT_MS` | |
| 31 | * later, once its gap is past `stallMs`. | |
| 32 | */ | |
| 33 | export function judgeSilence(now: number, beat: number, lastTick: number, stallMs: number): { silent: boolean; beat: number } { | |
| 34 | if (now - lastTick > 2 * BEAT_MS) return { silent: false, beat: now }; | |
| 35 | return { silent: now - beat > stallMs, beat }; | |
| 36 | } | |
| 37 | ||
| 38 | const KEPT_JOBS = 3; | |
| 39 | ||
| 40 | /** | |
| 41 | * The beat message: `BEAT` and the jobs that held the engine's loop, longest first, at most three, | |
| 42 | * as `name=12ms`. | |
| 43 | */ | |
| 44 | export function beatMessage(jobs: readonly { name: string; ms: number }[]): string { | |
| 45 | return BEAT + jobs.slice(0, KEPT_JOBS).map((j) => `${j.name}=${Math.round(j.ms)}ms`).join(","); | |
| 46 | } | |
| 47 | ||
| 48 | /** What a beat message says the engine was doing: its jobs as `name=12ms,...`, or `none`. */ | |
| 49 | export function jobsOfBeat(data: string): string { | |
| 50 | return data.startsWith(BEAT) ? data.slice(BEAT.length) || "none" : "none"; | |
| 51 | } |
src/engine/host.ts +13 −5
| @@ -22,7 +22,7 @@ import type { ForgeConfig } from "../app.ts"; | ||
| 22 | 22 | import type { EventTail } from "../core/event_tail.ts"; |
| 23 | 23 | import { createLog } from "../kernel/log.ts"; |
| 24 | 24 | import { during } from "../perf/jobs.ts"; |
| 25 | import { BEAT_MS, FAILED, STALL_MS } from "./beat.ts"; | |
| 25 | import { BEAT, BEAT_MS, FAILED, jobsOfBeat, judgeSilence, STALL_MS } from "./beat.ts"; | |
| 26 | 26 | import { engineConfig } from "./crossing.ts"; |
| 27 | 27 | import type { EngineInit } from "./worker.ts"; |
| 28 | 28 | |
| @@ -119,8 +119,12 @@ export function startEngine( | ||
| 119 | 119 | } |
| 120 | 120 | worker = w; |
| 121 | 121 | let beat = Date.now(); |
| 122 | let lastTick = beat; | |
| 123 | /** What the engine said it was doing in its last beat: the jobs that held its loop. */ | |
| 124 | let doing = "none"; | |
| 122 | 125 | w.onmessage = (event: MessageEvent<string>) => { |
| 123 | 126 | beat = Date.now(); |
| 127 | if (typeof event.data === "string" && event.data.startsWith(BEAT)) doing = jobsOfBeat(event.data); | |
| 124 | 128 | if (event.data === "a") tail.poke(); |
| 125 | 129 | else if (event.data === "ready") { |
| 126 | 130 | log.info("the engine is up", { restarts }); |
| @@ -132,7 +136,7 @@ export function startEngine( | ||
| 132 | 136 | } |
| 133 | 137 | }; |
| 134 | 138 | let ended = false; |
| 135 | const died = (why: string): void => { | |
| 139 | const died = (why: string, more: Record<string, unknown> = {}): void => { | |
| 136 | 140 | if (ended || stopping || gaveUp !== undefined) return; |
| 137 | 141 | ended = true; |
| 138 | 142 | clearInterval(watch); |
| @@ -151,7 +155,7 @@ export function startEngine( | ||
| 151 | 155 | } |
| 152 | 156 | deaths = now - started > STABLE_MS ? 1 : deaths + 1; |
| 153 | 157 | const wait = Math.min(RESTART_MAX_MS, (options.restartMinMs ?? RESTART_MIN_MS) * 2 ** (deaths - 1)); |
| 154 | log.error("the engine stopped; serving goes on, it starts again", { why, inMs: wait, restarts: restarts + 1 }); | |
| 158 | log.error("the engine stopped; serving goes on, it starts again", { why, inMs: wait, restarts: restarts + 1, ...more }); | |
| 155 | 159 | timer = setTimeout( |
| 156 | 160 | () => |
| 157 | 161 | during("engine-restart", () => { |
| @@ -165,10 +169,14 @@ export function startEngine( | ||
| 165 | 169 | const watch = setInterval(() => { |
| 166 | 170 | during("engine-watch", () => { |
| 167 | 171 | if (stopping || gaveUp !== undefined) return clearInterval(watch); |
| 168 | if (Date.now() - beat <= (options.stallMs ?? STALL_MS)) return; | |
| 172 | const now = Date.now(); | |
| 173 | const verdict = judgeSilence(now, beat, lastTick, options.stallMs ?? STALL_MS); | |
| 174 | lastTick = now; | |
| 175 | beat = verdict.beat; | |
| 176 | if (!verdict.silent) return; | |
| 169 | 177 | clearInterval(watch); |
| 170 | 178 | w.terminate(); |
| 171 | died("silent"); | |
| 179 | died("silent", { quietMs: now - beat, lastJobs: doing }); | |
| 172 | 180 | }, undefined, options.forge); |
| 173 | 181 | }, BEAT_MS); |
| 174 | 182 | watch.unref(); |
src/engine/worker.ts +6 −3
| @@ -13,9 +13,9 @@ import { loadFlags } from "../core/flags.ts"; | ||
| 13 | 13 | import { startDuties } from "./duties.ts"; |
| 14 | 14 | import { timeStatements } from "../kernel/slow_sql.ts"; |
| 15 | 15 | import { createLog } from "../kernel/log.ts"; |
| 16 | import { during } from "../perf/jobs.ts"; | |
| 16 | import { during, jobsOf } from "../perf/jobs.ts"; | |
| 17 | 17 | import { workerMemory } from "../perf/memory_watch.ts"; |
| 18 | import { BEAT_MS, FAILED } from "./beat.ts"; | |
| 18 | import { BEAT_MS, beatMessage, FAILED } from "./beat.ts"; | |
| 19 | 19 | |
| 20 | 20 | declare const self: Worker; |
| 21 | 21 | |
| @@ -65,7 +65,10 @@ async function boot(init: EngineInit): Promise<void> { | ||
| 65 | 65 | app.close(); |
| 66 | 66 | }; |
| 67 | 67 | // Alive: a thread stuck in one statement stops saying so, and the serving thread ends it (`host.ts`). |
| 68 | const beat = setInterval(() => during("engine-beat", () => postMessage("h"), undefined, app), BEAT_MS); | |
| 68 | const beat = setInterval(() => during("engine-beat", () => { | |
| 69 | const t = performance.now(); | |
| 70 | postMessage(beatMessage(jobsOf(app).held(t - 2 * BEAT_MS, t))); | |
| 71 | }, undefined, app), BEAT_MS); | |
| 69 | 72 | const stopMemory = workerMemory("engine"); |
| 70 | 73 | const was = stop; |
| 71 | 74 | stop = () => { |
test/engine-stall.test.ts +34 −1
| @@ -1,5 +1,5 @@ | ||
| 1 | 1 | import { expect, test } from "bun:test"; |
| 2 | import { BEAT_MS, STALL_MS } from "../src/engine/beat.ts"; | |
| 2 | import { BEAT_MS, beatMessage, BEAT, jobsOfBeat, judgeSilence, STALL_MS } from "../src/engine/beat.ts"; | |
| 3 | 3 | import { BUSY_TIMEOUT_MS } from "../src/kernel/db.ts"; |
| 4 | 4 | |
| 5 | 5 | test("the engine is not called silent while a statement waits out the busy timeout", () => { |
| @@ -7,3 +7,36 @@ test("the engine is not called silent while a statement waits out the busy timeo | ||
| 7 | 7 | // A tick of the duties can wait out two locks in a row and still be alive. |
| 8 | 8 | expect(STALL_MS).toBeGreaterThanOrEqual(2 * BUSY_TIMEOUT_MS + BEAT_MS); |
| 9 | 9 | }); |
| 10 | ||
| 11 | test("an engine that has beaten within the window is not silent", () => { | |
| 12 | expect(judgeSilence(10_000, 9_000, 9_000, STALL_MS).silent).toBe(false); | |
| 13 | }); | |
| 14 | ||
| 15 | test("an engine silent past the window on a watcher that kept time is silent", () => { | |
| 16 | expect(judgeSilence(100_000, 9_000, 100_000 - BEAT_MS, STALL_MS).silent).toBe(true); | |
| 17 | }); | |
| 18 | ||
| 19 | test("a serving thread that was itself held does not end the engine for the gap", () => { | |
| 20 | // The watcher's last tick was 40 s ago: its own loop was held, the beats waited behind it. | |
| 21 | const v = judgeSilence(100_000, 9_000, 60_000, STALL_MS); | |
| 22 | expect(v.silent).toBe(false); | |
| 23 | expect(v.beat).toBe(100_000); | |
| 24 | }); | |
| 25 | ||
| 26 | test("a beat names at most three jobs, longest first as given, and the host reads them back", () => { | |
| 27 | const jobs = [ | |
| 28 | { name: "a", ms: 120.4 }, | |
| 29 | { name: "b", ms: 80 }, | |
| 30 | { name: "c", ms: 5.6 }, | |
| 31 | { name: "d", ms: 1 }, | |
| 32 | ]; | |
| 33 | const message = beatMessage(jobs); | |
| 34 | expect(message).toBe("ha=120ms,b=80ms,c=6ms"); | |
| 35 | expect(jobsOfBeat(message)).toBe("a=120ms,b=80ms,c=6ms"); | |
| 36 | }); | |
| 37 | ||
| 38 | test("a beat with no jobs reads as none, and so does a word that is not a beat", () => { | |
| 39 | expect(beatMessage([])).toBe(BEAT); | |
| 40 | expect(jobsOfBeat(beatMessage([]))).toBe("none"); | |
| 41 | expect(jobsOfBeat("ready")).toBe("none"); | |
| 42 | }); |
test/job-slices.test.ts +2 −2
| @@ -326,8 +326,8 @@ const UNTIMED: Record<string, string> = { | ||
| 326 | 326 | "asks-ledger": "records the asks ledger and the database's size on the engine thread when there is one; both are timed as part of the asks tick's pieces elsewhere, and need the whole big forge", |
| 327 | 327 | "observed-publish": "writes what changed in the thread's observations of the pulls to engine_observation, every two seconds; a diff of small maps", |
| 328 | 328 | checkpoint: "a passive WAL checkpoint run by the engine thread every two seconds: I/O on the engine's thread, nothing to time on the serving one", |
| 329 | "engine-beat": "a one-byte message from the engine thread to the serving thread, every second", | |
| 330 | "engine-watch": "compares the engine's last word with the clock, every second; ends a thread silent for 15 s", | |
| 329 | "engine-beat": "a short message (the beat and the engine's longest-held jobs) from the engine thread to the serving thread, every second", | |
| 330 | "engine-watch": "compares the engine's last word with the clock, every second; ends a thread silent past the stall limit, forgiving a tick of its own that came late", | |
| 331 | 331 | "engine-restart": "starts the engine thread again after it died: a thread start; no fixture's engine dies", |
| 332 | 332 | "event-tail": "a PRAGMA data_version read and, when the other thread wrote, an indexed read of the new events; no fixture has a second thread", |
| 333 | 333 | "spawn-due-wake": "a timer in the spawn host that wakes a due run: needs a spawned run; the fixture has none", |
new above
offline
Non-blocking: quietMs and lastJobs are assembled here with no test. Cover them with a host test using a small stallMs and a worker that goes quiet.