Public: anyone can read this; the crew can change it.

The engine's thread goes silent and is restarted two or three times in a few hours on production #14208

Merged opened this in answer to

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/14202-the-engine-s-thread-goes-silent-and-is-r → main 4 commits fork 1 thread · 1 unresolved

  • Merged. mason/14202-the-engine-s-thread-goes-silent-and-is-r was merged into main as 0a909e7.

Review

Sign in to review this change.
6 files changed +102 −11
docs/features/engine-shape.md +13
@@ -36,6 +36,19 @@ error level, the serving thread takes the duties back, and the forge runs one
3636 thread until its next start. `/system` then reads one thread with what
3737 happened, and an admin's needs list carries one line linking there.
3838
39 ## How the serving thread knows the engine is alive
40
41 The engine beats every second (`beat.ts`): a message of `h` and the jobs that held its loop
42 longest in the last two seconds, at most three, as `name=12ms`. Any message from the engine
43 counts as a word; one silent past `STALL_MS` is ended and started again, and the log line says
44 `why=silent` with `quietMs` and `lastJobs`, the jobs of its last beat. `inMs` on that line is the
45 wait before the restart, not the window.
46
47 The watch runs on the serving thread, so a stall there delays the engine's beats behind it. A watch
48 tick that comes more than two beats after the one before is the serving thread's own lateness: the
49 gap is forgiven and the engine's window starts again. This is bounded: it applies to a late tick
50 only, so an engine that is truly hung is ended on the next on-time tick.
51
3952 ## Which shape runs
4053
4154 Open `/system` and read "Engine shape", or open the System command in the
src/engine/beat.ts +34
@@ -15,3 +15,37 @@ export const STALL_MS = 3 * BUSY_TIMEOUT_MS;
1515 * thread counts it a death at once.
1616 */
1717 export const FAILED = "failed:";
18
19 /**
20 * What a beat message begins with; the rest is the jobs that held the engine's loop since the last
21 * (`worker.ts`).
22 */
23 export const BEAT = "h";
24
25 /**
26 * Is the engine silent? Silence is only the engine's when the watcher itself kept time: a serving
27 * thread whose loop was held past a tick had the engine's beats queued behind the stall, so the
28 * gap it measures is its own. Such a tick forgives the gap (the beat counts from now) instead of
29 * ending an engine for the serving thread's lateness. The forgiveness is bounded: it applies only
30 * to a late tick, so a truly hung engine is still ended on the next on-time tick, one `BEAT_MS`
31 * later, once its gap is past `stallMs`.
32 */
33 export function judgeSilence(now: number, beat: number, lastTick: number, stallMs: number): { silent: boolean; beat: number } {
34 if (now - lastTick > 2 * BEAT_MS) return { silent: false, beat: now };
35 return { silent: now - beat > stallMs, beat };
36 }
37
38 const KEPT_JOBS = 3;
39
40 /**
41 * The beat message: `BEAT` and the jobs that held the engine's loop, longest first, at most three,
42 * as `name=12ms`.
43 */
44 export function beatMessage(jobs: readonly { name: string; ms: number }[]): string {
45 return BEAT + jobs.slice(0, KEPT_JOBS).map((j) => `${j.name}=${Math.round(j.ms)}ms`).join(",");
46 }
47
48 /** What a beat message says the engine was doing: its jobs as `name=12ms,...`, or `none`. */
49 export function jobsOfBeat(data: string): string {
50 return data.startsWith(BEAT) ? data.slice(BEAT.length) || "none" : "none";
51 }
src/engine/host.ts +13 −5
@@ -22,7 +22,7 @@ import type { ForgeConfig } from "../app.ts";
2222 import type { EventTail } from "../core/event_tail.ts";
2323 import { createLog } from "../kernel/log.ts";
2424 import { during } from "../perf/jobs.ts";
25 import { BEAT_MS, FAILED, STALL_MS } from "./beat.ts";
25 import { BEAT, BEAT_MS, FAILED, jobsOfBeat, judgeSilence, STALL_MS } from "./beat.ts";
2626 import { engineConfig } from "./crossing.ts";
2727 import type { EngineInit } from "./worker.ts";
2828
@@ -119,8 +119,12 @@ export function startEngine(
119119 }
120120 worker = w;
121121 let beat = Date.now();
122 let lastTick = beat;
123 /** What the engine said it was doing in its last beat: the jobs that held its loop. */
124 let doing = "none";
122125 w.onmessage = (event: MessageEvent<string>) => {
123126 beat = Date.now();
127 if (typeof event.data === "string" && event.data.startsWith(BEAT)) doing = jobsOfBeat(event.data);
124128 if (event.data === "a") tail.poke();
125129 else if (event.data === "ready") {
126130 log.info("the engine is up", { restarts });
@@ -132,7 +136,7 @@ export function startEngine(
132136 }
133137 };
134138 let ended = false;
135 const died = (why: string): void => {
139 const died = (why: string, more: Record<string, unknown> = {}): void => {
136140 if (ended || stopping || gaveUp !== undefined) return;
137141 ended = true;
138142 clearInterval(watch);
@@ -151,7 +155,7 @@ export function startEngine(
151155 }
152156 deaths = now - started > STABLE_MS ? 1 : deaths + 1;
153157 const wait = Math.min(RESTART_MAX_MS, (options.restartMinMs ?? RESTART_MIN_MS) * 2 ** (deaths - 1));
154 log.error("the engine stopped; serving goes on, it starts again", { why, inMs: wait, restarts: restarts + 1 });
158 log.error("the engine stopped; serving goes on, it starts again", { why, inMs: wait, restarts: restarts + 1, ...more });
155159 timer = setTimeout(
156160 () =>
157161 during("engine-restart", () => {
@@ -165,10 +169,14 @@ export function startEngine(
165169 const watch = setInterval(() => {
166170 during("engine-watch", () => {
167171 if (stopping || gaveUp !== undefined) return clearInterval(watch);
168 if (Date.now() - beat <= (options.stallMs ?? STALL_MS)) return;
172 const now = Date.now();
173 const verdict = judgeSilence(now, beat, lastTick, options.stallMs ?? STALL_MS);
174 lastTick = now;
175 beat = verdict.beat;
176 if (!verdict.silent) return;
169177 clearInterval(watch);
170178 w.terminate();
171 died("silent");
179 died("silent", { quietMs: now - beat, lastJobs: doing });

Non-blocking: quietMs and lastJobs are assembled here with no test. Cover them with a host test using a small stallMs and a worker that goes quiet.

172180 }, undefined, options.forge);
173181 }, BEAT_MS);
174182 watch.unref();
src/engine/worker.ts +6 −3
@@ -13,9 +13,9 @@ import { loadFlags } from "../core/flags.ts";
1313 import { startDuties } from "./duties.ts";
1414 import { timeStatements } from "../kernel/slow_sql.ts";
1515 import { createLog } from "../kernel/log.ts";
16 import { during } from "../perf/jobs.ts";
16 import { during, jobsOf } from "../perf/jobs.ts";
1717 import { workerMemory } from "../perf/memory_watch.ts";
18 import { BEAT_MS, FAILED } from "./beat.ts";
18 import { BEAT_MS, beatMessage, FAILED } from "./beat.ts";
1919
2020 declare const self: Worker;
2121
@@ -65,7 +65,10 @@ async function boot(init: EngineInit): Promise<void> {
6565 app.close();
6666 };
6767 // Alive: a thread stuck in one statement stops saying so, and the serving thread ends it (`host.ts`).
68 const beat = setInterval(() => during("engine-beat", () => postMessage("h"), undefined, app), BEAT_MS);
68 const beat = setInterval(() => during("engine-beat", () => {
69 const t = performance.now();
70 postMessage(beatMessage(jobsOf(app).held(t - 2 * BEAT_MS, t)));
71 }, undefined, app), BEAT_MS);
6972 const stopMemory = workerMemory("engine");
7073 const was = stop;
7174 stop = () => {
test/engine-stall.test.ts +34 −1
@@ -1,5 +1,5 @@
11 import { expect, test } from "bun:test";
2 import { BEAT_MS, STALL_MS } from "../src/engine/beat.ts";
2 import { BEAT_MS, beatMessage, BEAT, jobsOfBeat, judgeSilence, STALL_MS } from "../src/engine/beat.ts";
33 import { BUSY_TIMEOUT_MS } from "../src/kernel/db.ts";
44
55 test("the engine is not called silent while a statement waits out the busy timeout", () => {
@@ -7,3 +7,36 @@ test("the engine is not called silent while a statement waits out the busy timeo
77 // A tick of the duties can wait out two locks in a row and still be alive.
88 expect(STALL_MS).toBeGreaterThanOrEqual(2 * BUSY_TIMEOUT_MS + BEAT_MS);
99 });
10
11 test("an engine that has beaten within the window is not silent", () => {
12 expect(judgeSilence(10_000, 9_000, 9_000, STALL_MS).silent).toBe(false);
13 });
14
15 test("an engine silent past the window on a watcher that kept time is silent", () => {
16 expect(judgeSilence(100_000, 9_000, 100_000 - BEAT_MS, STALL_MS).silent).toBe(true);
17 });
18
19 test("a serving thread that was itself held does not end the engine for the gap", () => {
20 // The watcher's last tick was 40 s ago: its own loop was held, the beats waited behind it.
21 const v = judgeSilence(100_000, 9_000, 60_000, STALL_MS);
22 expect(v.silent).toBe(false);
23 expect(v.beat).toBe(100_000);
24 });
25
26 test("a beat names at most three jobs, longest first as given, and the host reads them back", () => {
27 const jobs = [
28 { name: "a", ms: 120.4 },
29 { name: "b", ms: 80 },
30 { name: "c", ms: 5.6 },
31 { name: "d", ms: 1 },
32 ];
33 const message = beatMessage(jobs);
34 expect(message).toBe("ha=120ms,b=80ms,c=6ms");
35 expect(jobsOfBeat(message)).toBe("a=120ms,b=80ms,c=6ms");
36 });
37
38 test("a beat with no jobs reads as none, and so does a word that is not a beat", () => {
39 expect(beatMessage([])).toBe(BEAT);
40 expect(jobsOfBeat(beatMessage([]))).toBe("none");
41 expect(jobsOfBeat("ready")).toBe("none");
42 });
test/job-slices.test.ts +2 −2
@@ -326,8 +326,8 @@ const UNTIMED: Record<string, string> = {
326326 "asks-ledger": "records the asks ledger and the database's size on the engine thread when there is one; both are timed as part of the asks tick's pieces elsewhere, and need the whole big forge",
327327 "observed-publish": "writes what changed in the thread's observations of the pulls to engine_observation, every two seconds; a diff of small maps",
328328 checkpoint: "a passive WAL checkpoint run by the engine thread every two seconds: I/O on the engine's thread, nothing to time on the serving one",
329 "engine-beat": "a one-byte message from the engine thread to the serving thread, every second",
330 "engine-watch": "compares the engine's last word with the clock, every second; ends a thread silent for 15 s",
329 "engine-beat": "a short message (the beat and the engine's longest-held jobs) from the engine thread to the serving thread, every second",
330 "engine-watch": "compares the engine's last word with the clock, every second; ends a thread silent past the stall limit, forgiving a tick of its own that came late",
331331 "engine-restart": "starts the engine thread again after it died: a thread start; no fixture's engine dies",
332332 "event-tail": "a PRAGMA data_version read and, when the other thread wrote, an indexed read of the new events; no fixture has a second thread",
333333 "spawn-due-wake": "a timer in the spawn host that wakes a due run: needs a spawned run; the fixture has none",

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs