Public: anyone can read this; the crew can change it.

On the stage rig three background jobs meet a locked database at start, every release since the engine moved to its own thread #14206

Merged opened this in answer to

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/14203-on-the-stage-rig-three-background-jobs-m → main 3 commits fork

  • Merged. scout/14203-on-the-stage-rig-three-background-jobs-m was merged into main as 83079e5.

Review

Sign in to review this change.
2 files changed +81 −8
src/system/retention.ts +15 −7
@@ -35,7 +35,7 @@ import type { App } from "../app.ts";
3535 import { backfillLanded, LANDED_SLICE } from "../core/carried.ts";
3636 import { pillarWindowStart } from "../core/pillars.ts";
3737 import { quietly } from "../core/write_clock.ts";
38 import { withoutWaiting } from "../kernel/db.ts";
38 import { isBusy, withoutWaiting } from "../kernel/db.ts";
3939 import { during } from "../perf/jobs.ts";
4040 import { limitValue } from "../settings/limits.ts";
4141 import { deployRetention } from "./retention_deploys.ts";
@@ -58,6 +58,12 @@ export const IDLE_MS = 15_000;
5858 * SQLite refuses) is logged once and that job stays quiet for the hour; the other jobs and the process go on.
5959 */
6060 export const FAILED_MS = 60 * 60_000;
61 /**
62 * How long a job whose slice met another writer's lock waits to try again. The slices do not wait
63 * for a lock (`withoutWaiting`), and at start the engine's thread and the serving thread write
64 * together, so a locked database is the other thread's turn, not a fault: seconds, not the hour.
65 */
66 export const BUSY_RETRY_MS = 3_000;
6167 /** How long the backfill waits between passes, once a pass has found nothing more. */
6268 export const BACKFILL_EVERY_MS = 60 * 60_000;
6369
@@ -140,18 +146,15 @@ export function retentionJobs(app: App): Retention {
140146 samples: {
141147 idleMs: IDLE_MS,
142148 slice(now) {
143 const r = pruneSamples(db, now, ROW_CAP);
149 const r = withoutWaiting(db, () => pruneSamples(db, now, ROW_CAP));
144150 return { did: r.deleted + r.rolled, more: r.more };
145151 },
146152 },
147153 backfill: {
148154 idleMs: BACKFILL_EVERY_MS,
149155 slice(now) {
150 const r = backfillLanded(
151 db,
152 pillarWindowStart(new Date(now)),
153 LANDED_SLICE,
154 cursor,
156 const r = withoutWaiting(db, () =>
157 backfillLanded(db, pillarWindowStart(new Date(now)), LANDED_SLICE, cursor),
155158 );
156159 cursor = r.next;
157160 // The end of a pass is the end of the window's pulls: the ledger may go.
@@ -199,6 +202,11 @@ export function startRetention(app: App, which: { local: boolean; engine: boolea
199202 quietUntil = performance.now() + Math.max(MIN_GAP_MS, took * QUIET_X);
200203 return gapAfter(took, r.more, job.idleMs);
201204 } catch (error) {
205 if (isBusy(error)) {
206 app.log.info(`${what} met a locked database; trying again shortly`);
207 quietUntil = performance.now() + MIN_GAP_MS;
208 return BUSY_RETRY_MS;
209 }
202210 app.log.warn(`${what} failed; that job rests an hour`, { error: String(error) });
203211 quietUntil = performance.now() + MIN_GAP_MS;
204212 return FAILED_MS;
test/retention-jobs.test.ts +66 −1
@@ -8,7 +8,7 @@ import type { Database } from "bun:sqlite";
88 import type { App } from "../src/app.ts";
99 import { migrations } from "../src/core/schema.ts";
1010 import { openDb } from "../src/kernel/db.ts";
11 import { DUTY, FAILED_MS, gapAfter, IDLE_MS, MIN_GAP_MS, retentionJobs, startRetention } from "../src/system/retention.ts";
11 import { BUSY_RETRY_MS, DUTY, FAILED_MS, gapAfter, IDLE_MS, MIN_GAP_MS, retentionJobs, startRetention } from "../src/system/retention.ts";
1212
1313 const DAY = 86_400_000;
1414 const iso = (ms: number) => new Date(ms).toISOString();
@@ -106,6 +106,71 @@ describe("the jobs", () => {
106106 expect(FAILED_MS).toBe(3_600_000);
107107 });
108108
109 test("a slice that meets a locked database is scheduled again in BUSY_RETRY_MS, not an hour, and runs once the lock clears", () => {
110 const warned: string[] = [];
111 const db = openDb(":memory:", migrations);
112 db.exec("PRAGMA foreign_keys = OFF");
113 const app = { db, log: { warn: (m: string) => warned.push(m), info: () => {} } } as unknown as App;
114 let locked = true;
115 const real = db.query.bind(db);
116 (db as unknown as { query: unknown }).query = (sql: string) => {
117 if (locked && (sql.includes("deploy_landed") || sql.includes("deploy_pull")))
118 throw new Error("SQLiteError: database is locked");
119 return real(sql);
120 };
121 // Timers and the clock stepped by hand: a timer is (callback, delay), and time moves only when told to.
122 const timers: { fn: () => void; delay: number }[] = [];
123 let clock = 1_000;
124 const realSet = globalThis.setTimeout;
125 const realClear = globalThis.clearTimeout;
126 const realNow = performance.now.bind(performance);
127 (globalThis as { setTimeout: unknown }).setTimeout = (fn: () => void, delay: number) => {
128 timers.push({ fn, delay });
129 return timers.length;
130 };
131 (globalThis as { clearTimeout: unknown }).clearTimeout = () => {};
132 performance.now = () => clock;
133 try {
134 const stop = startRetention(app, { local: false, engine: true });
135 // The backfill (0 ms) and the event prune (3 s) are armed.
136 expect(timers.map((t) => t.delay)).toEqual([0, 3_000]);
137 timers.shift()!.fn();
138 expect(warned).toEqual([]);
139 // The next timer armed is the busy retry, never the hour.
140 expect(timers.at(-1)!.delay).toBe(BUSY_RETRY_MS);
141 expect(timers.at(-1)!.delay).not.toBe(FAILED_MS);
142 // The lock clears; the retry runs and the slice succeeds.
143 locked = false;
144 clock += BUSY_RETRY_MS;
145 ledger(db, 3, Date.now() - DAY);
146 timers.pop()!.fn();
147 expect(warned).toEqual([]);
148 expect(db.query("SELECT COUNT(*) AS n FROM deploy_landed").get()).toEqual({ n: 3 });
149 stop();
150 } finally {
151 (globalThis as { setTimeout: unknown }).setTimeout = realSet;
152 (globalThis as { clearTimeout: unknown }).clearTimeout = realClear;
153 performance.now = realNow;
154 }
155 });
156
157 test("the sample prune does not wait on a lock: busy_timeout is 0 inside the slice and put back after", () => {
158 const { db, app } = seeded();
159 const timeout = () => db.query<{ timeout: number }, []>("PRAGMA busy_timeout").get()!.timeout;
160 const before = timeout();
161 expect(before).toBeGreaterThan(0);
162 const seen: number[] = [];
163 const real = db.query.bind(db);
164 (db as unknown as { query: unknown }).query = (sql: string) => {
165 if (sql.includes("machine_sample")) seen.push(timeout());
166 return real(sql);
167 };
168 retentionJobs(app).samples.slice(Date.now());
169 expect(seen.length).toBeGreaterThan(0);
170 expect(seen.every((t) => t === 0)).toBe(true);
171 expect(timeout()).toBe(before);
172 });
173
109174 test("starting the jobs runs the backfill at once; stopping them leaves no timer to fire", async () => {
110175 const { db, app } = seeded();
111176 ledger(db, 5, Date.now() - DAY);

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs