Public: anyone can read this; the crew can change it.
Revert "On the stage rig three background jobs meet a locked database at start, every release since the engine moved to its own thread" #14244
Check · Review · Land · Live done — Landed and live in production.
revert-14206 → main 1 commit
2 files
changed +8
−81
src/system/retention.ts +7 −15
| @@ -35,7 +35,7 @@ import type { App } from "../app.ts"; | ||
| 35 | 35 | import { backfillLanded, LANDED_SLICE } from "../core/carried.ts"; |
| 36 | 36 | import { pillarWindowStart } from "../core/pillars.ts"; |
| 37 | 37 | import { quietly } from "../core/write_clock.ts"; |
| 38 | import { isBusy, withoutWaiting } from "../kernel/db.ts"; | |
| 38 | import { withoutWaiting } from "../kernel/db.ts"; | |
| 39 | 39 | import { during } from "../perf/jobs.ts"; |
| 40 | 40 | import { limitValue } from "../settings/limits.ts"; |
| 41 | 41 | import { deployRetention } from "./retention_deploys.ts"; |
| @@ -58,12 +58,6 @@ export const IDLE_MS = 15_000; | ||
| 58 | 58 | * SQLite refuses) is logged once and that job stays quiet for the hour; the other jobs and the process go on. |
| 59 | 59 | */ |
| 60 | 60 | export const FAILED_MS = 60 * 60_000; |
| 61 | /** | |
| 62 | * How long a job whose slice met another writer's lock waits to try again. The slices do not wait | |
| 63 | * for a lock (`withoutWaiting`), and at start the engine's thread and the serving thread write | |
| 64 | * together, so a locked database is the other thread's turn, not a fault: seconds, not the hour. | |
| 65 | */ | |
| 66 | export const BUSY_RETRY_MS = 3_000; | |
| 67 | 61 | /** How long the backfill waits between passes, once a pass has found nothing more. */ |
| 68 | 62 | export const BACKFILL_EVERY_MS = 60 * 60_000; |
| 69 | 63 | |
| @@ -146,15 +140,18 @@ export function retentionJobs(app: App): Retention { | ||
| 146 | 140 | samples: { |
| 147 | 141 | idleMs: IDLE_MS, |
| 148 | 142 | slice(now) { |
| 149 | const r = withoutWaiting(db, () => pruneSamples(db, now, ROW_CAP)); | |
| 143 | const r = pruneSamples(db, now, ROW_CAP); | |
| 150 | 144 | return { did: r.deleted + r.rolled, more: r.more }; |
| 151 | 145 | }, |
| 152 | 146 | }, |
| 153 | 147 | backfill: { |
| 154 | 148 | idleMs: BACKFILL_EVERY_MS, |
| 155 | 149 | slice(now) { |
| 156 | const r = withoutWaiting(db, () => | |
| 157 | backfillLanded(db, pillarWindowStart(new Date(now)), LANDED_SLICE, cursor), | |
| 150 | const r = backfillLanded( | |
| 151 | db, | |
| 152 | pillarWindowStart(new Date(now)), | |
| 153 | LANDED_SLICE, | |
| 154 | cursor, | |
| 158 | 155 | ); |
| 159 | 156 | cursor = r.next; |
| 160 | 157 | // The end of a pass is the end of the window's pulls: the ledger may go. |
| @@ -202,11 +199,6 @@ export function startRetention(app: App, which: { local: boolean; engine: boolea | ||
| 202 | 199 | quietUntil = performance.now() + Math.max(MIN_GAP_MS, took * QUIET_X); |
| 203 | 200 | return gapAfter(took, r.more, job.idleMs); |
| 204 | 201 | } catch (error) { |
| 205 | if (isBusy(error)) { | |
| 206 | app.log.info(`${what} met a locked database; trying again shortly`); | |
| 207 | quietUntil = performance.now() + MIN_GAP_MS; | |
| 208 | return BUSY_RETRY_MS; | |
| 209 | } | |
| 210 | 202 | app.log.warn(`${what} failed; that job rests an hour`, { error: String(error) }); |
| 211 | 203 | quietUntil = performance.now() + MIN_GAP_MS; |
| 212 | 204 | return FAILED_MS; |
test/retention-jobs.test.ts +1 −66
| @@ -8,7 +8,7 @@ import type { Database } from "bun:sqlite"; | ||
| 8 | 8 | import type { App } from "../src/app.ts"; |
| 9 | 9 | import { migrations } from "../src/core/schema.ts"; |
| 10 | 10 | import { openDb } from "../src/kernel/db.ts"; |
| 11 | import { BUSY_RETRY_MS, DUTY, FAILED_MS, gapAfter, IDLE_MS, MIN_GAP_MS, retentionJobs, startRetention } from "../src/system/retention.ts"; | |
| 11 | import { DUTY, FAILED_MS, gapAfter, IDLE_MS, MIN_GAP_MS, retentionJobs, startRetention } from "../src/system/retention.ts"; | |
| 12 | 12 | |
| 13 | 13 | const DAY = 86_400_000; |
| 14 | 14 | const iso = (ms: number) => new Date(ms).toISOString(); |
| @@ -106,71 +106,6 @@ describe("the jobs", () => { | ||
| 106 | 106 | expect(FAILED_MS).toBe(3_600_000); |
| 107 | 107 | }); |
| 108 | 108 | |
| 109 | test("a slice that meets a locked database is scheduled again in BUSY_RETRY_MS, not an hour, and runs once the lock clears", () => { | |
| 110 | const warned: string[] = []; | |
| 111 | const db = openDb(":memory:", migrations); | |
| 112 | db.exec("PRAGMA foreign_keys = OFF"); | |
| 113 | const app = { db, log: { warn: (m: string) => warned.push(m), info: () => {} } } as unknown as App; | |
| 114 | let locked = true; | |
| 115 | const real = db.query.bind(db); | |
| 116 | (db as unknown as { query: unknown }).query = (sql: string) => { | |
| 117 | if (locked && (sql.includes("deploy_landed") || sql.includes("deploy_pull"))) | |
| 118 | throw new Error("SQLiteError: database is locked"); | |
| 119 | return real(sql); | |
| 120 | }; | |
| 121 | // Timers and the clock stepped by hand: a timer is (callback, delay), and time moves only when told to. | |
| 122 | const timers: { fn: () => void; delay: number }[] = []; | |
| 123 | let clock = 1_000; | |
| 124 | const realSet = globalThis.setTimeout; | |
| 125 | const realClear = globalThis.clearTimeout; | |
| 126 | const realNow = performance.now.bind(performance); | |
| 127 | (globalThis as { setTimeout: unknown }).setTimeout = (fn: () => void, delay: number) => { | |
| 128 | timers.push({ fn, delay }); | |
| 129 | return timers.length; | |
| 130 | }; | |
| 131 | (globalThis as { clearTimeout: unknown }).clearTimeout = () => {}; | |
| 132 | performance.now = () => clock; | |
| 133 | try { | |
| 134 | const stop = startRetention(app, { local: false, engine: true }); | |
| 135 | // The backfill (0 ms) and the event prune (3 s) are armed. | |
| 136 | expect(timers.map((t) => t.delay)).toEqual([0, 3_000]); | |
| 137 | timers.shift()!.fn(); | |
| 138 | expect(warned).toEqual([]); | |
| 139 | // The next timer armed is the busy retry, never the hour. | |
| 140 | expect(timers.at(-1)!.delay).toBe(BUSY_RETRY_MS); | |
| 141 | expect(timers.at(-1)!.delay).not.toBe(FAILED_MS); | |
| 142 | // The lock clears; the retry runs and the slice succeeds. | |
| 143 | locked = false; | |
| 144 | clock += BUSY_RETRY_MS; | |
| 145 | ledger(db, 3, Date.now() - DAY); | |
| 146 | timers.pop()!.fn(); | |
| 147 | expect(warned).toEqual([]); | |
| 148 | expect(db.query("SELECT COUNT(*) AS n FROM deploy_landed").get()).toEqual({ n: 3 }); | |
| 149 | stop(); | |
| 150 | } finally { | |
| 151 | (globalThis as { setTimeout: unknown }).setTimeout = realSet; | |
| 152 | (globalThis as { clearTimeout: unknown }).clearTimeout = realClear; | |
| 153 | performance.now = realNow; | |
| 154 | } | |
| 155 | }); | |
| 156 | ||
| 157 | test("the sample prune does not wait on a lock: busy_timeout is 0 inside the slice and put back after", () => { | |
| 158 | const { db, app } = seeded(); | |
| 159 | const timeout = () => db.query<{ timeout: number }, []>("PRAGMA busy_timeout").get()!.timeout; | |
| 160 | const before = timeout(); | |
| 161 | expect(before).toBeGreaterThan(0); | |
| 162 | const seen: number[] = []; | |
| 163 | const real = db.query.bind(db); | |
| 164 | (db as unknown as { query: unknown }).query = (sql: string) => { | |
| 165 | if (sql.includes("machine_sample")) seen.push(timeout()); | |
| 166 | return real(sql); | |
| 167 | }; | |
| 168 | retentionJobs(app).samples.slice(Date.now()); | |
| 169 | expect(seen.length).toBeGreaterThan(0); | |
| 170 | expect(seen.every((t) => t === 0)).toBe(true); | |
| 171 | expect(timeout()).toBe(before); | |
| 172 | }); | |
| 173 | ||
| 174 | 109 | test("starting the jobs runs the backfill at once; stopping them leaves no timer to fire", async () => { |
| 175 | 110 | const { db, app } = seeded(); |
| 176 | 111 | ledger(db, 5, Date.now() - DAY); |
new above
offline