Public: anyone can read this; the crew can change it.

Revert "On the stage rig three background jobs meet a locked database at start, every release since the engine moved to its own thread" #14244

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

revert-14206 → main 1 commit

  • Merged. revert-14206 was merged into main as 5bf7638.

Review

Sign in to review this change.
2 files changed +8 −81
src/system/retention.ts +7 −15
@@ -35,7 +35,7 @@ import type { App } from "../app.ts";
3535 import { backfillLanded, LANDED_SLICE } from "../core/carried.ts";
3636 import { pillarWindowStart } from "../core/pillars.ts";
3737 import { quietly } from "../core/write_clock.ts";
38 import { isBusy, withoutWaiting } from "../kernel/db.ts";
38 import { withoutWaiting } from "../kernel/db.ts";
3939 import { during } from "../perf/jobs.ts";
4040 import { limitValue } from "../settings/limits.ts";
4141 import { deployRetention } from "./retention_deploys.ts";
@@ -58,12 +58,6 @@ export const IDLE_MS = 15_000;
5858 * SQLite refuses) is logged once and that job stays quiet for the hour; the other jobs and the process go on.
5959 */
6060 export const FAILED_MS = 60 * 60_000;
61 /**
62 * How long a job whose slice met another writer's lock waits to try again. The slices do not wait
63 * for a lock (`withoutWaiting`), and at start the engine's thread and the serving thread write
64 * together, so a locked database is the other thread's turn, not a fault: seconds, not the hour.
65 */
66 export const BUSY_RETRY_MS = 3_000;
6761 /** How long the backfill waits between passes, once a pass has found nothing more. */
6862 export const BACKFILL_EVERY_MS = 60 * 60_000;
6963
@@ -146,15 +140,18 @@ export function retentionJobs(app: App): Retention {
146140 samples: {
147141 idleMs: IDLE_MS,
148142 slice(now) {
149 const r = withoutWaiting(db, () => pruneSamples(db, now, ROW_CAP));
143 const r = pruneSamples(db, now, ROW_CAP);
150144 return { did: r.deleted + r.rolled, more: r.more };
151145 },
152146 },
153147 backfill: {
154148 idleMs: BACKFILL_EVERY_MS,
155149 slice(now) {
156 const r = withoutWaiting(db, () =>
157 backfillLanded(db, pillarWindowStart(new Date(now)), LANDED_SLICE, cursor),
150 const r = backfillLanded(
151 db,
152 pillarWindowStart(new Date(now)),
153 LANDED_SLICE,
154 cursor,
158155 );
159156 cursor = r.next;
160157 // The end of a pass is the end of the window's pulls: the ledger may go.
@@ -202,11 +199,6 @@ export function startRetention(app: App, which: { local: boolean; engine: boolea
202199 quietUntil = performance.now() + Math.max(MIN_GAP_MS, took * QUIET_X);
203200 return gapAfter(took, r.more, job.idleMs);
204201 } catch (error) {
205 if (isBusy(error)) {
206 app.log.info(`${what} met a locked database; trying again shortly`);
207 quietUntil = performance.now() + MIN_GAP_MS;
208 return BUSY_RETRY_MS;
209 }
210202 app.log.warn(`${what} failed; that job rests an hour`, { error: String(error) });
211203 quietUntil = performance.now() + MIN_GAP_MS;
212204 return FAILED_MS;
test/retention-jobs.test.ts +1 −66
@@ -8,7 +8,7 @@ import type { Database } from "bun:sqlite";
88 import type { App } from "../src/app.ts";
99 import { migrations } from "../src/core/schema.ts";
1010 import { openDb } from "../src/kernel/db.ts";
11 import { BUSY_RETRY_MS, DUTY, FAILED_MS, gapAfter, IDLE_MS, MIN_GAP_MS, retentionJobs, startRetention } from "../src/system/retention.ts";
11 import { DUTY, FAILED_MS, gapAfter, IDLE_MS, MIN_GAP_MS, retentionJobs, startRetention } from "../src/system/retention.ts";
1212
1313 const DAY = 86_400_000;
1414 const iso = (ms: number) => new Date(ms).toISOString();
@@ -106,71 +106,6 @@ describe("the jobs", () => {
106106 expect(FAILED_MS).toBe(3_600_000);
107107 });
108108
109 test("a slice that meets a locked database is scheduled again in BUSY_RETRY_MS, not an hour, and runs once the lock clears", () => {
110 const warned: string[] = [];
111 const db = openDb(":memory:", migrations);
112 db.exec("PRAGMA foreign_keys = OFF");
113 const app = { db, log: { warn: (m: string) => warned.push(m), info: () => {} } } as unknown as App;
114 let locked = true;
115 const real = db.query.bind(db);
116 (db as unknown as { query: unknown }).query = (sql: string) => {
117 if (locked && (sql.includes("deploy_landed") || sql.includes("deploy_pull")))
118 throw new Error("SQLiteError: database is locked");
119 return real(sql);
120 };
121 // Timers and the clock stepped by hand: a timer is (callback, delay), and time moves only when told to.
122 const timers: { fn: () => void; delay: number }[] = [];
123 let clock = 1_000;
124 const realSet = globalThis.setTimeout;
125 const realClear = globalThis.clearTimeout;
126 const realNow = performance.now.bind(performance);
127 (globalThis as { setTimeout: unknown }).setTimeout = (fn: () => void, delay: number) => {
128 timers.push({ fn, delay });
129 return timers.length;
130 };
131 (globalThis as { clearTimeout: unknown }).clearTimeout = () => {};
132 performance.now = () => clock;
133 try {
134 const stop = startRetention(app, { local: false, engine: true });
135 // The backfill (0 ms) and the event prune (3 s) are armed.
136 expect(timers.map((t) => t.delay)).toEqual([0, 3_000]);
137 timers.shift()!.fn();
138 expect(warned).toEqual([]);
139 // The next timer armed is the busy retry, never the hour.
140 expect(timers.at(-1)!.delay).toBe(BUSY_RETRY_MS);
141 expect(timers.at(-1)!.delay).not.toBe(FAILED_MS);
142 // The lock clears; the retry runs and the slice succeeds.
143 locked = false;
144 clock += BUSY_RETRY_MS;
145 ledger(db, 3, Date.now() - DAY);
146 timers.pop()!.fn();
147 expect(warned).toEqual([]);
148 expect(db.query("SELECT COUNT(*) AS n FROM deploy_landed").get()).toEqual({ n: 3 });
149 stop();
150 } finally {
151 (globalThis as { setTimeout: unknown }).setTimeout = realSet;
152 (globalThis as { clearTimeout: unknown }).clearTimeout = realClear;
153 performance.now = realNow;
154 }
155 });
156
157 test("the sample prune does not wait on a lock: busy_timeout is 0 inside the slice and put back after", () => {
158 const { db, app } = seeded();
159 const timeout = () => db.query<{ timeout: number }, []>("PRAGMA busy_timeout").get()!.timeout;
160 const before = timeout();
161 expect(before).toBeGreaterThan(0);
162 const seen: number[] = [];
163 const real = db.query.bind(db);
164 (db as unknown as { query: unknown }).query = (sql: string) => {
165 if (sql.includes("machine_sample")) seen.push(timeout());
166 return real(sql);
167 };
168 retentionJobs(app).samples.slice(Date.now());
169 expect(seen.length).toBeGreaterThan(0);
170 expect(seen.every((t) => t === 0)).toBe(true);
171 expect(timeout()).toBe(before);
172 });
173
174109 test("starting the jobs runs the backfill at once; stopping them leaves no timer to fire", async () => {
175110 const { db, app } = seeded();
176111 ledger(db, 5, Date.now() - DAY);

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs