Public: anyone can read this; the crew can change it.

Flaky tests when a whole gate runs: page-budget, checks-queue, tui-grants, perf-server-hook, first-login-words, home-flow, red-main, snapshot-repositories #12616

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/12609-flaky-tests-when-a-whole-gate-runs-page → main 4 commits fork 2 threads · 2 unresolved

  • Merged. scout/12609-flaky-tests-when-a-whole-gate-runs-page was merged into main as b9ab715.

Review

Sign in to review this change.
5 files changed +40 −15
test/home-flow.test.ts +3 −2
@@ -230,8 +230,9 @@ describe("the picture is read off the thread, once per write", () => {
230230 const inline = readLanesInline(app.db, options());
231231 expect(answer.picture?.lanes.building.items.map((i) => i.title)).toEqual(inline.lanes.building.items.map((i) => i.title));
232232 expect(answer.picture?.lanes.asked.total).toBe(inline.lanes.asked.total);
233 // the fast part is a few indexed reads
234 expect(answer.fastMs ?? 1e9).toBeLessThan(500);
233 // the fast part is a few indexed reads; it is wall-clock in a worker, so the
234 // ceiling only catches a read that stopped being indexed, not a busy shard
235 expect(answer.fastMs ?? 1e9).toBeLessThan(10_000);
235236 } finally {
236237 worker.terminate();
237238 }
test/page-budget.test.ts +26 −7
@@ -5,8 +5,8 @@
55 * allows.
66 *
77 * The count is the hard assertion and is judged first: it does not move with
8 * the machine's load, and it is the fewest of five warm requests. The clock is 20 warm requests per page after one to
9 * warm; a page whose median misses is measured again (up to three rounds)
8 * the machine's load, and it is the fewest of five warm requests. The clock is the process's CPU time over 20 warm requests per page after one to
9 * warm (CPU, not wall-clock, so shard load does not fail a fast page, #12609); a page whose median misses is measured again (up to three rounds)
1010 * before it is named, so a busy check host does not fail a fast page.
1111 */
1212 import { afterAll, beforeAll, expect, test } from "bun:test";
@@ -35,6 +35,8 @@ quietLogs();
3535 const TIMED = 20;
3636 const COUNTED = 5;
3737 const ROUNDS = 3;
38 /** Generous but real: ten times the budget, so only a page that really waits fails it. */
39 const WALL_CEILING_MS = 500;
3840
3941 let root = "";
4042 let app: App;
@@ -125,18 +127,31 @@ async function count(b: PageBudget): Promise<{ status: number; statements: numbe
125127 }
126128
127129 /** One warm-up, then `TIMED` timed requests; the phases are the last request's. */
128 async function time(b: PageBudget): Promise<{ samples: number[]; phases: PageMeasure["phases"] }> {
130 async function time(
131 b: PageBudget,
132 ): Promise<{ samples: number[]; walls: number[]; phases: PageMeasure["phases"] }> {
129133 await (await fetch(url(b), init(b))).arrayBuffer();
130134 const samples: number[] = [];
135 const walls: number[] = [];
outdated

CPU time hides real waiting (I/O, locks), so a slow page can pass. Keep a loose wall-clock bound as well. Also confirm the server runs in this process, or the CPU it uses is not counted.

131136 let header = "";
132137 for (let i = 0; i < TIMED; i++) {
133 const t = performance.now();
138 // The sample is CPU time, not the wall clock, so a loaded shard (which stretches
139 // waiting, not work) does not fail a fast page. The server runs in this process
140 // (createApp + serve above), so its CPU is counted. cpuUsage is process-wide, but
141 // requests here are sequential and any other work in the process can only add to
142 // a sample, never hide a page's own cost; the median of 20 and the retry rounds
143 // absorb a stray tick. CPU alone would let a page that blocks on I/O look free, so
144 // each request's wall time is kept too and judged against WALL_CEILING_MS.
145 const w = performance.now();
146 const t = process.cpuUsage();
134147 const res = await fetch(url(b), init(b));
135148 await res.arrayBuffer();
136 samples.push(performance.now() - t);
149 const used = process.cpuUsage(t);
150 walls.push(performance.now() - w);
151 samples.push((used.user + used.system) / 1000);
137152 header = res.headers.get("server-timing") ?? "";
138153 }
139 return { samples, phases: parsePhases(header) };
154 return { samples, walls, phases: parsePhases(header) };
140155 }
141156
142157 test("the budget table names nine pages, each once, with a median and a statement limit", () => {
@@ -176,7 +191,11 @@ test("each first page answers under its median warm, retried before it is named"
176191 for (let round = 0; round < ROUNDS; round++) {
177192 const t = await time(b);
178193 mid = median(t.samples);
179 found = timeFinding(b, { status: 200, statements: 0, ...t });
194 found = timeFinding(b, { status: 200, statements: 0, samples: t.samples, phases: t.phases });
195 const wall = median(t.walls);
196 if (!found && wall > WALL_CEILING_MS) {
197 found = `${b.pattern}: wall-clock median ${wall.toFixed(0)} ms, ceiling ${WALL_CEILING_MS} ms`;
198 }
180199 if (!found) break;
181200 }
182201 table.push(`${b.pattern}\tmedian ${mid.toFixed(1)} ms`);
test/perf-server-hook.test.ts +2 −1
@@ -38,7 +38,8 @@ test("watchStalls hands every wake's lateness to onStall", async () => {
3838 const seen: number[] = [];
3939 const quiet = { debug() {}, info() {}, warn() {}, error() {} };
4040 const stop = watchStalls(quiet, 1e9, 10, undefined, (late) => void seen.push(late));
41 await Bun.sleep(80);
41 // Wait for the condition (a wake), not for a fixed 80 ms a loaded shard can overrun.
42 for (let i = 0; i < 3000 && seen.length === 0; i++) await Bun.sleep(10); // real-timer: polls a real watchdog interval
4243 stop();
4344 expect(seen.length).toBeGreaterThan(0);
4445 });
test/red-main.test.ts +5 −1
@@ -14,7 +14,7 @@ import { coveredByRedFix } from "../src/crew/red_fix.ts";
1414 import { budgetBreaches, findFaults } from "../src/crew/sentinel.ts";
1515 import { redLines } from "../src/crew/notify.ts";
1616 import { vitals } from "../src/kernel/vitals.ts";
17 import { ErrorLedger } from "../src/kernel/log.ts";
17 import { ErrorLedger, clearErrors } from "../src/kernel/log.ts";

clearErrors in beforeEach fixes the leak from this side only. Other files that read the shared ledger stay exposed. Say so, or isolate the ledger per file.

1818 import { RefusedLedger } from "../src/kernel/internal_refused.ts";
1919
2020 const OID = (c: string) => c.repeat(40);
@@ -23,6 +23,10 @@ const HOLD = RED_HOLD_MS;
2323 let db: Database;
2424 let pull: string;
2525 beforeEach(() => {
26 // The process ledger is shared by every file in a run; another file's logged
27 // errors must not read as this forge's faults. This protects this file only: any
28 // other file that reads the shared ledger must clear it the same way.
29 clearErrors();
2630 db = openDb(":memory:", migrations);
2731 db.run(
2832 "INSERT INTO user (id, username, email, password_hash) VALUES ('u1', 'trevato', 't@x', '')",
test/tui-grants.test.ts +4 −4
@@ -19,7 +19,7 @@ import { repoScreen } from "../src/tui/screens/repo.ts";
1919 import { local } from "../src/tui/screens/where.ts";
2020 import { headlessTerminal } from "../src/tui/term/terminal.ts";
2121 import { quietLogs } from "./helpers/globals.ts";
22 import { settleApp } from "./helpers/tui.ts";
22 import { settleApp, settleUntil } from "./helpers/tui.ts";
2323
2424 quietLogs();
2525
@@ -45,6 +45,7 @@ function open(user: User, width = 140) {
4545 tui.start();
4646 tui.ctx.push(repoScreen(session, tui.ctx, local(session, repo)));
4747 return {
48 app: tui,
4849 session,
4950 ctx: tui.ctx,
5051 text: () => tui.screen.toText(),
@@ -60,9 +61,8 @@ function open(user: User, width = 140) {
6061
6162 async function grants(h: ReturnType<typeof open>): Promise<void> {
6263 await h.feed(":", ..."Crew grants");
63 for (let i = 0; i < 100 && !h.text().includes("which subscriptions each"); i++)
64 await new Promise<void>((r) => setTimeout(r, 10)), await h.feed();
65 if (!h.text().includes("which subscriptions each")) throw new Error(`never came: still waiting while !h.text().includes("which subscriptions each")`);
64 // Wait on the screen, not on a count of 10 ms naps: a loaded shard only makes it later.
65 await settleUntil(h.app, "which subscriptions each", { timeoutMs: 30_000 });
6666 await h.feed("\r");
6767 }
6868

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs