Public: anyone can read this; the crew can change it.

The flake admission rule catches a test that waits on time: close the gap that let tui-backups and conversation-queue in #9026

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/9017-the-flake-admission-rule-catches-a-test → main 3 commits fork 1 thread · 1 unresolved

  • Merged. mason/9017-the-flake-admission-rule-catches-a-test was merged into main as 37a4874.

Review

Sign in to review this change.
5 files changed +257 −17
docs/DEVELOP.md +32 −1
@@ -162,7 +162,38 @@ record and exits non-zero unless all ten are green.
162162 A brand-new test file does not need that record: it joins the gate on its
163163 own green check, not ten runs. That is accepted on purpose — a flaky new
164164 test can turn the gate red once before anyone sees it — and the nightly
165 flake hunt below is the backstop that files it and quarantines it.
165 flake hunt below is the backstop that files it and quarantines it. One
166 shape is refused on sight, new file or not, because ten runs on a fast box
167 miss it: a wait that runs out of time and carries on — a loop that polls a
168 condition at most N times or until a deadline, and whose next statement
169 neither checks the outcome nor returns it (`fallThroughWaits` in
170 `scripts/flake.ts`). `admit` refuses such a file before any run, naming the
171 line, and `test/flake.test.ts` refuses it in every test file except the 47
172 that held one when the rule began (`FALL_THROUGH_OWED`, owned by #9025; the
173 rule goes red on a listed file that no longer breaches, so the list only
174 shrinks). The fix is one line after the loop: fail saying what never came.
175
176 ### How tui-backups and conversation-queue got in (#9017)
177
178 Neither passed an admission run; neither was ever given one. Both were new
179 files, so each joined the gate on its pull's own green check: no seed (the
180 gate runs every file's tests in written order, without `--randomize`), the
181 check cell's own load, one run.
182
183 - `test/tui-backups.test.ts` joined with #7278 (6e48d8c0). Its tests read
184 the destination an earlier test sealed, and its key check read the frames
185 every earlier test drew, so it was green only in written order. Run
186 through `admit` as it landed, it is red in 9 of 10 runs (seeds 2899264984
187 … 2895068531, the one green seed 3988233074); as #8995 left it, 0 of 10,
188 and refused first for its prompt wait. #9017 made each test make its own
189 destination and check every frame as it is drawn: 10 of 10 green.
190 - `test/conversation-queue.test.ts` joined with #8893 (65654ceb). Its
191 `until` polled 200 × 20 ms and then went on as if the condition had
192 come, so on a slow box the test stopped a turn before the forge had read
193 its partial answer. Under `admit` as it landed it was red once in 10
194 (seed 795951491), so one admission run could have passed it; it is now
195 refused before any run, at its line 48. As #8995 left it, it passes:
196 10 of 10 green.
166197
167198 **No flaky tests.** Nightly at 02:17 UTC the forge itself (`src/nightly/`,
168199 a cell on a checkout of main) runs every gate file five times on a loaded box, each run a fresh seed that orders both
scripts/flake-hunt.ts +11
@@ -23,6 +23,8 @@
2323 * bun scripts/flake-hunt.ts admit test/x.test.ts [--runs 10]
2424 * The file ADMISSION_RUNS times the same way; prints the record the
2525 * admitting pull carries. Exits non-zero unless every run is green.
26 * A file with a wait that runs out of time and carries on
27 * (`fallThroughWaits`) is refused before any run (#9017).
2628 */
2729
2830 import { availableParallelism } from "node:os";
@@ -31,6 +33,7 @@ import {
3133 admissionRecord,
3234 admitted,
3335 ADMISSION_RUNS,
36 fallThroughWaits,
3437 findBroken,
3538 findFlakes,
3639 huntedFile,
@@ -358,6 +361,14 @@ async function admit(): Promise<number> {
358361 if (file === undefined || file.includes(".."))
359362 fail("admit needs a file: bun scripts/flake-hunt.ts admit test/x.test.ts");
360363 const runs = runsOf(ADMISSION_RUNS);
364 // A wait that runs out and carries on passes ten runs on a fast box (#9017): refused on sight.
365 const waits = fallThroughWaits(file, readFileSync(file, "utf8"));
366 if (waits.length > 0) {
367 console.log(`admission: ${file} — refused, a wait runs out of time and carries on:`);
368 for (const line of waits) console.log(` ${line}`);
369 console.log(" after the loop, fail when the condition never came (or return it), then run admit again");
370 return 1;
371 }
361372 const results = await underLoad(async () => {
362373 const out: FileRun[] = [];
363374 for (let i = 0; i < runs; i++) out.push(await runFile(file, freshSeed()));
scripts/flake.ts +102
@@ -313,6 +313,108 @@ export function rawBudgets(name: string, source: string): string[] {
313313 return found;
314314 }
315315
316 /** A loop that polls a condition a bounded number of times, or until a deadline. */
317 const POLL_HEAD =
318 /^\s*(?:for\s*\(\s*let\s+\w+\s*=\s*0\s*;\s*\w+\s*<\s*[\w_.]+\s*&&|while\s*\(.*\bDate\.now\(\)\s*<.*&&|while\s*\(.*&&.*\bDate\.now\(\)\s*<)/;
319 const WAITS = /\bawait\b/;
320 /** What may follow a poll: the poll's own verdict, said. */
321 const SAYS_SO = /^(?:if\s*\(|throw\b|expect\(|assert|return\b)/;
322
323 /**
324 * Every poll in `source` that runs out of time and carries on (#9017): a
325 * loop that waits while a condition is false, at most N times or until a
326 * deadline, and whose next statement neither checks nor returns the
327 * outcome. When the box is slow it runs out silently, and the test goes on
328 * to act on a state that never came — red far from the cause, or not at
329 * all in the runs that admitted it.
330 */
331 export function fallThroughWaits(name: string, source: string): string[] {
332 const lines = source.split("\n");
333 const found: string[] = [];
334 for (let i = 0; i < lines.length; i++) {
335 const head = lines[i]!;
336 if (!POLL_HEAD.test(head)) continue;
337 // The loop's last line: the head itself, or where its braces close.
338 let end = i;
339 let body = head;
340 if (head.trimEnd().endsWith("{")) {
341 let depth = 0;
342 for (let j = i; j < lines.length; j++) {
343 for (const ch of lines[j]!) depth += ch === "{" ? 1 : ch === "}" ? -1 : 0;
344 body += `\n${lines[j]}`;
345 end = j;
346 if (depth <= 0) break;
347 }
348 } else {
349 // One statement as the body, on the lines after the head.
350 while (!lines[end]!.trimEnd().endsWith(";") && end + 1 < lines.length)
351 body += `\n${lines[++end]}`;
352 }
353 if (!WAITS.test(body)) continue;
354 let next = end + 1;
355 while (next < lines.length && lines[next]!.trim() === "") next++;
356 if (SAYS_SO.test(lines[next]?.trim() ?? "")) continue;
357 found.push(`${name}:${i + 1}: ${head.trim()}`);
358 }
359 return found;
360 }
361
362 /**
363 * The test files that held a fall-through wait when the gate began refusing
364 * one (#9017); #9025 owns making each fail saying what never came. A file
365 * leaves in the pull that fixes it, and the rule goes red on a listed file
366 * that no longer breaches, so this only shrinks. A new file is never added.
367 */
368 export const FALL_THROUGH_OWED: ReadonlySet<string> = new Set([
369 "test/act-now.test.ts",
370 "test/apps-ssh.test.ts",
371 "test/backups-live.test.ts",
372 "test/cell-relays-reaped.test.ts",
373 "test/check-flake.test.ts",
374 "test/check-memory-warning.test.ts",
375 "test/check-pull-flake.test.ts",
376 "test/checks.test.ts",
377 "test/client-abort.test.ts",
378 "test/conflicts-tui.test.ts",
379 "test/conversation-queue-api.test.ts",
380 "test/conversation-talk.test.ts",
381 "test/crew-early.test.ts",
382 "test/crew-merge-conflict.test.ts",
383 "test/crew-parks-planner.test.ts",
384 "test/drill-board-line.test.ts",
385 "test/event-order.test.ts",
386 "test/explore-rank.test.ts",
387 "test/following-card.test.ts",
388 "test/forges-side-by-side.test.ts",
389 "test/live.test.ts",
390 "test/looks-owner.test.ts",
391 "test/merge-queue-clash.test.ts",
392 "test/merge-queue-parallel.test.ts",
393 "test/migrate-token-reach.test.ts",
394 "test/pull-red-alone-path.test.ts",
395 "test/quotas-storage.test.ts",
396 "test/roll-write-pause.test.ts",
397 "test/snapshot-restore-streamed.test.ts",
398 "test/spawn-cell-socket.test.ts",
399 "test/spawn-child-runs-apps.test.ts",
400 "test/studio-door.test.ts",
401 "test/system-stream.test.ts",
402 "test/tui-delete-repo.test.ts",
403 "test/tui-forks-take.test.ts",
404 "test/tui-grants.test.ts",
405 "test/tui-hooks.test.ts",
406 "test/tui-newapp.test.ts",
407 "test/tui-onyou.test.ts",
408 "test/tui-origins-forge.test.ts",
409 "test/tui-origins-guard.test.ts",
410 "test/tui-origins.test.ts",
411 "test/tui-resilience-drill.test.ts",
412 "test/tui-resilience-now.test.ts",
413 "test/tui-teams.test.ts",
414 "test/tui-working-yours.test.ts",
415 "test/web-apps.test.ts",
416 ]);
417
316418 /** The record an admitting pull carries. */
317419 export function admissionRecord(
318420 file: string,
test/flake.test.ts +64
@@ -14,6 +14,8 @@ import {
1414 admissionRecord,
1515 admitted,
1616 ADMISSION_RUNS,
17 fallThroughWaits,
18 FALL_THROUGH_OWED,
1719 findBroken,
1820 findFlakes,
1921 flakyFile,
@@ -276,6 +278,68 @@ describe("admission", () => {
276278 }, 120_000); // two `bun test` children beside busy loops: past the 5 s default
277279 });
278280
281 describe("a wait that runs out of time and carries on (#9017)", () => {
282 // test/conversation-queue.test.ts before #8995: ran out after 4 s and went on to steer a turn whose partial was unread.
283 const QUEUE_BEFORE = [
284 "const until = async (ok: () => boolean): Promise<void> => {",
285 " for (let i = 0; i < 200 && !ok(); i++) await Bun.sleep(20);",
286 "};",
287 ].join("\n");
288 const QUEUE_AFTER = [
289 "const until = async (ok: () => boolean, what = \"the condition\"): Promise<void> => {",
290 " for (let i = 0; i < 500 && !ok(); i++) await Bun.sleep(20);",
291 " if (!ok()) throw new Error(`timed out waiting for ${what}`);",
292 "};",
293 ].join("\n");
294
295 test("the detector sees a poll that falls through, in each shape", () => {
296 expect(fallThroughWaits("q", QUEUE_BEFORE)).toEqual(["q:2: for (let i = 0; i < 200 && !ok(); i++) await Bun.sleep(20);"]);
297 const braced = "for (let i = 0; i < 100 && !h.text().includes(what); i++) {\n await Bun.sleep(20);\n await h.press();\n}\n}";
298 expect(fallThroughWaits("t", braced)).toHaveLength(1);
299 expect(fallThroughWaits("d", "while (Date.now() < deadline && !done()) await Bun.sleep(20);\nact();")).toHaveLength(1);
300 expect(fallThroughWaits("n", "for (let i = 0; i < 9 && (await bytes()) < 5; i++)\n await Bun.sleep(20);\nact();")).toHaveLength(1);
301 });
302
303 test("a poll whose outcome is checked or handed back is no breach", () => {
304 expect(fallThroughWaits("q", QUEUE_AFTER)).toEqual([]);
305 expect(fallThroughWaits("e", "for (let i = 0; i < 9 && !ok(); i++) await Bun.sleep(20);\nexpect(ok()).toBe(true);")).toEqual([]);
306 expect(fallThroughWaits("r", "while (Date.now() < end && !ok()) await Bun.sleep(5);\nreturn ok();")).toEqual([]);
307 // A loop that never waits is no poll.
308 expect(fallThroughWaits("s", "for (let i = 0; i < 9 && n > 0; i++) n--;\nact();")).toEqual([]);
309 });
310
311 test("`admit` refuses it before any run, naming the line", async () => {
312 const dir = await scratch();
313 await mkdir(join(dir, "test"));
314 await writeFile(
315 join(dir, "test", "x.test.ts"),
316 `import { test } from "bun:test";\n${QUEUE_BEFORE}\ntest("one", async () => { await until(() => true); });\n`,
317 );
318 const child = Bun.spawn(["bun", join(root, "scripts/flake-hunt.ts"), "admit", "test/x.test.ts"], {
319 cwd: dir,
320 stdout: "pipe",
321 stderr: "pipe",
322 });
323 const [out, code] = await Promise.all([new Response(child.stdout).text(), child.exited]);
324 expect(out).toContain("admission: test/x.test.ts — refused, a wait runs out of time and carries on");
325 expect(out).toContain("test/x.test.ts:3: for (let i = 0; i < 200 && !ok(); i++)");
326 expect(out).not.toContain("run 1:");
327 expect(code).toBe(1);
328 }, 30_000); // one bun child, no runs
329 });
330
331 // A new file is refused here, on its own first check, not only by an `admit` it may never get.
332 breaches(
333 "no test file waits on a poll that runs out of time and carries on",

These breaches(...) calls sit outside any describe but are indented as if inside one. Wrap them in a describe("…(#9017)") so their failures are named.

334 scanned.filter((path) => !FALL_THROUGH_OWED.has(path)).flatMap((path) => fallThroughWaits(path, read(path))),
335 );
336 breaches(
337 "every file owed under #9025 still owes a fix: strike the ones fixed",
338 [...FALL_THROUGH_OWED]
339 .filter((path) => !scanned.includes(path) || fallThroughWaits(path, read(path)).length === 0)
340 .map((path) => `${path}: no fall-through wait left (or no such file); delete it from FALL_THROUGH_OWED in scripts/flake.ts`),
341 );
342
279343 describe("random order", () => {
280344 const files = Array.from({ length: 30 }, (_, i) => `test/f${i}.test.ts`);
281345
test/tui-backups.test.ts +48 −16
@@ -1,10 +1,11 @@
11 /**
22 * The snapshot destination in the terminal (#7269): the Resilience screen
33 * says where snapshots go; `d` seals a destination field by field, `f`
4 * forgets it after a confirm. Every frame drawn is kept, and none may hold
5 * the access key or the secret key.
4 * forgets it after a confirm. No frame drawn may hold the access key or
5 * the secret key. Each test makes the destination it reads, so any order
6 * passes (#9017).
67 */
7 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
8 import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
89 import { mkdtemp, rm } from "node:fs/promises";
910 import { tmpdir } from "node:os";
1011 import { join } from "node:path";
@@ -26,15 +27,35 @@ const SECRET = "s3cr3t/Key+ThatMustNeverShow7269";
2627 describe("the snapshot destination in the terminal", () => {
2728 let dataDir: string;
2829 let forge: Forge;
29 const frames: string[] = [];
3030 const saved = { ...process.env };
3131
32 /** Under a loaded machine a screen can land a turn after the first settle: settle again until it shows. */
32 /** Under a loaded machine a screen can land a turn after the first settle: settle again until it shows, and fail saying so when it never does. */
3333 async function until(h: { text(): string; press(...k: string[]): Promise<string> }, what: string) {
3434 for (let i = 0; i < 100 && !h.text().includes(what); i++) {
3535 await Bun.sleep(20);
3636 await h.press();
3737 }
38 if (!h.text().includes(what)) throw new Error(`the screen never showed ${JSON.stringify(what)}`);
39 }
40
41 /** A destination sealed as a person admin would, without the screen. */
42 function seal() {
43 const made = saveDestination(forge, getUserByName(forge.db, "alice"), {
44 endpoint: "https://s3.example.com",
45 bucket: "forge-snaps",
46 region: "eu-west-1",
47 accessKey: ACCESS,
48 secretKey: SECRET,
49 });
50 if (!made.ok) throw new Error(made.error);
51 }
52
53 /** Every frame is read as it is drawn: none may hold either key. */
54 function keyless(t: string): string {
55 expect(t).not.toContain(ACCESS);
56 expect(t).not.toContain(SECRET);
57 expect(t).not.toContain(SECRET.slice(0, 10));
58 return t;
3859 }
3960
4061 function boot(username: string) {
@@ -43,11 +64,7 @@ describe("the snapshot destination in the terminal", () => {
4364 const identity: TuiIdentity = { sshUsername: "t", remoteAddress: "127.0.0.1", user };
4465 const { app } = createForgeApp(forge, identity, { terminal, exitHooks: false });
4566 app.start();
46 const frame = () => {
47 const t = app.screen.toText();
48 frames.push(t);
49 return t;
50 };
67 const frame = () => keyless(app.screen.toText());
5168 const key = (k: string, ctrl = false) => {
5269 app.dispatch({ type: "key", key: k, ctrl, alt: false, shift: false });
5370 frame();
@@ -90,6 +107,11 @@ describe("the snapshot destination in the terminal", () => {
90107 forge.db.query("UPDATE user SET is_admin = 1, is_agent = 1 WHERE username = 'carol'").run();
91108 });
92109
110 beforeEach(() => {
111 const cleared = forgetDestination(forge, getUserByName(forge.db, "alice"));
112 if (!cleared.ok) throw new Error(cleared.error);
113 });
114
93115 afterAll(async () => {
94116 forge.close();
95117 await rm(dataDir, { recursive: true, force: true });
@@ -148,6 +170,7 @@ describe("the snapshot destination in the terminal", () => {
148170 });
149171
150172 test("an agent admin reads the destination but cannot change it", async () => {
173 seal();
151174 const h = boot("carol");
152175 const t = await h.open();
153176 expect(t).toContain("s3.example.com/forge-snaps");
@@ -164,6 +187,7 @@ describe("the snapshot destination in the terminal", () => {
164187 });
165188
166189 test("x forgets only after the confirm; f is the mirror's", async () => {
190 seal();
167191 const h = boot("alice");
168192 await h.open();
169193 // `f` forgets the mirror (#7281), never the destination.
@@ -187,12 +211,20 @@ describe("the snapshot destination in the terminal", () => {
187211 expect(mayChangeDestination(undefined)).toBe(false);
188212 });
189213
190 test("no frame ever held the access key or the secret key", () => {
191 expect(frames.length).toBeGreaterThan(50);
192 for (const f of frames) {
193 expect(f).not.toContain(ACCESS);
194 expect(f).not.toContain(SECRET);
195 expect(f).not.toContain(SECRET.slice(0, 10));
214 test("no frame ever held the access key or the secret key", async () => {
215 // Sealed, then typed through the screen, then opened afresh: every frame is checked as it is drawn.
216 seal();
217 const h = boot("alice");
218 await h.open();
219 await h.press("d");
220 for (const field of ["https://s3.example.com", "forge-snaps", "eu-west-1", ACCESS, SECRET]) {
221 await h.type(field);
222 await h.press("enter");
196223 }
224 h.stop();
225 const again = boot("alice");
226 expect(await again.open()).toContain("s3.example.com/forge-snaps");
227 again.stop();
228 expect(sealedSet(forge.db).length).toBeGreaterThanOrEqual(4);
197229 });
198230 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs