Public: anyone can read this; the crew can change it.

A full-suite file "red alone" on the runner says how it was run, and one red only under the runner's load is held, not handed to a builder #15241

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

torque/forge:torque/15240-a-full-suite-file-red-alone-on-the-runne → main 1 commit fork

  • Merged. torque/15240-a-full-suite-file-red-alone-on-the-runne was merged into main as ffb82ff.

Review

Sign in to review this change.
5 files changed +135 −28
src/checks/alone_label.ts +2 −1
@@ -1,6 +1,7 @@
11 /**
22 * The label on a full-suite report issue whose red files all passed when rerun alone on the same
3 * commit (#14404): a flake under load. The report (`report.ts`) puts it on and takes it off when a
3 * commit (#14404), or failed alone while other checks shared the runner (#15240): a flake under
4 * load, or maybe one. The report (`report.ts`) puts it on and takes it off when a
45 * second full-suite run is red for the same file; the crew (`crew/held.ts`, `full_suite_flake`)
56 * does not start an issue that carries it.
67 */
src/checks/execute.ts +30 −10
@@ -9,7 +9,8 @@ import { withDisk } from "./disk_red.ts";
99 import { proposeRevert } from "./revert_red.ts";
1010 import { afterVerify, closeOnGreen } from "./culprit.ts";
1111 import { noteLookPages } from "./look_pages.ts";
12 import { reportGreen, reportRed } from "./report.ts";
12 import { reportGreen, reportRed, type AloneAnswer } from "./report.ts";
13 import type { Database } from "bun:sqlite";
1314 import { bisectOrigin, inVerifyLane, safeFile, VERIFY_ENV } from "./verify.ts";
1415 import { getIssueById } from "../core/issues.ts";
1516 import { BUDGETS, budgetSummary, checkLimitMs } from "../kernel/budgets.ts";
@@ -206,7 +207,10 @@ export async function execute(
206207 // A red verification after landing names its culprit (#7786).
207208 if (done && run.env === VERIFY_ENV) {
208209 // Each red file is rerun alone on the same commit, once, before it is filed (#14404).
209 const alone = done.state === "failed" ? await rerunAlone(lane, done, write, signal) : new Map<string, boolean>();
210 const alone =
211 done.state === "failed"
212 ? await rerunAlone(lane, done, write, signal, () => otherChecksRunning(app.db, done.id))
213 : new Map<string, AloneAnswer>();
210214 reportRed(app, repo, done, alone).forEach(write);
211215 // A file the run printed passing closes its open report issue (#9524).
212216 reportGreen(app, repo, done, outcome.passed ?? []).forEach(write);
@@ -223,34 +227,50 @@ export async function execute(
223227 }
224228
225229 /**
226 * Each red file of a red verification run alone on the same commit, once (#14404): true when it
227 * passed alone. A file the rerun could not answer (stopped, the runner refused) is left out, and
228 * is filed as any red is. At most {@link ALONE_MAX} files are rerun; the rest are filed as today.
230 * Each red file of a red verification run alone on the same commit, once (#14404): whether it
231 * passed alone, its wall time and how many other checks were running when it started (#15240), so
232 * a red alone on a shared runner is told from one on an idle runner. A file the rerun could not
233 * answer (stopped, the runner refused) is left out, and is filed as any red is. At most {@link
234 * ALONE_MAX} files are rerun; the rest are filed as today.
229235 */
230236 export async function rerunAlone(
231237 lane: Pick<LaneState, "runner">,
232238 run: Run,
233239 write: (line: string) => void,
234240 signal: AbortSignal,
235 ): Promise<Map<string, boolean>> {
241 others: () => number = () => 0,
242 now: () => number = Date.now,
243 ): Promise<Map<string, { passed: boolean; ms: number; others: number }>> {
236244 const files = [...new Set(run.checkDetail?.files ?? [])].filter(safeFile).slice(0, ALONE_MAX);
237 const alone = new Map<string, boolean>();
245 const alone = new Map<string, { passed: boolean; ms: number; others: number }>();
238246 for (const file of files) {
239247 if (signal.aborted) break;
240 write(`rerun alone: bun test ./${file} on ${run.commitOid.slice(0, 7)}`);
248 const beside = Math.max(0, Math.trunc(others()));
249 write(`rerun alone: bun test ./${file} on ${run.commitOid.slice(0, 7)}, ${beside} other check${beside === 1 ? "" : "s"} running`);
250 const began = now();
241251 const outcome = await lane.runner
242252 .run(aloneJob(run, file), { write: () => {}, signal })
243253 .catch((error): CheckOutcome => ({ ok: false, error: String(error), infrastructure: true }));
254 const ms = Math.max(0, now() - began);
244255 if (!outcome.ok && (outcome.infrastructure === true || outcome.killed === true || outcome.cpuStop !== undefined)) {
245256 write(`rerun alone: ${file} not answered (${outcome.error})`);
246257 continue;
247258 }
248 alone.set(file, outcome.ok);
249 write(`rerun alone: ${file} ${outcome.ok ? "passed alone" : "failed alone"}`);
259 alone.set(file, { passed: outcome.ok, ms, others: beside });
260 write(`rerun alone: ${file} ${outcome.ok ? "passed alone" : "failed alone"} in ${(ms / 1000).toFixed(1)} s`);
250261 }
251262 return alone;
252263 }
253264
265 /**
266 * How many checks other than `runId` this forge has running now: the company a rerun alone keeps
267 * on the runner (#15240).
268 */
269 export const otherChecksRunning = (db: Database, runId: string): number =>
270 db
271 .query<{ n: number }, [string]>("SELECT COUNT(*) AS n FROM run WHERE kind = 'check' AND state = 'running' AND id != ?1")
272 .get(runId)?.n ?? 0;
273
254274 /** The most red files a verification reruns alone: one each, once. */
255275 export const ALONE_MAX = 10;
256276
src/checks/report.ts +41 −12
@@ -286,7 +286,7 @@ function fileSection(run: Run, file: string): string {
286286 * is more, so a run that goes red for one cause starts one builder, and the same run reported again
287287 * comments.
288288 */
289 export function reportRed(app: App, repo: Repo, run: Run, alone: ReadonlyMap<string, boolean> = new Map()): string[] {
289 export function reportRed(app: App, repo: Repo, run: Run, alone: ReadonlyMap<string, AloneAnswer> = new Map()): string[] {
290290 if (run.state !== "failed") return [];
291291 const files = [...new Set(run.checkDetail?.files ?? [])].filter(safeFile);
292292 if (files.length === 0) return [];
@@ -328,9 +328,10 @@ export function reportRed(app: App, repo: Repo, run: Run, alone: ReadonlyMap<str
328328 lines.push(`report: added to #${issue.number} for ${issue.files.join(", ")}`);
329329 }
330330 if (fresh.length === 0) return lines;
331 // A file that passed when rerun alone on this commit is filed apart and held (#14404).
332 const flaky = fresh.filter((f) => alone.get(f) === true);
333 const real = fresh.filter((f) => alone.get(f) !== true);
331 // A file that passed when rerun alone on this commit is filed apart and held (#14404), and so is
332 // one red alone while other checks shared the runner: load may be its red too (#15240).
333 const flaky = fresh.filter((f) => heldAlone(alone.get(f)));
334 const real = fresh.filter((f) => !heldAlone(alone.get(f)));
334335 if (real.length > 0) lines.push(fileFresh(app, repo, run, real, label, alone));
335336 if (flaky.length > 0) {
336337 const hold = createLabel(app.db, repo.id, ALONE_GREEN_LABEL, ALONE_GREEN_COLOR);
@@ -341,15 +342,43 @@ export function reportRed(app: App, repo: Repo, run: Run, alone: ReadonlyMap<str
341342 }
342343
343344 /**
344 * The one line that says which of the two cases a file is, naming main's commit (#14404); none
345 * for a file that was not rerun alone.
345 * What one file's rerun alone said (#14404, #15240): whether it passed, how long it took in
346 * milliseconds and how many other checks were running when it started. A bare boolean is a rerun
347 * whose time and company were not recorded.
346348 */
347 export function aloneLine(file: string, commitOid: string, branch: string, passedAlone: boolean | undefined): string {
349 export type AloneAnswer = boolean | { readonly passed: boolean; readonly ms: number; readonly others: number };
350
351 const passedOf = (a: AloneAnswer | undefined): boolean | undefined => (typeof a === "object" ? a.passed : a);
352
353 /**
354 * Whether a file is filed held: it passed alone (a flake under load), or it failed alone while
355 * other checks shared the runner, so its red alone may be load too (#15240). Red alone on an idle
356 * runner is a real red, filed startable.
357 */
358 export const heldAlone = (a: AloneAnswer | undefined): boolean =>
359 passedOf(a) === true || (typeof a === "object" && !a.passed && a.others > 0);
360
361 /** `in 14.2 s, with 5 other checks running`: how the rerun ran, when it was recorded. */
362 function howRun(a: AloneAnswer): string {
363 if (typeof a !== "object") return "";
364 const others = a.others === 0 ? "no other check running" : `${a.others} other check${a.others === 1 ? "" : "s"} running`;
365 return ` in ${(a.ms / 1000).toFixed(1)} s, with ${others}`;
366 }
367
368 /**
369 * The one line that says which case a file is, naming main's commit (#14404) and how the rerun
370 * ran (#15240); none for a file that was not rerun alone.
371 */
372 export function aloneLine(file: string, commitOid: string, branch: string, answer: AloneAnswer | undefined): string {
348373 const at = `${branch}'s commit ${commitOid.slice(0, 7)}`;
349 if (passedAlone === true)
350 return `Red only in the full suite: \`${file}\` passed when rerun alone on ${at} (a flake under load), so it is held until a second full-suite run is red for it.`;
351 if (passedAlone === false) return `Red alone too: \`${file}\` failed when rerun alone on ${at}.`;
352 return "";
374 if (answer === undefined) return "";
375 const how = howRun(answer);
376 const hold = "so it is held until a second full-suite run is red for it.";
377 if (passedOf(answer) === true)
378 return `Red only in the full suite: \`${file}\` passed when rerun alone on ${at}${how} (a flake under load), ${hold}`;
379 if (heldAlone(answer))
380 return `Red alone too: \`${file}\` failed when rerun alone on ${at}${how}. The runner was shared, so the red may be load, and ${hold}`;
381 return `Red alone too: \`${file}\` failed when rerun alone on ${at}${how}.`;
353382 }
354383
355384 /** File `fresh` as one issue for the run, with the hold label when given; its log line. */
@@ -359,7 +388,7 @@ function fileFresh(
359388 run: Run,
360389 fresh: readonly string[],
361390 label: Label | undefined,
362 alone: ReadonlyMap<string, boolean>,
391 alone: ReadonlyMap<string, AloneAnswer>,
363392 hold?: Label,
364393 ): string {
365394 const where = `check run ${run.id}, ${run.commitOid.slice(0, 7)}`;
src/crew/held.ts +2 −1
@@ -24,7 +24,8 @@
2424 * - `crew_off`: the repository's admin switched `crew builds here` off
2525 * (`core/crew_switch.ts`): no builder or planner run starts on it.
2626 * - `full_suite_flake`: a full-suite report issue whose red files all passed when
27 * rerun alone on the same commit (`checks/report.ts`, #14404); it starts when a
27 * rerun alone on the same commit, or failed alone while other checks shared the
28 * runner (`checks/report.ts`, #14404, #15240); it starts when a
2829 * second full-suite run is red for the same file and the report lifts its label.
2930 * - `stalled`: a builder ran on it and left without a pull, and nobody
3031 * who directs the crew has spoken since. The next tick parks it in the
test/full-suite-report.test.ts +60 −4
@@ -21,7 +21,7 @@ import { createRun, getRun, type Run } from "../src/core/runs.ts";
2121 import { createUser } from "../src/core/users.ts";
2222 import { heldBy } from "../src/crew/held.ts";
2323 import type { Ranked } from "../src/core/dependencies.ts";
24 import { aloneJob, rerunAlone } from "../src/checks/execute.ts";
24 import { aloneJob, otherChecksRunning, rerunAlone } from "../src/checks/execute.ts";
2525 import type { CheckJob, CheckOutcome, Runner } from "../src/checks/runner.ts";
2626
2727 /**
@@ -242,6 +242,53 @@ describe("the report", () => {
242242 expect(hold(issue!.number)).toBe(false);
243243 });
244244
245 test("a file red alone on an idle runner is filed startable, naming its time and that nothing else ran (#15240)", () => {
246 const run = red(VERIFY_ENV, "a".repeat(40), one);
247 const said = reportRed(app, repo, run, new Map([["test/a.test.ts", { passed: false, ms: 14_200, others: 0 }]]));
248 expect(said[0]).toMatch(/^report: filed #\d+ for test\/a\.test\.ts$/);
249 const [issue] = reports();
250 expect(labels(issue!.id)).toEqual([REPORT_LABEL]);
251 expect(issue!.body).toContain(
252 `Red alone too: \`test/a.test.ts\` failed when rerun alone on ${repo.defaultBranch}'s commit aaaaaaa in 14.2 s, with no other check running.`,
253 );
254 expect(hold(issue!.number)).toBe(false);
255 });
256
257 test("a file red alone while other checks shared the runner is held until a second full-suite red (#15240)", () => {
258 const run = red(VERIFY_ENV, "a".repeat(40), one);
259 const said = reportRed(app, repo, run, new Map([["test/a.test.ts", { passed: false, ms: 14_200, others: 5 }]]));
260 expect(said[0]).toEndWith(`held as ${ALONE_GREEN_LABEL}`);
261 const [issue] = reports();
262 expect(labels(issue!.id).sort()).toEqual([REPORT_LABEL, ALONE_GREEN_LABEL].sort());
263 expect(issue!.body).toContain(
264 `Red alone too: \`test/a.test.ts\` failed when rerun alone on ${repo.defaultBranch}'s commit aaaaaaa in 14.2 s, with 5 other checks running. ` +
265 "The runner was shared, so the red may be load, and so it is held until a second full-suite run is red for it.",
266 );
267 expect(hold(issue!.number)).toBe(true);
268 reportRed(app, repo, red(VERIFY_ENV, "b".repeat(40), one), new Map([["test/a.test.ts", { passed: false, ms: 900, others: 0 }]]));
269 expect(labels(issue!.id)).toEqual([REPORT_LABEL]);
270 expect(hold(issue!.number)).toBe(false);
271 });
272
273 test("one other check is said in the singular, and a pass alone names its time too", () => {
274 reportRed(app, repo, red(VERIFY_ENV, "a".repeat(40), one), new Map([["test/a.test.ts", { passed: true, ms: 2_100, others: 1 }]]));
275 expect(reports()[0]!.body).toContain(`passed when rerun alone on ${repo.defaultBranch}'s commit aaaaaaa in 2.1 s, with 1 other check running (a flake under load)`);
276 });
277
278 test("other checks running counts running checks but the run itself and anything not running", () => {
279 const mk = (state: string) => {
280 const r = createRun(app.db, { repoId: repo.id, kind: "check", env: VERIFY_ENV, commitOid: "c".repeat(40) });
281 app.db.query("UPDATE run SET state = ?2 WHERE id = ?1").run(r.id, state);
282 return r.id;
283 };
284 const self = mk("running");
285 mk("running");
286 mk("running");
287 mk("queued");
288 mk("ok");
289 expect(otherChecksRunning(app.db, self)).toBe(2);
290 });
291
245292 test("a file that passes alone is held until a second full-suite red, which lifts the hold", () => {
246293 const run = red(VERIFY_ENV, "a".repeat(40), one);
247294 expect(reportRed(app, repo, run, new Map([["test/a.test.ts", true]]))[0]).toEndWith(`held as ${ALONE_GREEN_LABEL}`);
@@ -286,11 +333,20 @@ describe("the report", () => {
286333 };
287334 const runner: Runner = { name: "fake", run: async (job: CheckJob) => (jobs.push(job), answers[job.only![0]!]!) };
288335 const said: string[] = [];
289 const alone = await rerunAlone({ runner }, run, (l) => said.push(l), new AbortController().signal);
290 expect([...alone]).toEqual([["test/a.test.ts", true], ["test/b.test.ts", false]]);
336 let clock = 0;
337 let busy = 5;
338 const timed: Runner = { name: "fake", run: async (job, o) => ((clock += 14_200), runner.run(job, o)) };
339 const alone = await rerunAlone({ runner: timed }, run, (l) => said.push(l), new AbortController().signal, () => busy--, () => clock);
340 // Each answer carries its wall time and the checks running beside it as it started (#15240).
341 expect([...alone]).toEqual([
342 ["test/a.test.ts", { passed: true, ms: 14_200, others: 5 }],
343 ["test/b.test.ts", { passed: false, ms: 14_200, others: 4 }],
344 ]);
345 expect(said).toContain("rerun alone: bun test ./test/b.test.ts on aaaaaaa, 4 other checks running");
346 expect(said).toContain("rerun alone: test/b.test.ts failed alone in 14.2 s");
291347 expect(jobs).toEqual(["test/a.test.ts", "test/b.test.ts", "test/c.test.ts"].map((f) => aloneJob(run, f)));
292348 expect(jobs.every((j) => j.commitOid === run.commitOid && j.commandsFrom === run.commitOid && j.only!.length === 1)).toBe(true);
293 expect(said).toContain("rerun alone: test/a.test.ts passed alone");
349 expect(said).toContain("rerun alone: test/a.test.ts passed alone in 14.2 s");
294350 });
295351 });
296352

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs