Public: anyone can read this; the crew can change it.
A full-suite file "red alone" on the runner says how it was run, and one red only under the runner's load is held, not handed to a builder #15241
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/15240-a-full-suite-file-red-alone-on-the-runne → main 1 commit fork
5 files
changed +135
−28
src/checks/alone_label.ts +2 −1
| @@ -1,6 +1,7 @@ | ||
| 1 | 1 | /** |
| 2 | 2 | * The label on a full-suite report issue whose red files all passed when rerun alone on the same |
| 3 | * commit (#14404): a flake under load. The report (`report.ts`) puts it on and takes it off when a | |
| 3 | * commit (#14404), or failed alone while other checks shared the runner (#15240): a flake under | |
| 4 | * load, or maybe one. The report (`report.ts`) puts it on and takes it off when a | |
| 4 | 5 | * second full-suite run is red for the same file; the crew (`crew/held.ts`, `full_suite_flake`) |
| 5 | 6 | * does not start an issue that carries it. |
| 6 | 7 | */ |
src/checks/execute.ts +30 −10
| @@ -9,7 +9,8 @@ import { withDisk } from "./disk_red.ts"; | ||
| 9 | 9 | import { proposeRevert } from "./revert_red.ts"; |
| 10 | 10 | import { afterVerify, closeOnGreen } from "./culprit.ts"; |
| 11 | 11 | import { noteLookPages } from "./look_pages.ts"; |
| 12 | import { reportGreen, reportRed } from "./report.ts"; | |
| 12 | import { reportGreen, reportRed, type AloneAnswer } from "./report.ts"; | |
| 13 | import type { Database } from "bun:sqlite"; | |
| 13 | 14 | import { bisectOrigin, inVerifyLane, safeFile, VERIFY_ENV } from "./verify.ts"; |
| 14 | 15 | import { getIssueById } from "../core/issues.ts"; |
| 15 | 16 | import { BUDGETS, budgetSummary, checkLimitMs } from "../kernel/budgets.ts"; |
| @@ -206,7 +207,10 @@ export async function execute( | ||
| 206 | 207 | // A red verification after landing names its culprit (#7786). |
| 207 | 208 | if (done && run.env === VERIFY_ENV) { |
| 208 | 209 | // Each red file is rerun alone on the same commit, once, before it is filed (#14404). |
| 209 | const alone = done.state === "failed" ? await rerunAlone(lane, done, write, signal) : new Map<string, boolean>(); | |
| 210 | const alone = | |
| 211 | done.state === "failed" | |
| 212 | ? await rerunAlone(lane, done, write, signal, () => otherChecksRunning(app.db, done.id)) | |
| 213 | : new Map<string, AloneAnswer>(); | |
| 210 | 214 | reportRed(app, repo, done, alone).forEach(write); |
| 211 | 215 | // A file the run printed passing closes its open report issue (#9524). |
| 212 | 216 | reportGreen(app, repo, done, outcome.passed ?? []).forEach(write); |
| @@ -223,34 +227,50 @@ export async function execute( | ||
| 223 | 227 | } |
| 224 | 228 | |
| 225 | 229 | /** |
| 226 | * Each red file of a red verification run alone on the same commit, once (#14404): true when it | |
| 227 | * passed alone. A file the rerun could not answer (stopped, the runner refused) is left out, and | |
| 228 | * is filed as any red is. At most {@link ALONE_MAX} files are rerun; the rest are filed as today. | |
| 230 | * Each red file of a red verification run alone on the same commit, once (#14404): whether it | |
| 231 | * passed alone, its wall time and how many other checks were running when it started (#15240), so | |
| 232 | * a red alone on a shared runner is told from one on an idle runner. A file the rerun could not | |
| 233 | * answer (stopped, the runner refused) is left out, and is filed as any red is. At most {@link | |
| 234 | * ALONE_MAX} files are rerun; the rest are filed as today. | |
| 229 | 235 | */ |
| 230 | 236 | export async function rerunAlone( |
| 231 | 237 | lane: Pick<LaneState, "runner">, |
| 232 | 238 | run: Run, |
| 233 | 239 | write: (line: string) => void, |
| 234 | 240 | signal: AbortSignal, |
| 235 | ): Promise<Map<string, boolean>> { | |
| 241 | others: () => number = () => 0, | |
| 242 | now: () => number = Date.now, | |
| 243 | ): Promise<Map<string, { passed: boolean; ms: number; others: number }>> { | |
| 236 | 244 | const files = [...new Set(run.checkDetail?.files ?? [])].filter(safeFile).slice(0, ALONE_MAX); |
| 237 | const alone = new Map<string, boolean>(); | |
| 245 | const alone = new Map<string, { passed: boolean; ms: number; others: number }>(); | |
| 238 | 246 | for (const file of files) { |
| 239 | 247 | if (signal.aborted) break; |
| 240 | write(`rerun alone: bun test ./${file} on ${run.commitOid.slice(0, 7)}`); | |
| 248 | const beside = Math.max(0, Math.trunc(others())); | |
| 249 | write(`rerun alone: bun test ./${file} on ${run.commitOid.slice(0, 7)}, ${beside} other check${beside === 1 ? "" : "s"} running`); | |
| 250 | const began = now(); | |
| 241 | 251 | const outcome = await lane.runner |
| 242 | 252 | .run(aloneJob(run, file), { write: () => {}, signal }) |
| 243 | 253 | .catch((error): CheckOutcome => ({ ok: false, error: String(error), infrastructure: true })); |
| 254 | const ms = Math.max(0, now() - began); | |
| 244 | 255 | if (!outcome.ok && (outcome.infrastructure === true || outcome.killed === true || outcome.cpuStop !== undefined)) { |
| 245 | 256 | write(`rerun alone: ${file} not answered (${outcome.error})`); |
| 246 | 257 | continue; |
| 247 | 258 | } |
| 248 | alone.set(file, outcome.ok); | |
| 249 | write(`rerun alone: ${file} ${outcome.ok ? "passed alone" : "failed alone"}`); | |
| 259 | alone.set(file, { passed: outcome.ok, ms, others: beside }); | |
| 260 | write(`rerun alone: ${file} ${outcome.ok ? "passed alone" : "failed alone"} in ${(ms / 1000).toFixed(1)} s`); | |
| 250 | 261 | } |
| 251 | 262 | return alone; |
| 252 | 263 | } |
| 253 | 264 | |
| 265 | /** | |
| 266 | * How many checks other than `runId` this forge has running now: the company a rerun alone keeps | |
| 267 | * on the runner (#15240). | |
| 268 | */ | |
| 269 | export const otherChecksRunning = (db: Database, runId: string): number => | |
| 270 | db | |
| 271 | .query<{ n: number }, [string]>("SELECT COUNT(*) AS n FROM run WHERE kind = 'check' AND state = 'running' AND id != ?1") | |
| 272 | .get(runId)?.n ?? 0; | |
| 273 | ||
| 254 | 274 | /** The most red files a verification reruns alone: one each, once. */ |
| 255 | 275 | export const ALONE_MAX = 10; |
| 256 | 276 |
src/checks/report.ts +41 −12
| @@ -286,7 +286,7 @@ function fileSection(run: Run, file: string): string { | ||
| 286 | 286 | * is more, so a run that goes red for one cause starts one builder, and the same run reported again |
| 287 | 287 | * comments. |
| 288 | 288 | */ |
| 289 | export function reportRed(app: App, repo: Repo, run: Run, alone: ReadonlyMap<string, boolean> = new Map()): string[] { | |
| 289 | export function reportRed(app: App, repo: Repo, run: Run, alone: ReadonlyMap<string, AloneAnswer> = new Map()): string[] { | |
| 290 | 290 | if (run.state !== "failed") return []; |
| 291 | 291 | const files = [...new Set(run.checkDetail?.files ?? [])].filter(safeFile); |
| 292 | 292 | if (files.length === 0) return []; |
| @@ -328,9 +328,10 @@ export function reportRed(app: App, repo: Repo, run: Run, alone: ReadonlyMap<str | ||
| 328 | 328 | lines.push(`report: added to #${issue.number} for ${issue.files.join(", ")}`); |
| 329 | 329 | } |
| 330 | 330 | if (fresh.length === 0) return lines; |
| 331 | // A file that passed when rerun alone on this commit is filed apart and held (#14404). | |
| 332 | const flaky = fresh.filter((f) => alone.get(f) === true); | |
| 333 | const real = fresh.filter((f) => alone.get(f) !== true); | |
| 331 | // A file that passed when rerun alone on this commit is filed apart and held (#14404), and so is | |
| 332 | // one red alone while other checks shared the runner: load may be its red too (#15240). | |
| 333 | const flaky = fresh.filter((f) => heldAlone(alone.get(f))); | |
| 334 | const real = fresh.filter((f) => !heldAlone(alone.get(f))); | |
| 334 | 335 | if (real.length > 0) lines.push(fileFresh(app, repo, run, real, label, alone)); |
| 335 | 336 | if (flaky.length > 0) { |
| 336 | 337 | const hold = createLabel(app.db, repo.id, ALONE_GREEN_LABEL, ALONE_GREEN_COLOR); |
| @@ -341,15 +342,43 @@ export function reportRed(app: App, repo: Repo, run: Run, alone: ReadonlyMap<str | ||
| 341 | 342 | } |
| 342 | 343 | |
| 343 | 344 | /** |
| 344 | * The one line that says which of the two cases a file is, naming main's commit (#14404); none | |
| 345 | * for a file that was not rerun alone. | |
| 345 | * What one file's rerun alone said (#14404, #15240): whether it passed, how long it took in | |
| 346 | * milliseconds and how many other checks were running when it started. A bare boolean is a rerun | |
| 347 | * whose time and company were not recorded. | |
| 346 | 348 | */ |
| 347 | export function aloneLine(file: string, commitOid: string, branch: string, passedAlone: boolean | undefined): string { | |
| 349 | export type AloneAnswer = boolean | { readonly passed: boolean; readonly ms: number; readonly others: number }; | |
| 350 | ||
| 351 | const passedOf = (a: AloneAnswer | undefined): boolean | undefined => (typeof a === "object" ? a.passed : a); | |
| 352 | ||
| 353 | /** | |
| 354 | * Whether a file is filed held: it passed alone (a flake under load), or it failed alone while | |
| 355 | * other checks shared the runner, so its red alone may be load too (#15240). Red alone on an idle | |
| 356 | * runner is a real red, filed startable. | |
| 357 | */ | |
| 358 | export const heldAlone = (a: AloneAnswer | undefined): boolean => | |
| 359 | passedOf(a) === true || (typeof a === "object" && !a.passed && a.others > 0); | |
| 360 | ||
| 361 | /** `in 14.2 s, with 5 other checks running`: how the rerun ran, when it was recorded. */ | |
| 362 | function howRun(a: AloneAnswer): string { | |
| 363 | if (typeof a !== "object") return ""; | |
| 364 | const others = a.others === 0 ? "no other check running" : `${a.others} other check${a.others === 1 ? "" : "s"} running`; | |
| 365 | return ` in ${(a.ms / 1000).toFixed(1)} s, with ${others}`; | |
| 366 | } | |
| 367 | ||
| 368 | /** | |
| 369 | * The one line that says which case a file is, naming main's commit (#14404) and how the rerun | |
| 370 | * ran (#15240); none for a file that was not rerun alone. | |
| 371 | */ | |
| 372 | export function aloneLine(file: string, commitOid: string, branch: string, answer: AloneAnswer | undefined): string { | |
| 348 | 373 | const at = `${branch}'s commit ${commitOid.slice(0, 7)}`; |
| 349 | if (passedAlone === true) | |
| 350 | return `Red only in the full suite: \`${file}\` passed when rerun alone on ${at} (a flake under load), so it is held until a second full-suite run is red for it.`; | |
| 351 | if (passedAlone === false) return `Red alone too: \`${file}\` failed when rerun alone on ${at}.`; | |
| 352 | return ""; | |
| 374 | if (answer === undefined) return ""; | |
| 375 | const how = howRun(answer); | |
| 376 | const hold = "so it is held until a second full-suite run is red for it."; | |
| 377 | if (passedOf(answer) === true) | |
| 378 | return `Red only in the full suite: \`${file}\` passed when rerun alone on ${at}${how} (a flake under load), ${hold}`; | |
| 379 | if (heldAlone(answer)) | |
| 380 | return `Red alone too: \`${file}\` failed when rerun alone on ${at}${how}. The runner was shared, so the red may be load, and ${hold}`; | |
| 381 | return `Red alone too: \`${file}\` failed when rerun alone on ${at}${how}.`; | |
| 353 | 382 | } |
| 354 | 383 | |
| 355 | 384 | /** File `fresh` as one issue for the run, with the hold label when given; its log line. */ |
| @@ -359,7 +388,7 @@ function fileFresh( | ||
| 359 | 388 | run: Run, |
| 360 | 389 | fresh: readonly string[], |
| 361 | 390 | label: Label | undefined, |
| 362 | alone: ReadonlyMap<string, boolean>, | |
| 391 | alone: ReadonlyMap<string, AloneAnswer>, | |
| 363 | 392 | hold?: Label, |
| 364 | 393 | ): string { |
| 365 | 394 | const where = `check run ${run.id}, ${run.commitOid.slice(0, 7)}`; |
src/crew/held.ts +2 −1
| @@ -24,7 +24,8 @@ | ||
| 24 | 24 | * - `crew_off`: the repository's admin switched `crew builds here` off |
| 25 | 25 | * (`core/crew_switch.ts`): no builder or planner run starts on it. |
| 26 | 26 | * - `full_suite_flake`: a full-suite report issue whose red files all passed when |
| 27 | * rerun alone on the same commit (`checks/report.ts`, #14404); it starts when a | |
| 27 | * rerun alone on the same commit, or failed alone while other checks shared the | |
| 28 | * runner (`checks/report.ts`, #14404, #15240); it starts when a | |
| 28 | 29 | * second full-suite run is red for the same file and the report lifts its label. |
| 29 | 30 | * - `stalled`: a builder ran on it and left without a pull, and nobody |
| 30 | 31 | * who directs the crew has spoken since. The next tick parks it in the |
test/full-suite-report.test.ts +60 −4
| @@ -21,7 +21,7 @@ import { createRun, getRun, type Run } from "../src/core/runs.ts"; | ||
| 21 | 21 | import { createUser } from "../src/core/users.ts"; |
| 22 | 22 | import { heldBy } from "../src/crew/held.ts"; |
| 23 | 23 | import type { Ranked } from "../src/core/dependencies.ts"; |
| 24 | import { aloneJob, rerunAlone } from "../src/checks/execute.ts"; | |
| 24 | import { aloneJob, otherChecksRunning, rerunAlone } from "../src/checks/execute.ts"; | |
| 25 | 25 | import type { CheckJob, CheckOutcome, Runner } from "../src/checks/runner.ts"; |
| 26 | 26 | |
| 27 | 27 | /** |
| @@ -242,6 +242,53 @@ describe("the report", () => { | ||
| 242 | 242 | expect(hold(issue!.number)).toBe(false); |
| 243 | 243 | }); |
| 244 | 244 | |
| 245 | test("a file red alone on an idle runner is filed startable, naming its time and that nothing else ran (#15240)", () => { | |
| 246 | const run = red(VERIFY_ENV, "a".repeat(40), one); | |
| 247 | const said = reportRed(app, repo, run, new Map([["test/a.test.ts", { passed: false, ms: 14_200, others: 0 }]])); | |
| 248 | expect(said[0]).toMatch(/^report: filed #\d+ for test\/a\.test\.ts$/); | |
| 249 | const [issue] = reports(); | |
| 250 | expect(labels(issue!.id)).toEqual([REPORT_LABEL]); | |
| 251 | expect(issue!.body).toContain( | |
| 252 | `Red alone too: \`test/a.test.ts\` failed when rerun alone on ${repo.defaultBranch}'s commit aaaaaaa in 14.2 s, with no other check running.`, | |
| 253 | ); | |
| 254 | expect(hold(issue!.number)).toBe(false); | |
| 255 | }); | |
| 256 | ||
| 257 | test("a file red alone while other checks shared the runner is held until a second full-suite red (#15240)", () => { | |
| 258 | const run = red(VERIFY_ENV, "a".repeat(40), one); | |
| 259 | const said = reportRed(app, repo, run, new Map([["test/a.test.ts", { passed: false, ms: 14_200, others: 5 }]])); | |
| 260 | expect(said[0]).toEndWith(`held as ${ALONE_GREEN_LABEL}`); | |
| 261 | const [issue] = reports(); | |
| 262 | expect(labels(issue!.id).sort()).toEqual([REPORT_LABEL, ALONE_GREEN_LABEL].sort()); | |
| 263 | expect(issue!.body).toContain( | |
| 264 | `Red alone too: \`test/a.test.ts\` failed when rerun alone on ${repo.defaultBranch}'s commit aaaaaaa in 14.2 s, with 5 other checks running. ` + | |
| 265 | "The runner was shared, so the red may be load, and so it is held until a second full-suite run is red for it.", | |
| 266 | ); | |
| 267 | expect(hold(issue!.number)).toBe(true); | |
| 268 | reportRed(app, repo, red(VERIFY_ENV, "b".repeat(40), one), new Map([["test/a.test.ts", { passed: false, ms: 900, others: 0 }]])); | |
| 269 | expect(labels(issue!.id)).toEqual([REPORT_LABEL]); | |
| 270 | expect(hold(issue!.number)).toBe(false); | |
| 271 | }); | |
| 272 | ||
| 273 | test("one other check is said in the singular, and a pass alone names its time too", () => { | |
| 274 | reportRed(app, repo, red(VERIFY_ENV, "a".repeat(40), one), new Map([["test/a.test.ts", { passed: true, ms: 2_100, others: 1 }]])); | |
| 275 | expect(reports()[0]!.body).toContain(`passed when rerun alone on ${repo.defaultBranch}'s commit aaaaaaa in 2.1 s, with 1 other check running (a flake under load)`); | |
| 276 | }); | |
| 277 | ||
| 278 | test("other checks running counts running checks but the run itself and anything not running", () => { | |
| 279 | const mk = (state: string) => { | |
| 280 | const r = createRun(app.db, { repoId: repo.id, kind: "check", env: VERIFY_ENV, commitOid: "c".repeat(40) }); | |
| 281 | app.db.query("UPDATE run SET state = ?2 WHERE id = ?1").run(r.id, state); | |
| 282 | return r.id; | |
| 283 | }; | |
| 284 | const self = mk("running"); | |
| 285 | mk("running"); | |
| 286 | mk("running"); | |
| 287 | mk("queued"); | |
| 288 | mk("ok"); | |
| 289 | expect(otherChecksRunning(app.db, self)).toBe(2); | |
| 290 | }); | |
| 291 | ||
| 245 | 292 | test("a file that passes alone is held until a second full-suite red, which lifts the hold", () => { |
| 246 | 293 | const run = red(VERIFY_ENV, "a".repeat(40), one); |
| 247 | 294 | expect(reportRed(app, repo, run, new Map([["test/a.test.ts", true]]))[0]).toEndWith(`held as ${ALONE_GREEN_LABEL}`); |
| @@ -286,11 +333,20 @@ describe("the report", () => { | ||
| 286 | 333 | }; |
| 287 | 334 | const runner: Runner = { name: "fake", run: async (job: CheckJob) => (jobs.push(job), answers[job.only![0]!]!) }; |
| 288 | 335 | const said: string[] = []; |
| 289 | const alone = await rerunAlone({ runner }, run, (l) => said.push(l), new AbortController().signal); | |
| 290 | expect([...alone]).toEqual([["test/a.test.ts", true], ["test/b.test.ts", false]]); | |
| 336 | let clock = 0; | |
| 337 | let busy = 5; | |
| 338 | const timed: Runner = { name: "fake", run: async (job, o) => ((clock += 14_200), runner.run(job, o)) }; | |
| 339 | const alone = await rerunAlone({ runner: timed }, run, (l) => said.push(l), new AbortController().signal, () => busy--, () => clock); | |
| 340 | // Each answer carries its wall time and the checks running beside it as it started (#15240). | |
| 341 | expect([...alone]).toEqual([ | |
| 342 | ["test/a.test.ts", { passed: true, ms: 14_200, others: 5 }], | |
| 343 | ["test/b.test.ts", { passed: false, ms: 14_200, others: 4 }], | |
| 344 | ]); | |
| 345 | expect(said).toContain("rerun alone: bun test ./test/b.test.ts on aaaaaaa, 4 other checks running"); | |
| 346 | expect(said).toContain("rerun alone: test/b.test.ts failed alone in 14.2 s"); | |
| 291 | 347 | expect(jobs).toEqual(["test/a.test.ts", "test/b.test.ts", "test/c.test.ts"].map((f) => aloneJob(run, f))); |
| 292 | 348 | expect(jobs.every((j) => j.commitOid === run.commitOid && j.commandsFrom === run.commitOid && j.only!.length === 1)).toBe(true); |
| 293 | expect(said).toContain("rerun alone: test/a.test.ts passed alone"); | |
| 349 | expect(said).toContain("rerun alone: test/a.test.ts passed alone in 14.2 s"); | |
| 294 | 350 | }); |
| 295 | 351 | }); |
| 296 | 352 |
new above
offline