Public: anyone can read this; the crew can change it.
A full-suite red that passes on main's tip alone is not handed to a builder: the report reruns its red files before an issue becomes startable #14408
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/14404-a-full-suite-red-that-passes-on-main-s-t → main 2 commits fork
6 files
changed +224
−17
src/checks/alone_label.ts added +7
| @@ -0,0 +1,7 @@ | ||
| 1 | /** | |
| 2 | * The label on a full-suite report issue whose red files all passed when rerun alone on the same | |
| 3 | * commit (#14404): a flake under load. The report (`report.ts`) puts it on and takes it off when a | |
| 4 | * second full-suite run is red for the same file; the crew (`crew/held.ts`, `full_suite_flake`) | |
| 5 | * does not start an issue that carries it. | |
| 6 | */ | |
| 7 | export const ALONE_GREEN_LABEL = "red-only-in-full-suite"; |
src/checks/execute.ts +54 −3
| @@ -10,7 +10,7 @@ import { proposeRevert } from "./revert_red.ts"; | ||
| 10 | 10 | import { afterVerify, closeOnGreen } from "./culprit.ts"; |
| 11 | 11 | import { noteLookPages } from "./look_pages.ts"; |
| 12 | 12 | import { reportGreen, reportRed } from "./report.ts"; |
| 13 | import { bisectOrigin, inVerifyLane, VERIFY_ENV } from "./verify.ts"; | |
| 13 | import { bisectOrigin, inVerifyLane, safeFile, VERIFY_ENV } from "./verify.ts"; | |
| 14 | 14 | import { getIssueById } from "../core/issues.ts"; |
| 15 | 15 | import { BUDGETS, budgetSummary, checkLimitMs } from "../kernel/budgets.ts"; |
| 16 | 16 | import { createLog } from "../kernel/log.ts"; |
| @@ -20,7 +20,7 @@ import { cancelledLine } from "../runs/cancelled.ts"; | ||
| 20 | 20 | import { requeueCpuStop } from "./cpu_stop.ts"; |
| 21 | 21 | import { isUsedRefusal } from "../apps/disk_guard.ts"; |
| 22 | 22 | import { JOB_VANISHED } from "./job_step.ts"; |
| 23 | import { describeJob, type CheckOutcome } from "./runner.ts"; | |
| 23 | import { describeJob, type CheckJob, type CheckOutcome } from "./runner.ts"; | |
| 24 | 24 | import type { LaneState } from "./lane_state.ts"; |
| 25 | 25 | |
| 26 | 26 | const log = createLog("checks"); |
| @@ -205,7 +205,9 @@ export async function execute( | ||
| 205 | 205 | ); |
| 206 | 206 | // A red verification after landing names its culprit (#7786). |
| 207 | 207 | if (done && run.env === VERIFY_ENV) { |
| 208 | reportRed(app, repo, done).forEach(write); | |
| 208 | // Each red file is rerun alone on the same commit, once, before it is filed (#14404). | |
| 209 | const alone = done.state === "failed" ? await rerunAlone(lane, done, write, signal) : new Map<string, boolean>(); | |
| 210 | reportRed(app, repo, done, alone).forEach(write); | |
| 209 | 211 | // A file the run printed passing closes its open report issue (#9524). |
| 210 | 212 | reportGreen(app, repo, done, outcome.passed ?? []).forEach(write); |
| 211 | 213 | // A green that ran every file a culprit's issue names closes it (#11395). |
| @@ -220,6 +222,55 @@ export async function execute( | ||
| 220 | 222 | } |
| 221 | 223 | } |
| 222 | 224 | |
| 225 | /** | |
| 226 | * Each red file of a red verification run alone on the same commit, once (#14404): true when it | |
| 227 | * passed alone. A file the rerun could not answer (stopped, the runner refused) is left out, and | |
| 228 | * is filed as any red is. At most {@link ALONE_MAX} files are rerun; the rest are filed as today. | |
| 229 | */ | |
| 230 | export async function rerunAlone( | |
| 231 | lane: Pick<LaneState, "runner">, | |
| 232 | run: Run, | |
| 233 | write: (line: string) => void, | |
| 234 | signal: AbortSignal, | |
| 235 | ): Promise<Map<string, boolean>> { | |
| 236 | const files = [...new Set(run.checkDetail?.files ?? [])].filter(safeFile).slice(0, ALONE_MAX); | |
| 237 | const alone = new Map<string, boolean>(); | |
| 238 | for (const file of files) { | |
| 239 | if (signal.aborted) break; | |
| 240 | write(`rerun alone: bun test ./${file} on ${run.commitOid.slice(0, 7)}`); | |
| 241 | const outcome = await lane.runner | |
| 242 | .run(aloneJob(run, file), { write: () => {}, signal }) | |
| 243 | .catch((error): CheckOutcome => ({ ok: false, error: String(error), infrastructure: true })); | |
| 244 | if (!outcome.ok && (outcome.infrastructure === true || outcome.killed === true || outcome.cpuStop !== undefined)) { | |
| 245 | write(`rerun alone: ${file} not answered (${outcome.error})`); | |
| 246 | continue; | |
| 247 | } | |
| 248 | alone.set(file, outcome.ok); | |
| 249 | write(`rerun alone: ${file} ${outcome.ok ? "passed alone" : "failed alone"}`); | |
| 250 | } | |
| 251 | return alone; | |
| 252 | } | |
| 253 | ||
| 254 | /** The most red files a verification reruns alone: one each, once. */ | |
| 255 | export const ALONE_MAX = 10; | |
| 256 | ||
| 257 | /** The job that runs one file of a verification alone, with the run's own commands. */ | |
| 258 | export function aloneJob(run: Pick<Run, "id" | "repoId" | "commitOid">, file: string): CheckJob { | |
| 259 | return { | |
| 260 | runId: run.id, | |
| 261 | repoId: run.repoId, | |
| 262 | commitOid: run.commitOid, | |
| 263 | issueId: null, | |
| 264 | tier: "verify", | |
| 265 | only: [file], | |
| 266 | commandsFrom: run.commitOid, | |
| 267 | keep: false, | |
| 268 | keepImpact: false, | |
| 269 | fileOrderDependent: false, | |
| 270 | isMain: true, | |
| 271 | }; | |
| 272 | } | |
| 273 | ||
| 223 | 274 | /** Describe a claimed check as data and hand it to the runner. */ |
| 224 | 275 | async function check( |
| 225 | 276 | lane: LaneState, |
src/checks/flake.ts +3 −2
| @@ -279,7 +279,7 @@ export function fileIssueOnce( | ||
| 279 | 279 | db: Database, |
| 280 | 280 | bus: EventBus, |
| 281 | 281 | repo: Pick<Repo, "id" | "ownerId">, |
| 282 | issue: { title: string; body: string; labelId?: string }, | |
| 282 | issue: { title: string; body: string; labelId?: string; labelIds?: readonly string[] }, | |
| 283 | 283 | ): { number: number; opened: boolean } { |
| 284 | 284 | const open = db |
| 285 | 285 | .query<{ number: number }, [string, string]>( |
| @@ -289,7 +289,8 @@ export function fileIssueOnce( | ||
| 289 | 289 | if (open !== null) return { number: open.number, opened: false }; |
| 290 | 290 | const made = createIssue(db, { repoId: repo.id, authorId: repo.ownerId, title: issue.title, body: issue.body }); |
| 291 | 291 | if (!made.ok) return { number: 0, opened: false }; |
| 292 | if (issue.labelId !== undefined) setIssueLabels(db, made.value.id, [issue.labelId]); | |
| 292 | const labelIds = [...(issue.labelId === undefined ? [] : [issue.labelId]), ...(issue.labelIds ?? [])]; | |
| 293 | if (labelIds.length > 0) setIssueLabels(db, made.value.id, labelIds); | |
| 293 | 294 | bus.append({ |
| 294 | 295 | kind: "issue.opened", |
| 295 | 296 | repoId: repo.id, |
src/checks/report.ts +67 −10
| @@ -11,17 +11,20 @@ | ||
| 11 | 11 | |
| 12 | 12 | import type { App } from "../app.ts"; |
| 13 | 13 | import type { Database } from "bun:sqlite"; |
| 14 | import { addComment, createLabel, getLabel, listIssues, setIssueState, type Label } from "../core/issues.ts"; | |
| 14 | import { addComment, createLabel, getLabel, labelsFor, listIssues, setIssueLabels, setIssueState, type Label } from "../core/issues.ts"; | |
| 15 | 15 | import { getRepo, type Repo } from "../core/repos.ts"; |
| 16 | 16 | import { ownRepoPath } from "../kernel/own_repo.ts"; |
| 17 | 17 | import type { CheckDetail, Run } from "../core/runs.ts"; |
| 18 | 18 | import { failedTests, fileIssueOnce } from "./flake.ts"; |
| 19 | 19 | import { isSummaryLine } from "./bun-test.ts"; |
| 20 | 20 | import { safeFile } from "./verify.ts"; |
| 21 | import { ALONE_GREEN_LABEL } from "./alone_label.ts"; | |
| 21 | 22 | |
| 22 | 23 | /** The label every report issue carries. */ |
| 23 | 24 | export const REPORT_LABEL = "full-suite"; |
| 24 | 25 | const REPORT_COLOR = "d93f0b"; |
| 26 | export { ALONE_GREEN_LABEL }; | |
| 27 | const ALONE_GREEN_COLOR = "fbca04"; | |
| 25 | 28 | /** The most failing tests a run carries for the report. */ |
| 26 | 29 | export const MAX_FAILURES = 40; |
| 27 | 30 | /** The most tests one issue names. */ |
| @@ -283,7 +286,7 @@ function fileSection(run: Run, file: string): string { | ||
| 283 | 286 | * is more, so a run that goes red for one cause starts one builder, and the same run reported again |
| 284 | 287 | * comments. |
| 285 | 288 | */ |
| 286 | export function reportRed(app: App, repo: Repo, run: Run): string[] { | |
| 289 | export function reportRed(app: App, repo: Repo, run: Run, alone: ReadonlyMap<string, boolean> = new Map()): string[] { | |
| 287 | 290 | if (run.state !== "failed") return []; |
| 288 | 291 | const files = [...new Set(run.checkDetail?.files ?? [])].filter(safeFile); |
| 289 | 292 | if (files.length === 0) return []; |
| @@ -304,11 +307,16 @@ export function reportRed(app: App, repo: Repo, run: Run): string[] { | ||
| 304 | 307 | held.set(issue.id, at); |
| 305 | 308 | } |
| 306 | 309 | for (const issue of held.values()) { |
| 307 | const body = | |
| 310 | // A second full-suite red for a file held as a flake makes its issue startable (#14404). | |
| 311 | const lifted = unhold(app, issue.id); | |
| 312 | const said = | |
| 308 | 313 | issue.files.length === 1 |
| 309 | 314 | ? `Still red in the full suite (${where}).${fileSection(run, issue.files[0]!)}` |
| 310 | 315 | : `Still red in the full suite (${where}): ${issue.files.map((f) => `\`${f}\``).join(", ")}.\n\n` + |
| 311 | 316 | issue.files.map((f) => `# \`${f}\`${fileSection(run, f)}`).join("\n\n"); |
| 317 | const body = lifted | |
| 318 | ? `A second full-suite run is red for it, so it is no longer held as \`${ALONE_GREEN_LABEL}\`: a builder may start it.\n\n${said}` | |
| 319 | : said; | |
| 312 | 320 | const noted = addComment(app.db, { issueId: issue.id, authorId: repo.ownerId, body }); |
| 313 | 321 | if (noted.ok) |
| 314 | 322 | app.bus.append({ |
| @@ -320,23 +328,72 @@ export function reportRed(app: App, repo: Repo, run: Run): string[] { | ||
| 320 | 328 | lines.push(`report: added to #${issue.number} for ${issue.files.join(", ")}`); |
| 321 | 329 | } |
| 322 | 330 | if (fresh.length === 0) return lines; |
| 331 | // A file that passed when rerun alone on this commit is filed apart and held (#14404). | |
| 332 | const flaky = fresh.filter((f) => alone.get(f) === true); | |
| 333 | const real = fresh.filter((f) => alone.get(f) !== true); | |
| 334 | if (real.length > 0) lines.push(fileFresh(app, repo, run, real, label, alone)); | |
| 335 | if (flaky.length > 0) { | |
| 336 | const hold = createLabel(app.db, repo.id, ALONE_GREEN_LABEL, ALONE_GREEN_COLOR); | |
| 337 | const holdLabel = hold.ok ? hold.value : getLabel(app.db, repo.id, ALONE_GREEN_LABEL); | |
| 338 | lines.push(fileFresh(app, repo, run, flaky, label, alone, holdLabel)); | |
| 339 | } | |
| 340 | return lines; | |
| 341 | } | |
| 342 | ||
| 343 | /** | |
| 344 | * The one line that says which of the two cases a file is, naming main's commit (#14404); none | |
| 345 | * for a file that was not rerun alone. | |
| 346 | */ | |
| 347 | export function aloneLine(file: string, commitOid: string, branch: string, passedAlone: boolean | undefined): string { | |
| 348 | const at = `${branch}'s commit ${commitOid.slice(0, 7)}`; | |
| 349 | if (passedAlone === true) | |
| 350 | return `Red only in the full suite: \`${file}\` passed when rerun alone on ${at} (a flake under load), so it is held until a second full-suite run is red for it.`; | |
| 351 | if (passedAlone === false) return `Red alone too: \`${file}\` failed when rerun alone on ${at}.`; | |
| 352 | return ""; | |
| 353 | } | |
| 354 | ||
| 355 | /** File `fresh` as one issue for the run, with the hold label when given; its log line. */ | |
| 356 | function fileFresh( | |
| 357 | app: App, | |
| 358 | repo: Repo, | |
| 359 | run: Run, | |
| 360 | fresh: readonly string[], | |
| 361 | label: Label | undefined, | |
| 362 | alone: ReadonlyMap<string, boolean>, | |
| 363 | hold?: Label, | |
| 364 | ): string { | |
| 365 | const where = `check run ${run.id}, ${run.commitOid.slice(0, 7)}`; | |
| 323 | 366 | const opening = `${FILED_BY} It holds no pull, batch or release._\n\n`; |
| 367 | const section = (f: string): string => { | |
| 368 | const line = aloneLine(f, run.commitOid, repo.defaultBranch, alone.get(f)); | |
| 369 | return `${line === "" ? "" : `\n\n${line}`}${fileSection(run, f)}`; | |
| 370 | }; | |
| 324 | 371 | const body = |
| 325 | 372 | fresh.length === 1 |
| 326 | ? `${opening}\`${fresh[0]}\` failed in the full suite on ${repo.defaultBranch} (${where}).${fileSection(run, fresh[0]!)}` | |
| 373 | ? `${opening}\`${fresh[0]}\` failed in the full suite on ${repo.defaultBranch} (${where}).${section(fresh[0]!)}` | |
| 327 | 374 | : `${opening}${fresh.length} files failed in the full suite on ${repo.defaultBranch} (${where}), filed as one issue: ` + |
| 328 | 375 | `one run red for one cause starts one builder.\n\n${fresh.map((f) => `- \`${f}\``).join("\n")}\n\n` + |
| 329 | fresh.map((f) => `# \`${f}\`${fileSection(run, f)}`).join("\n\n"); | |
| 376 | fresh.map((f) => `# \`${f}\`${section(f)}`).join("\n\n"); | |
| 377 | const labelIds = [label?.id, hold?.id].filter((id): id is string => id !== undefined); | |
| 330 | 378 | const filed = fileIssueOnce(app.db, app.bus, repo, { |
| 331 | 379 | title: fresh.length === 1 ? reportTitle(fresh[0]!) : runReportTitle(run.id, fresh.length), |
| 332 | 380 | body: `${body}\n\nFiles: ${fresh.join(", ")}`, |
| 333 | ...(label === undefined ? {} : { labelId: label.id }), | |
| 381 | labelIds, | |
| 334 | 382 | }); |
| 335 | 383 | const named = fresh.join(", "); |
| 336 | if (filed.number === 0) lines.push(`report: ${named} not filed`); | |
| 337 | else if (filed.opened) lines.push(`report: filed #${filed.number} for ${named}`); | |
| 338 | else lines.push(`report: #${filed.number} already open for ${named}`); | |
| 339 | return lines; | |
| 384 | const held = hold === undefined ? "" : `, held as ${ALONE_GREEN_LABEL}`; | |
| 385 | if (filed.number === 0) return `report: ${named} not filed`; | |
| 386 | if (filed.opened) return `report: filed #${filed.number} for ${named}${held}`; | |
| 387 | return `report: #${filed.number} already open for ${named}`; | |
| 388 | } | |
| 389 | ||
| 390 | /** Take the hold label off a report issue; whether it had it. */ | |
| 391 | function unhold(app: App, issueId: string): boolean { | |
| 392 | const labels = labelsFor(app.db, issueId); | |
| 393 | const kept = labels.filter((l) => l.name !== ALONE_GREEN_LABEL); | |
| 394 | if (kept.length === labels.length) return false; | |
| 395 | setIssueLabels(app.db, issueId, kept.map((l) => l.id)); | |
| 396 | return true; | |
| 340 | 397 | } |
| 341 | 398 | |
| 342 | 399 | /** |
src/crew/held.ts +14 −1
| @@ -23,6 +23,9 @@ | ||
| 23 | 23 | * until a key is added. |
| 24 | 24 | * - `crew_off`: the repository's admin switched `crew builds here` off |
| 25 | 25 | * (`core/crew_switch.ts`): no builder or planner run starts on it. |
| 26 | * - `full_suite_flake`: a full-suite report issue whose red files all passed when | |
| 27 | * rerun alone on the same commit (`checks/report.ts`, #14404); it starts when a | |
| 28 | * second full-suite run is red for the same file and the report lifts its label. | |
| 26 | 29 | * - `stalled`: a builder ran on it and left without a pull, and nobody |
| 27 | 30 | * who directs the crew has spoken since. The next tick parks it in the |
| 28 | 31 | * crew's name, which frees it (`stalls.ts`). |
| @@ -46,7 +49,8 @@ import type { App } from "../app.ts"; | ||
| 46 | 49 | import { listCrewRoles, type Agent, type CrewRole } from "../core/agents.ts"; |
| 47 | 50 | import { readyNow, repoGraph, type Graph, type Ranked } from "../core/dependencies.ts"; |
| 48 | 51 | import { alreadyDone, BY_CREW, DONE_LIKE } from "../core/done_on_main.ts"; |
| 49 | import { getIssuesByIds, getIssuesByNumbers, type Issue } from "../core/issues.ts"; | |
| 52 | import { getIssuesByIds, getIssuesByNumbers, labelsForIssues, type Issue } from "../core/issues.ts"; | |
| 53 | import { ALONE_GREEN_LABEL } from "../checks/alone_label.ts"; | |
| 50 | 54 | import { withPlanner } from "../core/limb_replan.ts"; |
| 51 | 55 | import { isGo, type Repo } from "../core/repos.ts"; |
| 52 | 56 | import { sealedRowByNumber } from "../core/sealed.ts"; |
| @@ -78,6 +82,7 @@ export const HOLDS = [ | ||
| 78 | 82 | "landed", |
| 79 | 83 | NO_KEY_HELD, |
| 80 | 84 | CREW_OFF_HELD, |
| 85 | "full_suite_flake", | |
| 81 | 86 | "stalled", |
| 82 | 87 | "run_cap", |
| 83 | 88 | "parked_twice", |
| @@ -260,6 +265,12 @@ export function heldBy( | ||
| 260 | 265 | )); |
| 261 | 266 | const lastRuns = lastAgentRuns(app, ready.map((r) => r.issue.id)); |
| 262 | 267 | let shared: Set<string> | undefined; |
| 268 | // The ready issues the full-suite report holds as red only under load, read once for the pass. | |
| 269 | const flakes = new Set( | |
| 270 | [...labelsForIssues(app.db, ready.map((r) => r.issue.id))] | |
| 271 | .filter(([, labels]) => labels.some((l) => l.name === ALONE_GREEN_LABEL)) | |
| 272 | .map(([id]) => id), | |
| 273 | ); | |
| 263 | 274 | const holdOf = (issue: Issue): HoldName | null => { |
| 264 | 275 | if (busyIssues.has(issue.id)) return "running"; |
| 265 | 276 | if (proposed.has(issue.number)) return "open_pull"; |
| @@ -269,6 +280,7 @@ export function heldBy( | ||
| 269 | 280 | if (landedUnanswered(app.db, repo, issue) !== undefined) return "landed"; |
| 270 | 281 | if (noKey) return NO_KEY_HELD; |
| 271 | 282 | if (crewOff) return CREW_OFF_HELD; |
| 283 | if (flakes.has(issue.id)) return "full_suite_flake"; | |
| 272 | 284 | if (stalled(app, repo, issue, lastRuns)) return "stalled"; |
| 273 | 285 | if (capped().get(issue.id) ?? atRunCap(app, repo, issue)) return "run_cap"; |
| 274 | 286 | if (!isGo(repo) && parkedCount(app.db, repo, issue.id) >= PARK_STRIKES && retryReason(app.db, repo, issue, now) === undefined) |
| @@ -374,6 +386,7 @@ const HOLD_WORDS: Readonly<Record<HoldName, string>> = { | ||
| 374 | 386 | landed: "a merged pull closes it with boxes unticked; waits for a person", |
| 375 | 387 | [NO_KEY_HELD]: "this forge has no model key; it starts when one is added", |
| 376 | 388 | [CREW_OFF_HELD]: CREW_OFF_WORDS, |
| 389 | full_suite_flake: "red only in the full suite; it waits for a second full-suite red", | |
| 377 | 390 | stalled: "a builder ran on it and left without a pull", |
| 378 | 391 | run_cap: "it has used its runs for the day", |
| 379 | 392 | parked_twice: "parked twice; it is the planner's", |
test/full-suite-report.test.ts +79 −1
| @@ -11,7 +11,7 @@ import { join } from "node:path"; | ||
| 11 | 11 | import { createApp, type App } from "../src/app.ts"; |
| 12 | 12 | import { createRepo } from "../src/actions/repos.ts"; |
| 13 | 13 | import { flowLines } from "../src/tui/screens/flow.ts"; |
| 14 | import { RERUN, EXCERPT_LINES, LINE_MAX, MAX_FAILURES, redBody, redDetail, reportGreen, reportRed, reportTitle, runReportTitle, REPORT_LABEL } from "../src/checks/report.ts"; | |
| 14 | import { ALONE_GREEN_LABEL, RERUN, EXCERPT_LINES, LINE_MAX, MAX_FAILURES, redBody, redDetail, reportGreen, reportRed, reportTitle, runReportTitle, REPORT_LABEL } from "../src/checks/report.ts"; | |
| 15 | 15 | import { failedTests } from "../src/checks/flake.ts"; |
| 16 | 16 | import { heldByRed, redMain } from "../src/checks/red_main.ts"; |
| 17 | 17 | import { failedFiles, passedFiles, VERIFY_ENV } from "../src/checks/verify.ts"; |
| @@ -19,6 +19,10 @@ import { createIssue, getLabel, listIssues, listComments, labelsFor, setIssueLab | ||
| 19 | 19 | import type { Repo } from "../src/core/repos.ts"; |
| 20 | 20 | import { createRun, getRun, type Run } from "../src/core/runs.ts"; |
| 21 | 21 | import { createUser } from "../src/core/users.ts"; |
| 22 | import { heldBy } from "../src/crew/held.ts"; | |
| 23 | import type { Ranked } from "../src/core/dependencies.ts"; | |
| 24 | import { aloneJob, rerunAlone } from "../src/checks/execute.ts"; | |
| 25 | import type { CheckJob, CheckOutcome, Runner } from "../src/checks/runner.ts"; | |
| 22 | 26 | |
| 23 | 27 | /** |
| 24 | 28 | * What `bun test` 1.3 prints for three files, two with a failure, the green |
| @@ -216,6 +220,80 @@ describe("the report", () => { | ||
| 216 | 220 | expect(reports()).toHaveLength(2); |
| 217 | 221 | }); |
| 218 | 222 | |
| 223 | describe("each red file rerun alone before it is filed (#14404)", () => { | |
| 224 | const one = ["test/a.test.ts"]; | |
| 225 | const hold = (number: number) => { | |
| 226 | const issue = reports().find((i) => i.number === number)!; | |
| 227 | const held = heldBy(app, repo, [{ issue } as unknown as Ranked], [], []); | |
| 228 | const flake = held.by.full_suite_flake.map((i) => i.number).includes(number); | |
| 229 | // Not held as a flake is startable: no other hold applies in this repository. | |
| 230 | expect(held.startable.map((r) => r.issue.number).includes(number)).toBe(!flake); | |
| 231 | return flake; | |
| 232 | }; | |
| 233 | const labels = (id: string) => labelsFor(app.db, id).map((l) => l.name); | |
| 234 | ||
| 235 | test("a file that fails alone is filed as today, startable, and says so in one line naming main's commit", () => { | |
| 236 | const run = red(VERIFY_ENV, "a".repeat(40), one); | |
| 237 | expect(reportRed(app, repo, run, new Map([["test/a.test.ts", false]]))[0]).toMatch(/^report: filed #\d+ for test\/a\.test\.ts$/); | |
| 238 | const [issue] = reports(); | |
| 239 | expect(labels(issue!.id)).toEqual([REPORT_LABEL]); | |
| 240 | expect(issue!.body).toContain(`Red alone too: \`test/a.test.ts\` failed when rerun alone on ${repo.defaultBranch}'s commit aaaaaaa.`); | |
| 241 | expect(issue!.body).not.toContain("Red only in the full suite"); | |
| 242 | expect(hold(issue!.number)).toBe(false); | |
| 243 | }); | |
| 244 | ||
| 245 | test("a file that passes alone is held until a second full-suite red, which lifts the hold", () => { | |
| 246 | const run = red(VERIFY_ENV, "a".repeat(40), one); | |
| 247 | expect(reportRed(app, repo, run, new Map([["test/a.test.ts", true]]))[0]).toEndWith(`held as ${ALONE_GREEN_LABEL}`); | |
| 248 | const [issue] = reports(); | |
| 249 | expect(labels(issue!.id).sort()).toEqual([REPORT_LABEL, ALONE_GREEN_LABEL].sort()); | |
| 250 | expect(issue!.body).toContain( | |
| 251 | `Red only in the full suite: \`test/a.test.ts\` passed when rerun alone on ${repo.defaultBranch}'s commit aaaaaaa (a flake under load)`, | |
| 252 | ); | |
| 253 | expect(hold(issue!.number)).toBe(true); | |
| 254 | // A second full-suite run red for the same file: the issue becomes startable. | |
| 255 | reportRed(app, repo, red(VERIFY_ENV, "b".repeat(40), one), new Map([["test/a.test.ts", true]])); | |
| 256 | expect(reports()).toHaveLength(1); | |
| 257 | expect(labels(issue!.id)).toEqual([REPORT_LABEL]); | |
| 258 | expect(listComments(app.db, issue!.id)[0]!.body).toStartWith("A second full-suite run is red for it"); | |
| 259 | expect(hold(issue!.number)).toBe(false); | |
| 260 | }); | |
| 261 | ||
| 262 | test("a held flake closes as green again when a later run passes the file", () => { | |
| 263 | reportRed(app, repo, red(VERIFY_ENV, "a".repeat(40), one), new Map([["test/a.test.ts", true]])); | |
| 264 | expect(reportGreen(app, repo, green("e".repeat(40)), one)).toHaveLength(1); | |
| 265 | const [issue] = reports(); | |
| 266 | expect(issue!.state).toBe("closed"); | |
| 267 | expect(listComments(app.db, issue!.id)[0]!.body).toStartWith("Green again in the full suite"); | |
| 268 | }); | |
| 269 | ||
| 270 | test("files of one run split: the flakes are held apart, the real red is startable", () => { | |
| 271 | const run = red(VERIFY_ENV, "a".repeat(40)); | |
| 272 | expect(reportRed(app, repo, run, new Map([["test/a.test.ts", false], ["test/b.test.ts", true]]))).toHaveLength(2); | |
| 273 | const [real, flake] = reports().sort((x, y) => x.number - y.number); | |
| 274 | expect([real!.title, flake!.title]).toEqual([reportTitle("test/a.test.ts"), reportTitle("test/b.test.ts")]); | |
| 275 | expect(hold(real!.number)).toBe(false); | |
| 276 | expect(hold(flake!.number)).toBe(true); | |
| 277 | }); | |
| 278 | ||
| 279 | test("the rerun runs each red file alone on the run's commit, once, and leaves out what it could not answer", async () => { | |
| 280 | const run = red(VERIFY_ENV, "a".repeat(40), ["test/a.test.ts", "test/b.test.ts", "test/c.test.ts"]); | |
| 281 | const jobs: CheckJob[] = []; | |
| 282 | const answers: Record<string, CheckOutcome> = { | |
| 283 | "test/a.test.ts": { ok: true }, | |
| 284 | "test/b.test.ts": { ok: false, error: "1 failed" }, | |
| 285 | "test/c.test.ts": { ok: false, error: "cell lost", infrastructure: true }, | |
| 286 | }; | |
| 287 | const runner: Runner = { name: "fake", run: async (job: CheckJob) => (jobs.push(job), answers[job.only![0]!]!) }; | |
| 288 | const said: string[] = []; | |
| 289 | const alone = await rerunAlone({ runner }, run, (l) => said.push(l), new AbortController().signal); | |
| 290 | expect([...alone]).toEqual([["test/a.test.ts", true], ["test/b.test.ts", false]]); | |
| 291 | expect(jobs).toEqual(["test/a.test.ts", "test/b.test.ts", "test/c.test.ts"].map((f) => aloneJob(run, f))); | |
| 292 | expect(jobs.every((j) => j.commitOid === run.commitOid && j.commandsFrom === run.commitOid && j.only!.length === 1)).toBe(true); | |
| 293 | expect(said).toContain("rerun alone: test/a.test.ts passed alone"); | |
| 294 | }); | |
| 295 | }); | |
| 296 | ||
| 219 | 297 | test("a green verification files nothing", () => { |
| 220 | 298 | const run = createRun(app.db, { repoId: repo.id, kind: "check", env: VERIFY_ENV, commitOid: "c".repeat(40) }); |
| 221 | 299 | app.db.query("UPDATE run SET state = 'ok' WHERE id = ?1").run(run.id); |
new above
offline