Public: anyone can read this; the crew can change it.

A full-suite red that passes on main's tip alone is not handed to a builder: the report reruns its red files before an issue becomes startable #14408

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/14404-a-full-suite-red-that-passes-on-main-s-t → main 2 commits fork

  • Merged. scout/14404-a-full-suite-red-that-passes-on-main-s-t was merged into main as 464d760.

Review

Sign in to review this change.
6 files changed +224 −17
src/checks/alone_label.ts added +7
@@ -0,0 +1,7 @@
1 /**
2 * The label on a full-suite report issue whose red files all passed when rerun alone on the same
3 * commit (#14404): a flake under load. The report (`report.ts`) puts it on and takes it off when a
4 * second full-suite run is red for the same file; the crew (`crew/held.ts`, `full_suite_flake`)
5 * does not start an issue that carries it.
6 */
7 export const ALONE_GREEN_LABEL = "red-only-in-full-suite";
src/checks/execute.ts +54 −3
@@ -10,7 +10,7 @@ import { proposeRevert } from "./revert_red.ts";
1010 import { afterVerify, closeOnGreen } from "./culprit.ts";
1111 import { noteLookPages } from "./look_pages.ts";
1212 import { reportGreen, reportRed } from "./report.ts";
13 import { bisectOrigin, inVerifyLane, VERIFY_ENV } from "./verify.ts";
13 import { bisectOrigin, inVerifyLane, safeFile, VERIFY_ENV } from "./verify.ts";
1414 import { getIssueById } from "../core/issues.ts";
1515 import { BUDGETS, budgetSummary, checkLimitMs } from "../kernel/budgets.ts";
1616 import { createLog } from "../kernel/log.ts";
@@ -20,7 +20,7 @@ import { cancelledLine } from "../runs/cancelled.ts";
2020 import { requeueCpuStop } from "./cpu_stop.ts";
2121 import { isUsedRefusal } from "../apps/disk_guard.ts";
2222 import { JOB_VANISHED } from "./job_step.ts";
23 import { describeJob, type CheckOutcome } from "./runner.ts";
23 import { describeJob, type CheckJob, type CheckOutcome } from "./runner.ts";
2424 import type { LaneState } from "./lane_state.ts";
2525
2626 const log = createLog("checks");
@@ -205,7 +205,9 @@ export async function execute(
205205 );
206206 // A red verification after landing names its culprit (#7786).
207207 if (done && run.env === VERIFY_ENV) {
208 reportRed(app, repo, done).forEach(write);
208 // Each red file is rerun alone on the same commit, once, before it is filed (#14404).
209 const alone = done.state === "failed" ? await rerunAlone(lane, done, write, signal) : new Map<string, boolean>();
210 reportRed(app, repo, done, alone).forEach(write);
209211 // A file the run printed passing closes its open report issue (#9524).
210212 reportGreen(app, repo, done, outcome.passed ?? []).forEach(write);
211213 // A green that ran every file a culprit's issue names closes it (#11395).
@@ -220,6 +222,55 @@ export async function execute(
220222 }
221223 }
222224
225 /**
226 * Each red file of a red verification run alone on the same commit, once (#14404): true when it
227 * passed alone. A file the rerun could not answer (stopped, the runner refused) is left out, and
228 * is filed as any red is. At most {@link ALONE_MAX} files are rerun; the rest are filed as today.
229 */
230 export async function rerunAlone(
231 lane: Pick<LaneState, "runner">,
232 run: Run,
233 write: (line: string) => void,
234 signal: AbortSignal,
235 ): Promise<Map<string, boolean>> {
236 const files = [...new Set(run.checkDetail?.files ?? [])].filter(safeFile).slice(0, ALONE_MAX);
237 const alone = new Map<string, boolean>();
238 for (const file of files) {
239 if (signal.aborted) break;
240 write(`rerun alone: bun test ./${file} on ${run.commitOid.slice(0, 7)}`);
241 const outcome = await lane.runner
242 .run(aloneJob(run, file), { write: () => {}, signal })
243 .catch((error): CheckOutcome => ({ ok: false, error: String(error), infrastructure: true }));
244 if (!outcome.ok && (outcome.infrastructure === true || outcome.killed === true || outcome.cpuStop !== undefined)) {
245 write(`rerun alone: ${file} not answered (${outcome.error})`);
246 continue;
247 }
248 alone.set(file, outcome.ok);
249 write(`rerun alone: ${file} ${outcome.ok ? "passed alone" : "failed alone"}`);
250 }
251 return alone;
252 }
253
254 /** The most red files a verification reruns alone: one each, once. */
255 export const ALONE_MAX = 10;
256
257 /** The job that runs one file of a verification alone, with the run's own commands. */
258 export function aloneJob(run: Pick<Run, "id" | "repoId" | "commitOid">, file: string): CheckJob {
259 return {
260 runId: run.id,
261 repoId: run.repoId,
262 commitOid: run.commitOid,
263 issueId: null,
264 tier: "verify",
265 only: [file],
266 commandsFrom: run.commitOid,
267 keep: false,
268 keepImpact: false,
269 fileOrderDependent: false,
270 isMain: true,
271 };
272 }
273
223274 /** Describe a claimed check as data and hand it to the runner. */
224275 async function check(
225276 lane: LaneState,
src/checks/flake.ts +3 −2
@@ -279,7 +279,7 @@ export function fileIssueOnce(
279279 db: Database,
280280 bus: EventBus,
281281 repo: Pick<Repo, "id" | "ownerId">,
282 issue: { title: string; body: string; labelId?: string },
282 issue: { title: string; body: string; labelId?: string; labelIds?: readonly string[] },
283283 ): { number: number; opened: boolean } {
284284 const open = db
285285 .query<{ number: number }, [string, string]>(
@@ -289,7 +289,8 @@ export function fileIssueOnce(
289289 if (open !== null) return { number: open.number, opened: false };
290290 const made = createIssue(db, { repoId: repo.id, authorId: repo.ownerId, title: issue.title, body: issue.body });
291291 if (!made.ok) return { number: 0, opened: false };
292 if (issue.labelId !== undefined) setIssueLabels(db, made.value.id, [issue.labelId]);
292 const labelIds = [...(issue.labelId === undefined ? [] : [issue.labelId]), ...(issue.labelIds ?? [])];
293 if (labelIds.length > 0) setIssueLabels(db, made.value.id, labelIds);
293294 bus.append({
294295 kind: "issue.opened",
295296 repoId: repo.id,
src/checks/report.ts +67 −10
@@ -11,17 +11,20 @@
1111
1212 import type { App } from "../app.ts";
1313 import type { Database } from "bun:sqlite";
14 import { addComment, createLabel, getLabel, listIssues, setIssueState, type Label } from "../core/issues.ts";
14 import { addComment, createLabel, getLabel, labelsFor, listIssues, setIssueLabels, setIssueState, type Label } from "../core/issues.ts";
1515 import { getRepo, type Repo } from "../core/repos.ts";
1616 import { ownRepoPath } from "../kernel/own_repo.ts";
1717 import type { CheckDetail, Run } from "../core/runs.ts";
1818 import { failedTests, fileIssueOnce } from "./flake.ts";
1919 import { isSummaryLine } from "./bun-test.ts";
2020 import { safeFile } from "./verify.ts";
21 import { ALONE_GREEN_LABEL } from "./alone_label.ts";
2122
2223 /** The label every report issue carries. */
2324 export const REPORT_LABEL = "full-suite";
2425 const REPORT_COLOR = "d93f0b";
26 export { ALONE_GREEN_LABEL };
27 const ALONE_GREEN_COLOR = "fbca04";
2528 /** The most failing tests a run carries for the report. */
2629 export const MAX_FAILURES = 40;
2730 /** The most tests one issue names. */
@@ -283,7 +286,7 @@ function fileSection(run: Run, file: string): string {
283286 * is more, so a run that goes red for one cause starts one builder, and the same run reported again
284287 * comments.
285288 */
286 export function reportRed(app: App, repo: Repo, run: Run): string[] {
289 export function reportRed(app: App, repo: Repo, run: Run, alone: ReadonlyMap<string, boolean> = new Map()): string[] {
287290 if (run.state !== "failed") return [];
288291 const files = [...new Set(run.checkDetail?.files ?? [])].filter(safeFile);
289292 if (files.length === 0) return [];
@@ -304,11 +307,16 @@ export function reportRed(app: App, repo: Repo, run: Run): string[] {
304307 held.set(issue.id, at);
305308 }
306309 for (const issue of held.values()) {
307 const body =
310 // A second full-suite red for a file held as a flake makes its issue startable (#14404).
311 const lifted = unhold(app, issue.id);
312 const said =
308313 issue.files.length === 1
309314 ? `Still red in the full suite (${where}).${fileSection(run, issue.files[0]!)}`
310315 : `Still red in the full suite (${where}): ${issue.files.map((f) => `\`${f}\``).join(", ")}.\n\n` +
311316 issue.files.map((f) => `### \`${f}\`${fileSection(run, f)}`).join("\n\n");
317 const body = lifted
318 ? `A second full-suite run is red for it, so it is no longer held as \`${ALONE_GREEN_LABEL}\`: a builder may start it.\n\n${said}`
319 : said;
312320 const noted = addComment(app.db, { issueId: issue.id, authorId: repo.ownerId, body });
313321 if (noted.ok)
314322 app.bus.append({
@@ -320,23 +328,72 @@ export function reportRed(app: App, repo: Repo, run: Run): string[] {
320328 lines.push(`report: added to #${issue.number} for ${issue.files.join(", ")}`);
321329 }
322330 if (fresh.length === 0) return lines;
331 // A file that passed when rerun alone on this commit is filed apart and held (#14404).
332 const flaky = fresh.filter((f) => alone.get(f) === true);
333 const real = fresh.filter((f) => alone.get(f) !== true);
334 if (real.length > 0) lines.push(fileFresh(app, repo, run, real, label, alone));
335 if (flaky.length > 0) {
336 const hold = createLabel(app.db, repo.id, ALONE_GREEN_LABEL, ALONE_GREEN_COLOR);
337 const holdLabel = hold.ok ? hold.value : getLabel(app.db, repo.id, ALONE_GREEN_LABEL);
338 lines.push(fileFresh(app, repo, run, flaky, label, alone, holdLabel));
339 }
340 return lines;
341 }
342
343 /**
344 * The one line that says which of the two cases a file is, naming main's commit (#14404); none
345 * for a file that was not rerun alone.
346 */
347 export function aloneLine(file: string, commitOid: string, branch: string, passedAlone: boolean | undefined): string {
348 const at = `${branch}'s commit ${commitOid.slice(0, 7)}`;
349 if (passedAlone === true)
350 return `Red only in the full suite: \`${file}\` passed when rerun alone on ${at} (a flake under load), so it is held until a second full-suite run is red for it.`;
351 if (passedAlone === false) return `Red alone too: \`${file}\` failed when rerun alone on ${at}.`;
352 return "";
353 }
354
355 /** File `fresh` as one issue for the run, with the hold label when given; its log line. */
356 function fileFresh(
357 app: App,
358 repo: Repo,
359 run: Run,
360 fresh: readonly string[],
361 label: Label | undefined,
362 alone: ReadonlyMap<string, boolean>,
363 hold?: Label,
364 ): string {
365 const where = `check run ${run.id}, ${run.commitOid.slice(0, 7)}`;
323366 const opening = `${FILED_BY} It holds no pull, batch or release._\n\n`;
367 const section = (f: string): string => {
368 const line = aloneLine(f, run.commitOid, repo.defaultBranch, alone.get(f));
369 return `${line === "" ? "" : `\n\n${line}`}${fileSection(run, f)}`;
370 };
324371 const body =
325372 fresh.length === 1
326 ? `${opening}\`${fresh[0]}\` failed in the full suite on ${repo.defaultBranch} (${where}).${fileSection(run, fresh[0]!)}`
373 ? `${opening}\`${fresh[0]}\` failed in the full suite on ${repo.defaultBranch} (${where}).${section(fresh[0]!)}`
327374 : `${opening}${fresh.length} files failed in the full suite on ${repo.defaultBranch} (${where}), filed as one issue: ` +
328375 `one run red for one cause starts one builder.\n\n${fresh.map((f) => `- \`${f}\``).join("\n")}\n\n` +
329 fresh.map((f) => `### \`${f}\`${fileSection(run, f)}`).join("\n\n");
376 fresh.map((f) => `### \`${f}\`${section(f)}`).join("\n\n");
377 const labelIds = [label?.id, hold?.id].filter((id): id is string => id !== undefined);
330378 const filed = fileIssueOnce(app.db, app.bus, repo, {
331379 title: fresh.length === 1 ? reportTitle(fresh[0]!) : runReportTitle(run.id, fresh.length),
332380 body: `${body}\n\nFiles: ${fresh.join(", ")}`,
333 ...(label === undefined ? {} : { labelId: label.id }),
381 labelIds,
334382 });
335383 const named = fresh.join(", ");
336 if (filed.number === 0) lines.push(`report: ${named} not filed`);
337 else if (filed.opened) lines.push(`report: filed #${filed.number} for ${named}`);
338 else lines.push(`report: #${filed.number} already open for ${named}`);
339 return lines;
384 const held = hold === undefined ? "" : `, held as ${ALONE_GREEN_LABEL}`;
385 if (filed.number === 0) return `report: ${named} not filed`;
386 if (filed.opened) return `report: filed #${filed.number} for ${named}${held}`;
387 return `report: #${filed.number} already open for ${named}`;
388 }
389
390 /** Take the hold label off a report issue; whether it had it. */
391 function unhold(app: App, issueId: string): boolean {
392 const labels = labelsFor(app.db, issueId);
393 const kept = labels.filter((l) => l.name !== ALONE_GREEN_LABEL);
394 if (kept.length === labels.length) return false;
395 setIssueLabels(app.db, issueId, kept.map((l) => l.id));
396 return true;
340397 }
341398
342399 /**
src/crew/held.ts +14 −1
@@ -23,6 +23,9 @@
2323 * until a key is added.
2424 * - `crew_off`: the repository's admin switched `crew builds here` off
2525 * (`core/crew_switch.ts`): no builder or planner run starts on it.
26 * - `full_suite_flake`: a full-suite report issue whose red files all passed when
27 * rerun alone on the same commit (`checks/report.ts`, #14404); it starts when a
28 * second full-suite run is red for the same file and the report lifts its label.
2629 * - `stalled`: a builder ran on it and left without a pull, and nobody
2730 * who directs the crew has spoken since. The next tick parks it in the
2831 * crew's name, which frees it (`stalls.ts`).
@@ -46,7 +49,8 @@ import type { App } from "../app.ts";
4649 import { listCrewRoles, type Agent, type CrewRole } from "../core/agents.ts";
4750 import { readyNow, repoGraph, type Graph, type Ranked } from "../core/dependencies.ts";
4851 import { alreadyDone, BY_CREW, DONE_LIKE } from "../core/done_on_main.ts";
49 import { getIssuesByIds, getIssuesByNumbers, type Issue } from "../core/issues.ts";
52 import { getIssuesByIds, getIssuesByNumbers, labelsForIssues, type Issue } from "../core/issues.ts";
53 import { ALONE_GREEN_LABEL } from "../checks/alone_label.ts";
5054 import { withPlanner } from "../core/limb_replan.ts";
5155 import { isGo, type Repo } from "../core/repos.ts";
5256 import { sealedRowByNumber } from "../core/sealed.ts";
@@ -78,6 +82,7 @@ export const HOLDS = [
7882 "landed",
7983 NO_KEY_HELD,
8084 CREW_OFF_HELD,
85 "full_suite_flake",
8186 "stalled",
8287 "run_cap",
8388 "parked_twice",
@@ -260,6 +265,12 @@ export function heldBy(
260265 ));
261266 const lastRuns = lastAgentRuns(app, ready.map((r) => r.issue.id));
262267 let shared: Set<string> | undefined;
268 // The ready issues the full-suite report holds as red only under load, read once for the pass.
269 const flakes = new Set(
270 [...labelsForIssues(app.db, ready.map((r) => r.issue.id))]
271 .filter(([, labels]) => labels.some((l) => l.name === ALONE_GREEN_LABEL))
272 .map(([id]) => id),
273 );
263274 const holdOf = (issue: Issue): HoldName | null => {
264275 if (busyIssues.has(issue.id)) return "running";
265276 if (proposed.has(issue.number)) return "open_pull";
@@ -269,6 +280,7 @@ export function heldBy(
269280 if (landedUnanswered(app.db, repo, issue) !== undefined) return "landed";
270281 if (noKey) return NO_KEY_HELD;
271282 if (crewOff) return CREW_OFF_HELD;
283 if (flakes.has(issue.id)) return "full_suite_flake";
272284 if (stalled(app, repo, issue, lastRuns)) return "stalled";
273285 if (capped().get(issue.id) ?? atRunCap(app, repo, issue)) return "run_cap";
274286 if (!isGo(repo) && parkedCount(app.db, repo, issue.id) >= PARK_STRIKES && retryReason(app.db, repo, issue, now) === undefined)
@@ -374,6 +386,7 @@ const HOLD_WORDS: Readonly<Record<HoldName, string>> = {
374386 landed: "a merged pull closes it with boxes unticked; waits for a person",
375387 [NO_KEY_HELD]: "this forge has no model key; it starts when one is added",
376388 [CREW_OFF_HELD]: CREW_OFF_WORDS,
389 full_suite_flake: "red only in the full suite; it waits for a second full-suite red",
377390 stalled: "a builder ran on it and left without a pull",
378391 run_cap: "it has used its runs for the day",
379392 parked_twice: "parked twice; it is the planner's",
test/full-suite-report.test.ts +79 −1
@@ -11,7 +11,7 @@ import { join } from "node:path";
1111 import { createApp, type App } from "../src/app.ts";
1212 import { createRepo } from "../src/actions/repos.ts";
1313 import { flowLines } from "../src/tui/screens/flow.ts";
14 import { RERUN, EXCERPT_LINES, LINE_MAX, MAX_FAILURES, redBody, redDetail, reportGreen, reportRed, reportTitle, runReportTitle, REPORT_LABEL } from "../src/checks/report.ts";
14 import { ALONE_GREEN_LABEL, RERUN, EXCERPT_LINES, LINE_MAX, MAX_FAILURES, redBody, redDetail, reportGreen, reportRed, reportTitle, runReportTitle, REPORT_LABEL } from "../src/checks/report.ts";
1515 import { failedTests } from "../src/checks/flake.ts";
1616 import { heldByRed, redMain } from "../src/checks/red_main.ts";
1717 import { failedFiles, passedFiles, VERIFY_ENV } from "../src/checks/verify.ts";
@@ -19,6 +19,10 @@ import { createIssue, getLabel, listIssues, listComments, labelsFor, setIssueLab
1919 import type { Repo } from "../src/core/repos.ts";
2020 import { createRun, getRun, type Run } from "../src/core/runs.ts";
2121 import { createUser } from "../src/core/users.ts";
22 import { heldBy } from "../src/crew/held.ts";
23 import type { Ranked } from "../src/core/dependencies.ts";
24 import { aloneJob, rerunAlone } from "../src/checks/execute.ts";
25 import type { CheckJob, CheckOutcome, Runner } from "../src/checks/runner.ts";
2226
2327 /**
2428 * What `bun test` 1.3 prints for three files, two with a failure, the green
@@ -216,6 +220,80 @@ describe("the report", () => {
216220 expect(reports()).toHaveLength(2);
217221 });
218222
223 describe("each red file rerun alone before it is filed (#14404)", () => {
224 const one = ["test/a.test.ts"];
225 const hold = (number: number) => {
226 const issue = reports().find((i) => i.number === number)!;
227 const held = heldBy(app, repo, [{ issue } as unknown as Ranked], [], []);
228 const flake = held.by.full_suite_flake.map((i) => i.number).includes(number);
229 // Not held as a flake is startable: no other hold applies in this repository.
230 expect(held.startable.map((r) => r.issue.number).includes(number)).toBe(!flake);
231 return flake;
232 };
233 const labels = (id: string) => labelsFor(app.db, id).map((l) => l.name);
234
235 test("a file that fails alone is filed as today, startable, and says so in one line naming main's commit", () => {
236 const run = red(VERIFY_ENV, "a".repeat(40), one);
237 expect(reportRed(app, repo, run, new Map([["test/a.test.ts", false]]))[0]).toMatch(/^report: filed #\d+ for test\/a\.test\.ts$/);
238 const [issue] = reports();
239 expect(labels(issue!.id)).toEqual([REPORT_LABEL]);
240 expect(issue!.body).toContain(`Red alone too: \`test/a.test.ts\` failed when rerun alone on ${repo.defaultBranch}'s commit aaaaaaa.`);
241 expect(issue!.body).not.toContain("Red only in the full suite");
242 expect(hold(issue!.number)).toBe(false);
243 });
244
245 test("a file that passes alone is held until a second full-suite red, which lifts the hold", () => {
246 const run = red(VERIFY_ENV, "a".repeat(40), one);
247 expect(reportRed(app, repo, run, new Map([["test/a.test.ts", true]]))[0]).toEndWith(`held as ${ALONE_GREEN_LABEL}`);
248 const [issue] = reports();
249 expect(labels(issue!.id).sort()).toEqual([REPORT_LABEL, ALONE_GREEN_LABEL].sort());
250 expect(issue!.body).toContain(
251 `Red only in the full suite: \`test/a.test.ts\` passed when rerun alone on ${repo.defaultBranch}'s commit aaaaaaa (a flake under load)`,
252 );
253 expect(hold(issue!.number)).toBe(true);
254 // A second full-suite run red for the same file: the issue becomes startable.
255 reportRed(app, repo, red(VERIFY_ENV, "b".repeat(40), one), new Map([["test/a.test.ts", true]]));
256 expect(reports()).toHaveLength(1);
257 expect(labels(issue!.id)).toEqual([REPORT_LABEL]);
258 expect(listComments(app.db, issue!.id)[0]!.body).toStartWith("A second full-suite run is red for it");
259 expect(hold(issue!.number)).toBe(false);
260 });
261
262 test("a held flake closes as green again when a later run passes the file", () => {
263 reportRed(app, repo, red(VERIFY_ENV, "a".repeat(40), one), new Map([["test/a.test.ts", true]]));
264 expect(reportGreen(app, repo, green("e".repeat(40)), one)).toHaveLength(1);
265 const [issue] = reports();
266 expect(issue!.state).toBe("closed");
267 expect(listComments(app.db, issue!.id)[0]!.body).toStartWith("Green again in the full suite");
268 });
269
270 test("files of one run split: the flakes are held apart, the real red is startable", () => {
271 const run = red(VERIFY_ENV, "a".repeat(40));
272 expect(reportRed(app, repo, run, new Map([["test/a.test.ts", false], ["test/b.test.ts", true]]))).toHaveLength(2);
273 const [real, flake] = reports().sort((x, y) => x.number - y.number);
274 expect([real!.title, flake!.title]).toEqual([reportTitle("test/a.test.ts"), reportTitle("test/b.test.ts")]);
275 expect(hold(real!.number)).toBe(false);
276 expect(hold(flake!.number)).toBe(true);
277 });
278
279 test("the rerun runs each red file alone on the run's commit, once, and leaves out what it could not answer", async () => {
280 const run = red(VERIFY_ENV, "a".repeat(40), ["test/a.test.ts", "test/b.test.ts", "test/c.test.ts"]);
281 const jobs: CheckJob[] = [];
282 const answers: Record<string, CheckOutcome> = {
283 "test/a.test.ts": { ok: true },
284 "test/b.test.ts": { ok: false, error: "1 failed" },
285 "test/c.test.ts": { ok: false, error: "cell lost", infrastructure: true },
286 };
287 const runner: Runner = { name: "fake", run: async (job: CheckJob) => (jobs.push(job), answers[job.only![0]!]!) };
288 const said: string[] = [];
289 const alone = await rerunAlone({ runner }, run, (l) => said.push(l), new AbortController().signal);
290 expect([...alone]).toEqual([["test/a.test.ts", true], ["test/b.test.ts", false]]);
291 expect(jobs).toEqual(["test/a.test.ts", "test/b.test.ts", "test/c.test.ts"].map((f) => aloneJob(run, f)));
292 expect(jobs.every((j) => j.commitOid === run.commitOid && j.commandsFrom === run.commitOid && j.only!.length === 1)).toBe(true);
293 expect(said).toContain("rerun alone: test/a.test.ts passed alone");
294 });
295 });
296
219297 test("a green verification files nothing", () => {
220298 const run = createRun(app.db, { repoId: repo.id, kind: "check", env: VERIFY_ENV, commitOid: "c".repeat(40) });
221299 app.db.query("UPDATE run SET state = 'ok' WHERE id = ?1").run(run.id);

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs