Public: anyone can read this; the crew can change it.
One main run that goes red for one cause files one issue naming every file, not one issue per file #12859
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/12849-one-main-run-that-goes-red-for-one-cause → main 4 commits fork 1 thread · 1 unresolved
3 files
changed +199
−110
src/checks/report.ts +122 −60
| @@ -1,11 +1,12 @@ | ||
| 1 | 1 | /** |
| 2 | 2 | * The full suite as a scheduled report (#7780). A verification after landing |
| 3 | 3 | * (`checks/verify.ts`) runs every shard against main's tip; when it is red, |
| 4 | * each failing test file gets one issue, labelled {@link REPORT_LABEL} so it | |
| 5 | * is found: the open one is added to when a later run fails the same file, | |
| 6 | * and none is made twice. The report only files. Nothing reads a verification | |
| 7 | * when it decides whether a pull, a batch or a release may go ahead (those | |
| 8 | * look at the default branch's own check), so a red report holds none of them. | |
| 4 | * its failing files get one issue for the run (#12849), labelled {@link | |
| 5 | * REPORT_LABEL} so it is found: the open one is added to when a later run | |
| 6 | * fails a file it names, and none is made twice. The report only files. | |
| 7 | * Nothing reads a verification when it decides whether a pull, a batch or a | |
| 8 | * release may go ahead (those look at the default branch's own check), so a | |
| 9 | * red report holds none of them. | |
| 9 | 10 | */ |
| 10 | 11 | |
| 11 | 12 | import type { App } from "../app.ts"; |
| @@ -226,60 +227,25 @@ function labelOf(app: App, repo: Repo): Label | undefined { | ||
| 226 | 227 | return made.ok ? made.value : getLabel(app.db, repo.id, REPORT_LABEL); |
| 227 | 228 | } |
| 228 | 229 | |
| 229 | /** The lines for a red verification's log: one per failing file, filed or added to. Empty when green. */ | |
| 230 | export function reportRed(app: App, repo: Repo, run: Run): string[] { | |
| 231 | if (run.state !== "failed") return []; | |
| 232 | const files = run.checkDetail?.files ?? []; | |
| 233 | const failures = run.checkDetail?.failures ?? []; | |
| 234 | const label = files.length > 0 ? labelOf(app, repo) : undefined; | |
| 235 | const lines: string[] = []; | |
| 236 | for (const file of files) { | |
| 237 | const names = failures.filter((f) => f.file === file).map((f) => f.name); | |
| 238 | const said = names.length === 0 ? "" : `\n\n${names.slice(0, NAMED).map((n) => `- \`${n}\``).join("\n")}`; | |
| 239 | const more = names.length > NAMED ? `\n- and ${names.length - NAMED} more` : ""; | |
| 240 | const where = `check run ${run.id}, ${run.commitOid.slice(0, 7)}`; | |
| 241 | const title = reportTitle(file); | |
| 242 | const told = redBody(run.checkDetail, file); | |
| 243 | const filed = fileIssueOnce(app.db, app.bus, repo, { | |
| 244 | title, | |
| 245 | body: | |
| 246 | `${FILED_BY} It holds no pull, batch or release._\n\n` + | |
| 247 | `\`${file}\` failed in the full suite on ${repo.defaultBranch} (${where}).` + | |
| 248 | `${said}${more}\n\n${told}\n\nFiles: ${file}`, | |
| 249 | ...(label === undefined ? {} : { labelId: label.id }), | |
| 250 | }); | |
| 251 | if (filed.number === 0) lines.push(`report: ${file} not filed`); | |
| 252 | else if (filed.opened) lines.push(`report: filed #${filed.number} for ${file}`); | |
| 253 | else { | |
| 254 | const open = app.db | |
| 255 | .query<{ id: string }, [string, number]>("SELECT id FROM issue WHERE repo_id = ?1 AND number = ?2") | |
| 256 | .get(repo.id, filed.number); | |
| 257 | if (open !== null) | |
| 258 | addComment(app.db, { | |
| 259 | issueId: open.id, | |
| 260 | authorId: repo.ownerId, | |
| 261 | body: `Still red in the full suite (${where}).${said}${more}\n\n${told}`, | |
| 262 | }); | |
| 263 | lines.push(`report: added to #${filed.number} for ${file}`); | |
| 264 | } | |
| 265 | } | |
| 266 | return lines; | |
| 230 | /** | |
| 231 | * The title of the one issue a check run files when two or more of its files are red and none is | |
| 232 | * filed yet (#12849). | |
| 233 | */ | |
| 234 | export const runReportTitle = (runId: string, count: number): string => `Full suite red: ${count} files in check run ${runId}`; | |
| 235 | const RUN_TITLE = /^Full suite red: \d+ files in check run (\S+)$/; | |
| 236 | ||
| 237 | /** The files a report issue's body names on its closing `Files:` line. */ | |
| 238 | function filesOf(body: string): string[] { | |
| 239 | const line = body.split("\n").reverse().find((l) => l.startsWith("Files: ")); | |
| 240 | return line === undefined ? [] : line.slice("Files: ".length).split(", ").filter(safeFile); | |
| 267 | 241 | } |
| 268 | 242 | |
| 269 | 243 | /** |
| 270 | * The lines for a finished verification's log: one per open report issue | |
| 271 | * whose file ran and passed every test in it (#9524), commented on and | |
| 272 | * closed as done. `passed` is what the run's own log showed | |
| 273 | * (`verify.ts` `passedFiles`); a file it never printed (a shard that did | |
| 274 | * not run, a run cut short) keeps its issue open. Only an issue the report | |
| 275 | * filed is closed: its label, its exact title, the owner as author and the | |
| 276 | * report's own opening line, never one a person wrote. | |
| 244 | * The open issues the report filed: its label, the owner as author and its own opening line, never | |
| 245 | * one a person wrote. | |
| 277 | 246 | */ |
| 278 | export function reportGreen(app: App, repo: Repo, run: Run, passed: readonly string[]): string[] { | |
| 279 | if (run.state !== "ok" && run.state !== "failed") return []; | |
| 280 | if (passed.length === 0) return []; | |
| 281 | const ran = new Map(passed.map((file) => [reportTitle(file), file])); | |
| 282 | const open = app.db | |
| 247 | function openReports(app: App, repo: Repo): { id: string; number: number; title: string; files: string[] }[] { | |
| 248 | return app.db | |
| 283 | 249 | .query<{ id: string; number: number; title: string; body: string }, [string, string, string]>( |
| 284 | 250 | `SELECT issue.id, issue.number, issue.title, issue.body FROM issue |
| 285 | 251 | WHERE issue.repo_id = ?1 AND issue.kind = 'issue' AND issue.state = 'open' |
| @@ -288,12 +254,108 @@ export function reportGreen(app: App, repo: Repo, run: Run, passed: readonly str | ||
| 288 | 254 | WHERE issue_label.issue_id = issue.id AND label.name = ?3) |
| 289 | 255 | ORDER BY issue.number`, |
| 290 | 256 | ) |
| 291 | .all(repo.id, repo.ownerId, REPORT_LABEL); | |
| 257 | .all(repo.id, repo.ownerId, REPORT_LABEL) | |
| 258 | .filter((i) => i.body.startsWith(FILED_BY)) | |
| 259 | .map((i) => { | |
| 260 | const files = filesOf(i.body); | |
| 261 | // A file's own issue holds that file only; a run's issue the files its body names. | |
| 262 | const own = files.length === 1 && i.title === reportTitle(files[0]!); | |
| 263 | return { id: i.id, number: i.number, title: i.title, files: own || RUN_TITLE.test(i.title) ? files : [] }; | |
| 264 | }) | |
| 265 | .filter((i) => i.files.length > 0); | |
| 266 | } | |
| 267 | ||
| 268 | /** | |
| 269 | * What a report says of one red file: its failing tests by name, then the quoted failures and how | |
| 270 | * to reproduce it. | |
| 271 | */ | |
| 272 | function fileSection(run: Run, file: string): string { | |
| 273 | const names = (run.checkDetail?.failures ?? []).filter((f) => f.file === file).map((f) => f.name); | |
| 274 | const said = names.length === 0 ? "" : `\n\n${names.slice(0, NAMED).map((n) => `- ${inline(n)}`).join("\n")}`; | |
| 275 | const more = names.length > NAMED ? `\n- and ${names.length - NAMED} more` : ""; | |
| 276 | return `${said}${more}\n\n${redBody(run.checkDetail, file)}`; | |
| 277 | } | |
| 278 | ||
| 279 | /** | |
| 280 | * The lines for a red verification's log, filed or added to. Empty when green. A red file that an | |
| 281 | * open report issue already names is added to that issue, one comment per issue; the rest are | |
| 282 | * filed as one issue for the run (#12849): titled for the file when it is one, for the run when it | |
| 283 | * is more, so a run that goes red for one cause starts one builder, and the same run reported again | |
| 284 | * comments. | |
| 285 | */ | |
| 286 | export function reportRed(app: App, repo: Repo, run: Run): string[] { | |
| 287 | if (run.state !== "failed") return []; | |
| 288 | const files = [...new Set(run.checkDetail?.files ?? [])].filter(safeFile); | |
| 289 | if (files.length === 0) return []; | |
| 290 | const label = labelOf(app, repo); | |
| 292 | 291 | const where = `check run ${run.id}, ${run.commitOid.slice(0, 7)}`; |
| 293 | 292 | const lines: string[] = []; |
| 294 | for (const issue of open) { | |
| 295 | const file = ran.get(issue.title); | |
| 296 | if (file === undefined || !issue.body.startsWith(FILED_BY)) continue; | |
| 293 | const held = new Map<string, { id: string; number: number; files: string[] }>(); | |
| 294 | const open = openReports(app, repo); | |
| 295 | const fresh: string[] = []; | |
| 296 | for (const file of files) { | |
| 297 | const issue = open.find((i) => i.title === reportTitle(file)) ?? open.find((i) => i.files.includes(file)); | |
| 298 | if (issue === undefined) { | |
| 299 | fresh.push(file); | |
| 300 | continue; | |
| 301 | } | |
| 302 | const at = held.get(issue.id) ?? { id: issue.id, number: issue.number, files: [] }; | |
| 303 | at.files.push(file); | |
| 304 | held.set(issue.id, at); | |
| 305 | } | |
| 306 | for (const issue of held.values()) { | |
| 307 | const body = | |
| 308 | issue.files.length === 1 | |
| 309 | ? `Still red in the full suite (${where}).${fileSection(run, issue.files[0]!)}` | |
| 310 | : `Still red in the full suite (${where}): ${issue.files.map((f) => `\`${f}\``).join(", ")}.\n\n` + | |
| 311 | issue.files.map((f) => `# \`${f}\`${fileSection(run, f)}`).join("\n\n"); | |
| 312 | const noted = addComment(app.db, { issueId: issue.id, authorId: repo.ownerId, body }); | |
| 313 | if (noted.ok) | |
| 314 | app.bus.append({ | |
| 315 | kind: "issue.commented", | |
| 316 | repoId: repo.id, | |
| 317 | actorId: repo.ownerId, | |
| 318 | payload: { number: issue.number, body, commentId: noted.value.id }, | |
| 319 | }); | |
| 320 | lines.push(`report: added to #${issue.number} for ${issue.files.join(", ")}`); | |
| 321 | } | |
| 322 | if (fresh.length === 0) return lines; | |
| 323 | const opening = `${FILED_BY} It holds no pull, batch or release._\n\n`; | |
| 324 | const body = | |
| 325 | fresh.length === 1 | |
| 326 | ? `${opening}\`${fresh[0]}\` failed in the full suite on ${repo.defaultBranch} (${where}).${fileSection(run, fresh[0]!)}` | |
| 327 | : `${opening}${fresh.length} files failed in the full suite on ${repo.defaultBranch} (${where}), filed as one issue: ` + | |
| 328 | `one run red for one cause starts one builder.\n\n${fresh.map((f) => `- \`${f}\``).join("\n")}\n\n` + | |
| 329 | fresh.map((f) => `# \`${f}\`${fileSection(run, f)}`).join("\n\n"); | |
| 330 | const filed = fileIssueOnce(app.db, app.bus, repo, { | |
| 331 | title: fresh.length === 1 ? reportTitle(fresh[0]!) : runReportTitle(run.id, fresh.length), | |
| 332 | body: `${body}\n\nFiles: ${fresh.join(", ")}`, | |
| 333 | ...(label === undefined ? {} : { labelId: label.id }), | |
| 334 | }); | |
| 335 | const named = fresh.join(", "); | |
| 336 | if (filed.number === 0) lines.push(`report: ${named} not filed`); | |
| 337 | else if (filed.opened) lines.push(`report: filed #${filed.number} for ${named}`); | |
| 338 | else lines.push(`report: #${filed.number} already open for ${named}`); | |
| 339 | return lines; | |
| 340 | } | |
| 341 | ||
| 342 | /** | |
| 343 | * The lines for a finished verification's log: one per open report issue | |
| 344 | * every file of which ran and passed every test in it (#9524), commented on and | |
| 345 | * closed as done. `passed` is what the run's own log showed | |
| 346 | * (`verify.ts` `passedFiles`); a file it never printed (a shard that did | |
| 347 | * not run, a run cut short) keeps its issue open. Only an issue the report | |
| 348 | * filed is closed: its label, its own title, the owner as author and the | |
| 349 | * report's own opening line, never one a person wrote. | |
| 350 | */ | |
| 351 | export function reportGreen(app: App, repo: Repo, run: Run, passed: readonly string[]): string[] { | |
| 352 | if (run.state !== "ok" && run.state !== "failed") return []; | |
| 353 | if (passed.length === 0) return []; | |
| 354 | const ran = new Set(passed); | |
| 355 | const where = `check run ${run.id}, ${run.commitOid.slice(0, 7)}`; | |
| 356 | const lines: string[] = []; | |
| 357 | for (const issue of openReports(app, repo)) { | |
| 358 | if (!issue.files.every((f) => ran.has(f))) continue; | |
| 297 | 359 | const body = `Green again in the full suite (${where}).`; |
| 298 | 360 | const noted = addComment(app.db, { issueId: issue.id, authorId: repo.ownerId, body }); |
| 299 | 361 | if (noted.ok) |
| @@ -310,7 +372,7 @@ export function reportGreen(app: App, repo: Repo, run: Run, passed: readonly str | ||
| 310 | 372 | actorId: repo.ownerId, |
| 311 | 373 | payload: { number: issue.number }, |
| 312 | 374 | }); |
| 313 | lines.push(`report: closed #${issue.number}, ${file} green again`); | |
| 375 | lines.push(`report: closed #${issue.number}, ${issue.files.join(", ")} green again`); | |
| 314 | 376 | } |
| 315 | 377 | return lines; |
| 316 | 378 | } |
src/web/pages/pipeline.ts +1 −1
| @@ -107,7 +107,7 @@ export function pipelinePage(ctx: Ctx): Response { | ||
| 107 | 107 | </p> |
| 108 | 108 | <p class="small" id="full-suite"> |
| 109 | 109 | The whole suite still runs, as a report: after a landing, on a schedule, against the default branch's tip. |
| 110 | Each test file it finds failing gets one issue labelled <span class="mono">full-suite</span>, listed | |
| 110 | When a run fails several test files, the report opens one issue naming all of them, labelled <span class="mono">full-suite</span>, listed | |
| 111 | 111 | ${reportRepo(ctx.app.db) ? html`<a href="${reportPath()}">here</a>` : "on the forge's own repository, which this forge does not hold"}. The report holds no pull, batch or release. |
| 112 | 112 | </p> |
| 113 | 113 | <p class="muted small">For agents: the MCP tool <span class="mono">pull</span> answers <span class="mono">step: { step, why, done }</span> with these four words.</p> |
test/full-suite-report.test.ts +76 −49
| @@ -1,7 +1,7 @@ | ||
| 1 | 1 | /** |
| 2 | 2 | * The full suite as a scheduled report (#7780): a red verification files one |
| 3 | * labelled issue per failing file, a rerun adds to it instead of filing | |
| 4 | * again, and the red holds nothing. | |
| 3 | * labelled issue for its failing files (#12849), a rerun adds to it instead | |
| 4 | * of filing again, and the red holds nothing. | |
| 5 | 5 | */ |
| 6 | 6 | |
| 7 | 7 | import { afterEach, beforeEach, describe, expect, test } from "bun:test"; |
| @@ -11,7 +11,7 @@ import { join } from "node:path"; | ||
| 11 | 11 | import { createApp, type App } from "../src/app.ts"; |
| 12 | 12 | import { createRepo } from "../src/actions/repos.ts"; |
| 13 | 13 | import { flowLines } from "../src/tui/screens/flow.ts"; |
| 14 | import { RERUN, EXCERPT_LINES, LINE_MAX, MAX_FAILURES, redBody, redDetail, reportGreen, reportRed, reportTitle, REPORT_LABEL } from "../src/checks/report.ts"; | |
| 14 | import { RERUN, EXCERPT_LINES, LINE_MAX, MAX_FAILURES, redBody, redDetail, reportGreen, reportRed, reportTitle, runReportTitle, REPORT_LABEL } from "../src/checks/report.ts"; | |
| 15 | 15 | import { failedTests } from "../src/checks/flake.ts"; |
| 16 | 16 | import { heldByRed, redMain } from "../src/checks/red_main.ts"; |
| 17 | 17 | import { failedFiles, passedFiles, VERIFY_ENV } from "../src/checks/verify.ts"; |
| @@ -136,57 +136,84 @@ describe("the report", () => { | ||
| 136 | 136 | await rm(dataDir, { recursive: true, force: true }); |
| 137 | 137 | }); |
| 138 | 138 | |
| 139 | const red = (env: string, commit: string): Run => { | |
| 139 | const red = (env: string, commit: string, files = ["test/a.test.ts", "test/b.test.ts"]): Run => { | |
| 140 | 140 | const run = createRun(app.db, { repoId: repo.id, kind: "check", env, commitOid: commit, reason: "schedule" }); |
| 141 | const detail = { | |
| 142 | passed: 10, | |
| 143 | failed: 3, | |
| 144 | names: ["a > one", "a > two", "b > three"], | |
| 145 | files: ["test/a.test.ts", "test/b.test.ts"], | |
| 146 | failures: [ | |
| 147 | { file: "test/a.test.ts", name: "a > one" }, | |
| 148 | { file: "test/a.test.ts", name: "a > two" }, | |
| 149 | { file: "test/b.test.ts", name: "b > three" }, | |
| 150 | ], | |
| 151 | }; | |
| 141 | const failures = [ | |
| 142 | { file: "test/a.test.ts", name: "a > one" }, | |
| 143 | { file: "test/a.test.ts", name: "a > two" }, | |
| 144 | { file: "test/b.test.ts", name: "b > three" }, | |
| 145 | ].filter((f) => files.includes(f.file)); | |
| 146 | const detail = { passed: 10, failed: failures.length, names: failures.map((f) => f.name), files, failures }; | |
| 152 | 147 | app.db |
| 153 | 148 | .query("UPDATE run SET state = 'failed', check_detail = ?2, finished_at = ?3 WHERE id = ?1") |
| 154 | 149 | .run(run.id, JSON.stringify(detail), new Date().toISOString()); |
| 155 | 150 | return getRun(app.db, run.id)!; |
| 156 | 151 | }; |
| 157 | 152 | |
| 153 | const green = (commit: string): Run => { | |
| 154 | const run = createRun(app.db, { repoId: repo.id, kind: "check", env: VERIFY_ENV, commitOid: commit, reason: "schedule" }); | |
| 155 | app.db.query("UPDATE run SET state = 'ok', finished_at = ?2 WHERE id = ?1").run(run.id, new Date().toISOString()); | |
| 156 | return getRun(app.db, run.id)!; | |
| 157 | }; | |
| 158 | 158 | const reports = () => listIssues(app.db, repo.id, { label: REPORT_LABEL, state: "all" }); |
| 159 | 159 | |
| 160 | test("one labelled issue per failing file, naming its failing tests", () => { | |
| 161 | const lines = reportRed(app, repo, red(VERIFY_ENV, "a".repeat(40))); | |
| 162 | expect(lines).toHaveLength(2); | |
| 160 | test("one run red in two files files one labelled issue naming both, each with its tests and repro (#12849)", () => { | |
| 161 | const run = red(VERIFY_ENV, "a".repeat(40)); | |
| 162 | const lines = reportRed(app, repo, run); | |
| 163 | expect(lines).toHaveLength(1); | |
| 163 | 164 | const issues = reports(); |
| 164 | expect(issues.map((i) => i.title).sort()).toEqual([ | |
| 165 | reportTitle("test/a.test.ts"), | |
| 166 | reportTitle("test/b.test.ts"), | |
| 167 | ]); | |
| 168 | const a = issues.find((i) => i.title === reportTitle("test/a.test.ts"))!; | |
| 169 | expect(a.body).toContain("a > one"); | |
| 170 | expect(a.body).toContain("a > two"); | |
| 171 | expect(a.body).not.toContain("b > three"); | |
| 172 | expect(a.body).toContain("Files: test/a.test.ts"); | |
| 173 | expect(labelsFor(app.db, a.id).map((l) => l.name)).toEqual([REPORT_LABEL]); | |
| 165 | expect(issues.map((i) => i.title)).toEqual([runReportTitle(run.id, 2)]); | |
| 166 | const [issue] = issues; | |
| 167 | for (const name of ["a > one", "a > two", "b > three"]) expect(issue!.body).toContain(name); | |
| 168 | expect(issue!.body).toContain(`bun test ./test/a.test.ts ${RERUN}`); | |
| 169 | expect(issue!.body).toContain(`bun test ./test/b.test.ts ${RERUN}`); | |
| 170 | expect(issue!.body).toEndWith("Files: test/a.test.ts, test/b.test.ts"); | |
| 171 | expect(labelsFor(app.db, issue!.id).map((l) => l.name)).toEqual([REPORT_LABEL]); | |
| 174 | 172 | }); |
| 175 | 173 | |
| 176 | test("a rerun files nothing new: it adds to the open issues", () => { | |
| 177 | reportRed(app, repo, red(VERIFY_ENV, "a".repeat(40))); | |
| 178 | const again = reportRed(app, repo, red(VERIFY_ENV, "b".repeat(40))); | |
| 174 | test("the same run reported again comments on its issue instead of opening another (#12849)", () => { | |
| 175 | const run = red(VERIFY_ENV, "a".repeat(40)); | |
| 176 | reportRed(app, repo, run); | |
| 177 | const again = reportRed(app, repo, run); | |
| 178 | expect(again).toEqual([`report: added to #${reports()[0]!.number} for test/a.test.ts, test/b.test.ts`]); | |
| 179 | expect(reports()).toHaveLength(1); | |
| 180 | expect(listComments(app.db, reports()[0]!.id)).toHaveLength(1); | |
| 181 | }); | |
| 182 | ||
| 183 | test("a second run red in other files opens a second issue; files already filed are added to (#12849)", () => { | |
| 184 | const first = red(VERIFY_ENV, "a".repeat(40)); | |
| 185 | reportRed(app, repo, first); | |
| 186 | const second = red(VERIFY_ENV, "b".repeat(40), ["test/b.test.ts", "test/c.test.ts", "test/d.test.ts"]); | |
| 187 | const lines = reportRed(app, repo, second); | |
| 188 | const [one, two] = reports().sort((x, y) => x.number - y.number); | |
| 189 | expect([one!.title, two!.title]).toEqual([runReportTitle(first.id, 2), runReportTitle(second.id, 2)]); | |
| 190 | expect(lines[0]).toBe(`report: added to #${one!.number} for test/b.test.ts`); | |
| 191 | expect(two!.body).toEndWith("Files: test/c.test.ts, test/d.test.ts"); | |
| 192 | expect(listComments(app.db, one!.id)[0]!.body).toContain("b > three"); | |
| 193 | }); | |
| 194 | ||
| 195 | test("a file's own issue from an earlier run is still added to, by its title", () => { | |
| 196 | reportRed(app, repo, red(VERIFY_ENV, "a".repeat(40), ["test/a.test.ts"])); | |
| 197 | expect(reports().map((i) => i.title)).toEqual([reportTitle("test/a.test.ts")]); | |
| 198 | const again = reportRed(app, repo, red(VERIFY_ENV, "b".repeat(40), ["test/a.test.ts"])); | |
| 179 | 199 | expect(again.every((l) => l.startsWith("report: added to #"))).toBe(true); |
| 180 | expect(reports()).toHaveLength(2); | |
| 181 | const a = reports().find((i) => i.title === reportTitle("test/a.test.ts"))!; | |
| 182 | expect(listComments(app.db, a.id)).toHaveLength(1); | |
| 200 | expect(reports()).toHaveLength(1); | |
| 201 | expect(listComments(app.db, reports()[0]!.id)).toHaveLength(1); | |
| 202 | }); | |
| 203 | ||
| 204 | test("a run's issue closes only when every file it names passed", () => { | |
| 205 | reportRed(app, repo, red(VERIFY_ENV, "a".repeat(40))); | |
| 206 | expect(reportGreen(app, repo, green("e".repeat(40)), ["test/a.test.ts"])).toEqual([]); | |
| 207 | expect(reports()[0]!.state).toBe("open"); | |
| 208 | expect(reportGreen(app, repo, green("f".repeat(40)), ["test/a.test.ts", "test/b.test.ts"])).toHaveLength(1); | |
| 209 | expect(reports()[0]!.state).toBe("closed"); | |
| 183 | 210 | }); |
| 184 | 211 | |
| 185 | 212 | test("a closed issue is filed anew when the file fails again", () => { |
| 186 | 213 | reportRed(app, repo, red(VERIFY_ENV, "a".repeat(40))); |
| 187 | 214 | app.db.query("UPDATE issue SET state = 'closed' WHERE repo_id = ?1").run(repo.id); |
| 188 | 215 | reportRed(app, repo, red(VERIFY_ENV, "b".repeat(40))); |
| 189 | expect(reports()).toHaveLength(4); | |
| 216 | expect(reports()).toHaveLength(2); | |
| 190 | 217 | }); |
| 191 | 218 | |
| 192 | 219 | test("a green verification files nothing", () => { |
| @@ -221,21 +248,21 @@ describe("the report", () => { | ||
| 221 | 248 | .run(run.id, JSON.stringify(detail)); |
| 222 | 249 | reportRed(app, repo, getRun(app.db, run.id)!); |
| 223 | 250 | const issues = reports(); |
| 224 | expect(issues.map((i) => i.title).sort()).toEqual([reportTitle("test/a.test.ts"), reportTitle("test/b.test.ts")]); | |
| 225 | const b = issues.find((i) => i.title === reportTitle("test/b.test.ts"))!; | |
| 226 | expect(b.body).toContain("b > two"); | |
| 227 | expect(b.body).not.toContain("a > one"); | |
| 251 | expect(issues.map((i) => i.title)).toEqual([runReportTitle(run.id, 2)]); | |
| 252 | expect(issues[0]!.body).toContain("b > two"); | |
| 253 | expect(issues[0]!.body).not.toContain("test/c.test.ts"); | |
| 228 | 254 | }); |
| 229 | 255 | |
| 230 | const green = (commit: string): Run => { | |
| 231 | const run = createRun(app.db, { repoId: repo.id, kind: "check", env: VERIFY_ENV, commitOid: commit, reason: "schedule" }); | |
| 232 | app.db.query("UPDATE run SET state = 'ok', finished_at = ?2 WHERE id = ?1").run(run.id, new Date().toISOString()); | |
| 233 | return getRun(app.db, run.id)!; | |
| 234 | }; | |
| 235 | 256 | const issueFor = (file: string) => reports().find((i) => i.title === reportTitle(file))!; |
| 236 | 257 | |
| 258 | /** Each file red in its own run: each gets its own issue, titled for the file. */ | |
| 259 | const redEach = (): void => { | |
| 260 | reportRed(app, repo, red(VERIFY_ENV, "a".repeat(40), ["test/a.test.ts"])); | |
| 261 | reportRed(app, repo, red(VERIFY_ENV, "c".repeat(40), ["test/b.test.ts"])); | |
| 262 | }; | |
| 263 | ||
| 237 | 264 | test("red, then a green run that passed the file closes its issue, then a run without the file leaves the other open (#9524)", () => { |
| 238 | reportRed(app, repo, red(VERIFY_ENV, "a".repeat(40))); | |
| 265 | redEach(); | |
| 239 | 266 | const ok = green("e".repeat(40)); |
| 240 | 267 | expect(reportGreen(app, repo, ok, ["test/a.test.ts", "test/z.test.ts"])).toEqual([ |
| 241 | 268 | `report: closed #${issueFor("test/a.test.ts").number}, test/a.test.ts green again`, |
| @@ -254,7 +281,7 @@ describe("the report", () => { | ||
| 254 | 281 | }); |
| 255 | 282 | |
| 256 | 283 | test("a red run closes the issue of a file it passed, and keeps the one it failed", () => { |
| 257 | reportRed(app, repo, red(VERIFY_ENV, "a".repeat(40))); | |
| 284 | redEach(); | |
| 258 | 285 | const run = createRun(app.db, { repoId: repo.id, kind: "check", env: VERIFY_ENV, commitOid: "d".repeat(40) }); |
| 259 | 286 | app.db.query("UPDATE run SET state = 'failed' WHERE id = ?1").run(run.id); |
| 260 | 287 | expect(reportGreen(app, repo, getRun(app.db, run.id)!, ["test/b.test.ts"])).toHaveLength(1); |
| @@ -263,7 +290,7 @@ describe("the report", () => { | ||
| 263 | 290 | }); |
| 264 | 291 | |
| 265 | 292 | test("an issue a person wrote, with the report's label and title, is never closed", () => { |
| 266 | reportRed(app, repo, red(VERIFY_ENV, "a".repeat(40))); | |
| 293 | redEach(); | |
| 267 | 294 | const label = getLabel(app.db, repo.id, REPORT_LABEL)!; |
| 268 | 295 | const made = createIssue(app.db, { repoId: repo.id, authorId: repo.ownerId, title: reportTitle("test/p.test.ts"), body: "I saw this fail." }); |
| 269 | 296 | if (!made.ok) throw new Error("issue"); |
| @@ -273,7 +300,7 @@ describe("the report", () => { | ||
| 273 | 300 | }); |
| 274 | 301 | |
| 275 | 302 | test("a run that is still going closes nothing", () => { |
| 276 | reportRed(app, repo, red(VERIFY_ENV, "a".repeat(40))); | |
| 303 | redEach(); | |
| 277 | 304 | const run = createRun(app.db, { repoId: repo.id, kind: "check", env: VERIFY_ENV, commitOid: "e".repeat(40) }); |
| 278 | 305 | expect(reportGreen(app, repo, run, ["test/a.test.ts"])).toEqual([]); |
| 279 | 306 | expect(issueFor("test/a.test.ts").state).toBe("open"); |
| @@ -379,7 +406,7 @@ describe("the report", () => { | ||
| 379 | 406 | try { |
| 380 | 407 | expect(flowLines(app.db).join("\n")).toContain("Full suite report: 0 open issues"); |
| 381 | 408 | reportRed(app, repo, red(VERIFY_ENV, "a".repeat(40))); |
| 382 | expect(flowLines(app.db).join("\n")).toContain("Full suite report: 2 open issues labelled full-suite"); | |
| 409 | expect(flowLines(app.db).join("\n")).toContain("Full suite report: 1 open issue labelled full-suite"); | |
| 383 | 410 | } finally { |
| 384 | 411 | delete process.env.FORGE_SENTINEL_REPO; |
| 385 | 412 | } |
new above
offline
Principle 8: 'A run that finds files failing files one issue for them all' reads as a typo, with 'files' as both noun and verb. Say: 'When a run fails several test files, the report opens one issue naming all of them, labelled full-suite, listed ...'. Keep it to one sentence.