Public: anyone can read this; the crew can change it.

The engine went idle for an hour with 17 red pulls: builders sent back to a red are shown a healthy preview log, find nothing, and the pull stays red for good #12969

Merged opened this in answer to

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/12968-the-engine-went-idle-for-an-hour-with-17 → main 5 commits fork 3 threads · 3 unresolved

  • Merged. mason/12968-the-engine-went-idle-for-an-hour-with-17 was merged into main as 168fd4a.

Review

Sign in to review this change.
11 files changed +411 −26
src/core/runs.ts +18
@@ -1227,6 +1227,24 @@ export function checkSummary(
12271227 : { state: "none", run: null };
12281228 }
12291229
1230 /**
1231 * The run a hand-back to a builder may quote as the failure (#12968): the newest check or deploy
1232 * of this head that failed, and was not cut by the forge. A deploy that succeeded, a check that
1233 * was cut and a check still to come are never one; without one there is no failure to quote.
1234 */
1235 export function failedRunOf(db: Database, issueId: string, headOid: string): Run | undefined {
1236 const rows = db
1237 .query<RunRow, [string, string]>(
1238 "SELECT * FROM run WHERE kind IN ('deploy','check') AND issue_id = ?1 AND commit_oid = ?2 AND state = 'failed' ORDER BY id DESC LIMIT 5",
1239 )
1240 .all(issueId, headOid);
1241 for (const row of rows) {
1242 const run = toRun(row);
1243 if (!isCut(run)) return run;
1244 }
1245 return undefined;
1246 }
1247
12301248 /**
12311249 * An issue's newest agent run in one of these states (#12277): one seek on
12321250 * `run_issue` (issue_id, id DESC). `ids` are time-ordered, so the newest id is the latest.
src/crew/autonomy.ts +17 −2
@@ -143,6 +143,8 @@ import { isFirstChange, reviewComing, sayFirstLanded } from "./first_change.ts";
143143 import { recheckRedApproved } from "./red_approved.ts";
144144 import { sweepStaleReds } from "./stale_reds.ts";
145145 import { sweepStranded } from "./stranded.ts";
146 import { afterNoChange } from "./no_change.ts";
147 import { redUnattended } from "./red_unattended.ts";
146148 import { parkPull, strikeNote } from "./park_pull.ts";
147149 import { keepVerdict } from "../flow/culprits.ts";
148150 import { sweepRepeats } from "./repeat_checks.ts";
@@ -430,7 +432,12 @@ export function inSlotOrder(app: App, repo: Repo, ready: readonly Ranked[], now:
430432 return pickOrder(allowed, ranked, (r) => r.issue.id, [], fromAppIssues(app.db, allowed.map((r) => r.issue.id)));
431433 }
432434
433 export type Drove = "off" | "paused" | "waiting" | "full" | "picked" | "planned" | "idle";
435 /**
436 * What one turn of the loop did, as a word. `stalled` (#12968): nothing to start, and open crew
437 * pulls are red with nobody working them. The only reader is the tick's `did` field, which logs
438 * the word as it is: nothing branches on a value, so a new one is never an error.
439 */
440 export type Drove = "off" | "paused" | "waiting" | "full" | "picked" | "planned" | "idle" | "stalled";
434441
435442 /**
436443 * One turn of the loop for one repository. Returns what it did, for the
@@ -565,7 +572,8 @@ async function driveOnce(
565572 // plate nor its own unanswered decline holds it: nothing else will move them (#2244).
566573 const stuck = replannable(held).length > 0 ? held : null;
567574 if (await planIfThin(app, repo, by, crew, active, 0, now, stuck)) return "planned";
568 return "idle";
575 // Red pulls nobody is working are a stall, not idleness (#12968): the tick says so.
576 return redUnattended(app, repo, active) > 0 ? "stalled" : "idle";
569577 }
570578
571579 /**
@@ -1055,6 +1063,13 @@ export function settleAgentRun(
10551063 if (run.reason === "review") return false;
10561064 if (/opened pull #|updated pull #/.test(run.summary)) return false;
10571065 if (interruptionOf(run.summary) !== undefined) return false;
1066 // A builder who changed nothing on a red pull: check it again (#12968). Only a run that ended
1067 // well said "no change"; one that errored said nothing, and is no answer to re-check on.
1068 if (run.reason !== "feedback" && run.state === "ok") {
1069 // Not a test: this queues a re-check or posts the failure, and the run is settled by it.
1070 const move = afterNoChange(app, repo, run.issueId);
1071 if (move !== undefined) return true;
1072 }
10581073 if (answeredWithoutChange(app, repo, run)) return true;
10591074 const carried = alreadyDoneOids(run.summary);
10601075 if (run.state === "ok" && carried.length > 0) {
src/crew/brief.ts +14
@@ -263,6 +263,10 @@ export interface TaskInput extends BriefInput {
263263 /** For a failed check: the tail of the deploy log. */
264264 readonly checkLog?: readonly string[] | undefined;
265265 readonly checkSummary?: string | undefined;
266 /** The failing test, its assertion and its output, quoted from the check that failed (#12968). */
267 readonly checkFailing?: string | undefined;
268 /** No check or deploy of this head is recorded as failed: it was cut, or is being run again. */
269 readonly noFailure?: boolean | undefined;
266270 /** What failed: the preview's deploy, or the repository's own check. */
267271 readonly checkKind?: "deploy" | "check" | undefined;
268272 /** For `conflict` and `converge`: the files both sides changed. */
@@ -668,10 +672,20 @@ function taskBody(input: TaskInput): string {
668672 return out.join("\n");
669673 }
670674
675 if (input.reason === "check_failed" && input.noFailure === true) {
676 out.push(
677 `Your pull #${issue.number} was sent back for a red check, but no check of its head is recorded as failed: the check was cut by the forge or is being run again, and a preview deploy that came up is not a failure.`,
678 "",
679 "Change nothing for this. The forge checks the head again by itself; if it is red on a named test you will be sent that test's assertion. If you find a real defect in the pull, fix it and push; otherwise push nothing and say so.",
680 );
681 return out.join("\n");
682 }
671683 if (input.reason === "check_failed") {
672684 out.push(
673685 `${input.checkKind === "check" ? "The check on" : "The preview deploy of"} your pull #${issue.number} failed${input.checkSummary ? `: ${input.checkSummary}` : "."}`,
674686 );
687 if (input.checkFailing !== undefined && input.checkFailing !== "")
688 out.push("", "The failing step and its assertion:", ...fenced(input.checkFailing));
675689 if (input.checkLog !== undefined && input.checkLog.length > 0) {
676690 out.push(
677691 "",
src/crew/failure_brief.ts added +27
@@ -0,0 +1,27 @@
1 /**
2 * What a builder sent back for a red is told about the red (#12968): the brief's failure fields,
3 * from the head's own failed check or deploy. A deploy that came up and a check that was cut are
4 * never the failure; without one the brief says no failure is recorded.
5 */
6
7 import type { Database } from "bun:sqlite";
8 import { failedRunOf } from "../core/runs.ts";
9 import { readRunLog } from "../runs/log.ts";
10 import { failingLines, failureQuote } from "./failure_quote.ts";
11 import type { TaskInput } from "./brief.ts";
12
13 export type FailureFields = Pick<TaskInput, "checkSummary" | "checkFailing" | "noFailure" | "checkKind" | "checkLog">;
14
15 /** The brief's failure fields for `reason` on this pull's head; empty for any other reason. */
16 export function failureBrief(db: Database, dataDir: string, reason: string, pullId: string | undefined, head: string): FailureFields {
17 if (reason !== "check_failed" || pullId === undefined) return {};
18 const run = failedRunOf(db, pullId, head);
19 if (run === undefined) return { noFailure: true };
20 return {
21 checkSummary: run.summary,
22 checkFailing: failureQuote(run, failingLines(readRunLog(dataDir, run.id, 32 * 1024).lines)),
23 noFailure: false,
24 checkKind: run.kind === "check" || run.kind === "deploy" ? run.kind : undefined,
25 checkLog: readRunLog(dataDir, run.id, 8 * 1024).lines.slice(-40),
26 };
27 }
src/crew/failure_quote.ts added +28
@@ -0,0 +1,28 @@
1 /**
2 * What a failed check says, quoted for a person or a builder (#12968): the failing test, its
3 * assertion and the end of its own output. Plain data in, one string out; a run that named no
4 * test is quoted by its summary.
5 */
6
7 import type { Run } from "../core/runs.ts";
8
9 /** The first failing test with its assertion and up to ten lines of its output. */
10 export function failureQuote(run: Run, logLines: readonly string[] = []): string {
11 const first = run.checkDetail?.failures?.[0];
12 const name = first?.name ?? run.checkDetail?.names[0] ?? "";
13 const head = name !== "" ? name : run.summary.trim().split("\n")[0] ?? "";
14 const out = [head.slice(0, 200) || "the check failed"];
15 if (first?.assertion) out.push(first.assertion.slice(0, 300));
16 if (first?.excerpt && first.excerpt.length > 0) out.push(...first.excerpt.slice(-10).map((l) => l.slice(0, 200)));
17 // The check's own lines (`gate: rule …`, `(fail) …`, `failed: …`) are quoted every time.
18 for (const l of logLines) if (!out.includes(l)) out.push(l);
19 return out.join("\n");
20 }
21
22 /** Lines a check's log says about its own failure: gate rules, failing tests, the final verdict. */
23 const FAILING_LINE = /^(?:gate: |\(fail\) |failed: |error: )/;
24
25 /** The check's own failing lines, from the end of its log, at most `max`; never a deploy's. */
26 export function failingLines(log: readonly string[], max = 20): string[] {
27 return log.filter((l) => FAILING_LINE.test(l.trim())).slice(-max);
28 }
src/crew/idle_reason.ts +10
@@ -22,6 +22,7 @@ import { weekSpent } from "./week_read.ts";
2222 import { governorFor } from "./governed.ts";
2323 import { heldNow, HOLDS } from "./held.ts";
2424 import { pace } from "./pacing.ts";
25 import { redUnattended } from "./red_unattended.ts";
2526
2627 export interface IdleReason {
2728 readonly word: string;
@@ -73,6 +74,9 @@ export function idleReason(app: App, now: Date): IdleReason | null {
7374
7475 const held = heldNow(app, repo);
7576 if (held.startable.length > 0) return null;
77 // Red pulls nobody is working are a stall, never idleness (#12968).
78 const stalled = redUnattended(app, repo, active);
79 if (stalled > 0) return { word: stallWord(stalled), href: "/flow" };
7680 // "Nothing ready" is true only when the map's ready list is empty; otherwise the ready issues are all held.
7781 const ready = HOLDS.reduce((n, h) => n + held.by[h].length, 0);
7882 return { word: ready === 0 ? "nothing ready" : "none startable", href: "/map" };
@@ -80,6 +84,12 @@ export function idleReason(app: App, now: Date): IdleReason | null {
8084 return null;
8185 }
8286
87 export { redUnattended };
88
89 /** The stall as one line with its count. */
90 export const stallWord = (n: number): string =>
91 `stalled: ${n} open pull${n === 1 ? " is" : "s are"} red with nobody working ${n === 1 ? "it" : "them"}`;
92
8393 /** What home says while the governor holds the next build: what is held, why, and what is not. */
8494 export const governedWord = (slots: number, ceiling: number): string =>
8595 `new builds are held to ${slots} of ${ceiling} until open pulls land; fixes, updates and reviews of open pulls keep every slot`;
src/crew/no_change.ts added +97
@@ -0,0 +1,97 @@
1 /**
2 * A builder sent back to a red pull who finds nothing to change (#12968) leaves the pull red
3 * with nobody next. The honest next step is cheap: check the same head again.
4 *
5 * - First "no change": a re-check of that head is queued (reason `no_change`). Green, the pull
6 * moves on; red on a named test, the red goes back to the builder with that test's assertion.
7 * - Second "no change" on a head whose re-check was red: the failure is quoted once, in a
8 * comment that names the head, and the pull waits for a person (the asks carry it). A later
9 * "no change" on that head finds the comment and says nothing again. It is never closed.
10 * - A re-check the forge cut short proved nothing: it is queued again and is no try.
11 */
12
13 import type { App } from "../app.ts";
14 import { addComment } from "../actions/issues.ts";
15 import { checksFor } from "../checks/service.ts";
16 import { getHold } from "../core/holds.ts";
17 import { getIssueById, listComments } from "../core/issues.ts";
18 import { getPull } from "../core/pulls.ts";
19 import type { Repo } from "../core/repos.ts";
20 import { checkSummary, getRun, isCut, type Run } from "../core/runs.ts";
21 import { getUserById } from "../core/users.ts";
22 import { createLog } from "../kernel/log.ts";
23 import { failingLines, failureQuote } from "./failure_quote.ts";
24 import { readRunLog } from "../runs/log.ts";
25
26 const log = createLog("no-change");
27
28 /** The reason the re-check carries. */
29 export const NO_CHANGE_REASON = "no_change";
30
31 export type NoChangeMove = "re-check queued" | "quoted";
32
33 /** How the quote opens; with the head's short id it marks the head as quoted. */
34 const QUOTED = "The builder found nothing to change, and the check of this head";
35
36 /** The words that say this head's red was quoted already. */
37 const quotedMark = (head: string): string => `${QUOTED} (${head.slice(0, 12)})`;
38
39 /** Whether `by` already quoted the red of this head on this pull. */
40 function alreadyQuoted(app: App, pullId: string, by: string, head: string): boolean {
41 const mark = quotedMark(head);
42 return listComments(app.db, pullId, { kind: "comment" }).some((c) => c.authorId === by && c.body.startsWith(mark));
43 }
44
45 /** The newest re-check of this head that this module queued. */
46 function lastRecheck(app: App, pullId: string, head: string): Run | undefined {
47 const row = app.db
48 .query<{ id: string }, [string, string, string]>(
49 "SELECT id FROM run WHERE kind = 'check' AND issue_id = ?1 AND commit_oid = ?2 AND reason = ?3 ORDER BY id DESC LIMIT 1",
50 )
51 .get(pullId, head, NO_CHANGE_REASON);
52 return row === null ? undefined : getRun(app.db, row.id);
53 }
54
55 /**
56 * A builder's run ended on `issueId` having pushed nothing. Returns what was done, or
57 * undefined when the pull is not red (or not the crew's to answer).
58 */
59 export function afterNoChange(app: App, repo: Repo, issueId: string): NoChangeMove | undefined {
60 const issue = getIssueById(app.db, issueId);
61 if (!issue || issue.kind !== "pull" || issue.state !== "open") return undefined;
62 const pull = getPull(app.db, repo.id, issue.number);
63 if (!pull || pull.mergedAt !== null || pull.headOid === "" || !pull.authorIsAgent) return undefined;
64 if (getHold(app.db, pull.id) !== undefined) return undefined;
65 const now = checkSummary(app.db, pull.id, pull.headOid);
66 if (now.run === null || (now.state !== "failed" && now.state !== "cancelled")) return undefined;
67
68 const prior = lastRecheck(app, pull.id, pull.headOid);
69 if (prior !== undefined && (prior.state === "queued" || prior.state === "running")) return undefined;
70 if (prior !== undefined && prior.state === "failed" && !isCut(prior)) {
71 // The head was checked again after a "no change" and is red on the same head: say it once.
72 // The comment speaks as the autonomy owner: the loop acts as them everywhere (assigning,
73 // parking), and a builder's account must not answer its own hand-back.
74 if (repo.autonomyBy === null) return undefined;
75 const by = getUserById(app.db, repo.autonomyBy);
76 if (!by) return undefined;
77 // Quoted once per head: the comment is the record, and a third "no change" finds it.
78 if (alreadyQuoted(app, pull.id, by.id, pull.headOid)) return undefined;
79 addComment(
80 app,
81 by,
82 repo,
83 issue,
84 `${quotedMark(pull.headOid)} is red again after a re-check:\n\n${failureQuote(prior, failingLines(readRunLog(app.config.dataDir, prior.id, 32 * 1024).lines))}\n\nIt stays open for a person or a new head.`,
85 );
86 log.info("no change twice on a red", { pull: pull.number });
87 return "quoted";
88 }
89 if (prior !== undefined && prior.state === "ok") return undefined;
90
91 const checks = checksFor(app);
92 if (checks === undefined) return undefined;
93 void checks
94 .request(repo, `pull-${pull.number}`, pull.headOid, NO_CHANGE_REASON, pull.id)
95 .catch((error) => log.warn("the re-check could not be queued", { pull: pull.number, error: String(error) }));
96 return "re-check queued";
97 }
src/crew/red_unattended.ts added +22
@@ -0,0 +1,22 @@
1 /**
2 * Open crew pulls that are red with nobody on them (#12968): the stall the engine must name
3 * where it would otherwise read idle. Asked by the idle reason and by the loop's tick.
4 */
5
6 import type { App } from "../app.ts";
7 import { getHold } from "../core/holds.ts";
8 import { listOpenPulls } from "../core/pulls.ts";
9 import type { Repo } from "../core/repos.ts";
10 import { checkSummary, type Run } from "../core/runs.ts";
11
12 /** Open crew pulls whose check is red, that nobody holds and no run is working. */
13 export function redUnattended(app: App, repo: Repo, active: readonly Run[]): number {
14 const busy = new Set(active.map((r) => r.issueId));
15 let n = 0;
16 for (const pull of listOpenPulls(app.db, repo.id)) {
17 if (!pull.authorIsAgent || pull.headOid === "" || busy.has(pull.id)) continue;
18 if (getHold(app.db, pull.id) !== undefined) continue;
19 if (checkSummary(app.db, pull.id, pull.headOid).state === "failed") n++;
20 }
21 return n;
22 }
src/crew/runner.ts +12 −24
@@ -11,6 +11,7 @@
1111 * what the forge has.
1212 */
1313
14 import { failureBrief } from "./failure_brief.ts";
1415 import { holdsNaming } from "./desk_posts.ts";
1516 import { isStrangerPull } from "./stranger.ts";
1617 import { forgetMeasure, noteMeasure } from "../core/measure.ts";
@@ -85,7 +86,6 @@ import {
8586 import {
8687 answerCancelled,
8788 cancelRun,
88 checkSummary,
8989 claimRun,
9090 finishRun,
9191 getRun,
@@ -121,7 +121,7 @@ import { BUDGETS, budgetSummary } from "../kernel/budgets.ts";
121121 import { createLog, holdValues, redactValues, releaseValues } from "../kernel/log.ts";
122122 import { err, ok, type Result } from "../kernel/result.ts";
123123 import { holdLease } from "../runs/lease.ts";
124 import { appendRunLog, readRunLog } from "../runs/log.ts";
124 import { appendRunLog } from "../runs/log.ts";
125125 import { cancelledLine } from "../runs/cancelled.ts";
126126 import {
127127 branchFor,
@@ -1257,15 +1257,15 @@ async function work(
12571257 resuming: !reviewing && resumption.resume,
12581258 role,
12591259 };
1260 const failedCheck =
1261 reason === "check_failed" && pull
1262 ? checkSummary(
1263 app.db,
1264 pull.id,
1265 (await resolveOid(forkBare, `refs/heads/${branch}`)) ??
1266 pull.headOid,
1267 )
1268 : undefined;
1260 // Only a check or deploy that failed is handed back as the failure (#12968): never the
1261 // success summary of a preview that came up.
1262 const failure = failureBrief(
1263 app.db,
1264 app.config.dataDir,
1265 reason,
1266 pull?.id,
1267 pull ? ((await resolveOid(forkBare, `refs/heads/${branch}`)) ?? pull.headOid) : "",
1268 );
12691269 // Imported words speak as their origin author (`voiceOf`), so the
12701270 // brief marks a stranger's as a reader's, not the connecting user's.
12711271 const comments = listComments(app.db, issue.id, { kind: "comment" }).map(
@@ -1303,7 +1303,7 @@ async function work(
13031303 reason === "feedback" && run.summary === RETRY_SUMMARY
13041304 ? lastTryOf(app, issue.id, run, reviews)
13051305 : undefined,
1306 checkSummary: failedCheck?.run?.summary,
1306 ...failure,
13071307 foreignPull: foreign
13081308 ? {
13091309 branch: foreign.headBranch,
@@ -1363,18 +1363,6 @@ async function work(
13631363 owed: owed(app.db, repo.id),
13641364 }
13651365 : {}),
1366 checkKind:
1367 failedCheck?.run?.kind === "check" ||
1368 failedCheck?.run?.kind === "deploy"
1369 ? failedCheck.run.kind
1370 : undefined,
1371 checkLog: failedCheck?.run
1372 ? readRunLog(
1373 app.config.dataDir,
1374 failedCheck.run.id,
1375 8 * 1024,
1376 ).lines.slice(-40)
1377 : undefined,
13781366 });
13791367
13801368 // The account this run works on: the nearest grant's, rotating; the
test/failure-quote.test.ts added +7
@@ -0,0 +1,7 @@
1 import { expect, test } from "bun:test";
2 import { failingLines } from "../src/crew/failure_quote.ts";
3
4 test("a check's own failing lines are picked from its log, a deploy's summary is not", () => {
5 const log = ["tier fast", 'gate: rule "x": a.ts:1 casts', "running at https://x · no tests declared, none run", "(fail) a > b", "failed: gate: rule x"];
6 expect(failingLines(log)).toEqual(['gate: rule "x": a.ts:1 casts', "(fail) a > b", "failed: gate: rule x"]);
7 });
test/no_change_recheck.test.ts added +159
@@ -0,0 +1,159 @@
1 /**
2 * A builder who returns "no change" on a red pull gets the head checked again (#12968), is never
3 * handed a deploy's success as a failure, and the engine does not read idle beside such a pull.
4 */
5 import { afterAll, beforeAll, expect, test } from "bun:test";
6 import { mkdtemp, rm } from "node:fs/promises";
7 import { tmpdir } from "node:os";
8 import { join } from "node:path";
9 import { createApp, type App } from "../src/app.ts";
10 import { addToCrew, createAgent } from "../src/actions/crew.ts";
11 import { openIssue } from "../src/actions/issues.ts";
12 import { openPull } from "../src/actions/pulls.ts";
13 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
14 import { provideChecksForTest, type ChecksHandle } from "../src/checks/service.ts";
15 import { listComments } from "../src/core/issues.ts";
16 import { getPull } from "../src/core/pulls.ts";
17 import { getRepoById, type Repo } from "../src/core/repos.ts";
18 import { createRun, failedRunOf, listRuns } from "../src/core/runs.ts";
19 import { createUser, getUserByName, type User } from "../src/core/users.ts";
20 import { driveRepo, settleAgentRun } from "../src/crew/autonomy.ts";
21 import { taskPrompt, type TaskInput } from "../src/crew/brief.ts";
22 import { failureBrief } from "../src/crew/failure_brief.ts";
23 import { idleReason, redUnattended } from "../src/crew/idle_reason.ts";
24 import { NO_CHANGE_REASON } from "../src/crew/no_change.ts";
25
26 let dataDir: string;
27 let app: App;
28 let alice: User;
29 let repo: Repo;
30 let pull: NonNullable<ReturnType<typeof getPull>>;
31
32 beforeAll(async () => {
33 dataDir = await mkdtemp(join(tmpdir(), "forge-no-change-"));
34 app = createApp({ dataDir, port: 0 });
35 const a = await createUser(app.db, {
36 origin: "seed", trusted: true, username: "alice", email: "[email protected]",
37 password: "a-good-passphrase", adminIfFirst: true,
38 });
39 if (!a.ok) throw new Error(a.error);
40 alice = a.value;
41 const made = await createRepo(app, alice, { name: "engine", visibility: "private" });
42 if (!made.ok) throw new Error(String(made.error));
43 repo = made.value;
44 await commitToBranch(app, alice, repo, { message: "Start", files: [{ path: "a.ts", content: "export const a = 0;\n" }] });
45 await createAgent(app, alice, { username: "mason" });
46 addToCrew(app, alice, repo, "mason");
47 const mason = getUserByName(app.db, "mason")!;
48 app.db.query("UPDATE repo SET autonomy = 'autonomous', autonomy_by = ?2 WHERE id = ?1").run(repo.id, alice.id);
49 repo = getRepoById(app.db, repo.id)!;
50 const ask = openIssue(app, alice, repo, { title: "Ask" });
51 if (!ask.ok) throw new Error(String(ask.error));
52 await commitToBranch(app, alice, repo, { branch: "feature/x", base: "main", message: "X", files: [{ path: "x.ts", content: "export const x = 1;\n" }] });
53 const opened = await openPull(app, mason, repo, { title: "X", body: `Closes #${ask.value.number}`, baseBranch: "main", headBranch: "feature/x" });
54 if (!opened.ok) throw new Error(String(opened.error));
55 pull = getPull(app.db, repo.id, opened.value.number)!;
56 provideChecksForTest(app, {
57 request: async (r: Repo, env: string, oid: string, reason: string, issueId?: string) =>
58 createRun(app.db, { repoId: r.id, kind: "check", env, commitOid: oid, reason, issueId }),
59 } as unknown as ChecksHandle);
60 });
61
62 afterAll(async () => {
63 app.db.close();
64 await rm(dataDir, { recursive: true, force: true });
65 });
66
67 const mark = (id: string, state: string, summary: string, detail = "") =>
68 app.db.query("UPDATE run SET state = ?2, summary = ?3, check_detail = ?4, finished_at = strftime('%Y-%m-%dT%H:%M:%fZ','now') WHERE id = ?1").run(id, state, summary, detail);
69 const run = (kind: "check" | "deploy") => createRun(app.db, { repoId: repo.id, kind, issueId: pull.id, commitOid: pull.headOid, reason: "" });
70 const noChange = () =>
71 settleAgentRun(app, repo, { issueId: pull.id, state: "ok", summary: "no change needed", reason: "check_failed", actorId: getUserByName(app.db, "mason")!.id });
72 const errored = () =>
73 settleAgentRun(app, repo, { issueId: pull.id, state: "failed", summary: "the agent exited with code 1", reason: "check_failed", actorId: getUserByName(app.db, "mason")!.id });
74 const rechecks = () => listRuns(app.db, repo.id, { issueId: pull.id, limit: 50 }).filter((r) => r.reason === NO_CHANGE_REASON);
75
76 test("a deploy that came up is never offered as the failure", () => {
77 const check = run("check");
78 mark(check.id, "failed", "failed: engine > adds up", JSON.stringify({ names: ["engine > adds up"], failures: [{ file: "a.test.ts", name: "engine > adds up", assertion: "expected 2, got 3" }] }));
79 const deploy = run("deploy");
80 mark(deploy.id, "ok", "running at https://x · no tests declared, none run");
81 expect(failedRunOf(app.db, pull.id, pull.headOid)?.id).toBe(check.id);
82 const base = { reason: "check_failed", issue: { number: pull.number, title: "X" } };
83 const quoted = taskPrompt({ ...base, checkKind: "check", checkSummary: "x", checkFailing: "engine > adds up\nexpected 2, got 3" } as unknown as TaskInput);
84 expect(quoted).toContain("expected 2, got 3");
85 expect(taskPrompt({ ...base, noFailure: true } as unknown as TaskInput)).toContain("no check of its head is recorded as failed");
86 });
87
88 test("a cut check is no failure; a builder's no change re-checks the head, and a second quotes the failure", () => {
89 app.db.query("DELETE FROM run WHERE issue_id = ?1").run(pull.id);
90 const cut = run("check");
91 mark(cut.id, "failed", "interrupted by a restart; queued again");
92 expect(failedRunOf(app.db, pull.id, pull.headOid)).toBeUndefined();
93 expect(redUnattended(app, repo, [])).toBe(0);
94
95 expect(noChange()).toBe(true);
96 expect(rechecks()).toHaveLength(1);
97 // Still running: nothing more is queued.
98 expect(noChange()).toBe(false);
99 expect(rechecks()).toHaveLength(1);
100
101 // It ends red on a named test: the stall is counted, and a second no change quotes it.
102 mark(rechecks()[0]!.id, "failed", "failed: engine > adds up", JSON.stringify({ names: ["engine > adds up"], failures: [{ file: "a.test.ts", name: "engine > adds up", assertion: "expected 2, got 3" }] }));
103 expect(redUnattended(app, repo, [])).toBe(1);
104 expect(noChange()).toBe(true);
105 expect(rechecks()).toHaveLength(1);
106 const said = listComments(app.db, pull.id).map((c) => c.body).join("\n");
107 expect(said).toContain("engine > adds up");
108 expect(said).toContain("expected 2, got 3");
109
110 // A third no change on the same head finds the quote and says nothing again.
111 const before = listComments(app.db, pull.id).length;
112 noChange();
113 noChange();
114 expect(listComments(app.db, pull.id)).toHaveLength(before);
115 expect(rechecks()).toHaveLength(1);
116 });
117
118 test("a builder run that errored is no answer: nothing is re-checked", () => {
119 app.db.query("DELETE FROM run WHERE issue_id = ?1").run(pull.id);
120 mark(run("check").id, "failed", "failed: engine > adds up", JSON.stringify({ names: ["engine > adds up"], failures: [{ file: "a.test.ts", name: "engine > adds up" }] }));
121 errored();
122 expect(rechecks()).toHaveLength(0);
123 });
124
125 test("the runner's brief for a head whose only run is a green deploy says no failure is recorded", () => {
126 app.db.query("DELETE FROM run WHERE issue_id = ?1").run(pull.id);
127 mark(run("deploy").id, "ok", "running at https://x · no tests declared, none run");
128 const fields = failureBrief(app.db, dataDir, "check_failed", pull.id, pull.headOid);
129 expect(fields).toEqual({ noFailure: true });
130 const brief = taskPrompt({ reason: "check_failed", issue: { number: pull.number, title: "X" }, ...fields } as unknown as TaskInput);
131 expect(brief).toContain("no check of its head is recorded as failed");
132 expect(brief).not.toContain("running at https://x");
133
134 // A failed check of the same head is quoted, by kind, with its assertion.
135 mark(run("check").id, "failed", "failed: engine > adds up", JSON.stringify({ names: ["engine > adds up"], failures: [{ file: "a.test.ts", name: "engine > adds up", assertion: "expected 2, got 3" }] }));
136 const red = failureBrief(app.db, dataDir, "check_failed", pull.id, pull.headOid);
137 expect(red.checkKind).toBe("check");
138 expect(red.noFailure).toBe(false);
139 expect(taskPrompt({ reason: "check_failed", issue: { number: pull.number, title: "X" }, ...red } as unknown as TaskInput)).toContain("expected 2, got 3");
140 expect(failureBrief(app.db, dataDir, "assigned", pull.id, pull.headOid)).toEqual({});
141 });
142
143 test("a re-check that is green moves the pull on: no second re-check, no comment", () => {
144 app.db.query("DELETE FROM run WHERE issue_id = ?1").run(pull.id);
145 mark(run("check").id, "failed", "the forge went away, and this commit no longer heads an open pull");
146 expect(noChange()).toBe(true);
147 mark(rechecks()[0]!.id, "ok", "ok");
148 expect(noChange()).toBe(false);
149 expect(rechecks()).toHaveLength(1);
150 });
151
152 test("idleReason names the stall, not idleness, beside a red pull nobody works; driveRepo says stalled", async () => {
153 app.db.query("DELETE FROM run WHERE issue_id = ?1").run(pull.id);
154 mark(run("check").id, "failed", "failed: engine > adds up", JSON.stringify({ names: ["engine > adds up"], failures: [{ file: "a.test.ts", name: "engine > adds up" }] }));
155 app.db.query("DELETE FROM run WHERE kind = 'agent'").run();
156 const reason = idleReason(app, new Date());
157 expect(reason?.word).toStartWith("stalled: 1 open pull is red");
158 expect(await driveRepo(app, getRepoById(app.db, repo.id)!)).toBe("stalled");
159 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs