Public: anyone can read this; the crew can change it.
The engine went idle for an hour with 17 red pulls: builders sent back to a red are shown a healthy preview log, find nothing, and the pull stays red for good #12969
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/12968-the-engine-went-idle-for-an-hour-with-17 → main 5 commits fork 3 threads · 3 unresolved
11 files
changed +411
−26
src/core/runs.ts +18
| @@ -1227,6 +1227,24 @@ export function checkSummary( | ||
| 1227 | 1227 | : { state: "none", run: null }; |
| 1228 | 1228 | } |
| 1229 | 1229 | |
| 1230 | /** | |
| 1231 | * The run a hand-back to a builder may quote as the failure (#12968): the newest check or deploy | |
| 1232 | * of this head that failed, and was not cut by the forge. A deploy that succeeded, a check that | |
| 1233 | * was cut and a check still to come are never one; without one there is no failure to quote. | |
| 1234 | */ | |
| 1235 | export function failedRunOf(db: Database, issueId: string, headOid: string): Run | undefined { | |
| 1236 | const rows = db | |
| 1237 | .query<RunRow, [string, string]>( | |
| 1238 | "SELECT * FROM run WHERE kind IN ('deploy','check') AND issue_id = ?1 AND commit_oid = ?2 AND state = 'failed' ORDER BY id DESC LIMIT 5", | |
| 1239 | ) | |
| 1240 | .all(issueId, headOid); | |
| 1241 | for (const row of rows) { | |
| 1242 | const run = toRun(row); | |
| 1243 | if (!isCut(run)) return run; | |
| 1244 | } | |
| 1245 | return undefined; | |
| 1246 | } | |
| 1247 | ||
| 1230 | 1248 | /** |
| 1231 | 1249 | * An issue's newest agent run in one of these states (#12277): one seek on |
| 1232 | 1250 | * `run_issue` (issue_id, id DESC). `ids` are time-ordered, so the newest id is the latest. |
src/crew/autonomy.ts +17 −2
| @@ -143,6 +143,8 @@ import { isFirstChange, reviewComing, sayFirstLanded } from "./first_change.ts"; | ||
| 143 | 143 | import { recheckRedApproved } from "./red_approved.ts"; |
| 144 | 144 | import { sweepStaleReds } from "./stale_reds.ts"; |
| 145 | 145 | import { sweepStranded } from "./stranded.ts"; |
| 146 | import { afterNoChange } from "./no_change.ts"; | |
| 147 | import { redUnattended } from "./red_unattended.ts"; | |
| 146 | 148 | import { parkPull, strikeNote } from "./park_pull.ts"; |
| 147 | 149 | import { keepVerdict } from "../flow/culprits.ts"; |
| 148 | 150 | import { sweepRepeats } from "./repeat_checks.ts"; |
| @@ -430,7 +432,12 @@ export function inSlotOrder(app: App, repo: Repo, ready: readonly Ranked[], now: | ||
| 430 | 432 | return pickOrder(allowed, ranked, (r) => r.issue.id, [], fromAppIssues(app.db, allowed.map((r) => r.issue.id))); |
| 431 | 433 | } |
| 432 | 434 | |
| 433 | export type Drove = "off" | "paused" | "waiting" | "full" | "picked" | "planned" | "idle"; | |
| 435 | /** | |
| 436 | * What one turn of the loop did, as a word. `stalled` (#12968): nothing to start, and open crew | |
| 437 | * pulls are red with nobody working them. The only reader is the tick's `did` field, which logs | |
| 438 | * the word as it is: nothing branches on a value, so a new one is never an error. | |
| 439 | */ | |
| 440 | export type Drove = "off" | "paused" | "waiting" | "full" | "picked" | "planned" | "idle" | "stalled"; | |
| 434 | 441 | |
| 435 | 442 | /** |
| 436 | 443 | * One turn of the loop for one repository. Returns what it did, for the |
| @@ -565,7 +572,8 @@ async function driveOnce( | ||
| 565 | 572 | // plate nor its own unanswered decline holds it: nothing else will move them (#2244). |
| 566 | 573 | const stuck = replannable(held).length > 0 ? held : null; |
| 567 | 574 | if (await planIfThin(app, repo, by, crew, active, 0, now, stuck)) return "planned"; |
| 568 | return "idle"; | |
| 575 | // Red pulls nobody is working are a stall, not idleness (#12968): the tick says so. | |
| 576 | return redUnattended(app, repo, active) > 0 ? "stalled" : "idle"; | |
| 569 | 577 | } |
| 570 | 578 | |
| 571 | 579 | /** |
| @@ -1055,6 +1063,13 @@ export function settleAgentRun( | ||
| 1055 | 1063 | if (run.reason === "review") return false; |
| 1056 | 1064 | if (/opened pull #|updated pull #/.test(run.summary)) return false; |
| 1057 | 1065 | if (interruptionOf(run.summary) !== undefined) return false; |
| 1066 | // A builder who changed nothing on a red pull: check it again (#12968). Only a run that ended | |
| 1067 | // well said "no change"; one that errored said nothing, and is no answer to re-check on. | |
| 1068 | if (run.reason !== "feedback" && run.state === "ok") { | |
| 1069 | // Not a test: this queues a re-check or posts the failure, and the run is settled by it. | |
| 1070 | const move = afterNoChange(app, repo, run.issueId); | |
| 1071 | if (move !== undefined) return true; | |
| 1072 | } | |
| 1058 | 1073 | if (answeredWithoutChange(app, repo, run)) return true; |
| 1059 | 1074 | const carried = alreadyDoneOids(run.summary); |
| 1060 | 1075 | if (run.state === "ok" && carried.length > 0) { |
src/crew/brief.ts +14
| @@ -263,6 +263,10 @@ export interface TaskInput extends BriefInput { | ||
| 263 | 263 | /** For a failed check: the tail of the deploy log. */ |
| 264 | 264 | readonly checkLog?: readonly string[] | undefined; |
| 265 | 265 | readonly checkSummary?: string | undefined; |
| 266 | /** The failing test, its assertion and its output, quoted from the check that failed (#12968). */ | |
| 267 | readonly checkFailing?: string | undefined; | |
| 268 | /** No check or deploy of this head is recorded as failed: it was cut, or is being run again. */ | |
| 269 | readonly noFailure?: boolean | undefined; | |
| 266 | 270 | /** What failed: the preview's deploy, or the repository's own check. */ |
| 267 | 271 | readonly checkKind?: "deploy" | "check" | undefined; |
| 268 | 272 | /** For `conflict` and `converge`: the files both sides changed. */ |
| @@ -668,10 +672,20 @@ function taskBody(input: TaskInput): string { | ||
| 668 | 672 | return out.join("\n"); |
| 669 | 673 | } |
| 670 | 674 | |
| 675 | if (input.reason === "check_failed" && input.noFailure === true) { | |
| 676 | out.push( | |
| 677 | `Your pull #${issue.number} was sent back for a red check, but no check of its head is recorded as failed: the check was cut by the forge or is being run again, and a preview deploy that came up is not a failure.`, | |
| 678 | "", | |
| 679 | "Change nothing for this. The forge checks the head again by itself; if it is red on a named test you will be sent that test's assertion. If you find a real defect in the pull, fix it and push; otherwise push nothing and say so.", | |
| 680 | ); | |
| 681 | return out.join("\n"); | |
| 682 | } | |
| 671 | 683 | if (input.reason === "check_failed") { |
| 672 | 684 | out.push( |
| 673 | 685 | `${input.checkKind === "check" ? "The check on" : "The preview deploy of"} your pull #${issue.number} failed${input.checkSummary ? `: ${input.checkSummary}` : "."}`, |
| 674 | 686 | ); |
| 687 | if (input.checkFailing !== undefined && input.checkFailing !== "") | |
| 688 | out.push("", "The failing step and its assertion:", ...fenced(input.checkFailing)); | |
| 675 | 689 | if (input.checkLog !== undefined && input.checkLog.length > 0) { |
| 676 | 690 | out.push( |
| 677 | 691 | "", |
src/crew/failure_brief.ts added +27
| @@ -0,0 +1,27 @@ | ||
| 1 | /** | |
| 2 | * What a builder sent back for a red is told about the red (#12968): the brief's failure fields, | |
| 3 | * from the head's own failed check or deploy. A deploy that came up and a check that was cut are | |
| 4 | * never the failure; without one the brief says no failure is recorded. | |
| 5 | */ | |
| 6 | ||
| 7 | import type { Database } from "bun:sqlite"; | |
| 8 | import { failedRunOf } from "../core/runs.ts"; | |
| 9 | import { readRunLog } from "../runs/log.ts"; | |
| 10 | import { failingLines, failureQuote } from "./failure_quote.ts"; | |
| 11 | import type { TaskInput } from "./brief.ts"; | |
| 12 | ||
| 13 | export type FailureFields = Pick<TaskInput, "checkSummary" | "checkFailing" | "noFailure" | "checkKind" | "checkLog">; | |
| 14 | ||
| 15 | /** The brief's failure fields for `reason` on this pull's head; empty for any other reason. */ | |
| 16 | export function failureBrief(db: Database, dataDir: string, reason: string, pullId: string | undefined, head: string): FailureFields { | |
| 17 | if (reason !== "check_failed" || pullId === undefined) return {}; | |
| 18 | const run = failedRunOf(db, pullId, head); | |
| 19 | if (run === undefined) return { noFailure: true }; | |
| 20 | return { | |
| 21 | checkSummary: run.summary, | |
| 22 | checkFailing: failureQuote(run, failingLines(readRunLog(dataDir, run.id, 32 * 1024).lines)), | |
| 23 | noFailure: false, | |
| 24 | checkKind: run.kind === "check" || run.kind === "deploy" ? run.kind : undefined, | |
| 25 | checkLog: readRunLog(dataDir, run.id, 8 * 1024).lines.slice(-40), | |
| 26 | }; | |
| 27 | } |
src/crew/failure_quote.ts added +28
| @@ -0,0 +1,28 @@ | ||
| 1 | /** | |
| 2 | * What a failed check says, quoted for a person or a builder (#12968): the failing test, its | |
| 3 | * assertion and the end of its own output. Plain data in, one string out; a run that named no | |
| 4 | * test is quoted by its summary. | |
| 5 | */ | |
| 6 | ||
| 7 | import type { Run } from "../core/runs.ts"; | |
| 8 | ||
| 9 | /** The first failing test with its assertion and up to ten lines of its output. */ | |
| 10 | export function failureQuote(run: Run, logLines: readonly string[] = []): string { | |
| 11 | const first = run.checkDetail?.failures?.[0]; | |
| 12 | const name = first?.name ?? run.checkDetail?.names[0] ?? ""; | |
| 13 | const head = name !== "" ? name : run.summary.trim().split("\n")[0] ?? ""; | |
| 14 | const out = [head.slice(0, 200) || "the check failed"]; | |
| 15 | if (first?.assertion) out.push(first.assertion.slice(0, 300)); | |
| 16 | if (first?.excerpt && first.excerpt.length > 0) out.push(...first.excerpt.slice(-10).map((l) => l.slice(0, 200))); | |
| 17 | // The check's own lines (`gate: rule …`, `(fail) …`, `failed: …`) are quoted every time. | |
| 18 | for (const l of logLines) if (!out.includes(l)) out.push(l); | |
| 19 | return out.join("\n"); | |
| 20 | } | |
| 21 | ||
| 22 | /** Lines a check's log says about its own failure: gate rules, failing tests, the final verdict. */ | |
| 23 | const FAILING_LINE = /^(?:gate: |\(fail\) |failed: |error: )/; | |
| 24 | ||
| 25 | /** The check's own failing lines, from the end of its log, at most `max`; never a deploy's. */ | |
| 26 | export function failingLines(log: readonly string[], max = 20): string[] { | |
| 27 | return log.filter((l) => FAILING_LINE.test(l.trim())).slice(-max); | |
| 28 | } |
src/crew/idle_reason.ts +10
| @@ -22,6 +22,7 @@ import { weekSpent } from "./week_read.ts"; | ||
| 22 | 22 | import { governorFor } from "./governed.ts"; |
| 23 | 23 | import { heldNow, HOLDS } from "./held.ts"; |
| 24 | 24 | import { pace } from "./pacing.ts"; |
| 25 | import { redUnattended } from "./red_unattended.ts"; | |
| 25 | 26 | |
| 26 | 27 | export interface IdleReason { |
| 27 | 28 | readonly word: string; |
| @@ -73,6 +74,9 @@ export function idleReason(app: App, now: Date): IdleReason | null { | ||
| 73 | 74 | |
| 74 | 75 | const held = heldNow(app, repo); |
| 75 | 76 | if (held.startable.length > 0) return null; |
| 77 | // Red pulls nobody is working are a stall, never idleness (#12968). | |
| 78 | const stalled = redUnattended(app, repo, active); | |
| 79 | if (stalled > 0) return { word: stallWord(stalled), href: "/flow" }; | |
| 76 | 80 | // "Nothing ready" is true only when the map's ready list is empty; otherwise the ready issues are all held. |
| 77 | 81 | const ready = HOLDS.reduce((n, h) => n + held.by[h].length, 0); |
| 78 | 82 | return { word: ready === 0 ? "nothing ready" : "none startable", href: "/map" }; |
| @@ -80,6 +84,12 @@ export function idleReason(app: App, now: Date): IdleReason | null { | ||
| 80 | 84 | return null; |
| 81 | 85 | } |
| 82 | 86 | |
| 87 | export { redUnattended }; | |
| 88 | ||
| 89 | /** The stall as one line with its count. */ | |
| 90 | export const stallWord = (n: number): string => | |
| 91 | `stalled: ${n} open pull${n === 1 ? " is" : "s are"} red with nobody working ${n === 1 ? "it" : "them"}`; | |
| 92 | ||
| 83 | 93 | /** What home says while the governor holds the next build: what is held, why, and what is not. */ |
| 84 | 94 | export const governedWord = (slots: number, ceiling: number): string => |
| 85 | 95 | `new builds are held to ${slots} of ${ceiling} until open pulls land; fixes, updates and reviews of open pulls keep every slot`; |
src/crew/no_change.ts added +97
| @@ -0,0 +1,97 @@ | ||
| 1 | /** | |
| 2 | * A builder sent back to a red pull who finds nothing to change (#12968) leaves the pull red | |
| 3 | * with nobody next. The honest next step is cheap: check the same head again. | |
| 4 | * | |
| 5 | * - First "no change": a re-check of that head is queued (reason `no_change`). Green, the pull | |
| 6 | * moves on; red on a named test, the red goes back to the builder with that test's assertion. | |
| 7 | * - Second "no change" on a head whose re-check was red: the failure is quoted once, in a | |
| 8 | * comment that names the head, and the pull waits for a person (the asks carry it). A later | |
| 9 | * "no change" on that head finds the comment and says nothing again. It is never closed. | |
| 10 | * - A re-check the forge cut short proved nothing: it is queued again and is no try. | |
| 11 | */ | |
| 12 | ||
| 13 | import type { App } from "../app.ts"; | |
| 14 | import { addComment } from "../actions/issues.ts"; | |
| 15 | import { checksFor } from "../checks/service.ts"; | |
| 16 | import { getHold } from "../core/holds.ts"; | |
| 17 | import { getIssueById, listComments } from "../core/issues.ts"; | |
| 18 | import { getPull } from "../core/pulls.ts"; | |
| 19 | import type { Repo } from "../core/repos.ts"; | |
| 20 | import { checkSummary, getRun, isCut, type Run } from "../core/runs.ts"; | |
| 21 | import { getUserById } from "../core/users.ts"; | |
| 22 | import { createLog } from "../kernel/log.ts"; | |
| 23 | import { failingLines, failureQuote } from "./failure_quote.ts"; | |
| 24 | import { readRunLog } from "../runs/log.ts"; | |
| 25 | ||
| 26 | const log = createLog("no-change"); | |
| 27 | ||
| 28 | /** The reason the re-check carries. */ | |
| 29 | export const NO_CHANGE_REASON = "no_change"; | |
| 30 | ||
| 31 | export type NoChangeMove = "re-check queued" | "quoted"; | |
| 32 | ||
| 33 | /** How the quote opens; with the head's short id it marks the head as quoted. */ | |
| 34 | const QUOTED = "The builder found nothing to change, and the check of this head"; | |
| 35 | ||
| 36 | /** The words that say this head's red was quoted already. */ | |
| 37 | const quotedMark = (head: string): string => `${QUOTED} (${head.slice(0, 12)})`; | |
| 38 | ||
| 39 | /** Whether `by` already quoted the red of this head on this pull. */ | |
| 40 | function alreadyQuoted(app: App, pullId: string, by: string, head: string): boolean { | |
| 41 | const mark = quotedMark(head); | |
| 42 | return listComments(app.db, pullId, { kind: "comment" }).some((c) => c.authorId === by && c.body.startsWith(mark)); | |
| 43 | } | |
| 44 | ||
| 45 | /** The newest re-check of this head that this module queued. */ | |
| 46 | function lastRecheck(app: App, pullId: string, head: string): Run | undefined { | |
| 47 | const row = app.db | |
| 48 | .query<{ id: string }, [string, string, string]>( | |
| 49 | "SELECT id FROM run WHERE kind = 'check' AND issue_id = ?1 AND commit_oid = ?2 AND reason = ?3 ORDER BY id DESC LIMIT 1", | |
| 50 | ) | |
| 51 | .get(pullId, head, NO_CHANGE_REASON); | |
| 52 | return row === null ? undefined : getRun(app.db, row.id); | |
| 53 | } | |
| 54 | ||
| 55 | /** | |
| 56 | * A builder's run ended on `issueId` having pushed nothing. Returns what was done, or | |
| 57 | * undefined when the pull is not red (or not the crew's to answer). | |
| 58 | */ | |
| 59 | export function afterNoChange(app: App, repo: Repo, issueId: string): NoChangeMove | undefined { | |
| 60 | const issue = getIssueById(app.db, issueId); | |
| 61 | if (!issue || issue.kind !== "pull" || issue.state !== "open") return undefined; | |
| 62 | const pull = getPull(app.db, repo.id, issue.number); | |
| 63 | if (!pull || pull.mergedAt !== null || pull.headOid === "" || !pull.authorIsAgent) return undefined; | |
| 64 | if (getHold(app.db, pull.id) !== undefined) return undefined; | |
| 65 | const now = checkSummary(app.db, pull.id, pull.headOid); | |
| 66 | if (now.run === null || (now.state !== "failed" && now.state !== "cancelled")) return undefined; | |
| 67 | ||
| 68 | const prior = lastRecheck(app, pull.id, pull.headOid); | |
| 69 | if (prior !== undefined && (prior.state === "queued" || prior.state === "running")) return undefined; | |
| 70 | if (prior !== undefined && prior.state === "failed" && !isCut(prior)) { | |
| 71 | // The head was checked again after a "no change" and is red on the same head: say it once. | |
| 72 | // The comment speaks as the autonomy owner: the loop acts as them everywhere (assigning, | |
| 73 | // parking), and a builder's account must not answer its own hand-back. | |
| 74 | if (repo.autonomyBy === null) return undefined; | |
| 75 | const by = getUserById(app.db, repo.autonomyBy); | |
| 76 | if (!by) return undefined; | |
| 77 | // Quoted once per head: the comment is the record, and a third "no change" finds it. | |
| 78 | if (alreadyQuoted(app, pull.id, by.id, pull.headOid)) return undefined; | |
| 79 | addComment( | |
| 80 | app, | |
| 81 | by, | |
| 82 | repo, | |
| 83 | issue, | |
| 84 | `${quotedMark(pull.headOid)} is red again after a re-check:\n\n${failureQuote(prior, failingLines(readRunLog(app.config.dataDir, prior.id, 32 * 1024).lines))}\n\nIt stays open for a person or a new head.`, | |
| 85 | ); | |
| 86 | log.info("no change twice on a red", { pull: pull.number }); | |
| 87 | return "quoted"; | |
| 88 | } | |
| 89 | if (prior !== undefined && prior.state === "ok") return undefined; | |
| 90 | ||
| 91 | const checks = checksFor(app); | |
| 92 | if (checks === undefined) return undefined; | |
| 93 | void checks | |
| 94 | .request(repo, `pull-${pull.number}`, pull.headOid, NO_CHANGE_REASON, pull.id) | |
| 95 | .catch((error) => log.warn("the re-check could not be queued", { pull: pull.number, error: String(error) })); | |
| 96 | return "re-check queued"; | |
| 97 | } |
src/crew/red_unattended.ts added +22
| @@ -0,0 +1,22 @@ | ||
| 1 | /** | |
| 2 | * Open crew pulls that are red with nobody on them (#12968): the stall the engine must name | |
| 3 | * where it would otherwise read idle. Asked by the idle reason and by the loop's tick. | |
| 4 | */ | |
| 5 | ||
| 6 | import type { App } from "../app.ts"; | |
| 7 | import { getHold } from "../core/holds.ts"; | |
| 8 | import { listOpenPulls } from "../core/pulls.ts"; | |
| 9 | import type { Repo } from "../core/repos.ts"; | |
| 10 | import { checkSummary, type Run } from "../core/runs.ts"; | |
| 11 | ||
| 12 | /** Open crew pulls whose check is red, that nobody holds and no run is working. */ | |
| 13 | export function redUnattended(app: App, repo: Repo, active: readonly Run[]): number { | |
| 14 | const busy = new Set(active.map((r) => r.issueId)); | |
| 15 | let n = 0; | |
| 16 | for (const pull of listOpenPulls(app.db, repo.id)) { | |
| 17 | if (!pull.authorIsAgent || pull.headOid === "" || busy.has(pull.id)) continue; | |
| 18 | if (getHold(app.db, pull.id) !== undefined) continue; | |
| 19 | if (checkSummary(app.db, pull.id, pull.headOid).state === "failed") n++; | |
| 20 | } | |
| 21 | return n; | |
| 22 | } |
src/crew/runner.ts +12 −24
| @@ -11,6 +11,7 @@ | ||
| 11 | 11 | * what the forge has. |
| 12 | 12 | */ |
| 13 | 13 | |
| 14 | import { failureBrief } from "./failure_brief.ts"; | |
| 14 | 15 | import { holdsNaming } from "./desk_posts.ts"; |
| 15 | 16 | import { isStrangerPull } from "./stranger.ts"; |
| 16 | 17 | import { forgetMeasure, noteMeasure } from "../core/measure.ts"; |
| @@ -85,7 +86,6 @@ import { | ||
| 85 | 86 | import { |
| 86 | 87 | answerCancelled, |
| 87 | 88 | cancelRun, |
| 88 | checkSummary, | |
| 89 | 89 | claimRun, |
| 90 | 90 | finishRun, |
| 91 | 91 | getRun, |
| @@ -121,7 +121,7 @@ import { BUDGETS, budgetSummary } from "../kernel/budgets.ts"; | ||
| 121 | 121 | import { createLog, holdValues, redactValues, releaseValues } from "../kernel/log.ts"; |
| 122 | 122 | import { err, ok, type Result } from "../kernel/result.ts"; |
| 123 | 123 | import { holdLease } from "../runs/lease.ts"; |
| 124 | import { appendRunLog, readRunLog } from "../runs/log.ts"; | |
| 124 | import { appendRunLog } from "../runs/log.ts"; | |
| 125 | 125 | import { cancelledLine } from "../runs/cancelled.ts"; |
| 126 | 126 | import { |
| 127 | 127 | branchFor, |
| @@ -1257,15 +1257,15 @@ async function work( | ||
| 1257 | 1257 | resuming: !reviewing && resumption.resume, |
| 1258 | 1258 | role, |
| 1259 | 1259 | }; |
| 1260 | const failedCheck = | |
| 1261 | reason === "check_failed" && pull | |
| 1262 | ? checkSummary( | |
| 1263 | app.db, | |
| 1264 | pull.id, | |
| 1265 | (await resolveOid(forkBare, `refs/heads/${branch}`)) ?? | |
| 1266 | pull.headOid, | |
| 1267 | ) | |
| 1268 | : undefined; | |
| 1260 | // Only a check or deploy that failed is handed back as the failure (#12968): never the | |
| 1261 | // success summary of a preview that came up. | |
| 1262 | const failure = failureBrief( | |
| 1263 | app.db, | |
| 1264 | app.config.dataDir, | |
| 1265 | reason, | |
| 1266 | pull?.id, | |
| 1267 | pull ? ((await resolveOid(forkBare, `refs/heads/${branch}`)) ?? pull.headOid) : "", | |
| 1268 | ); | |
| 1269 | 1269 | // Imported words speak as their origin author (`voiceOf`), so the |
| 1270 | 1270 | // brief marks a stranger's as a reader's, not the connecting user's. |
| 1271 | 1271 | const comments = listComments(app.db, issue.id, { kind: "comment" }).map( |
| @@ -1303,7 +1303,7 @@ async function work( | ||
| 1303 | 1303 | reason === "feedback" && run.summary === RETRY_SUMMARY |
| 1304 | 1304 | ? lastTryOf(app, issue.id, run, reviews) |
| 1305 | 1305 | : undefined, |
| 1306 | checkSummary: failedCheck?.run?.summary, | |
| 1306 | ...failure, | |
| 1307 | 1307 | foreignPull: foreign |
| 1308 | 1308 | ? { |
| 1309 | 1309 | branch: foreign.headBranch, |
| @@ -1363,18 +1363,6 @@ async function work( | ||
| 1363 | 1363 | owed: owed(app.db, repo.id), |
| 1364 | 1364 | } |
| 1365 | 1365 | : {}), |
| 1366 | checkKind: | |
| 1367 | failedCheck?.run?.kind === "check" || | |
| 1368 | failedCheck?.run?.kind === "deploy" | |
| 1369 | ? failedCheck.run.kind | |
| 1370 | : undefined, | |
| 1371 | checkLog: failedCheck?.run | |
| 1372 | ? readRunLog( | |
| 1373 | app.config.dataDir, | |
| 1374 | failedCheck.run.id, | |
| 1375 | 8 * 1024, | |
| 1376 | ).lines.slice(-40) | |
| 1377 | : undefined, | |
| 1378 | 1366 | }); |
| 1379 | 1367 | |
| 1380 | 1368 | // The account this run works on: the nearest grant's, rotating; the |
test/failure-quote.test.ts added +7
| @@ -0,0 +1,7 @@ | ||
| 1 | import { expect, test } from "bun:test"; | |
| 2 | import { failingLines } from "../src/crew/failure_quote.ts"; | |
| 3 | ||
| 4 | test("a check's own failing lines are picked from its log, a deploy's summary is not", () => { | |
| 5 | const log = ["tier fast", 'gate: rule "x": a.ts:1 casts', "running at https://x · no tests declared, none run", "(fail) a > b", "failed: gate: rule x"]; | |
| 6 | expect(failingLines(log)).toEqual(['gate: rule "x": a.ts:1 casts', "(fail) a > b", "failed: gate: rule x"]); | |
| 7 | }); |
test/no_change_recheck.test.ts added +159
| @@ -0,0 +1,159 @@ | ||
| 1 | /** | |
| 2 | * A builder who returns "no change" on a red pull gets the head checked again (#12968), is never | |
| 3 | * handed a deploy's success as a failure, and the engine does not read idle beside such a pull. | |
| 4 | */ | |
| 5 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 6 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 7 | import { tmpdir } from "node:os"; | |
| 8 | import { join } from "node:path"; | |
| 9 | import { createApp, type App } from "../src/app.ts"; | |
| 10 | import { addToCrew, createAgent } from "../src/actions/crew.ts"; | |
| 11 | import { openIssue } from "../src/actions/issues.ts"; | |
| 12 | import { openPull } from "../src/actions/pulls.ts"; | |
| 13 | import { commitToBranch, createRepo } from "../src/actions/repos.ts"; | |
| 14 | import { provideChecksForTest, type ChecksHandle } from "../src/checks/service.ts"; | |
| 15 | import { listComments } from "../src/core/issues.ts"; | |
| 16 | import { getPull } from "../src/core/pulls.ts"; | |
| 17 | import { getRepoById, type Repo } from "../src/core/repos.ts"; | |
| 18 | import { createRun, failedRunOf, listRuns } from "../src/core/runs.ts"; | |
| 19 | import { createUser, getUserByName, type User } from "../src/core/users.ts"; | |
| 20 | import { driveRepo, settleAgentRun } from "../src/crew/autonomy.ts"; | |
| 21 | import { taskPrompt, type TaskInput } from "../src/crew/brief.ts"; | |
| 22 | import { failureBrief } from "../src/crew/failure_brief.ts"; | |
| 23 | import { idleReason, redUnattended } from "../src/crew/idle_reason.ts"; | |
| 24 | import { NO_CHANGE_REASON } from "../src/crew/no_change.ts"; | |
| 25 | ||
| 26 | let dataDir: string; | |
| 27 | let app: App; | |
| 28 | let alice: User; | |
| 29 | let repo: Repo; | |
| 30 | let pull: NonNullable<ReturnType<typeof getPull>>; | |
| 31 | ||
| 32 | beforeAll(async () => { | |
| 33 | dataDir = await mkdtemp(join(tmpdir(), "forge-no-change-")); | |
| 34 | app = createApp({ dataDir, port: 0 }); | |
| 35 | const a = await createUser(app.db, { | |
| 36 | origin: "seed", trusted: true, username: "alice", email: "[email protected]", | |
| 37 | password: "a-good-passphrase", adminIfFirst: true, | |
| 38 | }); | |
| 39 | if (!a.ok) throw new Error(a.error); | |
| 40 | alice = a.value; | |
| 41 | const made = await createRepo(app, alice, { name: "engine", visibility: "private" }); | |
| 42 | if (!made.ok) throw new Error(String(made.error)); | |
| 43 | repo = made.value; | |
| 44 | await commitToBranch(app, alice, repo, { message: "Start", files: [{ path: "a.ts", content: "export const a = 0;\n" }] }); | |
| 45 | await createAgent(app, alice, { username: "mason" }); | |
| 46 | addToCrew(app, alice, repo, "mason"); | |
| 47 | const mason = getUserByName(app.db, "mason")!; | |
| 48 | app.db.query("UPDATE repo SET autonomy = 'autonomous', autonomy_by = ?2 WHERE id = ?1").run(repo.id, alice.id); | |
| 49 | repo = getRepoById(app.db, repo.id)!; | |
| 50 | const ask = openIssue(app, alice, repo, { title: "Ask" }); | |
| 51 | if (!ask.ok) throw new Error(String(ask.error)); | |
| 52 | await commitToBranch(app, alice, repo, { branch: "feature/x", base: "main", message: "X", files: [{ path: "x.ts", content: "export const x = 1;\n" }] }); | |
| 53 | const opened = await openPull(app, mason, repo, { title: "X", body: `Closes #${ask.value.number}`, baseBranch: "main", headBranch: "feature/x" }); | |
| 54 | if (!opened.ok) throw new Error(String(opened.error)); | |
| 55 | pull = getPull(app.db, repo.id, opened.value.number)!; | |
| 56 | provideChecksForTest(app, { | |
| 57 | request: async (r: Repo, env: string, oid: string, reason: string, issueId?: string) => | |
| 58 | createRun(app.db, { repoId: r.id, kind: "check", env, commitOid: oid, reason, issueId }), | |
| 59 | } as unknown as ChecksHandle); | |
| 60 | }); | |
| 61 | ||
| 62 | afterAll(async () => { | |
| 63 | app.db.close(); | |
| 64 | await rm(dataDir, { recursive: true, force: true }); | |
| 65 | }); | |
| 66 | ||
| 67 | const mark = (id: string, state: string, summary: string, detail = "") => | |
| 68 | app.db.query("UPDATE run SET state = ?2, summary = ?3, check_detail = ?4, finished_at = strftime('%Y-%m-%dT%H:%M:%fZ','now') WHERE id = ?1").run(id, state, summary, detail); | |
| 69 | const run = (kind: "check" | "deploy") => createRun(app.db, { repoId: repo.id, kind, issueId: pull.id, commitOid: pull.headOid, reason: "" }); | |
| 70 | const noChange = () => | |
| 71 | settleAgentRun(app, repo, { issueId: pull.id, state: "ok", summary: "no change needed", reason: "check_failed", actorId: getUserByName(app.db, "mason")!.id }); | |
| 72 | const errored = () => | |
| 73 | settleAgentRun(app, repo, { issueId: pull.id, state: "failed", summary: "the agent exited with code 1", reason: "check_failed", actorId: getUserByName(app.db, "mason")!.id }); | |
| 74 | const rechecks = () => listRuns(app.db, repo.id, { issueId: pull.id, limit: 50 }).filter((r) => r.reason === NO_CHANGE_REASON); | |
| 75 | ||
| 76 | test("a deploy that came up is never offered as the failure", () => { | |
| 77 | const check = run("check"); | |
| 78 | mark(check.id, "failed", "failed: engine > adds up", JSON.stringify({ names: ["engine > adds up"], failures: [{ file: "a.test.ts", name: "engine > adds up", assertion: "expected 2, got 3" }] })); | |
| 79 | const deploy = run("deploy"); | |
| 80 | mark(deploy.id, "ok", "running at https://x · no tests declared, none run"); | |
| 81 | expect(failedRunOf(app.db, pull.id, pull.headOid)?.id).toBe(check.id); | |
| 82 | const base = { reason: "check_failed", issue: { number: pull.number, title: "X" } }; | |
| 83 | const quoted = taskPrompt({ ...base, checkKind: "check", checkSummary: "x", checkFailing: "engine > adds up\nexpected 2, got 3" } as unknown as TaskInput); | |
| 84 | expect(quoted).toContain("expected 2, got 3"); | |
| 85 | expect(taskPrompt({ ...base, noFailure: true } as unknown as TaskInput)).toContain("no check of its head is recorded as failed"); | |
| 86 | }); | |
| 87 | ||
| 88 | test("a cut check is no failure; a builder's no change re-checks the head, and a second quotes the failure", () => { | |
| 89 | app.db.query("DELETE FROM run WHERE issue_id = ?1").run(pull.id); | |
| 90 | const cut = run("check"); | |
| 91 | mark(cut.id, "failed", "interrupted by a restart; queued again"); | |
| 92 | expect(failedRunOf(app.db, pull.id, pull.headOid)).toBeUndefined(); | |
| 93 | expect(redUnattended(app, repo, [])).toBe(0); | |
| 94 | ||
| 95 | expect(noChange()).toBe(true); | |
| 96 | expect(rechecks()).toHaveLength(1); | |
| 97 | // Still running: nothing more is queued. | |
| 98 | expect(noChange()).toBe(false); | |
| 99 | expect(rechecks()).toHaveLength(1); | |
| 100 | ||
| 101 | // It ends red on a named test: the stall is counted, and a second no change quotes it. | |
| 102 | mark(rechecks()[0]!.id, "failed", "failed: engine > adds up", JSON.stringify({ names: ["engine > adds up"], failures: [{ file: "a.test.ts", name: "engine > adds up", assertion: "expected 2, got 3" }] })); | |
| 103 | expect(redUnattended(app, repo, [])).toBe(1); | |
| 104 | expect(noChange()).toBe(true); | |
| 105 | expect(rechecks()).toHaveLength(1); | |
| 106 | const said = listComments(app.db, pull.id).map((c) => c.body).join("\n"); | |
| 107 | expect(said).toContain("engine > adds up"); | |
| 108 | expect(said).toContain("expected 2, got 3"); | |
| 109 | ||
| 110 | // A third no change on the same head finds the quote and says nothing again. | |
| 111 | const before = listComments(app.db, pull.id).length; | |
| 112 | noChange(); | |
| 113 | noChange(); | |
| 114 | expect(listComments(app.db, pull.id)).toHaveLength(before); | |
| 115 | expect(rechecks()).toHaveLength(1); | |
| 116 | }); | |
| 117 | ||
| 118 | test("a builder run that errored is no answer: nothing is re-checked", () => { | |
| 119 | app.db.query("DELETE FROM run WHERE issue_id = ?1").run(pull.id); | |
| 120 | mark(run("check").id, "failed", "failed: engine > adds up", JSON.stringify({ names: ["engine > adds up"], failures: [{ file: "a.test.ts", name: "engine > adds up" }] })); | |
| 121 | errored(); | |
| 122 | expect(rechecks()).toHaveLength(0); | |
| 123 | }); | |
| 124 | ||
| 125 | test("the runner's brief for a head whose only run is a green deploy says no failure is recorded", () => { | |
| 126 | app.db.query("DELETE FROM run WHERE issue_id = ?1").run(pull.id); | |
| 127 | mark(run("deploy").id, "ok", "running at https://x · no tests declared, none run"); | |
| 128 | const fields = failureBrief(app.db, dataDir, "check_failed", pull.id, pull.headOid); | |
| 129 | expect(fields).toEqual({ noFailure: true }); | |
| 130 | const brief = taskPrompt({ reason: "check_failed", issue: { number: pull.number, title: "X" }, ...fields } as unknown as TaskInput); | |
| 131 | expect(brief).toContain("no check of its head is recorded as failed"); | |
| 132 | expect(brief).not.toContain("running at https://x"); | |
| 133 | ||
| 134 | // A failed check of the same head is quoted, by kind, with its assertion. | |
| 135 | mark(run("check").id, "failed", "failed: engine > adds up", JSON.stringify({ names: ["engine > adds up"], failures: [{ file: "a.test.ts", name: "engine > adds up", assertion: "expected 2, got 3" }] })); | |
| 136 | const red = failureBrief(app.db, dataDir, "check_failed", pull.id, pull.headOid); | |
| 137 | expect(red.checkKind).toBe("check"); | |
| 138 | expect(red.noFailure).toBe(false); | |
| 139 | expect(taskPrompt({ reason: "check_failed", issue: { number: pull.number, title: "X" }, ...red } as unknown as TaskInput)).toContain("expected 2, got 3"); | |
| 140 | expect(failureBrief(app.db, dataDir, "assigned", pull.id, pull.headOid)).toEqual({}); | |
| 141 | }); | |
| 142 | ||
| 143 | test("a re-check that is green moves the pull on: no second re-check, no comment", () => { | |
| 144 | app.db.query("DELETE FROM run WHERE issue_id = ?1").run(pull.id); | |
| 145 | mark(run("check").id, "failed", "the forge went away, and this commit no longer heads an open pull"); | |
| 146 | expect(noChange()).toBe(true); | |
| 147 | mark(rechecks()[0]!.id, "ok", "ok"); | |
| 148 | expect(noChange()).toBe(false); | |
| 149 | expect(rechecks()).toHaveLength(1); | |
| 150 | }); | |
| 151 | ||
| 152 | test("idleReason names the stall, not idleness, beside a red pull nobody works; driveRepo says stalled", async () => { | |
| 153 | app.db.query("DELETE FROM run WHERE issue_id = ?1").run(pull.id); | |
| 154 | mark(run("check").id, "failed", "failed: engine > adds up", JSON.stringify({ names: ["engine > adds up"], failures: [{ file: "a.test.ts", name: "engine > adds up" }] })); | |
| 155 | app.db.query("DELETE FROM run WHERE kind = 'agent'").run(); | |
| 156 | const reason = idleReason(app, new Date()); | |
| 157 | expect(reason?.word).toStartWith("stalled: 1 open pull is red"); | |
| 158 | expect(await driveRepo(app, getRepoById(app.db, repo.id)!)).toBe("stalled"); | |
| 159 | }); |
new above
offline