Public: anyone can read this; the crew can change it.
A landing gate cut short by a roll is not red: the queue re-gates it on boot #3156
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/3155-a-landing-gate-cut-short-by-a-roll-is-no → main 3 commits fork
11 files
changed +318
−25
README.md +6 −1
| @@ -1028,7 +1028,12 @@ The full gate runs once on that candidate. Green lands every pull in order, | ||
| 1028 | 1028 | and each landing is refused unless main is still where the candidate was |
| 1029 | 1029 | built and the squash writes the tree that was gated, so main ends at the |
| 1030 | 1030 | gated tree. Red bisects: the front half is gated by itself, a green half |
| 1031 | lands, and the red pull is handed back with the failing tests named. Pulls | |
| 1031 | lands, and the red pull is handed back with the failing tests named. A red | |
| 1032 | waits for a push or a new main, except once: after 30 minutes with neither | |
| 1033 | side moved it is gated again, and lands if green; a red that reproduces | |
| 1034 | stays red and is not told twice. A gate cut off by the forge stopping (a | |
| 1035 | roll, a drain) or by the cluster losing its Job is no red at all: the next | |
| 1036 | sweep gates it again. Pulls | |
| 1032 | 1037 | that turn green while a candidate is gating wait for the next one. Each |
| 1033 | 1038 | queue's numbers — how many pulls are queued, how many are gating, and how |
| 1034 | 1039 | many landed in the last hour — show in three places: the flow panel on `/boards`, the |
docs/ARCHITECTURE.md +5 −3
| @@ -409,9 +409,11 @@ ends every open event stream with a resume hint (`retry: 1000`, then | ||
| 409 | 409 | `event: drain`, `http/sse.ts`), so each browser reconnects — to the new |
| 410 | 410 | process — from the last id it saw; the crew stops taking runs and lets the |
| 411 | 411 | ones it holds finish and land (`crew.stop({ drain: true })`, bounded); a |
| 412 | check in flight is cut and handed back to the queue for the new process; | |
| 413 | its app processes stop (the new process already runs its own); SSH sessions | |
| 414 | are kept until they leave. However little it holds, its listener stays open | |
| 412 | check in flight is cut and handed back to the queue for the new process. | |
| 413 | A gate on land in flight is cut too, and is no verdict: no red is told, and | |
| 414 | the new process's merge queue gates it again (#3155). Its app processes stop | |
| 415 | (the new process already runs its own), and SSH sessions are kept until | |
| 416 | they leave. However little it holds, its listener stays open | |
| 415 | 417 | for `FORGE_DRAIN_LINGER_S` (10 s) after the SIGTERM: routers hear that it |
| 416 | 418 | left rotation a probe period later, and a process already gone turns each |
| 417 | 419 | request sent in that beat into a 502. Then it exits. |
src/checks/landing.ts +2
| @@ -53,6 +53,8 @@ export async function gateCandidate( | ||
| 53 | 53 | if (!declared.ok || declared.value === undefined || declared.value.fast === null) |
| 54 | 54 | return { lands: true }; |
| 55 | 55 | const done = await checks.gate(repo, pull, commitOid); |
| 56 | // Cut off, not judged: no verdict, and not the gate log a red reads (#3155). | |
| 57 | if (done.interrupted === true) return { lands: false, interrupted: true }; | |
| 56 | 58 | keepForgeGate(app.config.dataDir, app.config.forgeRepo, `${repo.ownerName}/${repo.name}`, done.lines); |
| 57 | 59 | const failed = failuresOf(done.lines); |
| 58 | 60 | return landVerdict(done.ok, failed.tests, failed.files); |
src/checks/queue.ts +14 −2
| @@ -8,7 +8,9 @@ | ||
| 8 | 8 | * exactly the tree that was gated. Red bisects: the first half is gated on |
| 9 | 9 | * its own; green lands it and the culprit is in the rest, red keeps the |
| 10 | 10 | * search in the first half while the rest waits for the next candidate. A |
| 11 | * single red step is the culprit and is handed back. | |
| 11 | * single red step is the culprit and is handed back. A gate cut off by the | |
| 12 | * forge stopping is no verdict: nothing is handed back, and the forge that | |
| 13 | * carries on admits the line again (#3155). | |
| 12 | 14 | * |
| 13 | 15 | * What is queued is not stored: it is the pulls `landIfGreen` admitted, and |
| 14 | 16 | * every sweep admits them again. The queue only orders them and remembers |
| @@ -28,7 +30,7 @@ export interface Expect { | ||
| 28 | 30 | } |
| 29 | 31 | |
| 30 | 32 | /** A red verdict: what the culprit is told. */ |
| 31 | export type Red = Extract<LandVerdict, { lands: false }>; | |
| 33 | export type Red = Extract<LandVerdict, { reopen: true }>; | |
| 32 | 34 | |
| 33 | 35 | /** One pull in the line, and what landing or handing it back does. */ |
| 34 | 36 | export interface Queued { |
| @@ -222,6 +224,12 @@ async function round( | ||
| 222 | 224 | await land(candidate, steps); |
| 223 | 225 | return []; |
| 224 | 226 | } |
| 227 | // The gate was cut off (the forge stopping, the Job lost): nothing is | |
| 228 | // decided and no one is handed back; the next sweep admits them again (#3155). | |
| 229 | if ("interrupted" in verdict) { | |
| 230 | for (const w of steps) w.settle(false); | |
| 231 | return []; | |
| 232 | } | |
| 225 | 233 | if (steps.length === 1) { |
| 226 | 234 | steps[0]!.entry.handBack(verdict, candidate.baseOid); |
| 227 | 235 | steps[0]!.settle(false); |
| @@ -245,6 +253,10 @@ async function round( | ||
| 245 | 253 | // the rest on the new base is the red tree again. |
| 246 | 254 | return round(rest, gate, red); |
| 247 | 255 | } |
| 256 | if ("interrupted" in frontVerdict) { | |
| 257 | for (const w of steps) w.settle(false); | |
| 258 | return []; | |
| 259 | } | |
| 248 | 260 | const left = await round(front, gate, { treeOid: part.treeOid, red: frontVerdict }); |
| 249 | 261 | return [...left, ...rest]; |
| 250 | 262 | } |
src/checks/service.ts +16 −3
| @@ -89,6 +89,9 @@ import type { CheckDetail } from "../core/runs.ts"; | ||
| 89 | 89 | |
| 90 | 90 | const log = createLog("checks"); |
| 91 | 91 | |
| 92 | /** A gate this forge would not start, for it is stopping. */ | |
| 93 | const INTERRUPTED = "this forge is stopping"; | |
| 94 | ||
| 92 | 95 | export interface ChecksOptions { |
| 93 | 96 | /** How many checks may run at once. @default 1 */ |
| 94 | 97 | readonly concurrency?: number | undefined; |
| @@ -133,13 +136,15 @@ export interface ChecksHandle { | ||
| 133 | 136 | /** |
| 134 | 137 | * The full tier on a merge, before it is made (#1198): `commitOid` is the |
| 135 | 138 | * merge of the pull with its base, run in a clone of its own. Nothing is |
| 136 | * queued and no run is written; the answer is the verdict. | |
| 139 | * queued and no run is written; the answer is the verdict. `interrupted` | |
| 140 | * is a gate that did not finish for a reason that is not the code — this | |
| 141 | * forge stopping, or the cluster losing its Job — and is no verdict (#3155). | |
| 137 | 142 | */ |
| 138 | 143 | gate( |
| 139 | 144 | repo: Repo, |
| 140 | 145 | pull: Pull, |
| 141 | 146 | commitOid: string, |
| 142 | ): Promise<{ ok: boolean; error: string; lines: string[] }>; | |
| 147 | ): Promise<{ ok: boolean; error: string; lines: string[]; interrupted?: boolean }>; | |
| 143 | 148 | /** |
| 144 | 149 | * Stop checking. Checks under way get `drainMs` to finish unless `drain` |
| 145 | 150 | * is false (a person's Ctrl-C): then they are cut off and queued again now. |
| @@ -1111,6 +1116,9 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle { | ||
| 1111 | 1116 | request, |
| 1112 | 1117 | gate: async (repo, pull, commitOid) => { |
| 1113 | 1118 | const lines: string[] = []; |
| 1119 | // A gate asked of a forge already stopping is not started: the forge | |
| 1120 | // that carries on asks it again. | |
| 1121 | if (stopped) return { ok: false, error: INTERRUPTED, lines, interrupted: true }; | |
| 1114 | 1122 | const controller = new AbortController(); |
| 1115 | 1123 | gates.add(controller); |
| 1116 | 1124 | try { |
| @@ -1125,7 +1133,12 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle { | ||
| 1125 | 1133 | write: (line) => lines.push(line), |
| 1126 | 1134 | signal: controller.signal, |
| 1127 | 1135 | }); |
| 1128 | return { ok: done.ok, error: done.ok ? "" : done.error, lines }; | |
| 1136 | if (done.ok) return { ok: true, error: "", lines }; | |
| 1137 | // Cut off by this forge stopping, or the Job lost under it: the | |
| 1138 | // code was not judged, so this is no red. | |
| 1139 | if (stopped || controller.signal.aborted || done.error === JOB_VANISHED) | |
| 1140 | return { ok: false, error: done.error, lines, interrupted: true }; | |
| 1141 | return { ok: false, error: done.error, lines }; | |
| 1129 | 1142 | } finally { |
| 1130 | 1143 | gates.delete(controller); |
| 1131 | 1144 | } |
src/checks/tiers.ts +11 −2
| @@ -239,7 +239,8 @@ export function tierLine(state: TierState, seconds?: number): string { | ||
| 239 | 239 | } |
| 240 | 240 | } |
| 241 | 241 | |
| 242 | export type LandVerdict = | |
| 242 | /** What a gate that ran to its end found. */ | |
| 243 | export type Judged = | |
| 243 | 244 | | { readonly lands: true } |
| 244 | 245 | | { |
| 245 | 246 | readonly lands: false; |
| @@ -251,6 +252,14 @@ export type LandVerdict = | ||
| 251 | 252 | readonly learn: readonly string[]; |
| 252 | 253 | }; |
| 253 | 254 | |
| 255 | /** | |
| 256 | * What the gate on land answers: judged, or interrupted — the gate did not | |
| 257 | * finish for a reason that is not the code (the forge stopped under it, the | |
| 258 | * cluster lost its Job). Interrupted is no verdict; the queue gates it | |
| 259 | * again (#3155). | |
| 260 | */ | |
| 261 | export type LandVerdict = Judged | { readonly lands: false; readonly interrupted: true }; | |
| 262 | ||
| 254 | 263 | /** |
| 255 | 264 | * What the full gate on the merge with its base decides. Green lands the |
| 256 | 265 | * pull. Red reopens it, names the red tests, and teaches the fast tier |
| @@ -261,7 +270,7 @@ export function landVerdict( | ||
| 261 | 270 | green: boolean, |
| 262 | 271 | failedTests: readonly string[], |
| 263 | 272 | failedFiles: readonly string[] = [], |
| 264 | ): LandVerdict { | |
| 273 | ): Judged { | |
| 265 | 274 | if (green) return { lands: true }; |
| 266 | 275 | const names = failedTests.map((name) => `- ${name}`).join("\n"); |
| 267 | 276 | return { |
src/crew/autonomy.ts +48 −11
| @@ -39,6 +39,7 @@ import { | ||
| 39 | 39 | import { pickBuilder, routeIssue } from "./route.ts"; |
| 40 | 40 | import { readyNow, repoGraph, type Ranked } from "../core/dependencies.ts"; |
| 41 | 41 | import { |
| 42 | type Decision, | |
| 42 | 43 | latestDecision, |
| 43 | 44 | recordDecision, |
| 44 | 45 | resolveOpen, |
| @@ -1745,16 +1746,13 @@ async function admit( | ||
| 1745 | 1746 | const review = await reviewed(app, repo, pull); |
| 1746 | 1747 | if (review === "hold") return err("review"); |
| 1747 | 1748 | // The full tier, once, on the queue's candidate (#1198, #2036). Red is |
| 1748 | // told to the pull once per head and base, and the pull waits for a push. | |
| 1749 | // told to the pull once per head and base, and the pull waits for a push | |
| 1750 | // — but a red neither side has moved off is gated once more after | |
| 1751 | // REGATE_RED_MS, so a red that was the machine's and not the code's does | |
| 1752 | // not hold the queue for good; a red that reproduces stays red (#3155). | |
| 1749 | 1753 | const told = latestDecision(app.db, pull.id, "land"); |
| 1750 | 1754 | const gatedAt = `${pull.headOid}:${baseTip}`; |
| 1751 | if ( | |
| 1752 | told?.answer === "red" && | |
| 1753 | typeof told.state === "object" && | |
| 1754 | told.state !== null && | |
| 1755 | (told.state as { at?: unknown }).at === gatedAt | |
| 1756 | ) | |
| 1757 | return err("red"); | |
| 1755 | if (redAt(told, gatedAt) && !regateDue(told!, Date.now())) return err("red"); | |
| 1758 | 1756 | // Green only against a base that has moved since: the merge would hold |
| 1759 | 1757 | // it (`checksHold`), so the queue asks for the check again on the base |
| 1760 | 1758 | // as it is now rather than wait for a person to (#3055). The fresh green |
| @@ -1794,23 +1792,62 @@ async function admit( | ||
| 1794 | 1792 | }), |
| 1795 | 1793 | ), |
| 1796 | 1794 | handBack: (red, baseOid) => { |
| 1797 | // Told on the base it was found red on: a new base is a new try. | |
| 1795 | // Told on the base it was found red on: a new base is a new try. Red | |
| 1796 | // again on the same head and base is the re-gate reproducing it: it | |
| 1797 | // stays red for good, and the pull is not told twice. | |
| 1798 | const at = `${pull.headOid}:${baseOid}`; | |
| 1799 | const again = redAt(latestDecision(app.db, pull.id, "land"), at); | |
| 1800 | const state: RedState = again ? { head: pull.headOid, at, again } : { head: pull.headOid, at }; | |
| 1798 | 1801 | recordDecision(app.db, { |
| 1799 | 1802 | kind: "land", |
| 1800 | 1803 | repoId: repo.id, |
| 1801 | 1804 | issueId: pull.id, |
| 1802 | state: { head: pull.headOid, at: `${pull.headOid}:${baseOid}` }, | |
| 1805 | state, | |
| 1803 | 1806 | answer: "red", |
| 1804 | 1807 | p: 1, |
| 1805 | 1808 | by: "rule", |
| 1806 | 1809 | ms: 0, |
| 1807 | 1810 | }); |
| 1808 | 1811 | addLearned(app.config.dataDir, pull.id, red.learn); |
| 1809 | addComment(app, by, repo, getIssueById(app.db, pull.id) ?? pull, red.comment); | |
| 1812 | if (!again) addComment(app, by, repo, getIssueById(app.db, pull.id) ?? pull, red.comment); | |
| 1810 | 1813 | }, |
| 1811 | 1814 | }); |
| 1812 | 1815 | } |
| 1813 | 1816 | |
| 1817 | /** How long a red neither side has moved off waits before it is gated once more (#3155). */ | |
| 1818 | export const REGATE_RED_MS = 30 * 60_000; | |
| 1819 | ||
| 1820 | /** | |
| 1821 | * What a `red` land decision holds: the head it was red on, `head:base` | |
| 1822 | * as `at`, and `again` when it is the re-gate that reproduced it. | |
| 1823 | */ | |
| 1824 | interface RedState { | |
| 1825 | readonly head: string; | |
| 1826 | readonly at: string; | |
| 1827 | readonly again?: true; | |
| 1828 | } | |
| 1829 | ||
| 1830 | /** The red a land decision tells, read off the ledger's untyped state; undefined for any other. */ | |
| 1831 | function redOf(told: Decision | undefined): RedState | undefined { | |
| 1832 | if (told?.answer !== "red") return undefined; | |
| 1833 | const state = told.state; | |
| 1834 | if (typeof state !== "object" || state === null) return undefined; | |
| 1835 | const { head, at, again } = state as Record<string, unknown>; | |
| 1836 | if (typeof head !== "string" || typeof at !== "string") return undefined; | |
| 1837 | return again === true ? { head, at, again } : { head, at }; | |
| 1838 | } | |
| 1839 | ||
| 1840 | /** `told` is a red on this `head:base`. */ | |
| 1841 | function redAt(told: Decision | undefined, at: string): boolean { | |
| 1842 | return redOf(told)?.at === at; | |
| 1843 | } | |
| 1844 | ||
| 1845 | /** A red that is old enough, and not itself a re-gate, is gated once more. */ | |
| 1846 | export function regateDue(red: Decision, now: number): boolean { | |
| 1847 | if (redOf(red)?.again === true) return false; | |
| 1848 | return now - Date.parse(red.createdAt) >= REGATE_RED_MS; | |
| 1849 | } | |
| 1850 | ||
| 1814 | 1851 | /** The queue's gate: the full tier on the candidate's tip, asked as its first pull. */ |
| 1815 | 1852 | function gateFor(app: App, repo: Repo): GateCandidate { |
| 1816 | 1853 | return (candidate) => |
test/check-tiers.test.ts +1 −1
| @@ -613,7 +613,7 @@ describe("the tiers in the forge's own checks service", () => { | ||
| 613 | 613 | }, |
| 614 | 614 | ).stdout.toString().trim(); |
| 615 | 615 | const verdict = await gateCandidate(app, repo, pull, dir, commit); |
| 616 | if (verdict.lands) throw new Error("expected the full gate to be red"); | |
| 616 | if (verdict.lands || "interrupted" in verdict) throw new Error("expected the full gate to be red"); | |
| 617 | 617 | expect(verdict.comment).toContain("- lands red"); |
| 618 | 618 | expect(verdict.learn).toContain("test/y.test.ts"); |
| 619 | 619 | }); |
test/gate-interrupted.test.ts added +96
| @@ -0,0 +1,96 @@ | ||
| 1 | /** | |
| 2 | * A gate on land cut off by the forge stopping is no verdict (#3155): the | |
| 3 | * checks service answers it `interrupted`, the gate on land says so, and | |
| 4 | * the merge queue hands no one back (test/merge-queue.test.ts) — so the | |
| 5 | * forge that boots next gates it again and lands it without a push. | |
| 6 | */ | |
| 7 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 8 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 9 | import { tmpdir } from "node:os"; | |
| 10 | import { join } from "node:path"; | |
| 11 | import { createApp, type App } from "../src/app.ts"; | |
| 12 | import { openPull } from "../src/actions/pulls.ts"; | |
| 13 | import { commitToBranch, createRepo } from "../src/actions/repos.ts"; | |
| 14 | import { gateCandidate } from "../src/checks/landing.ts"; | |
| 15 | import { startChecks, type ChecksHandle } from "../src/checks/service.ts"; | |
| 16 | import { getPull } from "../src/core/pulls.ts"; | |
| 17 | import type { Repo } from "../src/core/repos.ts"; | |
| 18 | import { createUser, type User } from "../src/core/users.ts"; | |
| 19 | import { repoDir } from "../src/git/repos.ts"; | |
| 20 | ||
| 21 | describe("a gate on land cut off by the forge stopping", () => { | |
| 22 | let dataDir: string; | |
| 23 | let app: App; | |
| 24 | let checks: ChecksHandle; | |
| 25 | let repo: Repo; | |
| 26 | let alice: User; | |
| 27 | const cells = process.env["FORGE_CELLS"]; | |
| 28 | ||
| 29 | beforeAll(async () => { | |
| 30 | process.env["FORGE_CELLS"] = "off"; | |
| 31 | dataDir = await mkdtemp(join(tmpdir(), "forge-gate-cut-")); | |
| 32 | app = createApp({ dataDir, port: 0 }); | |
| 33 | checks = startChecks(app, { forgeUrl: "http://127.0.0.1:1", concurrency: 1 }); | |
| 34 | const a = await createUser(app.db, { | |
| 35 | origin: "seed", | |
| 36 | trusted: true, | |
| 37 | username: "alice", | |
| 38 | email: "[email protected]", | |
| 39 | password: "a-good-passphrase", | |
| 40 | adminIfFirst: true, | |
| 41 | }); | |
| 42 | if (!a.ok) throw new Error(a.error); | |
| 43 | alice = a.value; | |
| 44 | const made = await createRepo(app, alice, { name: "cut", visibility: "public" }); | |
| 45 | if (!made.ok) throw new Error(String(made.error)); | |
| 46 | repo = made.value; | |
| 47 | const seeded = await commitToBranch(app, alice, repo, { | |
| 48 | message: "Start", | |
| 49 | files: [ | |
| 50 | { | |
| 51 | path: "forge.json", | |
| 52 | content: JSON.stringify({ | |
| 53 | check: { run: "true", fast: "true", full: "echo test/slow.test.ts:; sleep 30; exit 1" }, | |
| 54 | }), | |
| 55 | }, | |
| 56 | ], | |
| 57 | }); | |
| 58 | if (!seeded.ok) throw new Error(JSON.stringify(seeded.error)); | |
| 59 | const branched = await commitToBranch(app, alice, repo, { | |
| 60 | branch: "feature", | |
| 61 | message: "Touch b", | |
| 62 | files: [{ path: "b.txt", content: "b\n" }], | |
| 63 | }); | |
| 64 | if (!branched.ok) throw new Error(JSON.stringify(branched.error)); | |
| 65 | }); | |
| 66 | ||
| 67 | afterAll(async () => { | |
| 68 | await checks.stop({ drain: false }); | |
| 69 | app.close(); | |
| 70 | if (cells === undefined) delete process.env["FORGE_CELLS"]; | |
| 71 | else process.env["FORGE_CELLS"] = cells; | |
| 72 | await rm(dataDir, { recursive: true, force: true }); | |
| 73 | }); | |
| 74 | ||
| 75 | test("answers interrupted, not red; a gate asked of a stopping forge is not started", async () => { | |
| 76 | const opened = await openPull(app, alice, repo, { | |
| 77 | title: "Feature", | |
| 78 | baseBranch: "main", | |
| 79 | headBranch: "feature", | |
| 80 | }); | |
| 81 | if (!opened.ok) throw new Error(String(opened.error)); | |
| 82 | const pull = getPull(app.db, repo.id, opened.value.number)!; | |
| 83 | const dir = repoDir(app.reposRoot, repo.ownerName, repo.name); | |
| 84 | const started = Date.now(); | |
| 85 | const gating = gateCandidate(app, repo, pull, dir, pull.headOid); | |
| 86 | await Bun.sleep(500); | |
| 87 | await checks.stop({ drain: false }); | |
| 88 | expect(await gating).toEqual({ lands: false, interrupted: true }); | |
| 89 | expect(Date.now() - started).toBeLessThan(20_000); | |
| 90 | // The roll is under way: nothing new is gated, and nothing is red. | |
| 91 | expect(await gateCandidate(app, repo, pull, dir, pull.headOid)).toEqual({ | |
| 92 | lands: false, | |
| 93 | interrupted: true, | |
| 94 | }); | |
| 95 | }, 30_000); | |
| 96 | }); |
test/helpers/chrome.ts +12
| @@ -172,6 +172,18 @@ async function evaluateLoaded( | ||
| 172 | 172 | const loaded = cdp.once("Page.loadEventFired"); |
| 173 | 173 | await cdp.send("Page.navigate", { url }); |
| 174 | 174 | await loaded; |
| 175 | // The load event can be the blank start page's, fired late on a busy | |
| 176 | // machine: the page measured is the navigated one, loaded, or none. | |
| 177 | const deadline = Date.now() + 10_000; | |
| 178 | for (;;) { | |
| 179 | const { result: at } = await cdp.send("Runtime.evaluate", { | |
| 180 | expression: `/^https?:/.test(location.href) && document.readyState === "complete"`, | |
| 181 | returnByValue: true, | |
| 182 | }); | |
| 183 | if (at.value === true) break; | |
| 184 | if (Date.now() > deadline) throw new Error(`chromium never finished loading ${url}`); | |
| 185 | await Bun.sleep(25); | |
| 186 | } | |
| 175 | 187 | const { result } = await cdp.send("Runtime.evaluate", { |
| 176 | 188 | expression, |
| 177 | 189 | returnByValue: true, |
test/merge-queue.test.ts +107 −2
| @@ -20,7 +20,8 @@ import { getPull, type Pull } from "../src/core/pulls.ts"; | ||
| 20 | 20 | import { getRepoById, type Repo } from "../src/core/repos.ts"; |
| 21 | 21 | import { claimRun, createRun, finishRun } from "../src/core/runs.ts"; |
| 22 | 22 | import { createUser, type User } from "../src/core/users.ts"; |
| 23 | import { landIfGreen, sweepGreen } from "../src/crew/autonomy.ts"; | |
| 23 | import { landIfGreen, REGATE_RED_MS, regateDue, sweepGreen } from "../src/crew/autonomy.ts"; | |
| 24 | import { listComments } from "../src/core/issues.ts"; | |
| 24 | 25 | import { readQueues } from "../src/flow/queue.ts"; |
| 25 | 26 | import { repoDir } from "../src/git/repos.ts"; |
| 26 | 27 | import { serve, type Servers } from "../src/http/server.ts"; |
| @@ -36,6 +37,10 @@ describe("the merge queue", () => { | ||
| 36 | 37 | let held: Promise<void> | undefined; |
| 37 | 38 | /** Every candidate the gate was asked about, with its verdict. */ |
| 38 | 39 | const gated: { repo: string; commit: string; ok: boolean }[] = []; |
| 40 | /** How many gates from now are cut off, as by the forge stopping under them. */ | |
| 41 | let cutOff = 0; | |
| 42 | /** How many gates from now go red whatever the tree, as a machine's red would. */ | |
| 43 | let flaky = 0; | |
| 39 | 44 | |
| 40 | 45 | const git = (repo: Repo, ...args: string[]) => |
| 41 | 46 | Bun.spawnSync(["git", ...args], { |
| @@ -60,8 +65,12 @@ describe("the merge queue", () => { | ||
| 60 | 65 | provideChecksForTest(app, { |
| 61 | 66 | gate: async (repo: Repo, _pull: Pull, commitOid: string) => { |
| 62 | 67 | if (held !== undefined) await held; |
| 68 | if (cutOff > 0) { | |
| 69 | cutOff--; | |
| 70 | return { ok: false, error: "aborted", lines: ["(fail) killed [1.00ms]"], interrupted: true }; | |
| 71 | } | |
| 63 | 72 | const files = git(repo, "ls-tree", "-r", "--name-only", commitOid).split("\n"); |
| 64 | const ok = !files.some((f) => f.startsWith("red-")); | |
| 73 | const ok = flaky > 0 ? (flaky--, false) : !files.some((f) => f.startsWith("red-")); | |
| 65 | 74 | gated.push({ repo: repo.id, commit: commitOid, ok }); |
| 66 | 75 | return ok |
| 67 | 76 | ? { ok, error: "", lines: [] } |
| @@ -328,4 +337,100 @@ describe("the merge queue", () => { | ||
| 328 | 337 | expect(terminalLine(repo)?.split(/\s+/)).toEqual([`alice/${repo.name}`, "0", "0", "3"]); |
| 329 | 338 | expect(await mcpQueue()).toContainEqual({ repo: `alice/${repo.name}`, queued: 0, gating: 0, landedHour: 3 }); |
| 330 | 339 | }); |
| 340 | ||
| 341 | /** Ages the pull's newest land decision past the re-gate wait. */ | |
| 342 | const age = (pull: Pull) => | |
| 343 | app.db | |
| 344 | .query( | |
| 345 | "UPDATE decision SET created_at = ?2 WHERE id = (SELECT id FROM decision WHERE issue_id = ?1 AND kind = 'land' ORDER BY created_at DESC, id DESC LIMIT 1)", | |
| 346 | ) | |
| 347 | .run(pull.id, new Date(Date.now() - REGATE_RED_MS - 1000).toISOString()); | |
| 348 | ||
| 349 | test("a gate cut off by the forge stopping is no red: the next sweep gates it again and it lands, no push (#3155)", async () => { | |
| 350 | const repo = await autonomousRepo("cut-off"); | |
| 351 | const pull = await greenPull(repo, "a.ts"); | |
| 352 | const base = git(repo, "rev-parse", "refs/heads/main"); | |
| 353 | cutOff = 1; | |
| 354 | expect(await sweepGreen(app, repo)).toBe(0); | |
| 355 | expect(cutOff).toBe(0); | |
| 356 | // Nothing told: no red on the ledger, no comment, nothing learned. | |
| 357 | expect(latestDecision(app.db, pull.id, "land")?.answer).not.toBe("red"); | |
| 358 | expect(listComments(app.db, pull.id)).toHaveLength(0); | |
| 359 | expect(readLearned(dataDir, pull.id)).toEqual([]); | |
| 360 | // The forge that carries on, on the same head and base, lands it. | |
| 361 | expect(git(repo, "rev-parse", "refs/heads/main")).toBe(base); | |
| 362 | expect(await sweepGreen(app, repo)).toBe(1); | |
| 363 | expect(merged(repo, pull)).toBe(true); | |
| 364 | }); | |
| 365 | ||
| 366 | test("a red cut off mid-bisection hands no one back; the line lands on the next sweep", async () => { | |
| 367 | const repo = await autonomousRepo("cut-off-bisect"); | |
| 368 | const pulls = []; | |
| 369 | for (const name of ["a.ts", "b.ts", "red-c.ts", "d.ts"]) pulls.push(await greenPull(repo, name)); | |
| 370 | const red = pulls[2]!; | |
| 371 | // The whole candidate is red; the gate of its front half is cut off. | |
| 372 | let calls = 0; | |
| 373 | const stopping = async () => { | |
| 374 | calls++; | |
| 375 | return calls === 1 | |
| 376 | ? ({ lands: false, reopen: true, comment: "red", learn: [] } as const) | |
| 377 | : ({ lands: false, interrupted: true } as const); | |
| 378 | }; | |
| 379 | const dir = repoDir(app.reposRoot, repo.ownerName, repo.name); | |
| 380 | const told: number[] = []; | |
| 381 | const answers = await enqueue( | |
| 382 | repo.id, | |
| 383 | pulls.map((pull) => ({ | |
| 384 | pull, | |
| 385 | dir, | |
| 386 | land: async () => undefined, | |
| 387 | release: () => {}, | |
| 388 | handBack: () => told.push(pull.number), | |
| 389 | })), | |
| 390 | stopping, | |
| 391 | ); | |
| 392 | expect(calls).toBe(2); | |
| 393 | expect(answers).toEqual([false, false, false, false]); | |
| 394 | expect(told).toEqual([]); | |
| 395 | expect(await sweepGreen(app, repo)).toBe(3); | |
| 396 | expect(merged(repo, red)).toBe(false); | |
| 397 | expect(latestDecision(app.db, red.id, "land")?.answer).toBe("red"); | |
| 398 | }); | |
| 399 | ||
| 400 | test("a red neither side moved off is gated once more after 30 minutes, and lands if it was not the code's (#3155)", async () => { | |
| 401 | const repo = await autonomousRepo("regate-green"); | |
| 402 | const pull = await greenPull(repo, "a.ts"); | |
| 403 | flaky = 1; | |
| 404 | expect(await sweepGreen(app, repo)).toBe(0); | |
| 405 | expect(latestDecision(app.db, pull.id, "land")?.answer).toBe("red"); | |
| 406 | // Fresh: the red stands, and is not gated again. | |
| 407 | const before = gatesOf(repo).length; | |
| 408 | expect(await sweepGreen(app, repo)).toBe(0); | |
| 409 | expect(gatesOf(repo)).toHaveLength(before); | |
| 410 | // Past the wait, with nothing moved: gated once more, green, landed. | |
| 411 | age(pull); | |
| 412 | expect(await sweepGreen(app, repo)).toBe(1); | |
| 413 | expect(gatesOf(repo)).toHaveLength(before + 1); | |
| 414 | expect(merged(repo, pull)).toBe(true); | |
| 415 | }); | |
| 416 | ||
| 417 | test("a red that reproduces on the re-gate stays red, told once", async () => { | |
| 418 | const repo = await autonomousRepo("regate-red"); | |
| 419 | const pull = await greenPull(repo, "red-a.ts"); | |
| 420 | expect(await sweepGreen(app, repo)).toBe(0); | |
| 421 | const comments = listComments(app.db, pull.id).length; | |
| 422 | expect(comments).toBe(1); | |
| 423 | age(pull); | |
| 424 | expect(await sweepGreen(app, repo)).toBe(0); | |
| 425 | const again = latestDecision(app.db, pull.id, "land"); | |
| 426 | expect(again?.answer).toBe("red"); | |
| 427 | expect((again?.state as { again?: boolean }).again).toBe(true); | |
| 428 | expect(regateDue(again!, Date.now() + 10 * REGATE_RED_MS)).toBe(false); | |
| 429 | expect(listComments(app.db, pull.id)).toHaveLength(comments); | |
| 430 | // However long it waits now, it is not gated again until a side moves. | |
| 431 | age(pull); | |
| 432 | const before = gatesOf(repo).length; | |
| 433 | expect(await sweepGreen(app, repo)).toBe(0); | |
| 434 | expect(gatesOf(repo)).toHaveLength(before); | |
| 435 | }); | |
| 331 | 436 | }); |
new above
offline