Public: anyone can read this; the crew can change it.
P0: the merge queue records and names the tests that failed each gate (not just the count), with a gates API, so a red batch can be acted on #6740
torque/forge:torque/6739-p0-the-merge-queue-records-and-names-the → main 1 commit fork
18 files
changed +797
−7
docs/MCP.md +1
| @@ -114,6 +114,7 @@ for a tunnel and a forge that sleeps. | ||
| 114 | 114 | | `propose_to_parent_forge` | `{ repo }`: offer this forge's landed change to its parent as a pull on `child/<name>/…`; the same tip answers with that pull | yes | |
| 115 | 115 | | `pull` | `{ repo, number }`: a pull's title, state, head and base, and for an open pull whether it merges into the base's current head (`mergeable`) and the paths that stop it (`conflicts`), and `next: { who, why }` (open only; who acts next: builder, reviewer, queue or person), and `check: { state, baseOid }`: the latest check's state and the base tip it was merged with (empty when it merged nothing); `mergeable` and `conflicts` are the same fields as `GET …/pulls/:number`, `check` is `state` and `run.baseOid` from `GET …/pulls/:number/checks`; draws the `ui://forge/pull` card: `#N title`, state with check and conflicts while open, `head → base` linked to the pull, and the preview link when the pull has an environment | | |
| 116 | 116 | | `pulls` | `{ repo, state?, limit? }`: a repository's pulls, newest first, one brief line each: `number`, `title`, `author`, `head`, `check` (state), `approvals`, `changesRequested`, `mergeable` (null once closed); `state` open (default) or closed, `limit` 1-50 (default 20); no bodies, no diffs | | |
| 117 | | `queue_gates` | `{ repo, limit? }`: the merge queue's recent gates, newest first (`limit` 1–100, default 20), as `GET …/queue/gates` answers: each gate's pulls, state, reason, counts and failing tests by name with their first error lines, a test red in two gates that share no pull flagged `flaky` | | | |
| 117 | 118 | | `quotas` | `{ key?, value?, dryRun? }` (forge admin): the per-account quotas `repositories`, `apps`, `storage` (MB), `previews`, `runs`, `checks`, each a number or null for none; enforced when the thing is made or started, never on a forge admin; with `key` and `value` (`""` clears), planned first (`dryRun` is true unless `false`); applying needs an admin-scope token | yes | |
| 118 | 119 | | `read_file` | a file (utf-8, or base64 for binary) or a directory listing at a ref, as GET …/contents | | |
| 119 | 120 | | `releases` | `{}`: the forge's kept releases newest first: short sha, version, the pull that landed it, when, and for the live one `live: true` and `since`; for whoever may read /releases, others get its refusal; no rollback, a person does that on the page | | |
docs/features/queue-gates.md added +33
| @@ -0,0 +1,33 @@ | ||
| 1 | # The merge queue's gates, by failing test | |
| 2 | ||
| 3 | Every gate the merge queue runs — a batch, a search prefix, a suspect alone — | |
| 4 | is kept for 7 days in `gate_result` (`src/checks/gate_result.ts`): its pulls, | |
| 5 | `green`, `red` or `cut_off`, the reason (`tests`, `typecheck`, `install`, …), | |
| 6 | how many tests passed and failed, up to 20 failing tests by name with their | |
| 7 | file and the first 5 lines of their error, and how long it took. | |
| 8 | ||
| 9 | The queue's log line names the tests: | |
| 10 | ||
| 11 | ||
| 12 | gated 8 pulls: red (2 failed: "a > b", "c > d") | |
| 13 | ||
| 14 | ||
| 15 | Read them: | |
| 16 | ||
| 17 | sh | |
| 18 | curl -sH "authorization: Bearer " \ | |
| 19 | "/api/v1/repos/<owner>/<name>/queue/gates?limit=20" | |
| 20 | ||
| 21 | ||
| 22 | `{ items, next_cursor }`, newest first; `limit` is 1–100 (default 20), and | |
| 23 | `cursor` is the last item's `id`. An agent asks the MCP tool `queue_gates` | |
| 24 | with `{ repo, limit? }` and gets the same gates. | |
| 25 | ||
| 26 | Each failing test carries `flaky`: `true` when the same test (file and name) | |
| 27 | failed in two red gates of the week that share no pull — no single pull's | |
| 28 | code explains both. | |
| 29 | ||
| 30 | The counts come from the gate's own output (`src/checks/gate_tests.ts`): | |
| 31 | `passed` sums each `bun test` process's ` N pass`; `failed` counts distinct | |
| 32 | `(fail)` names, so the shards' tally — which repeats every failure as | |
| 33 | `(fail) name [1ms] [shard 3]` — no longer counts a test twice. |
src/checks/failures.ts +4 −2
| @@ -1,11 +1,13 @@ | ||
| 1 | 1 | /** |
| 2 | 2 | * The red tests in a full gate's output: `bun test` prints the file as |
| 3 | 3 | * `test/x.test.ts:` before its results and `(fail) name` for each failure. |
| 4 | * Plain lines in, names and files out. | |
| 4 | * The shards' tally repeats each as `(fail) name [1ms] [shard 3]`: one test, | |
| 5 | * counted once (#6739). Plain lines in, names and files out. | |
| 5 | 6 | */ |
| 6 | 7 | |
| 8 | import { FAIL } from "./gate_tests.ts"; | |
| 9 | ||
| 7 | 10 | const HEADER = /^([A-Za-z0-9_./-]+\.test\.[cm]?[jt]sx?):$/; |
| 8 | const FAIL = /^\(fail\) (.+?)(?: \[[0-9.]+m?s\])?$/; | |
| 9 | 11 | |
| 10 | 12 | export function failuresOf(lines: readonly string[]): { |
| 11 | 13 | tests: string[]; |
src/checks/gate_reason.ts +18
| @@ -15,6 +15,8 @@ | ||
| 15 | 15 | * Plain data in, plain data out: no I/O here. |
| 16 | 16 | */ |
| 17 | 17 | |
| 18 | import type { GateTests } from "./gate_tests.ts"; | |
| 19 | ||
| 18 | 20 | /** Where in its run the gate stopped red, as the check service saw it. */ |
| 19 | 21 | export type GateStage = "setup" | "install" | "check" | "timeout"; |
| 20 | 22 | |
| @@ -57,6 +59,22 @@ export function reasonWords(reason: GateReason | undefined): string { | ||
| 57 | 59 | return reason.kind === "tests" ? `tests: ${reason.failed} failed` : reason.kind; |
| 58 | 60 | } |
| 59 | 61 | |
| 62 | /** The most failing tests the queue's log line names; the rest are counted. */ | |
| 63 | export const LOG_NAMES = 5; | |
| 64 | ||
| 65 | /** | |
| 66 | * A red in the queue's log (#6739): the failing tests by name when the gate | |
| 67 | * named any — `2 failed: "a > b", "c > d"`, `… and 3 more` past | |
| 68 | * `LOG_NAMES` — and otherwise its reason, as `reasonWords` says it. | |
| 69 | */ | |
| 70 | export function redWords(reason: GateReason | undefined, tests: GateTests | undefined): string { | |
| 71 | if (tests === undefined || tests.failing.length === 0) return reasonWords(reason); | |
| 72 | const failed = Math.max(tests.failed, tests.failing.length); | |
| 73 | const named = tests.failing.slice(0, LOG_NAMES).map((t) => JSON.stringify(t.name)); | |
| 74 | const more = failed - named.length; | |
| 75 | return `${failed} failed: ${named.join(", ")}${more > 0 ? ` and ${more} more` : ""}`; | |
| 76 | } | |
| 77 | ||
| 60 | 78 | /** |
| 61 | 79 | * A merge conflict in a batch, the earlier pull first and the one left out |
| 62 | 80 | * second: `merge-conflict (#5088 × #5090)`; `main × #5090` against the base. |
src/checks/gate_result.ts added +214
| @@ -0,0 +1,214 @@ | ||
| 1 | /** | |
| 2 | * What each merge-queue gate found (#6739), one `gate_result` row a gate: | |
| 3 | * the pulls its candidate carried, green, red or cut off, why it went red, | |
| 4 | * how many tests passed and failed, the failing tests by name with the | |
| 5 | * first lines of their errors (`checks/gate_tests.ts`), and how long it | |
| 6 | * took. Before this a red gate said only a count, and nobody could say | |
| 7 | * which tests; `GET …/queue/gates` and the `queue_gates` MCP tool read it. | |
| 8 | * | |
| 9 | * A test is flaky when it failed in two red gates that share no pull: no | |
| 10 | * one pull's code explains both. Each failing test read back carries that | |
| 11 | * flag, worked out from the week the table keeps. | |
| 12 | * | |
| 13 | * Plain rows in, plain records out; what is read back is checked here, the | |
| 14 | * boundary, so a row this build cannot read loses only the part it cannot. | |
| 15 | */ | |
| 16 | ||
| 17 | import type { Database } from "bun:sqlite"; | |
| 18 | import type { FailedTest } from "./gate_tests.ts"; | |
| 19 | import { MAX_ERROR_LINES, MAX_FAILING } from "./gate_tests.ts"; | |
| 20 | ||
| 21 | /** How long a gate's result is kept. */ | |
| 22 | export const KEEP_RESULT_MS = 7 * 24 * 3_600_000; | |
| 23 | ||
| 24 | /** The most red gates the flake check reads back: a busy week, bounded. */ | |
| 25 | const FLAKE_WINDOW = 500; | |
| 26 | ||
| 27 | export type GateState = "green" | "red" | "cut_off"; | |
| 28 | ||
| 29 | /** A gate as it finished, to be kept. */ | |
| 30 | export interface GateResult { | |
| 31 | readonly repoId: string; | |
| 32 | /** The check lane's run id: `gate-` and the tip's first 12. */ | |
| 33 | readonly runId: string; | |
| 34 | readonly tipOid: string; | |
| 35 | /** The pull numbers its candidate carried, in order. */ | |
| 36 | readonly pulls: readonly number[]; | |
| 37 | readonly state: GateState; | |
| 38 | /** Why it went red (`GateReason.kind`); none on green or cut off. */ | |
| 39 | readonly reason?: string; | |
| 40 | readonly passed: number; | |
| 41 | readonly failed: number; | |
| 42 | readonly failing: readonly FailedTest[]; | |
| 43 | readonly startedAt: string; | |
| 44 | readonly finishedAt: string; | |
| 45 | } | |
| 46 | ||
| 47 | /** A failing test read back, flagged when a red gate sharing no pull failed it too. */ | |
| 48 | export interface FailingTest extends FailedTest { | |
| 49 | readonly flaky: boolean; | |
| 50 | } | |
| 51 | ||
| 52 | /** A kept gate as the API answers it. */ | |
| 53 | export interface KeptResult { | |
| 54 | readonly id: string; | |
| 55 | readonly runId: string; | |
| 56 | readonly tipOid: string; | |
| 57 | readonly pulls: readonly number[]; | |
| 58 | readonly state: GateState; | |
| 59 | readonly reason: string | null; | |
| 60 | readonly passed: number; | |
| 61 | readonly failed: number; | |
| 62 | readonly failing: readonly FailingTest[]; | |
| 63 | readonly startedAt: string; | |
| 64 | readonly finishedAt: string; | |
| 65 | readonly durationMs: number; | |
| 66 | } | |
| 67 | ||
| 68 | interface Row { | |
| 69 | id: string; | |
| 70 | run_id: string; | |
| 71 | tip_oid: string; | |
| 72 | pulls: string; | |
| 73 | state: string; | |
| 74 | reason: string | null; | |
| 75 | passed: number; | |
| 76 | failed: number; | |
| 77 | tests: string; | |
| 78 | started_at: string; | |
| 79 | finished_at: string; | |
| 80 | duration_ms: number; | |
| 81 | } | |
| 82 | ||
| 83 | /** Keep one gate's result under `id`; results past `KEEP_RESULT_MS` go. */ | |
| 84 | export function keepGateResult(db: Database, id: string, result: GateResult, now = new Date()): void { | |
| 85 | db.query("DELETE FROM gate_result WHERE finished_at < ?1").run(new Date(now.getTime() - KEEP_RESULT_MS).toISOString()); | |
| 86 | const failing = result.failing.slice(0, MAX_FAILING).map((t) => ({ | |
| 87 | name: t.name, | |
| 88 | ...(t.file === undefined ? {} : { file: t.file }), | |
| 89 | errors: t.errors.slice(0, MAX_ERROR_LINES), | |
| 90 | })); | |
| 91 | const took = Math.max(0, Date.parse(result.finishedAt) - Date.parse(result.startedAt)) || 0; | |
| 92 | db.query( | |
| 93 | `INSERT OR REPLACE INTO gate_result | |
| 94 | (id, repo_id, run_id, tip_oid, pulls, state, reason, passed, failed, tests, started_at, finished_at, duration_ms) | |
| 95 | VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)`, | |
| 96 | ).run( | |
| 97 | id, | |
| 98 | result.repoId, | |
| 99 | result.runId, | |
| 100 | result.tipOid, | |
| 101 | JSON.stringify(result.pulls), | |
| 102 | result.state, | |
| 103 | result.reason ?? null, | |
| 104 | Math.max(0, Math.trunc(result.passed)), | |
| 105 | Math.max(0, Math.trunc(result.failed)), | |
| 106 | JSON.stringify(failing), | |
| 107 | result.startedAt, | |
| 108 | result.finishedAt, | |
| 109 | took, | |
| 110 | ); | |
| 111 | } | |
| 112 | ||
| 113 | /** | |
| 114 | * The repository's gates, newest first: at most `limit`, older than the | |
| 115 | * gate `before` names when it is given. Each failing test says whether it | |
| 116 | * is flaky. | |
| 117 | */ | |
| 118 | export function recentGates(db: Database, repoId: string, limit: number, before?: string): KeptResult[] { | |
| 119 | const n = Math.min(Math.max(Math.trunc(limit) || 20, 1), 100); | |
| 120 | const rows = | |
| 121 | before === undefined | |
| 122 | ? db | |
| 123 | .query<Row, [string, number]>("SELECT * FROM gate_result WHERE repo_id = ?1 ORDER BY id DESC LIMIT ?2") | |
| 124 | .all(repoId, n) | |
| 125 | : db | |
| 126 | .query<Row, [string, string, number]>( | |
| 127 | "SELECT * FROM gate_result WHERE repo_id = ?1 AND id < ?2 ORDER BY id DESC LIMIT ?3", | |
| 128 | ) | |
| 129 | .all(repoId, before, n); | |
| 130 | if (rows.length === 0) return []; | |
| 131 | const flaky = flakyTests(redGates(db, repoId)); | |
| 132 | return rows.map((row) => { | |
| 133 | const pulls = pullsOf(row.pulls); | |
| 134 | return { | |
| 135 | id: row.id, | |
| 136 | runId: row.run_id, | |
| 137 | tipOid: row.tip_oid, | |
| 138 | pulls, | |
| 139 | state: stateOf(row.state), | |
| 140 | reason: row.reason, | |
| 141 | passed: row.passed, | |
| 142 | failed: row.failed, | |
| 143 | failing: testsOf(row.tests).map((t) => ({ ...t, flaky: flaky.has(testKey(t)) })), | |
| 144 | startedAt: row.started_at, | |
| 145 | finishedAt: row.finished_at, | |
| 146 | durationMs: row.duration_ms, | |
| 147 | }; | |
| 148 | }); | |
| 149 | } | |
| 150 | ||
| 151 | /** The repository's last red gates that named tests: their pulls and failing tests. */ | |
| 152 | export function redGates(db: Database, repoId: string): { pulls: number[]; failing: FailedTest[] }[] { | |
| 153 | return db | |
| 154 | .query<{ pulls: string; tests: string }, [string, number]>( | |
| 155 | "SELECT pulls, tests FROM gate_result WHERE repo_id = ?1 AND state = 'red' AND failed > 0 ORDER BY id DESC LIMIT ?2", | |
| 156 | ) | |
| 157 | .all(repoId, FLAKE_WINDOW) | |
| 158 | .map((row) => ({ pulls: pullsOf(row.pulls), failing: testsOf(row.tests) })); | |
| 159 | } | |
| 160 | ||
| 161 | /** One test, in whichever gate: its file and its name. */ | |
| 162 | export function testKey(t: { readonly name: string; readonly file?: string }): string { | |
| 163 | return `${t.file ?? ""}\0${t.name}`; | |
| 164 | } | |
| 165 | ||
| 166 | /** | |
| 167 | * The tests that failed in two red gates sharing no pull (#6739): no one | |
| 168 | * pull's code explains both, so the test is flaky. By `testKey`. | |
| 169 | */ | |
| 170 | export function flakyTests(gates: readonly { pulls: readonly number[]; failing: readonly FailedTest[] }[]): Set<string> { | |
| 171 | const seen = new Map<string, (readonly number[])[]>(); | |
| 172 | const flaky = new Set<string>(); | |
| 173 | for (const gate of gates) { | |
| 174 | for (const t of gate.failing) { | |
| 175 | const key = testKey(t); | |
| 176 | const before = seen.get(key) ?? []; | |
| 177 | if (before.some((pulls) => !pulls.some((p) => gate.pulls.includes(p)))) flaky.add(key); | |
| 178 | before.push(gate.pulls); | |
| 179 | seen.set(key, before); | |
| 180 | } | |
| 181 | } | |
| 182 | return flaky; | |
| 183 | } | |
| 184 | ||
| 185 | function stateOf(text: string): GateState { | |
| 186 | return text === "green" || text === "red" ? text : "cut_off"; | |
| 187 | } | |
| 188 | ||
| 189 | function parsed(text: string): unknown { | |
| 190 | try { | |
| 191 | return JSON.parse(text); | |
| 192 | } catch { | |
| 193 | return undefined; | |
| 194 | } | |
| 195 | } | |
| 196 | ||
| 197 | function pullsOf(text: string): number[] { | |
| 198 | const list = parsed(text); | |
| 199 | return Array.isArray(list) ? list.filter((n): n is number => Number.isSafeInteger(n)) : []; | |
| 200 | } | |
| 201 | ||
| 202 | function testsOf(text: string): FailedTest[] { | |
| 203 | const list = parsed(text); | |
| 204 | if (!Array.isArray(list)) return []; | |
| 205 | const out: FailedTest[] = []; | |
| 206 | for (const item of list as unknown[]) { | |
| 207 | if (typeof item !== "object" || item === null) continue; | |
| 208 | const { name, file, errors } = item as Record<string, unknown>; | |
| 209 | if (typeof name !== "string") continue; | |
| 210 | const lines = Array.isArray(errors) ? errors.filter((l): l is string => typeof l === "string") : []; | |
| 211 | out.push({ name, ...(typeof file === "string" ? { file } : {}), errors: lines }); | |
| 212 | } | |
| 213 | return out; | |
| 214 | } |
src/checks/gate_tests.ts added +98
| @@ -0,0 +1,98 @@ | ||
| 1 | /** | |
| 2 | * What a gate's tests said (#6739): how many passed and failed, and each | |
| 3 | * failing test by name, with its file and the first lines of its error — | |
| 4 | * so a red gate names what to fix, not only how many. | |
| 5 | * | |
| 6 | * `bun test` prints a file as `test/x.test.ts:`, then for a failing test | |
| 7 | * its code frame, `error: …` and the stack, and last `(fail) name [1.2ms]`; | |
| 8 | * each process ends with ` N pass` / ` N fail`. `scripts/test-shards.ts` | |
| 9 | * runs several such processes and repeats every `(fail)` line in its tally | |
| 10 | * as `(fail) name [1.2ms] [shard 3]`: one test, named once here. | |
| 11 | * | |
| 12 | * Plain lines in, plain data out: no I/O. | |
| 13 | */ | |
| 14 | ||
| 15 | /** The most failing tests a gate keeps by name; the count is always whole. */ | |
| 16 | export const MAX_FAILING = 20; | |
| 17 | /** The most error lines kept per failing test. */ | |
| 18 | export const MAX_ERROR_LINES = 5; | |
| 19 | /** The longest error line kept, in characters. */ | |
| 20 | const MAX_LINE = 300; | |
| 21 | ||
| 22 | export interface FailedTest { | |
| 23 | readonly name: string; | |
| 24 | /** The test file it ran in, when the output said. */ | |
| 25 | readonly file?: string; | |
| 26 | /** The first lines of its error, from `error:` on; none when it printed none. */ | |
| 27 | readonly errors: readonly string[]; | |
| 28 | } | |
| 29 | ||
| 30 | export interface GateTests { | |
| 31 | /** Tests that passed, summed over every `bun test` process of the gate. */ | |
| 32 | readonly passed: number; | |
| 33 | /** Distinct tests that failed. */ | |
| 34 | readonly failed: number; | |
| 35 | /** The first `MAX_FAILING` of them, in the order the gate printed them. */ | |
| 36 | readonly failing: readonly FailedTest[]; | |
| 37 | } | |
| 38 | ||
| 39 | const HEADER = /^([A-Za-z0-9_./-]+\.test\.[cm]?[jt]sx?):$/; | |
| 40 | const RESULT = /^\((pass|fail|skip|todo)\) /; | |
| 41 | /** A failing test's line; its time and the tally's shard said apart from its name. */ | |
| 42 | export const FAIL = /^\(fail\) (.+?)(?: \[[0-9.]+m?s\])?(?: \[shard \d+\])?$/; | |
| 43 | const PASSED = /^\s*(\d+) pass$/; | |
| 44 | const SHARD_TALLY = / \[shard \d+\]$/; | |
| 45 | const ANSI = /\x1b\[[0-9;]*m/g; | |
| 46 | const FRAME = /^\s*\d+ \|/; | |
| 47 | const CARET = /^\s*\^\s*$/; | |
| 48 | const ERROR = /^\s*(error\b|\w*Error\b)/; | |
| 49 | ||
| 50 | /** The first lines of one failing test's error, read from the lines printed before its `(fail)`. */ | |
| 51 | function errorLines(block: readonly string[]): string[] { | |
| 52 | const start = block.findIndex((line) => ERROR.test(line)); | |
| 53 | const from = start < 0 ? block.filter((l) => !FRAME.test(l) && !CARET.test(l)) : block.slice(start); | |
| 54 | const lines = from.map((l) => l.trim()).filter((l) => l !== ""); | |
| 55 | const kept = start < 0 ? lines.slice(-MAX_ERROR_LINES) : lines.slice(0, MAX_ERROR_LINES); | |
| 56 | return kept.map((l) => (l.length > MAX_LINE ? `${l.slice(0, MAX_LINE - 1)}…` : l)); | |
| 57 | } | |
| 58 | ||
| 59 | export function gateTests(raw: readonly string[]): GateTests { | |
| 60 | let passed = 0; | |
| 61 | let file: string | undefined; | |
| 62 | let block: string[] = []; | |
| 63 | const failing = new Map<string, FailedTest>(); | |
| 64 | for (const text of raw) { | |
| 65 | const line = text.replace(ANSI, ""); | |
| 66 | const header = HEADER.exec(line); | |
| 67 | if (header) { | |
| 68 | file = header[1]; | |
| 69 | block = []; | |
| 70 | continue; | |
| 71 | } | |
| 72 | const pass = PASSED.exec(line); | |
| 73 | if (pass) { | |
| 74 | passed += Number(pass[1]); | |
| 75 | block = []; | |
| 76 | continue; | |
| 77 | } | |
| 78 | const fail = FAIL.exec(line); | |
| 79 | if (fail) { | |
| 80 | const name = fail[1]!; | |
| 81 | // The tally's repeat names a test already read with its error. | |
| 82 | if (!failing.has(name) && !SHARD_TALLY.test(line)) | |
| 83 | failing.set(name, { name, ...(file === undefined ? {} : { file }), errors: errorLines(block) }); | |
| 84 | else if (!failing.has(name)) failing.set(name, { name, errors: [] }); | |
| 85 | block = []; | |
| 86 | continue; | |
| 87 | } | |
| 88 | if (RESULT.test(line)) { | |
| 89 | block = []; | |
| 90 | continue; | |
| 91 | } | |
| 92 | block.push(line); | |
| 93 | // An error is printed just before its `(fail)`: a long stretch of other output is not it. | |
| 94 | if (block.length > 200) block.shift(); | |
| 95 | } | |
| 96 | const all = [...failing.values()]; | |
| 97 | return { passed, failed: all.length, failing: all.slice(0, MAX_FAILING) }; | |
| 98 | } |
src/checks/landing.ts +2 −1
| @@ -27,6 +27,7 @@ import type { Repo } from "../core/repos.ts"; | ||
| 27 | 27 | import { failuresOf } from "./failures.ts"; |
| 28 | 28 | import { keepForgeGate } from "./gate_last.ts"; |
| 29 | 29 | import { gateReason } from "./gate_reason.ts"; |
| 30 | import { gateTests } from "./gate_tests.ts"; | |
| 30 | 31 | import { keepGateLog } from "./gate_log.ts"; |
| 31 | 32 | import { GATE_DEADLINE_MS, type GateRecord } from "./queue.ts"; |
| 32 | 33 | import { gateRecorded, gateRunning, recordedVerdict } from "./queue_store.ts"; |
| @@ -108,7 +109,7 @@ function gateVerdict(app: App, repo: Repo, commitOid: string, done: GateDone): L | ||
| 108 | 109 | const link = kept === undefined ? undefined : `/api/v1/repos/${slug}/gates/${kept}/log`; |
| 109 | 110 | // Why it went red, said in the queue's log and read by its search (#5276). |
| 110 | 111 | const reason = done.ok ? undefined : gateReason(done.stage ?? "check", done.error, done.lines, failed.tests.length); |
| 111 | return landVerdict(done.ok, failed.tests, failed.files, link, reason); | |
| 112 | return landVerdict(done.ok, failed.tests, failed.files, link, reason, done.ok ? undefined : gateTests(done.lines)); | |
| 112 | 113 | } |
| 113 | 114 | |
| 114 | 115 | /** |
src/checks/queue.ts +31 −3
| @@ -48,6 +48,9 @@ | ||
| 48 | 48 | * red that way twice is infrastructure — no verdict, nothing handed back, |
| 49 | 49 | * nothing bisected. A pull that conflicts with an earlier one of its batch |
| 50 | 50 | * is never red: it is left out, the pair named, and the rest gate at once. |
| 51 | * A red with tests names them in the log — `red (2 failed: "a > b", "c")` — | |
| 52 | * and every finished gate is kept a week with its pulls, counts and failing | |
| 53 | * tests (#6739, `gate_result.ts`), read by `GET …/queue/gates`. | |
| 51 | 54 | * |
| 52 | 55 | * What is queued is not stored: it is the pulls `landIfGreen` admitted, and |
| 53 | 56 | * every sweep admits them again. The queue only orders them and remembers |
| @@ -60,7 +63,8 @@ import { runGit } from "../git/env.ts"; | ||
| 60 | 63 | import { mergeable, resolveOid, treeOf } from "../git/merge.ts"; |
| 61 | 64 | import { id } from "../kernel/id.ts"; |
| 62 | 65 | import { createLog } from "../kernel/log.ts"; |
| 63 | import { conflictWords, reasonWords, searched } from "./gate_reason.ts"; | |
| 66 | import { conflictWords, reasonWords, redWords, searched } from "./gate_reason.ts"; | |
| 67 | import type { GateResult } from "./gate_result.ts"; | |
| 64 | 68 | import type { LandVerdict } from "./tiers.ts"; |
| 65 | 69 | |
| 66 | 70 | /** The base a step must land on, and the tree it must write. */ |
| @@ -212,6 +216,8 @@ export interface QueueStore { | ||
| 212 | 216 | red(repoId: string, pullId: string, mark: Remembered | undefined): void; |
| 213 | 217 | /** The repository's red marks, by pull id. */ |
| 214 | 218 | reds(repoId: string): Map<string, Remembered>; |
| 219 | /** A gate finished — green, red or cut off: what it found, kept a week (#6739). */ | |
| 220 | judged?(id: string, result: GateResult): void; | |
| 215 | 221 | } |
| 216 | 222 | |
| 217 | 223 | /** The check lane's run id for a gate of `tipOid` (`checks/service.ts` names it so). */ |
| @@ -638,10 +644,31 @@ function hhmm(ms: number): string { | ||
| 638 | 644 | return new Date(ms).toISOString().slice(11, 16); |
| 639 | 645 | } |
| 640 | 646 | |
| 641 | /** A gate's verdict in a few words, for the log: a red says why (#5276). */ | |
| 647 | /** | |
| 648 | * A gate's verdict in a few words, for the log: a red says why (#5276), | |
| 649 | * naming its failing tests when it read any (#6739). | |
| 650 | */ | |
| 642 | 651 | function outcome(verdict: LandVerdict): string { |
| 643 | 652 | if (verdict.lands) return "green"; |
| 644 | return "interrupted" in verdict ? "cut off" : `red (${reasonWords(verdict.reason)})`; | |
| 653 | return "interrupted" in verdict ? "cut off" : `red (${redWords(verdict.reason, verdict.tests)})`; | |
| 654 | } | |
| 655 | ||
| 656 | /** What a finished gate found, as `gate_result` keeps it (#6739). */ | |
| 657 | function resultOf(repoId: string, record: GateRecord, candidate: Candidate, verdict: LandVerdict, since: number): GateResult { | |
| 658 | const red = !verdict.lands && !("interrupted" in verdict) ? verdict : undefined; | |
| 659 | return { | |
| 660 | repoId, | |
| 661 | runId: record.runId, | |
| 662 | tipOid: candidate.tipOid, | |
| 663 | pulls: candidate.steps.map((s) => s.entry.pull.number), | |
| 664 | state: verdict.lands ? "green" : red === undefined ? "cut_off" : "red", | |
| 665 | ...(red?.reason === undefined ? {} : { reason: red.reason.kind }), | |
| 666 | passed: red?.tests?.passed ?? 0, | |
| 667 | failed: red?.tests?.failed ?? (red?.reason?.kind === "tests" ? red.reason.failed : 0), | |
| 668 | failing: red?.tests?.failing ?? [], | |
| 669 | startedAt: new Date(since).toISOString(), | |
| 670 | finishedAt: new Date().toISOString(), | |
| 671 | }; | |
| 645 | 672 | } |
| 646 | 673 | |
| 647 | 674 | /** A red that names no pull's code — install, setup, timeout: gated once more, never searched. */ |
| @@ -743,6 +770,7 @@ function attempt(line: Line, gate: GateCandidate, adopt: QueueOptions["adopt"]): | ||
| 743 | 770 | took: `${Math.round((Date.now() - flight.since) / 1000)}s`, |
| 744 | 771 | ...(adopted === undefined ? {} : { adopted: record.runId }), |
| 745 | 772 | }); |
| 773 | kept(() => line.store?.judged?.(id(), resultOf(line.repoId, record, candidate, verdict, flight.since))); | |
| 746 | 774 | // A stopped line acts on nothing: a verdict is kept for the line that |
| 747 | 775 | // carries on, and so is a gate its forge stopping cut off. Anything |
| 748 | 776 | // else is acted on now, or was abandoned: not to resume. |
src/checks/queue_store.ts +27 −1
| @@ -12,6 +12,8 @@ | ||
| 12 | 12 | import type { Database } from "bun:sqlite"; |
| 13 | 13 | import type { GateRecord, QueueStore, RecordedStep, Remembered } from "./queue.ts"; |
| 14 | 14 | import type { GateReason } from "./gate_reason.ts"; |
| 15 | import { keepGateResult } from "./gate_result.ts"; | |
| 16 | import type { FailedTest, GateTests } from "./gate_tests.ts"; | |
| 15 | 17 | import type { LandVerdict } from "./tiers.ts"; |
| 16 | 18 | |
| 17 | 19 | /** How long a gate's record is kept unresumed: past it, its line has long moved on. */ |
| @@ -43,6 +45,7 @@ export function queueStore(db: Database, now: () => Date = () => new Date()): Qu | ||
| 43 | 45 | gates: (repoId) => gateRecords(db, repoId), |
| 44 | 46 | red: (repoId, pullId, mark) => keepRed(db, repoId, pullId, mark, now()), |
| 45 | 47 | reds: (repoId) => redMarks(db, repoId), |
| 48 | judged: (id, result) => keepGateResult(db, id, result, now()), | |
| 46 | 49 | }; |
| 47 | 50 | } |
| 48 | 51 | |
| @@ -143,7 +146,30 @@ function verdictOf(text: string): Judged | undefined { | ||
| 143 | 146 | if (!Array.isArray(learn) || !learn.every((f) => typeof f === "string")) return undefined; |
| 144 | 147 | const why = reasonOf(reason); |
| 145 | 148 | if (reason !== undefined && why === undefined) return undefined; |
| 146 | return { lands: false, reopen: true, comment, learn: learn as string[], ...(why === undefined ? {} : { reason: why }) }; | |
| 149 | const named = gateTestsOf((v as Record<string, unknown>)["tests"]); | |
| 150 | return { | |
| 151 | lands: false, | |
| 152 | reopen: true, | |
| 153 | comment, | |
| 154 | learn: learn as string[], | |
| 155 | ...(why === undefined ? {} : { reason: why }), | |
| 156 | ...(named === undefined ? {} : { tests: named }), | |
| 157 | }; | |
| 158 | } | |
| 159 | ||
| 160 | /** A recorded red's tests (#6739), read back; undefined when they do not read. */ | |
| 161 | function gateTestsOf(v: unknown): GateTests | undefined { | |
| 162 | if (typeof v !== "object" || v === null) return undefined; | |
| 163 | const { passed, failed, failing } = v as Record<string, unknown>; | |
| 164 | if (!Number.isSafeInteger(passed) || !Number.isSafeInteger(failed) || !Array.isArray(failing)) return undefined; | |
| 165 | const tests: FailedTest[] = []; | |
| 166 | for (const item of failing as unknown[]) { | |
| 167 | if (typeof item !== "object" || item === null) return undefined; | |
| 168 | const { name, file, errors } = item as Record<string, unknown>; | |
| 169 | if (typeof name !== "string" || !Array.isArray(errors) || !errors.every((l) => typeof l === "string")) return undefined; | |
| 170 | tests.push({ name, ...(typeof file === "string" ? { file } : {}), errors: errors as string[] }); | |
| 171 | } | |
| 172 | return { passed: passed as number, failed: failed as number, failing: tests }; | |
| 147 | 173 | } |
| 148 | 174 | |
| 149 | 175 | /** Why a recorded red went red (#5276), read back; undefined when it is not one. */ |
src/checks/tiers.ts +5
| @@ -10,6 +10,7 @@ | ||
| 10 | 10 | */ |
| 11 | 11 | |
| 12 | 12 | import type { GateReason } from "./gate_reason.ts"; |
| 13 | import type { GateTests } from "./gate_tests.ts"; | |
| 13 | 14 | import { GATE_FILE, needsFull, reach, reachLine, type Impact } from "./impact.ts"; |
| 14 | 15 | |
| 15 | 16 | /** A test under this many seconds is always in the fast tier. */ |
| @@ -253,6 +254,8 @@ export type Judged = | ||
| 253 | 254 | readonly learn: readonly string[]; |
| 254 | 255 | /** Why the gate went red (#5276); a red read without one is the code's. */ |
| 255 | 256 | readonly reason?: GateReason; |
| 257 | /** What its tests said: the counts and the failing tests by name (#6739). */ | |
| 258 | readonly tests?: GateTests; | |
| 256 | 259 | }; |
| 257 | 260 | |
| 258 | 261 | /** |
| @@ -275,6 +278,7 @@ export function landVerdict( | ||
| 275 | 278 | failedFiles: readonly string[] = [], |
| 276 | 279 | logLink?: string, |
| 277 | 280 | reason?: GateReason, |
| 281 | tests?: GateTests, | |
| 278 | 282 | ): Judged { |
| 279 | 283 | if (green) return { lands: true }; |
| 280 | 284 | const names = failedTests.map((name) => `- ${name}`).join("\n"); |
| @@ -288,5 +292,6 @@ export function landVerdict( | ||
| 288 | 292 | : `The full gate on land was red; the pull is open again. Red tests:\n${names}${kept}`, |
| 289 | 293 | learn: [...new Set(failedFiles)], |
| 290 | 294 | ...(reason === undefined ? {} : { reason }), |
| 295 | ...(tests === undefined ? {} : { tests }), | |
| 291 | 296 | }; |
| 292 | 297 | } |
src/contract/openapi/routes/getApiV1ReposOwnerNameQueueGates.ts added +3
| @@ -0,0 +1,3 @@ | ||
| 1 | import { REPO, type Route } from "../shared.ts"; | |
| 2 | ||
| 3 | export const route: Route = { method: "get", path: `${REPO}/queue/gates`, summary: "List the merge queue's recent gates, newest first: { items, next_cursor }; each with its pulls, state (green, red, cut_off), reason, passed and failed counts, the failing tests (up to 20) by name with their file, first error lines and a flaky flag (red in two gates that share no pull), and its duration; kept 7 days; limit 1 to 100 (default 20), cursor a gate id", auth: false, query: { limit: "integer", cursor: "string" } }; |
src/contract/openapi/routes/index.ts +1
| @@ -108,6 +108,7 @@ export { route as getApiV1ReposOwnerNamePullsNumberChecks } from "./getApiV1Repo | ||
| 108 | 108 | export { route as getApiV1ReposOwnerNamePullsNumberReviewStates } from "./getApiV1ReposOwnerNamePullsNumberReviewStates.ts"; |
| 109 | 109 | export { route as getApiV1ReposOwnerNamePullsNumberReviews } from "./getApiV1ReposOwnerNamePullsNumberReviews.ts"; |
| 110 | 110 | export { route as getApiV1ReposOwnerNamePullsNumberThreads } from "./getApiV1ReposOwnerNamePullsNumberThreads.ts"; |
| 111 | export { route as getApiV1ReposOwnerNameQueueGates } from "./getApiV1ReposOwnerNameQueueGates.ts"; | |
| 111 | 112 | export { route as getApiV1ReposOwnerNameReports } from "./getApiV1ReposOwnerNameReports.ts"; |
| 112 | 113 | export { route as getApiV1ReposOwnerNameRuns } from "./getApiV1ReposOwnerNameRuns.ts"; |
| 113 | 114 | export { route as getApiV1ReposOwnerNameRunsId } from "./getApiV1ReposOwnerNameRunsId.ts"; |
src/core/migrations/20261027041521-gate-result.ts added +25
| @@ -0,0 +1,25 @@ | ||
| 1 | // What each merge-queue gate found (#6739, checks/gate_result.ts): its pulls, | |
| 2 | // its state, why it went red, how many tests passed and failed, the failing | |
| 3 | // tests by name with the first lines of their errors (`tests`, JSON, at most | |
| 4 | // 20), and how long it took — so a red batch names what to fix, and a test | |
| 5 | // red in two batches that share no pull reads as flaky. Pruned past a week | |
| 6 | // as new rows are written. Additive. | |
| 7 | export default ` | |
| 8 | CREATE TABLE IF NOT EXISTS gate_result ( | |
| 9 | id TEXT PRIMARY KEY, | |
| 10 | repo_id TEXT NOT NULL REFERENCES repo(id) ON DELETE CASCADE, | |
| 11 | run_id TEXT NOT NULL, | |
| 12 | tip_oid TEXT NOT NULL, | |
| 13 | pulls TEXT NOT NULL DEFAULT '[]', | |
| 14 | state TEXT NOT NULL, | |
| 15 | reason TEXT, | |
| 16 | passed INTEGER NOT NULL DEFAULT 0, | |
| 17 | failed INTEGER NOT NULL DEFAULT 0, | |
| 18 | tests TEXT NOT NULL DEFAULT '[]', | |
| 19 | started_at TEXT NOT NULL, | |
| 20 | finished_at TEXT NOT NULL, | |
| 21 | duration_ms INTEGER NOT NULL DEFAULT 0 | |
| 22 | ); | |
| 23 | CREATE INDEX IF NOT EXISTS gate_result_repo ON gate_result (repo_id, id); | |
| 24 | CREATE INDEX IF NOT EXISTS gate_result_finished ON gate_result (finished_at); | |
| 25 | `; |
src/http/api.ts +16
| @@ -390,6 +390,7 @@ import { readReturns, reportSpend, RETURNS_DAYS } from "../actions/returns.ts"; | ||
| 390 | 390 | import { fleetSelection, readFleetGraph } from "../actions/fleet-graph.ts"; |
| 391 | 391 | import { carriedBy } from "../core/carried.ts"; |
| 392 | 392 | import { readGateLog } from "../checks/gate_log.ts"; |
| 393 | import { recentGates } from "../checks/gate_result.ts"; | |
| 393 | 394 | import { readRunLog, subscribeRunLog } from "../runs/log.ts"; |
| 394 | 395 | import { blobForRender, commitLog, listRefs, listTree } from "../git/read.ts"; |
| 395 | 396 | import { repoDir } from "../git/repos.ts"; |
| @@ -6288,6 +6289,21 @@ export function apiRoutes(router: Router<Handler>): void { | ||
| 6288 | 6289 | return timePhase(ctx.request, "render", () => json(found)); |
| 6289 | 6290 | }); |
| 6290 | 6291 | |
| 6292 | // The merge queue's recent gates (#6739, `checks/gate_result.ts`): newest | |
| 6293 | // first, each with its pulls, verdict, counts and failing tests by name, | |
| 6294 | // a test red in two gates that share no pull flagged `flaky`. | |
| 6295 | router.add("GET", "/api/v1/repos/:owner/:name/queue/gates", (ctx) => { | |
| 6296 | const repo = loadRepo(ctx); | |
| 6297 | if (!repo) return notFound(); | |
| 6298 | const q = ctx.url.searchParams; | |
| 6299 | const limit = Math.min(Math.max(Number(q.get("limit")) || 20, 1), 100); | |
| 6300 | const cursor = q.get("cursor") ?? undefined; | |
| 6301 | if (cursor !== undefined && !/^[0-9A-Za-z]{1,40}$/.test(cursor)) | |
| 6302 | return json({ error: "invalid", message: "`cursor` is a gate id from `next_cursor`" }, 422); | |
| 6303 | const items = recentGates(ctx.app.db, repo.id, limit, cursor); | |
| 6304 | return json(paged(items, limit, (g) => g.id), 200, { "cache-control": "no-store" }); | |
| 6305 | }); | |
| 6306 | ||
| 6291 | 6307 | // A kept gate's log (`checks/gate_log.ts`): the last 200 lines, as text. |
| 6292 | 6308 | router.add("GET", "/api/v1/repos/:owner/:name/gates/:id/log", (ctx) => { |
| 6293 | 6309 | const repo = loadRepo(ctx); |
src/http/mcp/tools/index.ts +1
| @@ -91,6 +91,7 @@ export { tool as proposal_tokens } from "./proposal_tokens.ts"; | ||
| 91 | 91 | export { tool as propose_to_parent_forge } from "./propose_to_parent_forge.ts"; |
| 92 | 92 | export { tool as pull } from "./pull.ts"; |
| 93 | 93 | export { tool as pulls } from "./pulls.ts"; |
| 94 | export { tool as queue_gates } from "./queue_gates.ts"; | |
| 94 | 95 | export { tool as quotas } from "./quotas.ts"; |
| 95 | 96 | export { tool as read_file } from "./read_file.ts"; |
| 96 | 97 | export { tool as releases } from "./releases.ts"; |
src/http/mcp/tools/queue_gates.ts added +27
| @@ -0,0 +1,27 @@ | ||
| 1 | import { recentGates } from "../../../checks/gate_result.ts"; | |
| 2 | import { good, bad, type Tool, REPO_ARG, repoArg, isAnswer } from "../shared.ts"; | |
| 3 | ||
| 4 | export const tool: Tool = { | |
| 5 | name: "queue_gates", | |
| 6 | doc: "`{ repo, limit? }`: the merge queue's recent gates, newest first (`limit` 1–100, default 20), as `GET …/queue/gates` answers: each gate's pulls, state, reason, counts and failing tests by name with their first error lines, a test red in two gates that share no pull flagged `flaky`", | |
| 7 | description: "Recent merge-queue gates: pulls, verdicts, failing tests.", | |
| 8 | inputSchema: { | |
| 9 | type: "object", | |
| 10 | properties: { | |
| 11 | repo: REPO_ARG, | |
| 12 | limit: { type: "integer", minimum: 1, maximum: 100 }, | |
| 13 | }, | |
| 14 | required: ["repo"], | |
| 15 | }, | |
| 16 | writes: false, | |
| 17 | run(ctx, user, args) { | |
| 18 | const repo = repoArg(ctx, user, args); | |
| 19 | if (isAnswer(repo)) return repo; | |
| 20 | const limit = args["limit"] ?? 20; | |
| 21 | if (typeof limit !== "number" || !Number.isInteger(limit) || limit < 1 || limit > 100) | |
| 22 | return bad("invalid", "`limit` is a whole number from 1 to 100"); | |
| 23 | const gates = recentGates(ctx.app.db, repo.id, limit); | |
| 24 | const red = gates.filter((g) => g.state === "red").length; | |
| 25 | return good({ says: `${gates.length} gate(s), ${red} red.`, gates }); | |
| 26 | }, | |
| 27 | }; |
test/mcp.test.ts +1
| @@ -192,6 +192,7 @@ describe("the forge over MCP", () => { | ||
| 192 | 192 | "pull", |
| 193 | 193 | "preview", |
| 194 | 194 | "pulls", |
| 195 | "queue_gates", | |
| 195 | 196 | "variant", |
| 196 | 197 | "edit", |
| 197 | 198 | "close", |
test/merge-queue-gate-result.test.ts added +290
| @@ -0,0 +1,290 @@ | ||
| 1 | /** | |
| 2 | * The merge queue names the tests that failed each gate (#6739), not just | |
| 3 | * how many: the gate's output is read for its counts and its failing tests | |
| 4 | * with their first error lines (`checks/gate_tests.ts`), the queue's log | |
| 5 | * line names them, every finished gate is kept a week in `gate_result`, | |
| 6 | * and `GET …/queue/gates` and the `queue_gates` MCP tool answer them. A test | |
| 7 | * red in two gates that share no pull is flagged flaky. | |
| 8 | */ | |
| 9 | import { afterAll, beforeAll, describe, expect, spyOn, test } from "bun:test"; | |
| 10 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 11 | import { tmpdir } from "node:os"; | |
| 12 | import { join } from "node:path"; | |
| 13 | import { createApp, type App } from "../src/app.ts"; | |
| 14 | import { openPull } from "../src/actions/pulls.ts"; | |
| 15 | import { commitToBranch, createRepo } from "../src/actions/repos.ts"; | |
| 16 | import { failuresOf } from "../src/checks/failures.ts"; | |
| 17 | import { redWords } from "../src/checks/gate_reason.ts"; | |
| 18 | import { flakyTests, keepGateResult, recentGates, testKey } from "../src/checks/gate_result.ts"; | |
| 19 | import { gateTests } from "../src/checks/gate_tests.ts"; | |
| 20 | import { stopQueue } from "../src/checks/queue.ts"; | |
| 21 | import { provideChecksForTest, type ChecksHandle, type GateDone } from "../src/checks/service.ts"; | |
| 22 | import { createToken } from "../src/core/tokens.ts"; | |
| 23 | import { getPull, type Pull } from "../src/core/pulls.ts"; | |
| 24 | import { getRepoById, type Repo } from "../src/core/repos.ts"; | |
| 25 | import { claimRun, createRun, finishRun } from "../src/core/runs.ts"; | |
| 26 | import { createUser, type User } from "../src/core/users.ts"; | |
| 27 | import { sweepGreen } from "../src/crew/autonomy.ts"; | |
| 28 | import { repoDir } from "../src/git/repos.ts"; | |
| 29 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 30 | import { mcpTool } from "./helpers/mcp.ts"; | |
| 31 | ||
| 32 | /** A sharded gate's output: two shards, a test red in each, and the tally repeating both. */ | |
| 33 | const RED_OUTPUT = [ | |
| 34 | "── shard 1/2 (exit 1, 3.0s): test/a.test.ts", | |
| 35 | "bun test v1.3.14", | |
| 36 | "", | |
| 37 | "test/a.test.ts:", | |
| 38 | "(pass) a > fine [0.10ms]", | |
| 39 | '2 | test("a > b", () => { expect(1).toBe(2); });', | |
| 40 | " ^", | |
| 41 | "error: expect(received).toBe(expected)", | |
| 42 | "", | |
| 43 | "Expected: 2", | |
| 44 | "Received: 1", | |
| 45 | "", | |
| 46 | " at <anonymous> (/w/test/a.test.ts:2:33)", | |
| 47 | "(fail) a > b [0.15ms]", | |
| 48 | "", | |
| 49 | " 1 pass", | |
| 50 | " 1 fail", | |
| 51 | "── shard 2/2 (exit 1, 2.0s): test/c.test.ts", | |
| 52 | "test/c.test.ts:", | |
| 53 | "(pass) c > one [0.10ms]", | |
| 54 | "(pass) c > two [0.10ms]", | |
| 55 | "error: boom", | |
| 56 | "second line", | |
| 57 | " at <anonymous> (/w/test/c.test.ts:3:49)", | |
| 58 | "(fail) c > d [0.05ms]", | |
| 59 | " 2 pass", | |
| 60 | " 1 fail", | |
| 61 | "", | |
| 62 | "2 of 2 shards failed:", | |
| 63 | "(fail) a > b [0.15ms] [shard 1]", | |
| 64 | "(fail) c > d [0.05ms] [shard 2]", | |
| 65 | "Ran 5 tests across 2 files in 5.0s; 2 failed.", | |
| 66 | ]; | |
| 67 | ||
| 68 | describe("a gate's tests, read from its output", () => { | |
| 69 | test("two failing tests: both named once, with their files, errors and the counts", () => { | |
| 70 | const read = gateTests(RED_OUTPUT); | |
| 71 | expect(read.passed).toBe(3); | |
| 72 | expect(read.failed).toBe(2); | |
| 73 | expect(read.failing).toEqual([ | |
| 74 | { | |
| 75 | name: "a > b", | |
| 76 | file: "test/a.test.ts", | |
| 77 | errors: ["error: expect(received).toBe(expected)", "Expected: 2", "Received: 1", "at <anonymous> (/w/test/a.test.ts:2:33)"], | |
| 78 | }, | |
| 79 | { name: "c > d", file: "test/c.test.ts", errors: ["error: boom", "second line", "at <anonymous> (/w/test/c.test.ts:3:49)"] }, | |
| 80 | ]); | |
| 81 | }); | |
| 82 | ||
| 83 | test("the shards' tally does not count a failing test twice", () => { | |
| 84 | expect(failuresOf(RED_OUTPUT).tests).toEqual(["a > b", "c > d"]); | |
| 85 | }); | |
| 86 | ||
| 87 | test("at most 20 failing tests are named; the count stays whole", () => { | |
| 88 | const lines = ["test/x.test.ts:"]; | |
| 89 | for (let i = 0; i < 25; i++) lines.push(`error: e${i}`, `(fail) t${i} [1.00ms]`); | |
| 90 | const read = gateTests(lines); | |
| 91 | expect(read.failed).toBe(25); | |
| 92 | expect(read.failing).toHaveLength(20); | |
| 93 | expect(read.failing[0]).toEqual({ name: "t0", file: "test/x.test.ts", errors: ["error: e0"] }); | |
| 94 | }); | |
| 95 | ||
| 96 | test("the log's words name the failing tests, the rest counted", () => { | |
| 97 | const read = gateTests(RED_OUTPUT); | |
| 98 | expect(redWords({ kind: "tests", failed: 2 }, read)).toBe('2 failed: "a > b", "c > d"'); | |
| 99 | const many = { passed: 0, failed: 7, failing: Array.from({ length: 7 }, (_, i) => ({ name: `t${i}`, errors: [] })) }; | |
| 100 | expect(redWords({ kind: "tests", failed: 7 }, many)).toBe('7 failed: "t0", "t1", "t2", "t3", "t4" and 2 more'); | |
| 101 | // No test named: the reason, as before. | |
| 102 | expect(redWords({ kind: "typecheck" }, { passed: 0, failed: 0, failing: [] })).toBe("typecheck"); | |
| 103 | expect(redWords({ kind: "tests", failed: 2 }, undefined)).toBe("tests: 2 failed"); | |
| 104 | }); | |
| 105 | ||
| 106 | test("a test red in two gates that share no pull is flaky; one shared pull explains it", () => { | |
| 107 | const x = { name: "x", file: "test/x.test.ts", errors: [] }; | |
| 108 | const y = { name: "y", errors: [] }; | |
| 109 | const flaky = flakyTests([ | |
| 110 | { pulls: [1, 2], failing: [x, y] }, | |
| 111 | { pulls: [3], failing: [x] }, | |
| 112 | { pulls: [2, 5], failing: [y] }, | |
| 113 | ]); | |
| 114 | expect(flaky.has(testKey(x))).toBe(true); | |
| 115 | expect(flaky.has(testKey(y))).toBe(false); | |
| 116 | }); | |
| 117 | }); | |
| 118 | ||
| 119 | describe("the merge queue keeps and names each gate's failing tests", () => { | |
| 120 | let dataDir: string; | |
| 121 | let app: App; | |
| 122 | let servers: Servers; | |
| 123 | let alice: User; | |
| 124 | let token: string; | |
| 125 | let logged: string[] = []; | |
| 126 | const spies: { mockRestore(): void }[] = []; | |
| 127 | ||
| 128 | const git = (repo: Repo, ...args: string[]) => | |
| 129 | Bun.spawnSync(["git", ...args], { cwd: repoDir(app.reposRoot, repo.ownerName, repo.name) }) | |
| 130 | .stdout.toString() | |
| 131 | .trim(); | |
| 132 | ||
| 133 | beforeAll(async () => { | |
| 134 | dataDir = await mkdtemp(join(tmpdir(), "forge-gate-result-")); | |
| 135 | app = createApp({ dataDir, port: 0 }); | |
| 136 | servers = await serve(app); | |
| 137 | const a = await createUser(app.db, { | |
| 138 | origin: "seed", | |
| 139 | trusted: true, | |
| 140 | username: "alice", | |
| 141 | email: "[email protected]", | |
| 142 | password: "a-good-passphrase", | |
| 143 | adminIfFirst: true, | |
| 144 | }); | |
| 145 | if (!a.ok) throw new Error(a.error); | |
| 146 | alice = a.value; | |
| 147 | token = createToken(app.db, alice.id, "t", "ro").token; | |
| 148 | // The lane: red with two failing tests whenever the tip carries bad.ts. | |
| 149 | provideChecksForTest(app, { | |
| 150 | gate: async (repo: Repo, _pull: Pull, commitOid: string): Promise<GateDone> => { | |
| 151 | const files = git(repo, "ls-tree", "--name-only", commitOid).split("\n"); | |
| 152 | return files.includes("bad.ts") | |
| 153 | ? { ok: false, error: "check exited with code 1", lines: RED_OUTPUT, stage: "check" } | |
| 154 | : { ok: true, error: "", lines: [] }; | |
| 155 | }, | |
| 156 | spareSlots: () => 0, | |
| 157 | } as unknown as ChecksHandle); | |
| 158 | for (const level of ["info", "warn"] as const) | |
| 159 | spies.push(spyOn(console, level).mockImplementation((...args: unknown[]) => void logged.push(args.join(" ")))); | |
| 160 | }); | |
| 161 | ||
| 162 | afterAll(async () => { | |
| 163 | for (const s of spies) s.mockRestore(); | |
| 164 | stopQueue(); | |
| 165 | servers.stop(); | |
| 166 | await app.close?.(); | |
| 167 | await rm(dataDir, { recursive: true, force: true }); | |
| 168 | }); | |
| 169 | ||
| 170 | async function autonomousRepo(name: string): Promise<Repo> { | |
| 171 | const made = await createRepo(app, alice, { name, visibility: "private" }); | |
| 172 | if (!made.ok) throw new Error(String(made.error)); | |
| 173 | const seeded = await commitToBranch(app, alice, made.value, { | |
| 174 | message: "Start", | |
| 175 | files: [ | |
| 176 | { path: "README.md", content: `# ${name}\n` }, | |
| 177 | { path: "forge.json", content: JSON.stringify({ check: { run: "true", fast: "true", full: "true" } }) }, | |
| 178 | ], | |
| 179 | }); | |
| 180 | if (!seeded.ok) throw new Error(JSON.stringify(seeded.error)); | |
| 181 | app.db.query("UPDATE repo SET autonomy = 'autonomous', autonomy_by = ?2 WHERE id = ?1").run(made.value.id, alice.id); | |
| 182 | return getRepoById(app.db, made.value.id)!; | |
| 183 | } | |
| 184 | ||
| 185 | async function greenPull(repo: Repo, path: string): Promise<Pull> { | |
| 186 | const pushed = await commitToBranch(app, alice, repo, { | |
| 187 | branch: `feature/${path}`, | |
| 188 | base: "main", | |
| 189 | message: path, | |
| 190 | files: [{ path, content: `export const n = "${path}";\n` }], | |
| 191 | }); | |
| 192 | if (!pushed.ok) throw new Error(JSON.stringify(pushed.error)); | |
| 193 | const opened = await openPull(app, alice, repo, { title: path, baseBranch: "main", headBranch: `feature/${path}` }); | |
| 194 | if (!opened.ok) throw new Error(String(opened.error)); | |
| 195 | const pull = getPull(app.db, repo.id, opened.value.number)!; | |
| 196 | const check = createRun(app.db, { | |
| 197 | repoId: repo.id, | |
| 198 | kind: "check", | |
| 199 | issueId: pull.id, | |
| 200 | env: `pull-${pull.number}`, | |
| 201 | commitOid: pull.headOid, | |
| 202 | reason: "pull", | |
| 203 | }); | |
| 204 | claimRun(app.db, check.id); | |
| 205 | finishRun(app.db, check.id, { state: "ok", summary: "check passed" }); | |
| 206 | return pull; | |
| 207 | } | |
| 208 | ||
| 209 | test("a red gate's two failing tests are kept, logged by name, and answered by the API and the MCP", async () => { | |
| 210 | const repo = await autonomousRepo("gate-result"); | |
| 211 | const good = await greenPull(repo, "good.ts"); | |
| 212 | const bad = await greenPull(repo, "bad.ts"); | |
| 213 | expect(await sweepGreen(app, repo)).toBe(1); | |
| 214 | ||
| 215 | // The log line names them. | |
| 216 | expect(logged.some((l) => l.includes('gated 2 pulls: red (2 failed: "a > b", "c > d")'))).toBe(true); | |
| 217 | ||
| 218 | // The API: newest first, the red batch with both names and its counts. | |
| 219 | const res = await fetch(`${servers.url}/api/v1/repos/alice/gate-result/queue/gates?limit=20`, { | |
| 220 | headers: { authorization: `Bearer ${token}` }, | |
| 221 | }); | |
| 222 | expect(res.status).toBe(200); | |
| 223 | const body = (await res.json()) as { items: any[]; next_cursor: unknown }; | |
| 224 | expect(body.next_cursor).toBeNull(); | |
| 225 | const red = body.items.find((g) => g.state === "red" && g.pulls.length === 2); | |
| 226 | expect(red).toBeDefined(); | |
| 227 | expect(red.pulls).toEqual([good.number, bad.number]); | |
| 228 | expect(red.reason).toBe("tests"); | |
| 229 | expect(red.passed).toBe(3); | |
| 230 | expect(red.failed).toBe(2); | |
| 231 | expect(red.failing.map((t: { name: string }) => t.name)).toEqual(["a > b", "c > d"]); | |
| 232 | expect(red.failed).toBe(red.failing.length); | |
| 233 | expect(red.failing[0].errors[0]).toBe("error: expect(received).toBe(expected)"); | |
| 234 | expect(red.failing[0].flaky).toBe(false); | |
| 235 | expect(red.runId).toMatch(/^gate-[0-9a-f]{12}$/); | |
| 236 | expect(red.durationMs).toBeGreaterThanOrEqual(0); | |
| 237 | // The search's own gates are kept too: good.ts alone went green. | |
| 238 | expect(body.items.some((g) => g.state === "green" && g.pulls[0] === good.number)).toBe(true); | |
| 239 | // Newest first. | |
| 240 | const ids = body.items.map((g) => g.id as string); | |
| 241 | expect([...ids].sort().reverse()).toEqual(ids); | |
| 242 | ||
| 243 | // A page of one has a cursor, and the cursor reads on. | |
| 244 | const one = (await ( | |
| 245 | await fetch(`${servers.url}/api/v1/repos/alice/gate-result/queue/gates?limit=1`, { | |
| 246 | headers: { authorization: `Bearer ${token}` }, | |
| 247 | }) | |
| 248 | ).json()) as { items: { id: string }[]; next_cursor: string }; | |
| 249 | expect(one.items).toHaveLength(1); | |
| 250 | const next = (await ( | |
| 251 | await fetch(`${servers.url}/api/v1/repos/alice/gate-result/queue/gates?limit=1&cursor=${one.next_cursor}`, { | |
| 252 | headers: { authorization: `Bearer ${token}` }, | |
| 253 | }) | |
| 254 | ).json()) as { items: { id: string }[] }; | |
| 255 | expect(next.items[0]!.id < one.items[0]!.id).toBe(true); | |
| 256 | ||
| 257 | // The MCP tool answers the same. | |
| 258 | const mcp = await mcpTool(servers.url, token, "queue_gates", { repo: "alice/gate-result" }); | |
| 259 | expect(mcp.error).toBeUndefined(); | |
| 260 | const viaMcp = (mcp.value.gates as any[]).find((g) => g.state === "red" && g.pulls.length === 2); | |
| 261 | expect(viaMcp.failing.map((t: { name: string }) => t.name)).toEqual(["a > b", "c > d"]); | |
| 262 | }); | |
| 263 | ||
| 264 | test("a private repository's gates are not found without access; a bad cursor is refused", async () => { | |
| 265 | const anon = await fetch(`${servers.url}/api/v1/repos/alice/gate-result/queue/gates`); | |
| 266 | expect(anon.status).toBe(404); | |
| 267 | const badCursor = await fetch(`${servers.url}/api/v1/repos/alice/gate-result/queue/gates?cursor=../x`, { | |
| 268 | headers: { authorization: `Bearer ${token}` }, | |
| 269 | }); | |
| 270 | expect(badCursor.status).toBe(422); | |
| 271 | }); | |
| 272 | ||
| 273 | test("the API flags a test red in two gates that share no pull, and prunes past a week", () => { | |
| 274 | const repo = getRepoById(app.db, app.db.query<{ id: string }, []>("SELECT id FROM repo WHERE name = 'gate-result'").get()!.id)!; | |
| 275 | const now = new Date(); | |
| 276 | const at = (ms: number) => new Date(now.getTime() - ms).toISOString(); | |
| 277 | const base = { repoId: repo.id, runId: "gate-000000000000", tipOid: "0".repeat(40), state: "red" as const, reason: "tests", passed: 1 }; | |
| 278 | const flake = { name: "sometimes", file: "test/s.test.ts", errors: ["error: timed out"] }; | |
| 279 | keepGateResult(app.db, "zz00000000000000000000001", { ...base, pulls: [901], failed: 1, failing: [flake], startedAt: at(2000), finishedAt: at(1000) }, now); | |
| 280 | keepGateResult(app.db, "zz00000000000000000000002", { ...base, pulls: [902], failed: 1, failing: [flake], startedAt: at(2000), finishedAt: at(1000) }, now); | |
| 281 | const gates = recentGates(app.db, repo.id, 2); | |
| 282 | expect(gates.map((g) => g.id)).toEqual(["zz00000000000000000000002", "zz00000000000000000000001"]); | |
| 283 | expect(gates[0]!.failing[0]).toMatchObject({ name: "sometimes", flaky: true }); | |
| 284 | expect(gates[0]!.durationMs).toBe(1000); | |
| 285 | // A week and a day later, a new gate's write prunes them. | |
| 286 | const later = new Date(now.getTime() + 8 * 24 * 3_600_000); | |
| 287 | keepGateResult(app.db, "zz00000000000000000000003", { ...base, pulls: [903], failed: 0, failing: [], startedAt: later.toISOString(), finishedAt: later.toISOString() }, later); | |
| 288 | expect(recentGates(app.db, repo.id, 100).map((g) => g.id)).toEqual(["zz00000000000000000000003"]); | |
| 289 | }); | |
| 290 | }); |
new above
offline