trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRuns12g rEventsFlows

Verify after landing: the full suite runs on main on a schedule, names the culprit, and proposes the revert #7813

Merged torquecrew opened this in answer to trevatoowner

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

torque/forge:torque/7786-verify-after-landing-the-full-suite-runs → main 4 commits fork

  • Merged. torque/7786-verify-after-landing-the-full-suite-runs was merged into main as 6ed2e67.

Review

Sign in to review this change.
21 files changed +1276 −20
docs/OPERATING.md +2

Diff not shown — a line 2,328 characters wide (limit 2,000). +2

View file
docs/design/areas/boards.md +4
@@ -8,3 +8,7 @@ Drawn from existing components only: `card` with an icon head, `row` per hold
88 repositories the viewer administers and shows when there are none, because it
99 holds the form. A person who administers a repository but is not a forge
1010 admin sees `/boards` as this card alone.
11
12 ## Verified after landing (#7786)
13
14 The card under the Pipeline table (`id="verified"`) adds no class: it is a `card()` with an `<h2>`, one `<p>` per repository in the default text colour, the repository and its run as plain `<a>` links, `small` for the run link. It is server-rendered, needs no JavaScript, and has no accent of its own.
forge.json +1
@@ -17,6 +17,7 @@
1717 "check": {
1818 "fast": "bun scripts/gate.ts && mkdir -p .forge-cache && GOMEMLIMIT=2GiB ./node_modules/typescript/bin/tsc --noEmit --incremental --tsBuildInfoFile .forge-cache/tsc.tsbuildinfo && bun scripts/test-shards.ts --timeout 60000 --tier fast --diff $FORGE_CHECK_BASE..HEAD --learned \"$FORGE_CHECK_LEARNED\"",
1919 "run": "bun scripts/gate.ts && mkdir -p .forge-cache && GOMEMLIMIT=2GiB ./node_modules/typescript/bin/tsc --noEmit --incremental --tsBuildInfoFile .forge-cache/tsc.tsbuildinfo",
20 "verify": "bun scripts/gate.ts && bun scripts/test-shards.ts --timeout 60000 --gate --impact --keep-going --nice",
2021 "timeout": 7200,
2122 "limits": {
2223 "memory": 8192,
scripts/test-shards.ts +56 −8
@@ -7,6 +7,7 @@
77 * bun scripts/test-shards.ts [bun test flags…] --tier fast --diff <base>..<head>
88 * bun scripts/test-shards.ts --record [bun test flags…] test/a.test.ts …
99 * bun scripts/test-shards.ts [bun test flags…] -- a b … (test/a.test.ts …)
10 * bun scripts/test-shards.ts [bun test flags…] --gate --impact --keep-going --nice
1011 *
1112 * `--gate` runs the files `scripts/gate.ts` admits (every test file not
1213 * excluded), which is how
@@ -26,6 +27,14 @@
2627 * clock — unless every red in it is a budget test, which the forge re-runs
2728 * once after the gate (#7354). Exits non-zero if any shard does.
2829 *
30 * `--keep-going` is the verification after landing (#7786): no shard stops
31 * the rest, a file red on its own is run once more alone (green then, it
32 * is named flaky and does not fail the run), and a measuring run writes
33 * the impact of every file that ended green even when another is red.
34 * `--nice` runs it all at the lowest CPU priority, so the checks a landing
35 * waits on go first. `FORGE_CHECK_ONLY` (comma-separated test files, set
36 * by the forge for a bisection step) keeps only those files.
37 *
2938 * The whole gate, unsliced (`--gate` on the full tier, as `check.run`),
3039 * runs one file per `bun test`, heaviest first into whichever worker is
3140 * free, so every file is timed alone; when it is green it writes those
@@ -40,7 +49,7 @@
4049 * table is rewritten from those times when every file is green.
4150 */
4251
43 import { availableParallelism, tmpdir, totalmem } from "node:os";
52 import { availableParallelism, setPriority, tmpdir, totalmem } from "node:os";
4453 import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
4554 import { join } from "node:path";
4655 import { resolveLimits } from "../src/apps/contain.ts";
@@ -74,6 +83,7 @@ import { importsReader, readImpact } from "./tier-input.ts";
7483 import { fastSet, duration } from "../src/checks/tiers.ts";
7584 import { CHECK_CACHE } from "../src/checks/cache.ts";
7685 import { parseCheck } from "../src/apps/manifest.ts";
86 import { FLAKY_ALONE, onlyFiles } from "../src/checks/verify.ts";
7787
7888 const TIMES = new URL("./test-times.json", import.meta.url).pathname;
7989 // The forge hands the map over in the checkout (`src/checks/impact-store.ts`):
@@ -90,6 +100,15 @@ const argv = process.argv.slice(2);
90100 const record = argv.includes("--record");
91101 const gate = argv.includes("--gate");
92102 const measure = argv.includes("--impact");
103 const keepGoing = argv.includes("--keep-going");
104 const nice = argv.includes("--nice");
105 // Every `bun test` this spawns inherits it.
106 if (nice)
107 try {
108 setPriority(19);
109 } catch (error) {
110 console.log(`test-shards: --nice refused (${String(error)}); running at normal priority`);
111 }
93112
94113 /**
95114 * Measured seconds where a table has them, bytes scaled to seconds where
@@ -175,7 +194,7 @@ function takeValue(
175194 }
176195
177196 const tierArg = takeValue(
178 argv.filter((arg) => arg !== "--record" && arg !== "--gate" && arg !== "--impact"),
197 argv.filter((arg) => !["--record", "--gate", "--impact", "--keep-going", "--nice"].includes(arg)),
179198 "--tier",
180199 );
181200 const diffArg = takeValue(tierArg.rest, "--diff");
@@ -288,6 +307,13 @@ if (tier === "fast") {
288307 files = plan.files;
289308 }
290309
310 const only = onlyFiles(process.env["FORGE_CHECK_ONLY"]);
311 if (only !== undefined) {
312 const named = new Set(only);
313 files = files.filter((file) => named.has(file) || named.has(file.replace(/^\.\//, "")));
314 console.log(`test-shards: FORGE_CHECK_ONLY keeps ${files.length} of ${only.length} named file(s)`);
315 }
316
291317 if (files.length === 0) {
292318 console.error("test-shards: name the test files to run, or --gate");
293319 process.exit(2);
@@ -505,7 +531,22 @@ async function runShard(
505531 : undefined;
506532 const first = await run(paths, cover, through);
507533 let code = first.code;
508 const output = first.output;
534 let output = first.output;
535 // Kept going, a file red on its own is run once more alone: green then,
536 // it was a flake, and its first output (the `(fail)` lines) is set aside.
537 if (code !== 0 && keepGoing && slice.length === 1) {
538 // The red run's coverage is not the green one's.
539 if (covDir) {
540 rmSync(covDir, { recursive: true, force: true });
541 mkdirSync(covDir);
542 }
543 const again = await run(paths, cover, through);
544 if (again.code === 0) {
545 code = 0;
546 output = [...again.output, `test-shards: ${FLAKY_ALONE} ${slice[0]}\n`];
547 through?.(output.at(-1)!);
548 } else output = again.output;
549 }
509550 // A sliced shard that went red: each failing file (the first few) is run
510551 // once alone, same env and flags. One that passes alone was red only
511552 // beside others, and does not fail the check; the log names it.
@@ -534,7 +575,10 @@ async function runShard(
534575 ? `${text.endsWith("\n") || text === "" ? "" : "\n"}── shard ${where} done (exit ${code}, ${seconds}s)\n`
535576 : `\n── shard ${where} (exit ${code}, ${seconds}s): ${slice.join(" ")}\n${text}${text.endsWith("\n") ? "" : "\n"}`,
536577 );
537 if (covDir) {
578 if (covDir && keepGoing && code !== 0) {
579 // A red file's coverage is not its impact; the map keeps its old row.
580 rmSync(covDir, { recursive: true, force: true });
581 } else if (covDir) {
538582 try {
539583 measured.set(slice[0]!, executedFiles(readFileSync(join(covDir, "lcov.info"), "utf8")));
540584 } catch {
@@ -563,8 +607,9 @@ async function pool(width: number): Promise<Timed[]> {
563607 const index = next++;
564608 if (stop) break;
565609 results[index] = await runShard(slices[index]!, index, width === 1);
566 // A shard red only on budget tests lets the rest run (#7354).
567 if (stopsTheRest(results[index]!)) {
610 // A shard red only on budget tests lets the rest run (#7354);
611 // a verification after landing runs everything (#7786).
612 if (!keepGoing && stopsTheRest(results[index]!)) {
568613 stop = true;
569614 halted = true;
570615 for (const child of children) child.kill("SIGTERM");
@@ -622,8 +667,11 @@ if (record || timed) {
622667 }
623668 }
624669 if (measure) {
625 if (ending.code !== 0 || unmeasured.size > 0) {
626 console.log(`test-shards: not writing ${IMPACT_NAME}: the run was not green and measured`);
670 if ((ending.code !== 0 && !keepGoing) || unmeasured.size > 0 || measured.size === 0) {
671 console.log(
672 `test-shards: not writing ${IMPACT_NAME}: the run was not green and measured` +
673 (unmeasured.size > 0 ? ` (no coverage from ${[...unmeasured].join(", ")})` : ""),
674 );
627675 } else {
628676 // A partial run refreshes what it ran and forgets nothing else.
629677 let old = new Map<string, string[]>();
src/apps/manifest.ts +9
@@ -311,6 +311,12 @@ export interface CheckManifest {
311311 readonly fast: string | null;
312312 /** The full tier, run once when a pull lands. Defaults to `run`. */
313313 readonly full: string;
314 /**
315 * The verification after landing (#7786): the whole suite, run on the
316 * default branch on a schedule, never as a landing's requirement. `null`:
317 * nothing verifies main after it lands.
318 */
319 readonly verify: string | null;
314320 /** Seconds the check may take before it fails. */
315321 readonly timeout: number;
316322 /** Hosts its steps may reach besides the forge and the npm registry. */
@@ -347,6 +353,8 @@ export function parseCheck(
347353 if (!fast.ok) return fast;
348354 const full = command(check, "full", "check");
349355 if (!full.ok) return full;
356 const verify = command(check, "verify", "check");
357 if (!verify.ok) return verify;
350358 const pkg = packageJson === undefined ? undefined : readPackage(packageJson);
351359 if (pkg !== undefined && !pkg.ok) return pkg;
352360 const timeout = check["timeout"];
@@ -368,6 +376,7 @@ export function parseCheck(
368376 run: run.value,
369377 fast: fast.value || null,
370378 full: full.value || run.value,
379 verify: verify.value || null,
371380 egress: egress.value,
372381 limits: limits.value,
373382 install:
src/checks/culprit.ts added +211
@@ -0,0 +1,211 @@
1 /**
2 * What a finished verification after landing does next (#7786,
3 * `checks/verify.ts`). Green: nothing. Red, with the test files it failed
4 * named: the range since the last green verification is bisected one step
5 * at a time — each step a run of only those files, in the verification
6 * lane — until one commit is left. That commit is the culprit: the landed
7 * pull it is gets its revert proposed (a person lands it, as every revert),
8 * a push no pull carried gets one issue. A red whose files were already red
9 * at the verification before it was named there and starts nothing.
10 *
11 * Called by the checks service with the way it asks for a run; the answer
12 * is the lines for the finished run's log.
13 */
14
15 import type { App } from "../app.ts";
16 import { revertPull } from "../actions/revert.ts";
17 import { createIssue } from "../core/issues.ts";
18 import { getPull, type Pull } from "../core/pulls.ts";
19 import type { Repo } from "../core/repos.ts";
20 import { getRun, type Run } from "../core/runs.ts";
21 import { getUserById } from "../core/users.ts";
22 import { runGit } from "../git/env.ts";
23 import { isSafeOid } from "../git/guard.ts";
24 import { repoDir } from "../git/repos.ts";
25 import { everReverted } from "./revert_red.ts";
26 import {
27 BISECT_ENV,
28 BISECT_PREFIX,
29 bisectOrigin,
30 MAX_RANGE,
31 maxSteps,
32 nextStep,
33 VERIFY_ENV,
34 } from "./verify.ts";
35
36 /** How the service asks for a bisection step. */
37 export type Ask = (repo: Repo, env: string, commitOid: string, reason: string) => Promise<Run | undefined>;
38
39 /** The machine says so: the author is the owner, the text is not theirs. */
40 const FILED_BY = "_Filed by the forge's verification after landing, not written by a person._";
41
42 const short = (oid: string): string => oid.slice(0, 7);
43
44
45 /** The last green verification before run `below`. */
46 function greenBefore(app: App, repoId: string, below: string): Run | undefined {
47 const row = app.db
48 .query<{ id: string }, [string, string]>(
49 `SELECT id FROM run WHERE repo_id = ?1 AND kind = 'check' AND env = '${VERIFY_ENV}'
50 AND state = 'ok' AND id < ?2 ORDER BY id DESC LIMIT 1`,
51 )
52 .get(repoId, below);
53 return row === null ? undefined : getRun(app.db, row.id);
54 }
55
56 /** The finished verification just before run `below`, green or red. */
57 function finishedBefore(app: App, repoId: string, below: string): Run | undefined {
58 const row = app.db
59 .query<{ id: string }, [string, string]>(
60 `SELECT id FROM run WHERE repo_id = ?1 AND kind = 'check' AND env = '${VERIFY_ENV}'
61 AND state IN ('ok','failed') AND id < ?2 ORDER BY id DESC LIMIT 1`,
62 )
63 .get(repoId, below);
64 return row === null ? undefined : getRun(app.db, row.id);
65 }
66
67 /** What each finished step of `origin`'s bisection said, by commit: green or not. */
68 function stepsOf(app: App, repoId: string, origin: string): { green: Map<string, boolean>; vague: Run | undefined } {
69 const rows = app.db
70 .query<{ id: string }, [string, string]>(
71 `SELECT id FROM run WHERE repo_id = ?1 AND kind = 'check' AND env = '${BISECT_ENV}'
72 AND reason = ?2 AND state IN ('ok','failed') ORDER BY id`,
73 )
74 .all(repoId, `${BISECT_PREFIX}${origin}`);
75 const green = new Map<string, boolean>();
76 let vague: Run | undefined;
77 for (const { id } of rows) {
78 const step = getRun(app.db, id);
79 if (step === undefined) continue;
80 // A step red for a reason that names no test (an install, the budget)
81 // says nothing about the files: the bisection cannot go on from it.
82 if (step.state === "failed" && (step.checkDetail?.files ?? []).length === 0) vague = step;
83 green.set(step.commitOid, step.state === "ok");
84 }
85 return { green, vague };
86 }
87
88 /** First-parent commits after `green` up to `red`, oldest first; undefined when git cannot say. */
89 async function rangeOf(app: App, repo: Repo, green: string, red: string): Promise<string[] | undefined> {
90 if (!isSafeOid(green) || !isSafeOid(red)) return undefined;
91 const log = await runGit(["rev-list", "--first-parent", "--reverse", `${green}..${red}`], {
92 cwd: repoDir(app.reposRoot, repo.ownerName, repo.name),
93 });
94 return log.ok ? log.value.split("\n").filter(Boolean) : undefined;
95 }
96
97 /** The pull whose landing is `oid` on the default branch, if one is. */
98 function landingOf(app: App, repo: Repo, oid: string): Pull | undefined {
99 const row = app.db
100 .query<{ number: number }, [string, string, string]>(
101 `SELECT i.number FROM pull p JOIN issue i ON i.id = p.issue_id
102 WHERE i.repo_id = ?1 AND p.merged_oid = ?2 AND p.base_branch = ?3`,
103 )
104 .get(repo.id, oid, repo.defaultBranch);
105 return row === null ? undefined : getPull(app.db, repo.id, row.number);
106 }
107
108 /** What the red is, in a sentence: its first failing test and how many more. */
109 function redSays(red: Run): string {
110 const names = red.checkDetail?.names ?? [];
111 const files = red.checkDetail?.files ?? [];
112 if (names[0] !== undefined)
113 return `"${names[0]}"${names.length > 1 ? ` and ${names.length - 1} more` : ""} in ${files.map((f) => `\`${f}\``).join(", ")}`;
114 return files.map((f) => `\`${f}\``).join(", ");
115 }
116
117 /** One open issue per title: a second red of the same thing adds nothing. */
118 function fileOnce(app: App, repo: Repo, title: string, body: string): string {
119 const open = app.db
120 .query<{ number: number }, [string, string]>(
121 "SELECT number FROM issue WHERE repo_id = ?1 AND title = ?2 AND state = 'open' LIMIT 1",
122 )
123 .get(repo.id, title);
124 if (open !== null) return `already filed: #${open.number}`;
125 const made = createIssue(app.db, { repoId: repo.id, authorId: repo.ownerId, title, body });
126 if (!made.ok) return `not filed: ${made.error}`;
127 app.bus.append({
128 kind: "issue.opened",
129 repoId: repo.id,
130 actorId: repo.ownerId,
131 payload: { number: made.value.number },
132 });
133 return `filed #${made.value.number}: ${title}`;
134 }
135
136 /** Names `culprit` as what turned `red` red: the revert of its pull, or one issue. */
137 async function name(app: App, repo: Repo, red: Run, culprit: string, green: string, tried: number): Promise<string> {
138 const files = red.checkDetail?.files ?? [];
139 const how =
140 tried === 0
141 ? `the one commit since main was last verified green at ${short(green)}`
142 : `found by bisecting ${short(green)}..${short(red.commitOid)} in ${tried} step${tried === 1 ? "" : "s"}`;
143 const pull = landingOf(app, repo, culprit);
144 if (pull !== undefined) {
145 if (everReverted(app, pull.id)) return `culprit #${pull.number} (${short(culprit)}): its revert was already proposed`;
146 const owner = getUserById(app.db, repo.ownerId);
147 const body =
148 `Reverts #${pull.number}: verifying main after landing found it broke ${redSays(red)} — ${how}.\n\n` +
149 `Main's verification ${red.id} is red at ${short(red.commitOid)}; each failed test fails alone too.\n\n` +
150 `Proposed by the forge; a person lands it.\n\nFiles: ${files.join(", ")}`;
151 const made = await revertPull(app, owner, repo, pull, body);
152 return made.ok
153 ? `culprit #${pull.number} (${short(culprit)}): proposed its revert, #${made.value.number}`
154 : `culprit #${pull.number} (${short(culprit)}): the revert could not be proposed (${made.error})`;
155 }
156 const title = `Red after landing: ${short(culprit)} broke ${files[0] ?? "main"}`;
157 return `culprit ${short(culprit)}, landed by no pull: ${fileOnce(
158 app,
159 repo,
160 title,
161 `${FILED_BY}\n\nCommit ${culprit} broke ${redSays(red)} — ${how}. No pull carried it, so there is no revert pull to propose: revert or fix it by hand.\n\nFiles: ${files.join(", ")}`,
162 )}`;
163 }
164
165 /** A range the bisection will not walk, named in one issue. */
166 function unbisected(app: App, repo: Repo, red: Run, why: string): string {
167 const files = red.checkDetail?.files ?? [];
168 return `not bisected (${why}): ${fileOnce(
169 app,
170 repo,
171 `Red after landing: ${files[0] ?? "main"} fails on ${repo.defaultBranch}`,
172 `${FILED_BY}\n\nMain's verification ${red.id} is red at ${short(red.commitOid)}: ${redSays(red)}, failing alone too. The forge did not name the landing that broke it: ${why}.\n\nFiles: ${files.join(", ")}`,
173 )}`;
174 }
175
176 /**
177 * The next thing after `run`, a finished verification or bisection step,
178 * as lines for its log. Never throws into the check.
179 */
180 export async function afterVerify(app: App, repo: Repo, run: Run, ask: Ask): Promise<string[]> {
181 const red = run.env === VERIFY_ENV ? run : getRun(app.db, bisectOrigin(run.reason) ?? "");
182 if (red === undefined || red.state !== "failed") return [];
183 const files = red.checkDetail?.files ?? [];
184 if (files.length === 0) return ["not bisected: the red names no failed test file"];
185 if (run === red) {
186 const before = finishedBefore(app, repo.id, red.id);
187 const known = before?.state === "failed" ? (before.checkDetail?.files ?? []) : [];
188 if (known.length > 0 && files.every((f) => known.includes(f)))
189 return [`not bisected: ${files.join(", ")} already failed at ${short(before!.commitOid)}, named there`];
190 }
191 const green = greenBefore(app, repo.id, red.id);
192 if (green === undefined) return [unbisected(app, repo, red, "main was never verified green before")];
193 const range = await rangeOf(app, repo, green.commitOid, red.commitOid);
194 if (range === undefined || range.length === 0)
195 return [unbisected(app, repo, red, `git could not list ${short(green.commitOid)}..${short(red.commitOid)}`)];
196 if (range.length > MAX_RANGE)
197 return [unbisected(app, repo, red, `${range.length} commits since the last green is over ${MAX_RANGE}`)];
198 const { green: tried, vague } = stepsOf(app, repo.id, red.id);
199 if (vague !== undefined)
200 return [unbisected(app, repo, red, `the step at ${short(vague.commitOid)} failed without naming a test`)];
201 const step = nextStep(range, tried);
202 if (step === undefined) return [];
203 if (step.kind === "culprit") return [await name(app, repo, red, step.commit, green.commitOid, tried.size)];
204 const asked = await ask(repo, BISECT_ENV, step.commit, `${BISECT_PREFIX}${red.id}`);
205 return asked === undefined
206 ? [`bisect: ${short(step.commit)} could not be asked for`]
207 : [
208 `bisect: trying ${short(step.commit)}, ${range.length} commit(s) since ${short(green.commitOid)}, ` +
209 `at most ${maxSteps(range.length)} step(s); run ${asked.id}`,
210 ];
211 }
src/checks/revert_red.ts +1 −1
@@ -35,7 +35,7 @@ function greenBefore(app: App, repoId: string, branch: string, below: string): s
3535 }
3636
3737 /** Whether any pull, in any state, already takes `pullId` back. */
38 function everReverted(app: App, pullId: string): boolean {
38 export function everReverted(app: App, pullId: string): boolean {
3939 return (
4040 app.db
4141 .query<{ n: number }, [string]>("SELECT COUNT(*) AS n FROM pull WHERE reverts_issue_id = ?1")
src/checks/service.ts +173 −10
@@ -64,6 +64,18 @@ import { ASKED, redMain, unheld } from "./red_main.ts";
6464 import { proposeRevert } from "./revert_red.ts";
6565 import { rebasesFirst } from "./finishing_first.ts";
6666 import { IMPACT_FILE, keepImpact, seedImpact } from "./impact-store.ts";
67 import { afterVerify } from "./culprit.ts";
68 import {
69 bisectOrigin,
70 failedFiles,
71 inVerifyLane,
72 onlyEnv,
73 VERIFY_ENV,
74 VERIFY_EVERY_MS,
75 VERIFY_REASON,
76 VERIFY_SWEEP_MS,
77 verifyDue,
78 } from "./verify.ts";
6779 import { getIssueById } from "../core/issues.ts";
6880 import { BUDGETS, budgetSummary } from "../kernel/budgets.ts";
6981 import { transact } from "../kernel/db.ts";
@@ -144,6 +156,12 @@ export interface ChecksOptions {
144156 readonly job?: JobOptions | undefined;
145157 /** How long a transient install failure waits for its retry. @default INSTALL_RETRY_MS */
146158 readonly installRetryMs?: number | undefined;
159 /**
160 * How often the default branch is verified after landing at most
161 * (`checks/verify.ts`). Unset, no timer asks; `verifyDue` still holds
162 * to `VERIFY_EVERY_MS`. The forge sets it to `VERIFY_EVERY_MS`.
163 */
164 readonly verifyEveryMs?: number | undefined;
147165 }
148166
149167 /** A gate's answer from the lane: `interrupted` is no verdict (#3155). */
@@ -200,6 +218,12 @@ export interface ChecksHandle {
200218 spareSlots(): number;
201219 /** The gate checks take their turn at; a hosted child's checks share it. */
202220 readonly lane: Gate;
221 /**
222 * Ask for every verification after landing that is due now
223 * (`checks/verify.ts`); the answer is the runs asked for. The forge's
224 * own timer calls it when `verifyEveryMs` is set.
225 */
226 verifyDue(now?: number): Promise<Run[]>;
203227 }
204228
205229 /**
@@ -421,6 +445,9 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
421445 >();
422446 /** Checks already told why they wait, so the log says it once. */
423447 const waiting = new Set<string>();
448 /** The verification-lane run in flight, which holds no check slot. */
449 let verifying: string | undefined;
450 const slotted = (): number => inFlight.size - (verifying === undefined ? 0 : 1);
424451 const settle = options.landSettleMs ?? 0;
425452 let stopped = false;
426453
@@ -455,6 +482,8 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
455482 ]);
456483 const declared = parseCheck(forgeJson, packageJson);
457484 if (declared.ok && declared.value === undefined) return undefined;
485 // Main is verified after landing only where it says how (#7786).
486 if (env === VERIFY_ENV && (!declared.ok || declared.value?.verify == null)) return undefined;
458487 // One check per environment at a time: a newer commit supersedes what
459488 // is still waiting, and the same commit is not checked twice. Looked for
460489 // and written in one write transaction, so a second forge process over
@@ -507,7 +536,9 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
507536 if (inFlight.has(run.id)) return;
508537 waiting.delete(run.id);
509538 const abort = new AbortController();
510 const promise = execute(run, repo, abort.signal)
539 const low = inVerifyLane(run.env);
540 if (low) verifying = run.id;
541 const promise = execute(run, repo, abort.signal, !low)
511542 .catch((error) =>
512543 log.error("check failed to execute", {
513544 run: run.id,
@@ -516,6 +547,7 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
516547 )
517548 .finally(() => {
518549 inFlight.delete(run.id);
550 if (verifying === run.id) verifying = undefined;
519551 // The slot this run held is free again: give it to whichever queued
520552 // check now deserves it most, not whoever asked first.
521553 schedule();
@@ -565,14 +597,16 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
565597 if (stopped) return;
566598 const limit = capacity();
567599 if (options.gate === undefined) gate.resize(limit);
600 startVerify();
568601 // Every merge-queue gate in flight holds a slot, a line's first too (#6376).
569 while (inFlight.size < limit - gatesInFlight() - proofsInFlight()) {
602 while (slotted() < limit - gatesInFlight() - proofsInFlight()) {
570603 // A pull's check waits while its main is red (`checks/red_main.ts`).
571604 const queued = unheld(
572605 listActive(app.db, "check").filter(
573606 (run) =>
574607 run.state === "queued" &&
575608 !inFlight.has(run.id) &&
609 !inVerifyLane(run.env) &&
576610 (run.notBefore === null || Date.parse(run.notBefore) <= Date.now()),
577611 ),
578612 (repoId) => redMain(app.db, repoId),
@@ -585,7 +619,7 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
585619 const { rebases, rest: others } = rebasesFirst(app.db, queued);
586620 // While a gate is about to start, the last free slot is its: a pull's
587621 // check waits for it, a rebase's and a main check do not.
588 const held = holdsLastSlot(limit - gatesInFlight() - proofsInFlight() - inFlight.size, gateQueued());
622 const held = holdsLastSlot(limit - gatesInFlight() - proofsInFlight() - slotted(), gateQueued());
589623 const next = startable(rebases) ?? startable(held ? others.filter((run) => run.issueId === null) : others);
590624 if (!next) return;
591625 const repo = getRepoById(app.db, next.repoId);
@@ -594,6 +628,65 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
594628 }
595629 }
596630
631 /**
632 * The verification lane (#7786): one run at a time, forge-wide, beside
633 * the check slots rather than in one, so no landing and no pull's check
634 * ever waits for it. A bisection step goes before a new verification.
635 */
636 function startVerify(): void {
637 if (verifying !== undefined) return;
638 const queued = listActive(app.db, "check").filter(
639 (run) => run.state === "queued" && inVerifyLane(run.env) && !inFlight.has(run.id),
640 );
641 const next =
642 queued.find((run) => bisectOrigin(run.reason) !== undefined) ?? queued[0];
643 if (next === undefined) return;
644 const repo = getRepoById(app.db, next.repoId);
645 if (repo !== undefined) start(next, repo);
646 }
647
648 /** Every verification due now: main moved since the last, and that one is old enough. */
649 async function askVerifications(now = Date.now()): Promise<Run[]> {
650 const everyMs = options.verifyEveryMs ?? VERIFY_EVERY_MS;
651 const since = new Date(now - 24 * 3_600_000).toISOString();
652 // The newest default-branch check of each repository that had one today.
653 const tips = app.db
654 .query<{ repo_id: string; commit_oid: string }, [string]>(
655 `SELECT r.repo_id, r.commit_oid FROM run r JOIN repo ON repo.id = r.repo_id
656 WHERE r.id IN (SELECT MAX(x.id) FROM run x JOIN repo y ON y.id = x.repo_id
657 WHERE x.kind = 'check' AND x.issue_id IS NULL
658 AND x.env = y.default_branch AND x.created_at >= ?1
659 GROUP BY x.repo_id)`,
660 )
661 .all(since);
662 const asked: Run[] = [];
663 for (const tip of tips) {
664 const last = app.db
665 .query<{ commit_oid: string; created_at: string }, [string]>(
666 `SELECT commit_oid, created_at FROM run
667 WHERE repo_id = ?1 AND kind = 'check' AND env = '${VERIFY_ENV}'
668 ORDER BY id DESC LIMIT 1`,
669 )
670 .get(tip.repo_id);
671 const active = listActive(app.db, "check").some(
672 (run) => run.repoId === tip.repo_id && inVerifyLane(run.env),
673 );
674 const due = verifyDue({
675 tip: tip.commit_oid,
676 last: last === null ? undefined : { commitOid: last.commit_oid, createdAt: last.created_at },
677 active,
678 now,
679 everyMs,
680 });
681 if (!due) continue;
682 const repo = getRepoById(app.db, tip.repo_id);
683 if (repo === undefined) continue;
684 const run = await request(repo, VERIFY_ENV, tip.commit_oid, VERIFY_REASON);
685 if (run !== undefined) asked.push(run);
686 }
687 return asked;
688 }
689
597690 /**
598691 * An open pull whose head was never checked — the event that should have
599692 * asked landed while a forge was on its way out, or before this forge
@@ -639,8 +732,10 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
639732 run: Run,
640733 repo: Repo,
641734 signal: AbortSignal,
735 slot = true,
642736 ): Promise<void> {
643 const release = await gate.acquire();
737 // The verification lane takes no slot of the gate (`startVerify`).
738 const release = slot ? await gate.acquire() : () => {};
644739 try {
645740 // A slot that opens while this forge is stopping is not taken: the run
646741 // stays queued for the forge that carries on.
@@ -755,6 +850,11 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
755850 proposeRevert(app, repo.id).catch((error) =>
756851 log.error("revert proposal failed", { error: String(error) }),
757852 );
853 // A red verification after landing names its culprit (#7786).
854 if (done && inVerifyLane(run.env))
855 await afterVerify(app, repo, done, (r, env, oid, reason) => request(r, env, oid, reason))
856 .then((lines) => lines.forEach(write))
857 .catch((error) => write(`culprit not named: ${String(error)}`));
758858 } finally {
759859 release();
760860 }
@@ -776,6 +876,28 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
776876 run.issueId === null ? undefined : getIssueById(app.db, run.issueId);
777877 const pull =
778878 issue?.kind === "pull" ? getPull(app.db, repo.id, issue.number) : undefined;
879 if (inVerifyLane(run.env)) {
880 // A bisection step runs only the red's failed files, with the red's
881 // own command: an older commit may not say how to verify (#7786).
882 const origin = run.env === VERIFY_ENV ? undefined : getRun(app.db, bisectOrigin(run.reason) ?? "");
883 if (run.env !== VERIFY_ENV && origin === undefined)
884 return { ok: false, error: "the red verification this step bisects is gone" };
885 return runTier({
886 bare,
887 repo,
888 id: run.id,
889 runId: run.id,
890 commitOid: run.commitOid,
891 tier: "verify",
892 ...(origin === undefined
893 ? { keepImpact: true }
894 : { only: origin.checkDetail?.files ?? [], commandsFrom: origin.commitOid }),
895 fileOrderDependent: origin === undefined,
896 isMain: true,
897 write,
898 signal,
899 });
900 }
779901 const outcome = await runTier({
780902 bare,
781903 repo,
@@ -802,8 +924,12 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
802924 readonly runId?: string;
803925 readonly commitOid: string;
804926 readonly pull?: Pull | undefined;
805 /** `run`: a push to a branch; `fast`: a pull's push; `full`: on land. */
806 readonly tier: "run" | "fast" | "full";
927 /** `run`: a push to a branch; `fast`: a pull's push; `full`: on land; `verify`: main after landing. */
928 readonly tier: "run" | "fast" | "full" | "verify";
929 /** A bisection step: only these test files run (`FORGE_CHECK_ONLY`). */
930 readonly only?: readonly string[];
931 /** A bisection step: the commit whose `check.verify` it runs. */
932 readonly commandsFrom?: string;
807933 /** A check of the default branch's own push: its cache is kept. */
808934 readonly keep?: boolean;
809935 /** A green run keeps the impact map it measured; `full` always does. */
@@ -902,12 +1028,20 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
9021028 );
9031029 }
9041030 }
1031 if (x.commandsFrom !== undefined) {
1032 const from = parseCheck(await readFileAt(bare, x.commandsFrom, "forge.json"), undefined);
1033 if (from.ok && from.value !== undefined) manifest = { ...manifest, verify: from.value.verify, timeout: from.value.timeout };
1034 }
1035 if (x.tier === "verify" && manifest.verify === null)
1036 return { ok: false, error: "this commit declares no check.verify" };
9051037 const command =
9061038 x.tier === "full"
9071039 ? manifest.full
9081040 : x.tier === "fast"
9091041 ? (manifest.fast ?? manifest.run)
910 : manifest.run;
1042 : x.tier === "verify"
1043 ? manifest.verify!
1044 : manifest.run;
9111045 // The fast tier diffs what the pull changes against the base's tip,
9121046 // which the clone is given as a repository of its own: `origin/main` is
9131047 // not a ref there.
@@ -917,7 +1051,8 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
9171051 const learned =
9181052 base === undefined ? "" : readLearned(app.config.dataDir, x.pull!.id).join(",");
9191053 const tree = await runGit(["rev-parse", `${run.commitOid}^{tree}`], { cwd: bare });
920 const key = tree.ok
1054 // A bisection step runs a slice: its green proves nothing of the whole.
1055 const key = tree.ok && x.only === undefined
9211056 ? checkKey({
9221057 repoId: repo.id,
9231058 treeOid: tree.value.trim(),
@@ -1023,6 +1158,7 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
10231158 FORGE_CHECK: run.id,
10241159 FORGE_COMMIT: x.commitOid,
10251160 FORGE_CHECK_IMPACT: IMPACT_FILE,
1161 ...(x.only === undefined ? {} : { FORGE_CHECK_ONLY: onlyEnv(x.only) }),
10261162 // The cell's CPU limit, which `availableParallelism()` may not
10271163 // report inside it; the shard count follows this (#2257).
10281164 FORGE_CELL_CPUS: String(cellCpus),
@@ -1255,7 +1391,24 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
12551391 ran.error,
12561392 );
12571393 const killed = killedStep(ran.error);
1258 return { ok: false, error: killed ? ran.error : summary, detail, stage: checkStage(ran.error), ...(killed ? { killed } : {}) };
1394 // A red verification keeps the map it measured all the same, and
1395 // names the files it failed in for the bisection (#7786).
1396 const files = x.tier === "verify" ? failedFiles(checkLines) : [];
1397 if (x.tier === "verify" && x.keepImpact === true) {
1398 try {
1399 const kept = keepImpact(app.config.dataDir, repo.id, dir);
1400 if (kept !== undefined) write(kept);
1401 } catch (error) {
1402 write(`impact map not kept: ${String(error)}`);
1403 }
1404 }
1405 return {
1406 ok: false,
1407 error: killed ? ran.error : summary,
1408 detail: files.length > 0 ? { ...(detail ?? { passed: 0, failed: files.length, names: [] }), files } : detail,
1409 stage: checkStage(ran.error),
1410 ...(killed ? { killed } : {}),
1411 };
12591412 }
12601413 // Only a green check of the default branch is kept for the next.
12611414 if (
@@ -1460,6 +1613,14 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
14601613 const timer = setInterval(() => {
14611614 if (!stopped) schedule();
14621615 }, TICK_MS);
1616 // Main is verified after landing on a schedule, when the forge says how often.
1617 const sweeper =
1618 options.verifyEveryMs === undefined
1619 ? undefined
1620 : setInterval(() => {
1621 if (!stopped)
1622 askVerifications().catch((error) => log.error("verification not asked", { error: String(error) }));
1623 }, Math.min(VERIFY_SWEEP_MS, options.verifyEveryMs));
14631624 // A gate's slot frees: the pull check held for it starts now, not at the tick.
14641625 const unwatch = watchGates(schedule);
14651626 schedule();
@@ -1529,11 +1690,13 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
15291690 for (const g of gates.values()) if (g.commitOid === commitOid) return g.done;
15301691 return undefined;
15311692 },
1532 spareSlots: () => Math.max(0, capacity() - PULL_RESERVE - inFlight.size - extraGates() - proofsInFlight()),
1693 spareSlots: () => Math.max(0, capacity() - PULL_RESERVE - slotted() - extraGates() - proofsInFlight()),
1694 verifyDue: (now) => askVerifications(now),
15331695 lane: gate,
15341696 stop: async (stopOptions) => {
15351697 stopped = true;
15361698 clearInterval(timer);
1699 if (sweeper !== undefined) clearInterval(sweeper);
15371700 unsubscribe();
15381701 unwatch();
15391702 // A roll waits for the checks under way, up to the budget, and only
src/checks/verify.ts added +139
@@ -0,0 +1,139 @@
1 /**
2 * Verify after landing (#7786). Landing is judged on lint and types; the
3 * whole suite runs after, on the default branch, on a schedule: at most
4 * once per {@link VERIFY_EVERY_MS}, only when main has moved since the last
5 * verification, in a lane of its own that no landing waits on. Its run
6 * measures the impact map (`--impact`) the fast tier selects by.
7 *
8 * A verification that stays red after its failed tests are re-run alone
9 * names the landing that broke it: the first-parent commits between the
10 * last green verification and the red one are bisected, each step running
11 * only the failed files, until one commit is left. That commit is a landed
12 * pull (its revert is proposed) or a push no pull carried (one issue).
13 *
14 * Plain data in, plain data out: when one is due, which commit to try
15 * next, which files a log says failed. The service runs them.
16 */
17
18 /** The environment a scheduled verification of the default branch runs in. */
19 export const VERIFY_ENV = "verify";
20 /** The environment a bisection step runs in. */
21 export const BISECT_ENV = "verify-bisect";
22 /** A bisection step's reason: this prefix and the id of the red verification it serves. */
23 export const BISECT_PREFIX = "bisect:";
24 /** The reason a scheduled verification carries. */
25 export const VERIFY_REASON = "schedule";
26
27 /** At most one verification per repository this often. */
28 export const VERIFY_EVERY_MS = 30 * 60_000;
29 /** How often the forge looks for a verification that is due. */
30 export const VERIFY_SWEEP_MS = 60_000;
31 /** A range wider than this is not bisected: it is named in one issue. */
32 export const MAX_RANGE = 256;
33 /** At most this many failed files are carried into a bisection. */
34 export const MAX_FILES = 20;
35
36 /** Whether a check run belongs to the verification lane. */
37 export const inVerifyLane = (env: string): boolean => env === VERIFY_ENV || env === BISECT_ENV;
38
39 /** The red verification a bisection step serves, from its reason. */
40 export function bisectOrigin(reason: string): string | undefined {
41 if (!reason.startsWith(BISECT_PREFIX)) return undefined;
42 const id = reason.slice(BISECT_PREFIX.length);
43 return /^[0-9A-Za-z]{1,40}$/.test(id) ? id : undefined;
44 }
45
46 export interface Due {
47 /** The default branch's newest checked commit. */
48 readonly tip: string;
49 /** The last verification asked for, if any. */
50 readonly last?: { readonly commitOid: string; readonly createdAt: string } | undefined;
51 /** A verification or bisection is queued or running for this repository. */
52 readonly active: boolean;
53 readonly now: number;
54 readonly everyMs: number;
55 }
56
57 /** Whether to ask for a verification of `tip` now. */
58 export function verifyDue(d: Due): boolean {
59 if (d.active || d.tip === "") return false;
60 if (d.last === undefined) return true;
61 if (d.last.commitOid === d.tip) return false;
62 return d.now - Date.parse(d.last.createdAt) >= d.everyMs;
63 }
64
65 export type Step =
66 | { readonly kind: "test"; readonly commit: string }
67 | { readonly kind: "culprit"; readonly commit: string };
68
69 /**
70 * The next step of a bisection. `range` is the first-parent commits after
71 * the last green verification, oldest first; its last is the red one.
72 * `green` maps a commit already tried to whether the failed files passed
73 * there. The culprit is the oldest red commit whose parent in the range is
74 * green (or is the green verification itself).
75 */
76 export function nextStep(range: readonly string[], green: ReadonlyMap<string, boolean>): Step | undefined {
77 if (range.length === 0) return undefined;
78 let lo = -1;
79 for (let i = 0; i < range.length - 1; i++) if (green.get(range[i]!) === true) lo = i;
80 let hi = range.length - 1;
81 for (let i = lo + 1; i < hi; i++)
82 if (green.get(range[i]!) === false) {
83 hi = i;
84 break;
85 }
86 if (hi - lo === 1) return { kind: "culprit", commit: range[hi]! };
87 return { kind: "test", commit: range[Math.floor((lo + hi) / 2)]! };
88 }
89
90 /** The most steps a range of `n` commits can take: ⌈log₂ n⌉. */
91 export const maxSteps = (n: number): number => (n <= 1 ? 0 : Math.ceil(Math.log2(n)));
92
93 const HEADER = /^(\S+\.test\.[cm]?[jt]sx?):$/;
94 const FAILED = /^\(fail\) /;
95 const PATH = /^[\w@.+][\w@./+-]{0,199}$/;
96
97 /** What `scripts/test-shards.ts --keep-going` says of a file red once and green alone. */
98 export const FLAKY_ALONE = "red once, green alone:";
99
100 /**
101 * The test files a `bun test` log printed a failure under, once each, in
102 * order, at most {@link MAX_FILES}. A path that could be read as a flag or
103 * climbs out of the checkout is left out: the log is repository output.
104 * A file the runner says went green alone is not one of them.
105 */
106 export function failedFiles(lines: readonly string[]): string[] {
107 const found: string[] = [];
108 const flaky = new Set<string>();
109 let current: string | undefined;
110 for (const line of lines) {
111 const at = line.indexOf(FLAKY_ALONE);
112 if (at >= 0) flaky.add(line.slice(at + FLAKY_ALONE.length).trim().replace(/^\.\//, ""));
113 }
114 for (const line of lines) {
115 const header = HEADER.exec(line);
116 if (header) current = header[1]!.replace(/^\.\//, "");
117 else if (FAILED.test(line) && current !== undefined && safeFile(current) && !flaky.has(current) && !found.includes(current)) {
118 found.push(current);
119 if (found.length >= MAX_FILES) break;
120 }
121 }
122 return found;
123 }
124
125 /** A test file path the forge hands a check to run alone. */
126 export const safeFile = (path: string): boolean => PATH.test(path) && !path.split("/").includes("..");
127
128 /** `FORGE_CHECK_ONLY`: the files a bisection step runs, comma-separated. */
129 export const onlyEnv = (files: readonly string[]): string => files.filter(safeFile).join(",");
130
131 /** The files `FORGE_CHECK_ONLY` names, the unsafe left out; undefined when unset. */
132 export function onlyFiles(value: string | undefined): string[] | undefined {
133 if (value === undefined) return undefined;
134 return value
135 .split(",")
136 .map((file) => file.trim())
137 .filter((file) => file !== "" && safeFile(file))
138 .slice(0, MAX_FILES);
139 }
src/core/migration-pins/20261101130000-run-verify.pin added +1
@@ -0,0 +1,1 @@
1 3b443eee32968424d43c9328895e24d661770c98baaa37864d24cbe182d73fc2
src/core/migrations/20261101130000-run-verify.ts added +7
@@ -0,0 +1,7 @@
1 // Main verified after landing (#7786): its runs are checks in the `verify`
2 // and `verify-bisect` environments. The scheduler and /boards' Pipeline ask
3 // for a repository's newest ones; this keeps that a seek, not a walk of
4 // every check the repository ever ran.
5 export default `
6 CREATE INDEX IF NOT EXISTS run_verify ON run(repo_id, env, id DESC);
7 `;
src/core/runs.ts +2
@@ -45,6 +45,8 @@ export interface CheckDetail {
4545 readonly passed: number;
4646 readonly failed: number;
4747 readonly names: readonly string[];
48 /** The test files the failures printed under, when a verification after landing failed (#7786). */
49 readonly files?: readonly string[];
4850 /** The check machine's free disk share (0..1) when a red finished (#5453). */
4951 readonly diskFree?: number;
5052 }
src/flow/verified.ts added +123
@@ -0,0 +1,123 @@
1 /**
2 * How long main has gone unverified (#7786): for each repository whose
3 * main is verified after landing (`checks/verify.ts`), the time since the
4 * first landing the last green verification does not cover. /boards'
5 * Pipeline card says it, one line per repository, linking the run.
6 *
7 * "Unverified since" is when the oldest default-branch check after the
8 * last green verification's commit was asked for — that commit landed
9 * then. Verified at the tip, it is nothing; never verified green, it is
10 * since the first verification was asked for.
11 */
12
13 import type { Database } from "bun:sqlite";
14 import type { App } from "../app.ts";
15 import { getRepoById } from "../core/repos.ts";
16
17 export interface VerifiedRow {
18 /** `owner/name`. */
19 readonly repo: string;
20 readonly public: boolean;
21 /** When main stopped being verified, or null when its tip is verified green. */
22 readonly unverifiedSince: string | null;
23 /** The last green verification: its run and commit. */
24 readonly green: { readonly runId: string; readonly commitOid: string; readonly at: string } | null;
25 /** The newest verification, any state. */
26 readonly last: { readonly runId: string; readonly state: string; readonly commitOid: string } | null;
27 }
28
29 interface RunRow {
30 id: string;
31 commit_oid: string;
32 state: string;
33 created_at: string;
34 finished_at: string | null;
35 }
36
37 /** Each repository with a verification, newest first; private ones for the owner only. */
38 export function readVerified(app: App, owner: boolean): VerifiedRow[] {
39 const ids = app.db
40 .query<{ repo_id: string }, []>(
41 `SELECT DISTINCT repo_id FROM run WHERE kind = 'check' AND env = 'verify'`,
42 )
43 .all();
44 const rows: VerifiedRow[] = [];
45 for (const { repo_id } of ids) {
46 const repo = getRepoById(app.db, repo_id);
47 if (repo === undefined || (!owner && repo.visibility !== "public")) continue;
48 rows.push(verifiedOf(app.db, repo.id, repo.defaultBranch, `${repo.ownerName}/${repo.name}`, repo.visibility === "public"));
49 }
50 return rows.sort((a, b) => (a.unverifiedSince ?? "~").localeCompare(b.unverifiedSince ?? "~"));
51 }
52
53 /** One repository's row. */
54 export function verifiedOf(db: Database, repoId: string, branch: string, name: string, isPublic: boolean): VerifiedRow {
55 // A state of "" is any state.
56 const verify = (state: string): RunRow | null =>
57 db
58 .query<RunRow, [string, string]>(
59 `SELECT id, commit_oid, state, created_at, finished_at FROM run
60 WHERE repo_id = ?1 AND kind = 'check' AND env = 'verify' AND (?2 = '' OR state = ?2)
61 ORDER BY id DESC LIMIT 1`,
62 )
63 .get(repoId, state);
64 const green = verify("ok");
65 const last = verify("");
66 // The newest default-branch check of the green commit; what came after it landed after.
67 const covered =
68 green === null
69 ? null
70 : db
71 .query<{ id: string }, [string, string, string]>(
72 `SELECT MAX(id) AS id FROM run WHERE repo_id = ?1 AND kind = 'check'
73 AND issue_id IS NULL AND env = ?2 AND commit_oid = ?3`,
74 )
75 .get(repoId, branch, green.commit_oid);
76 const since =
77 green === null
78 ? (db
79 .query<{ created_at: string }, [string]>(
80 `SELECT MIN(created_at) AS created_at FROM run
81 WHERE repo_id = ?1 AND kind = 'check' AND env = 'verify'`,
82 )
83 .get(repoId)?.created_at ?? null)
84 : (db
85 .query<{ created_at: string }, [string, string, string, string]>(
86 `SELECT created_at FROM run WHERE repo_id = ?1 AND kind = 'check' AND issue_id IS NULL
87 AND env = ?2 AND id > ?3 AND commit_oid != ?4 ORDER BY id LIMIT 1`,
88 )
89 .get(repoId, branch, covered?.id ?? green.id, green.commit_oid)?.created_at ?? null);
90 return {
91 repo: name,
92 public: isPublic,
93 unverifiedSince: since,
94 green: green === null ? null : { runId: green.id, commitOid: green.commit_oid, at: green.finished_at ?? green.created_at },
95 last: last === null ? null : { runId: last.id, state: last.state, commitOid: last.commit_oid },
96 };
97 }
98
99 const short = (oid: string): string => oid.slice(0, 7);
100
101 /** A span in whole minutes under an hour, else hours to one decimal: 7 min, 2.5 h. */
102 export const spanWords = (ms: number): string =>
103 ms < 3_600_000 ? `${Math.floor(Math.max(0, ms) / 60_000)} min` : `${(ms / 3_600_000).toFixed(1)} h`;
104
105 /** The sentence for a row: how long main has gone unverified, and what the last verification said. */
106 export function verifiedSays(row: VerifiedRow, now: number): string {
107 const lastSays =
108 row.last === null
109 ? ""
110 : row.last.state === "ok"
111 ? ""
112 : row.last.state === "failed"
113 ? `; last verification red at ${short(row.last.commitOid)}`
114 : row.last.state === "running" || row.last.state === "queued"
115 ? `; verifying ${short(row.last.commitOid)} now`
116 : "";
117 if (row.unverifiedSince === null && row.green !== null)
118 return `main verified at ${short(row.green.commitOid)}, ${spanWords(Math.max(0, now - Date.parse(row.green.at)))} ago${lastSays}`;
119 const span = row.unverifiedSince === null ? "—" : spanWords(Math.max(0, now - Date.parse(row.unverifiedSince)));
120 return row.green === null
121 ? `main never verified green, ${span} since the first verification${lastSays}`
122 : `main unverified for ${span} since ${short(row.green.commitOid)} was verified green${lastSays}`;
123 }
src/main.ts +3
@@ -35,6 +35,7 @@ import { listApps } from "./core/apps.ts";
3535 import { holdAppHosts, releaseAppHosts } from "./apps/holding.ts";
3636 import { resolveAppDomain } from "./apps/host.ts";
3737 import { LAND_SETTLE_MS, requeueAbandonedChecks, startChecks } from "./checks/service.ts";
38 import { VERIFY_EVERY_MS } from "./checks/verify.ts";
3839 import { RED_HOLD_MS } from "./checks/red_main.ts";
3940 import { stopQueue } from "./checks/queue.ts";
4041 import { startLooks } from "./looks/service.ts";
@@ -407,6 +408,8 @@ const checks =
407408 // before cutting them off: a full gate takes about ten minutes, and
408409 // the pod's grace (charts/app, 720 s) is longer still (#6722).
409410 drainMs: positive(process.env.FORGE_CHECKS_DRAIN_S, 660) * 1000,
411 // Main is verified after landing at most this often (#7786).
412 verifyEveryMs: VERIFY_EVERY_MS,
410413 });
411414 // Migrated repositories kept as a read-only mirror push to their source.
412415 const mirrorOut = startMirrorOut(app);
src/system/telemetry.ts +3
@@ -16,6 +16,7 @@ import { type QueueReading, readQueues } from "../flow/queue.ts";
1616 import { reviewersBusy, reviewingSays } from "../flow/reviewing.ts";
1717 import { requestedSays, requestedWaits } from "../flow/requested.ts";
1818 import { readThroughput, stageOut, throughputSays } from "../flow/throughput.ts";
19 import { readVerified, verifiedSays } from "../flow/verified.ts";
1920 import type { Reader } from "../actions/telemetry.ts";
2021 import { agentView, live } from "./live.ts";
2122 import type { SystemView } from "./view.ts";
@@ -38,6 +39,8 @@ export function flowFor(app: App, reader: Reader, now = new Date()) {
3839 const parked = readParked(app.db, now, reader.kind === "agent");
3940 return {
4041 says: verdict(stages, now.getTime()),
42 // How long each main has gone unverified after landing (#7786); private repositories for the owner only.
43 verified: readVerified(app, reader.kind === "owner").map((row) => ({ ...row, says: verifiedSays(row, now.getTime()) })),
4144 // Issues the crew parked in the last 24 h and 7 d, the newest three named (#6471).
4245 parked: { says: parkedSays(parked), ...parked },
4346 // Open pulls, opened and landed in 1 h and 6 h, and conflicted, against #5924's targets (#5935).
src/tui/forge.ts +2
@@ -56,6 +56,7 @@ import { capacityScreen, mayCapacity } from "./screens/capacity.ts";
5656 import { forgeCeiling } from "../flow/ceiling.ts";
5757 import { flowLines, flowScreen } from "./screens/flow.ts";
5858 import { readHeld } from "../flow/held.ts";
59 import { readVerified } from "../flow/verified.ts";
5960 import { memoryScreen } from "./screens/system.ts";
6061 import { noteRead, telemetryReader } from "../actions/telemetry.ts";
6162 import { queuesFor, systemFor } from "../system/telemetry.ts";
@@ -298,6 +299,7 @@ export function forgeCommands(session: Session): (ctx: Ctx) => Command[] {
298299 flowReader.kind === "agent" ? { queues: queuesFor(session.app, flowReader, new Date(), { batches: true }) } : undefined,
299300 readHeld(session.app, flowReader.kind === "owner"),
300301 forgeCeiling(session.app.config),
302 readVerified(session.app, flowReader.kind === "owner"),
301303 )));
302304 },
303305 });
src/tui/screens/flow.ts +8 −1
@@ -12,7 +12,8 @@
1212 * conflicted pulls, the idle crew pulls (#6479) and the pulls with
1313 * changes requested waiting on a builder and on a reviewer (`flow/requested.ts`, #2172); then each
1414 * autonomous repository's ready list, startable beside the count per
15 * filter that holds the rest (`flow/held.ts`, #2244), as the flow panel draws it.
15 * filter that holds the rest (`flow/held.ts`, #2244), as the flow panel draws it;
16 * last, how long each main has gone unverified after landing (`flow/verified.ts`, #7786).
1617 * Read-only; Enter, Esc or q closes it.
1718 */
1819
@@ -23,6 +24,7 @@ import { pullCounts } from "../../crew/hot_files.ts";
2324 import { warmingLine } from "../../crew/hot_history.ts";
2425 import { ago } from "../../feed/rank.ts";
2526 import type { HeldRow } from "../../flow/held.ts";
27 import { verifiedSays, type VerifiedRow } from "../../flow/verified.ts";
2628 import { conflictedSays, rebaseLaneSays } from "../../flow/conflicted.ts";
2729 import { IDLE_LIMIT, type IdlePull } from "../../flow/idle_pulls.ts";
2830 import { pileSays } from "../../flow/pile.ts";
@@ -60,6 +62,8 @@ export function flowLines(
6062 held?: readonly HeldRow[],
6163 /** The forge's daily ceiling, which the idle-crew alarm needs to know it is not paused on purpose. */
6264 ceilingUsd: number | null = null,
65 /** How long each main has gone unverified after landing (`readVerified`, already cut to what the reader may see). */
66 verified?: readonly VerifiedRow[],
6367 ): string[] {
6468 const stages = readPipeline(db, now);
6569 const flow = readThroughput(db, stages.map((s) => s.key), now);
@@ -122,6 +126,9 @@ export function flowLines(
122126 ...(reviewing === null ? [] : ["", ...wrap(reviewing, inner)]),
123127 ...(warming === "" ? [] : ["", ...warming.split("\n").flatMap((l) => wrap(l, inner))]),
124128 ...(held === undefined ? [] : ["", ...heldLines(held, inner)]),
129 ...(verified === undefined || verified.length === 0
130 ? []
131 : ["", ...verified.flatMap((r) => wrap(`${r.repo}: ${verifiedSays(r, now.getTime())}`, inner))]),
125132 "",
126133 ...duplicateLines(dups, inner),
127134 ];
src/web/pages/flow-panel.ts +23
@@ -43,6 +43,7 @@ import { buildYieldSays } from "../../flow/build_yield.ts";
4343 import { reviewLatencySays } from "../../flow/review_latency.ts";
4444 import { openerCounts } from "../../flow/openers.ts";
4545 import { readHeld } from "../../flow/held.ts";
46 import { readVerified, verifiedSays, type VerifiedRow } from "../../flow/verified.ts";
4647 import type { App } from "../../app.ts";
4748 import { html, type Html } from "../html.ts";
4849 import { gatesCard } from "./boards-gates.ts";
@@ -211,6 +212,7 @@ export function flowPanel(app: App, reader: Reader, url: URL, base: string): Htm
211212 </div>`,
212213 { head: html`<h2>Pipeline</h2><a class="muted small" href="/settings/capacity">Capacity</a>`, id: "pipeline", pad: false },
213214 )}
215 ${verifiedCard(readVerified(app, reader.kind === "owner"), now)}
214216 ${hotFilesCard(app, reader)}
215217 ${heldCard(readHeld(app, reader.kind === "owner", now))}
216218 ${queueCard(queuesFor(app, reader, new Date(now), { batches: true }))}
@@ -219,6 +221,27 @@ export function flowPanel(app: App, reader: Reader, url: URL, base: string): Htm
219221 ${listed === null || shown === undefined ? null : outList(shown, listed.hour, now)}`;
220222 }
221223
224 /**
225 * How long main has gone unverified, per repository whose main is verified
226 * after landing (#7786): the whole suite runs there on a schedule, after
227 * landings that were judged on lint and types.
228 */
229 function verifiedCard(rows: readonly VerifiedRow[], now: number): Html {
230 return card(
231 html`<h2>Verified after landing</h2>
232 ${rows.length === 0
233 ? html`<p class="muted">No repository verifies its main after landing yet: it declares <span class="mono">check.verify</span> in forge.json.</p>`
234 : rows.map(
235 (r) =>
236 html`<p data-verified="${r.repo}" data-unverified-since="${r.unverifiedSince ?? ""}">
237 <a href="/${r.repo}">${r.repo}</a>: ${verifiedSays(r, now)}
238 ${r.last === null ? null : html`· <a class="small" href="/${r.repo}/runs/${r.last.runId}">run</a>`}
239 </p>`,
240 )}`,
241 { id: "verified" },
242 );
243 }
244
222245 /** The count, `20+` when the read stopped at its limit. */
223246 const idleCount = (idle: readonly IdlePull[]): string => `${idle.length}${idle.length >= IDLE_LIMIT ? "+" : ""}`;
224247
test/check-verify.test.ts added +110
@@ -0,0 +1,110 @@
1 /**
2 * The checks service verifies main after landing (#7786): `verifyDue`
3 * asks for a run of the declared `check.verify` on main's checked tip, once
4 * per move of main; a red that fails alone too names the commit that broke
5 * it — here a push no pull carried, so one issue.
6 */
7 import { afterAll, beforeAll, expect, test } from "bun:test";
8 import { mkdtemp, rm } from "node:fs/promises";
9 import { tmpdir } from "node:os";
10 import { join } from "node:path";
11 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
12 import { createApp, type App } from "../src/app.ts";
13 import { startChecks, type ChecksHandle } from "../src/checks/service.ts";
14 import { VERIFY_ENV } from "../src/checks/verify.ts";
15 import type { Repo } from "../src/core/repos.ts";
16 import { getRun, listRuns, type Run } from "../src/core/runs.ts";
17 import { createUser, type User } from "../src/core/users.ts";
18 import { readRunLog } from "../src/runs/log.ts";
19 import { quietLogs } from "./helpers/globals.ts";
20
21 quietLogs();
22
23 let dir: string;
24 let app: App;
25 let alice: User;
26 let repo: Repo;
27 let checks: ChecksHandle;
28
29 const report = (fail: boolean): string =>
30 fail
31 ? `printf 't/a.test.ts:\\n(fail) grp > works\\n 0 pass\\n 1 fail\\n'; exit 1`
32 : `printf 't/a.test.ts:\\n(pass) grp > works\\n 1 pass\\n 0 fail\\n'`;
33
34 async function push(fail: boolean): Promise<string> {
35 const pushed = await commitToBranch(app, alice, repo, {
36 message: fail ? "breaks it" : "works",
37 files: [
38 { path: "forge.json", content: JSON.stringify({ check: { run: "true", verify: report(fail), install: "" } }) },
39 {
40 path: "t/a.test.ts",
41 content: `import { test, expect, describe } from "bun:test";\ndescribe("grp", () => { test("works", () => { expect(${fail ? 2 : 1}).toBe(1); }); });\n`,
42 },
43 ],
44 });
45 if (!pushed.ok) throw new Error(JSON.stringify(pushed.error));
46 return pushed.value.commitOid;
47 }
48
49 async function until<T>(read: () => T | undefined, ms = 40_000): Promise<T> {
50 const deadline = Date.now() + ms;
51 for (;;) {
52 const got = read();
53 if (got !== undefined) return got;
54 if (Date.now() > deadline) throw new Error("timed out");
55 await Bun.sleep(100);
56 }
57 }
58
59 const finished = (env: string, oid: string): Run | undefined =>
60 listRuns(app.db, repo.id, { kind: "check", limit: 50 }).find(
61 (r) => r.env === env && r.commitOid === oid && (r.state === "ok" || r.state === "failed"),
62 );
63
64 beforeAll(async () => {
65 process.env["FORGE_CELLS"] = "off";
66 dir = await mkdtemp(join(tmpdir(), "check-verify-"));
67 app = createApp({ dataDir: dir, port: 0, host: "forge.test" });
68 const u = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
69 if (!u.ok) throw new Error(u.error);
70 alice = u.value;
71 const made = await createRepo(app, alice, { name: "verified", visibility: "public" });
72 if (!made.ok) throw new Error("repo");
73 repo = made.value;
74 checks = startChecks(app, { forgeUrl: "http://forge.test", concurrency: 1 });
75 });
76 afterAll(async () => {
77 await checks.stop();
78 await rm(dir, { recursive: true, force: true });
79 });
80
81 test("a green main is verified once; asked again before it moves, it is not", async () => {
82 const oid = await push(false);
83 await until(() => finished(repo.defaultBranch, oid));
84 const asked = await checks.verifyDue();
85 expect(asked.map((r) => [r.env, r.commitOid])).toEqual([[VERIFY_ENV, oid]]);
86 const run = await until(() => finished(VERIFY_ENV, oid));
87 expect(run.state).toBe("ok");
88 expect(await checks.verifyDue()).toEqual([]);
89 }, 60_000);
90
91 test("a red verification that fails alone too names the push that broke it in one issue", async () => {
92 const oid = await push(true);
93 await until(() => finished(repo.defaultBranch, oid));
94 // Main moved, but the last verification is not 30 minutes old yet.
95 expect(await checks.verifyDue()).toEqual([]);
96 const asked = await checks.verifyDue(Date.now() + 31 * 60_000);
97 expect(asked).toHaveLength(1);
98 const run = await until(() => finished(VERIFY_ENV, oid));
99 expect(run.state).toBe("failed");
100 expect(getRun(app.db, run.id)?.checkDetail?.files).toEqual(["t/a.test.ts"]);
101 const issue = await until(() =>
102 app.db
103 .query<{ title: string; body: string }, [string]>("SELECT title, body FROM issue WHERE repo_id = ?1 AND kind = 'issue' AND title LIKE 'Red after landing:%'")
104 .get(repo.id) ?? undefined,
105 );
106 expect(issue.title).toContain(oid.slice(0, 7));
107 const log = readRunLog(app.config.dataDir, run.id).lines.join("\n");
108 expect(log).toContain("fails alone too");
109 expect(log).toContain("landed by no pull");
110 }, 60_000);
test/shards-keep-going.test.ts added +80
@@ -0,0 +1,80 @@
1 /**
2 * `scripts/test-shards.ts --keep-going`, the verification after landing
3 * (#7786): a red file stops nothing, a file red once and green alone is
4 * named flaky and passes, the impact of every green file is written though
5 * another is red, and `FORGE_CHECK_ONLY` keeps only the files it names.
6 */
7 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
8 import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
9 import { tmpdir } from "node:os";
10 import { join } from "node:path";
11 import { FLAKY_ALONE } from "../src/checks/verify.ts";
12
13 const script = new URL("../scripts/test-shards.ts", import.meta.url).pathname;
14 let dir: string;
15
16 beforeAll(() => {
17 dir = mkdtempSync(join(tmpdir(), "shards-keep-going-"));
18 mkdirSync(join(dir, "test"));
19 writeFileSync(join(dir, "lib.ts"), "export const one = (): number => 1;\n");
20 // The heaviest by bytes runs first, so the red is first and the rest still run.
21 writeFileSync(
22 join(dir, "test", "a.test.ts"),
23 `import { test, expect } from "bun:test";\n${"// padding so this file is heaviest\n".repeat(40)}test("red", () => { expect(1).toBe(2); });\n`,
24 );
25 writeFileSync(
26 join(dir, "test", "b.test.ts"),
27 `import { test, expect } from "bun:test";\nimport { one } from "../lib.ts";\ntest("green", () => { expect(one()).toBe(1); });\n`,
28 );
29 // Red the first time it runs, green after: a flake.
30 writeFileSync(
31 join(dir, "test", "c.test.ts"),
32 `import { test, expect } from "bun:test";\nimport { existsSync, writeFileSync } from "node:fs";\nimport { one } from "../lib.ts";\n` +
33 `test("flaky", () => { const seen = existsSync("seen"); writeFileSync("seen", ""); expect(seen && one() === 1).toBe(true); });\n`,
34 );
35 });
36 afterAll(() => rmSync(dir, { recursive: true, force: true }));
37
38 async function shards(args: string[], env: Record<string, string> = {}): Promise<{ code: number; out: string }> {
39 const child = Bun.spawn(["bun", script, ...args, "test/a.test.ts", "test/b.test.ts", "test/c.test.ts"], {
40 cwd: dir,
41 env: { ...process.env, FORGE_CHECK_SHARDS: "1", FORGE_CHECK_IMPACT: ".impact.json", ...env },
42 stdin: "ignore",
43 stdout: "pipe",
44 stderr: "pipe",
45 });
46 const [out, err] = await Promise.all([new Response(child.stdout).text(), new Response(child.stderr).text()]);
47 return { code: await child.exited, out: out + err };
48 }
49
50 describe("--keep-going", () => {
51 test("every file runs past the red, the flake passes alone, and the green files' impact is written", async () => {
52 rmSync(join(dir, "seen"), { force: true });
53 const { code, out } = await shards(["--keep-going", "--impact"]);
54 expect(code).toBe(1);
55 expect(out).not.toContain("fail-fast");
56 expect(out).toContain(`${FLAKY_ALONE} test/c.test.ts`);
57 expect(out).toContain("Ran 3 tests across 3 files");
58 const map = JSON.parse(readFileSync(join(dir, ".impact.json"), "utf8")) as Record<string, unknown>;
59 const tests = JSON.stringify(map);
60 expect(tests).toContain("test/b.test.ts");
61 expect(tests).toContain("test/c.test.ts");
62 expect(tests).not.toContain("test/a.test.ts");
63 }, 60_000);
64
65 test("without it, the first red stops the rest and no map is written", async () => {
66 rmSync(join(dir, "seen"), { force: true });
67 rmSync(join(dir, ".impact.json"), { force: true });
68 const { code, out } = await shards(["--impact"]);
69 expect(code).toBe(1);
70 expect(out).toContain("fail-fast");
71 expect(existsSync(join(dir, ".impact.json"))).toBe(false);
72 }, 60_000);
73
74 test("FORGE_CHECK_ONLY runs only the files it names", async () => {
75 const { code, out } = await shards(["--keep-going"], { FORGE_CHECK_ONLY: "test/b.test.ts,../x.test.ts" });
76 expect(code).toBe(0);
77 expect(out).toContain("FORGE_CHECK_ONLY keeps 1 of 1 named file(s)");
78 expect(out).not.toContain("(fail) red");
79 }, 60_000);
80 });
test/verify-after-landing.test.ts added +318
@@ -0,0 +1,318 @@
1 /**
2 * Verify after landing (#7786): main's whole suite runs on a schedule with
3 * `--impact`, a red that stays red alone is bisected to the landing that
4 * broke it, which gets one revert pull (or, landed by no pull, one issue),
5 * and /boards' Pipeline says how long main has gone unverified.
6 */
7
8 import { afterEach, beforeEach, describe, expect, test } from "bun:test";
9 import { readFileSync } from "node:fs";
10 import { mkdtemp, rm } from "node:fs/promises";
11 import { tmpdir } from "node:os";
12 import { join } from "node:path";
13 import { createApp, type App } from "../src/app.ts";
14 import { mergePull, openPull } from "../src/actions/pulls.ts";
15 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
16 import { parseCheck } from "../src/apps/manifest.ts";
17 import { afterVerify } from "../src/checks/culprit.ts";
18 import {
19 BISECT_ENV,
20 BISECT_PREFIX,
21 bisectOrigin,
22 FLAKY_ALONE,
23 failedFiles,
24 maxSteps,
25 nextStep,
26 onlyEnv,
27 onlyFiles,
28 safeFile,
29 VERIFY_ENV,
30 VERIFY_EVERY_MS,
31 verifyDue,
32 } from "../src/checks/verify.ts";
33 import { listPulls, getPull, type Pull } from "../src/core/pulls.ts";
34 import type { Repo } from "../src/core/repos.ts";
35 import { createRun, getRun, type Run } from "../src/core/runs.ts";
36 import { createUser, type User } from "../src/core/users.ts";
37 import { readVerified, spanWords, verifiedSays } from "../src/flow/verified.ts";
38 import { resolveOid } from "../src/git/merge.ts";
39 import { repoDir } from "../src/git/repos.ts";
40 import { flowPanel } from "../src/web/pages/flow-panel.ts";
41 import { flowLines } from "../src/tui/screens/flow.ts";
42 import { flowFor } from "../src/system/telemetry.ts";
43
44 describe("when a verification is due", () => {
45 const at = Date.parse("2026-11-01T12:00:00Z");
46 const base = { tip: "b", active: false, now: at, everyMs: VERIFY_EVERY_MS };
47 test("never verified: now", () => expect(verifyDue(base)).toBe(true));
48 test("main has not moved: not again", () =>
49 expect(verifyDue({ ...base, last: { commitOid: "b", createdAt: "2026-11-01T00:00:00Z" } })).toBe(false));
50 test("moved, 29 minutes after the last: not yet; 30: now", () => {
51 expect(verifyDue({ ...base, last: { commitOid: "a", createdAt: "2026-11-01T11:31:00Z" } })).toBe(false);
52 expect(verifyDue({ ...base, last: { commitOid: "a", createdAt: "2026-11-01T11:30:00Z" } })).toBe(true);
53 });
54 test("one already queued or running: never a second", () => expect(verifyDue({ ...base, active: true })).toBe(false));
55 test("every 30 minutes at most", () => expect(VERIFY_EVERY_MS).toBe(1_800_000));
56 });
57
58 describe("bisecting a range", () => {
59 // Eight commits after the green; c3 broke it. ⌈log₂ 8⌉ = 3 steps at most.
60 const range = ["c0", "c1", "c2", "c3", "c4", "c5", "c6", "c7"];
61 const broken = (c: string): boolean => range.indexOf(c) >= 3;
62
63 test("walks to the culprit in at most ⌈log₂ n⌉ steps, each a commit inside the range", () => {
64 const tried = new Map<string, boolean>();
65 let step = nextStep(range, tried);
66 while (step?.kind === "test") {
67 expect(range.slice(0, -1)).toContain(step.commit);
68 tried.set(step.commit, !broken(step.commit));
69 step = nextStep(range, tried);
70 }
71 expect(step).toEqual({ kind: "culprit", commit: "c3" });
72 expect(tried.size).toBeLessThanOrEqual(maxSteps(range.length));
73 expect(maxSteps(8)).toBe(3);
74 });
75
76 test("by hand: the first step is the middle, c3; red there, then c1; green, then c2; green: c3", () => {
77 const tried = new Map<string, boolean>();
78 expect(nextStep(range, tried)).toEqual({ kind: "test", commit: "c3" });
79 tried.set("c3", false);
80 expect(nextStep(range, tried)).toEqual({ kind: "test", commit: "c1" });
81 tried.set("c1", true);
82 expect(nextStep(range, tried)).toEqual({ kind: "test", commit: "c2" });
83 tried.set("c2", true);
84 expect(nextStep(range, tried)).toEqual({ kind: "culprit", commit: "c3" });
85 });
86
87 test("one commit since the green is the culprit at once", () =>
88 expect(nextStep(["c0"], new Map())).toEqual({ kind: "culprit", commit: "c0" }));
89
90 test("the culprit is the newest when every step is green", () => {
91 const tried = new Map([["c0", true], ["c1", true]]);
92 expect(nextStep(["c0", "c1", "c2"], tried)).toEqual({ kind: "culprit", commit: "c2" });
93 });
94
95 test("a step's reason names its red verification, and nothing else is read as one", () => {
96 expect(bisectOrigin(`${BISECT_PREFIX}01abc`)).toBe("01abc");
97 expect(bisectOrigin("bisect:../x")).toBeUndefined();
98 expect(bisectOrigin("schedule")).toBeUndefined();
99 });
100 });
101
102 describe("the files a red names", () => {
103 const log = [
104 "test/a.test.ts:",
105 "(pass) a > one",
106 "test/b.test.ts:",
107 "(fail) b > two",
108 "(fail) b > three",
109 "test/c.test.ts:",
110 "(fail) c > four",
111 "-rf.test.ts:",
112 "(fail) flag",
113 "test/d.test.ts:",
114 "(fail) d > flaky",
115 `test-shards: ${FLAKY_ALONE} ./test/d.test.ts`,
116 ];
117 test("each failing file once, in order; one green alone is not one; a flag-like path is dropped", () =>
118 expect(failedFiles(log)).toEqual(["test/b.test.ts", "test/c.test.ts"]));
119 test("a path that reads as a flag, climbs out, or carries a shell word is not a test file to run", () => {
120 for (const bad of ["-x.test.ts", "--preload=x.test.ts", "../a.test.ts", "test/../../a.test.ts", "a b.test.ts", "a;rm.test.ts", "$(x).test.ts", ""])
121 expect(safeFile(bad)).toBe(false);
122 expect(safeFile("test/a-b.test.ts")).toBe(true);
123 expect(failedFiles(["-rf.test.ts:", "(fail) x"])).toEqual([]);
124 });
125 test("FORGE_CHECK_ONLY carries only safe paths", () => {
126 expect(onlyEnv(["test/a.test.ts", "../etc/passwd", "-x.test.ts"])).toBe("test/a.test.ts");
127 expect(onlyFiles("test/a.test.ts, ../b.test.ts,,test/c.test.ts")).toEqual(["test/a.test.ts", "test/c.test.ts"]);
128 expect(onlyFiles(undefined)).toBeUndefined();
129 });
130 });
131
132 describe("the forge's own forge.json verifies main", () => {
133 const forgeJson = readFileSync(join(import.meta.dir, "..", "forge.json"), "utf8");
134 test("check.verify runs the whole gate with --impact, without stopping at the first red, at low priority", () => {
135 const parsed = parseCheck(forgeJson, undefined);
136 if (!parsed.ok || parsed.value === undefined) throw new Error("forge.json has no check");
137 const verify = parsed.value.verify ?? "";
138 for (const flag of ["--gate", "--impact", "--keep-going", "--nice"]) expect(verify).toContain(flag);
139 });
140 test("a check that does not say how verifies nothing", () => {
141 const parsed = parseCheck(JSON.stringify({ check: { run: "bun test" } }), undefined);
142 expect(parsed.ok && parsed.value?.verify).toBeNull();
143 });
144 });
145
146 describe("a red verification names its culprit", () => {
147 let dataDir: string;
148 let app: App;
149 let alice: User;
150 let repo: Repo;
151 const FILE = "test/thing.test.ts";
152
153 const tip = async (): Promise<string> =>
154 (await resolveOid(repoDir(app.reposRoot, repo.ownerName, repo.name), "refs/heads/main")) ?? "";
155
156 const land = async (branch: string): Promise<Pull> => {
157 await commitToBranch(app, alice, repo, {
158 branch,
159 base: "main",
160 message: branch,
161 files: [{ path: `${branch}.txt`, content: `${branch}\n` }],
162 });
163 const opened = await openPull(app, alice, repo, { title: `Change ${branch}`, baseBranch: "main", headBranch: branch });
164 if (!opened.ok) throw new Error(JSON.stringify(opened.error));
165 const merged = await mergePull(app, alice, repo, opened.value, "squash");
166 if (!merged.ok) throw new Error(JSON.stringify(merged.error));
167 return getPull(app.db, repo.id, opened.value.number)!;
168 };
169
170 const finish = (run: Run, state: "ok" | "failed"): Run => {
171 app.db
172 .query("UPDATE run SET state = ?2, check_detail = ?3, finished_at = ?4 WHERE id = ?1")
173 .run(
174 run.id,
175 state,
176 state === "ok" ? "" : JSON.stringify({ passed: 0, failed: 1, names: ["thing > works"], files: [FILE] }),
177 new Date().toISOString(),
178 );
179 return getRun(app.db, run.id)!;
180 };
181
182 const verification = async (state: "ok" | "failed"): Promise<Run> =>
183 finish(createRun(app.db, { repoId: repo.id, kind: "check", env: VERIFY_ENV, commitOid: await tip(), reason: "schedule" }), state);
184
185 /** Runs the bisection to its end; a step is red at and after `culprit`. */
186 const bisect = async (red: Run, culprit: string, order: readonly string[]): Promise<{ lines: string[]; steps: number }> => {
187 const lines: string[] = [];
188 let steps = 0;
189 let asked: Run | undefined;
190 const ask = async (_r: Repo, env: string, commitOid: string, reason: string): Promise<Run | undefined> => {
191 asked = createRun(app.db, { repoId: repo.id, kind: "check", env, commitOid, reason });
192 return asked;
193 };
194 let next: Run = red;
195 for (let i = 0; i < 10; i++) {
196 asked = undefined;
197 lines.push(...(await afterVerify(app, repo, next, ask)));
198 if (asked === undefined) break;
199 const step: Run = asked;
200 expect(step.env).toBe(BISECT_ENV);
201 steps++;
202 next = finish(step, order.indexOf(step.commitOid) >= order.indexOf(culprit) ? "failed" : "ok");
203 }
204 return { lines, steps };
205 };
206
207 beforeEach(async () => {
208 dataDir = await mkdtemp(join(tmpdir(), "forge-verify-"));
209 app = createApp({ dataDir, port: 0 });
210 const created = await createUser(app.db, {
211 origin: "seed",
212 trusted: true,
213 username: "alice",
214 email: "[email protected]",
215 password: "a-good-passphrase",
216 adminIfFirst: true,
217 });
218 if (!created.ok) throw new Error(created.error);
219 alice = created.value;
220 const made = await createRepo(app, alice, { name: "app", visibility: "public" });
221 if (!made.ok) throw new Error(String(made.error));
222 repo = made.value;
223 await commitToBranch(app, alice, repo, { branch: "main", message: "base", files: [{ path: "a.txt", content: "a\n" }] });
224 });
225
226 afterEach(async () => {
227 app.close();
228 await rm(dataDir, { recursive: true, force: true });
229 });
230
231 test("four landings since the green: the third is named in two steps, and its revert is proposed once", async () => {
232 await verification("ok");
233 const pulls: Pull[] = [];
234 const oids: string[] = [];
235 for (const b of ["one", "two", "three", "four"]) {
236 pulls.push(await land(b));
237 oids.push(await tip());
238 }
239 const red = await verification("failed");
240 const { lines, steps } = await bisect(red, oids[2]!, oids);
241 expect(steps).toBeLessThanOrEqual(maxSteps(4));
242 expect(lines.at(-1)).toContain(`culprit #${pulls[2]!.number}`);
243 const reverts = listPulls(app.db, repo.id, { state: "all" }).filter((p) => p.revertsPullId !== null);
244 expect(reverts.map((p) => p.revertsPullId)).toEqual([pulls[2]!.id]);
245 expect(reverts[0]!.body).toContain(FILE);
246 expect(reverts[0]!.body).toContain("thing > works");
247 expect(reverts[0]!.mergedAt).toBeNull();
248 // The same red asked again proposes nothing more.
249 expect((await afterVerify(app, repo, red, async () => undefined))[0]).toContain("already proposed");
250 expect(listPulls(app.db, repo.id, { state: "all" }).filter((p) => p.revertsPullId !== null)).toHaveLength(1);
251 });
252
253 test("a push no pull carried is named in one issue", async () => {
254 await verification("ok");
255 await commitToBranch(app, alice, repo, { branch: "main", message: "direct", files: [{ path: "b.txt", content: "b\n" }] });
256 const red = await verification("failed");
257 const lines = await afterVerify(app, repo, red, async () => undefined);
258 expect(lines[0]).toContain("landed by no pull");
259 const issues = app.db
260 .query<{ title: string; body: string }, [string]>("SELECT title, body FROM issue WHERE repo_id = ?1 AND kind = 'issue'")
261 .all(repo.id);
262 expect(issues).toHaveLength(1);
263 expect(issues[0]!.body).toContain(FILE);
264 // A second red of the same thing files nothing new.
265 await afterVerify(app, repo, red, async () => undefined);
266 expect(app.db.query("SELECT COUNT(*) AS n FROM issue WHERE repo_id = ?1 AND kind = 'issue'").get(repo.id)).toEqual({ n: 1 });
267 });
268
269 test("a red whose files were already red at the verification before starts nothing", async () => {
270 await verification("ok");
271 await land("one");
272 await verification("failed");
273 await land("two");
274 const again = await verification("failed");
275 let asked = 0;
276 const lines = await afterVerify(app, repo, again, async () => {
277 asked++;
278 return undefined;
279 });
280 expect(asked).toBe(0);
281 expect(lines[0]).toContain("already failed");
282 });
283
284 test("/boards says how long main has gone unverified, and a reader who cannot see a private repository does not", async () => {
285 createRun(app.db, { repoId: repo.id, kind: "check", env: "main", commitOid: await tip() });
286 const green = await verification("ok");
287 expect(verifiedSays(readVerified(app, true)[0]!, Date.now())).toContain("main verified at");
288 await land("one");
289 // The forge checks main's push of every landing; that check is when it landed.
290 createRun(app.db, { repoId: repo.id, kind: "check", env: "main", commitOid: await tip() });
291 const row = readVerified(app, true)[0]!;
292 expect(row.green?.runId).toBe(green.id);
293 expect(row.unverifiedSince).not.toBeNull();
294 const since = Date.parse(row.unverifiedSince!);
295 expect(verifiedSays(row, since + 45 * 60_000)).toContain("main unverified for 45 min");
296 expect(verifiedSays(row, since + 150 * 60_000)).toContain("main unverified for 2.5 h");
297 const page = String(flowPanel(app, { kind: "owner" }, new URL("http://x/boards"), "/boards"));
298 expect(page).toContain('id="verified"');
299 expect(page).toContain(`data-verified="alice/app"`);
300 expect(page).toContain("main unverified for");
301 // The API's and the MCP flow tool's reading says the same.
302 // The terminal's Flow dialog says the same line, for the same reader.
303 const said = (rows: ReturnType<typeof readVerified>): string =>
304 flowLines(app.db, new Date(), undefined, undefined, undefined, null, rows).join("\n");
305 expect(said(readVerified(app, true))).toContain("alice/app: main unverified for");
306 expect(flowFor(app, { kind: "owner" }).verified[0]?.says).toContain("main unverified for");
307 app.db.run("UPDATE repo SET visibility = 'private' WHERE id = ?1", [repo.id]);
308 expect(readVerified(app, false)).toEqual([]);
309 expect(flowFor(app, { kind: "agent" }).verified).toEqual([]);
310 expect(said(readVerified(app, false))).not.toContain("alice/app: main");
311 expect(String(flowPanel(app, { kind: "agent" }, new URL("http://x/boards"), "/boards"))).not.toContain('data-verified="alice/app"');
312 });
313
314 test("spans read in minutes under an hour and hours after", () => {
315 expect(spanWords(7 * 60_000 + 59_000)).toBe("7 min");
316 expect(spanWords(3_600_000)).toBe("1.0 h");
317 });
318 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs