Public: anyone can read this; the crew can change it.

Sentinel: queue:check is past its budget: 27 checks wait behind one lane; the oldest since 13:21; the lane runs #4346 for 3 min (limit 60). #4424

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/4391-sentinel-queue-check-is-past-its-budget → main 1 commit fork

  • Merged. mason/4391-sentinel-queue-check-is-past-its-budget was merged into main as 21c2fad.

Review

Sign in to review this change.
5 files changed +141 −11
README.md +3 −1
@@ -1299,7 +1299,9 @@ runs on main's tip with that pull merged, which is the proof a re-check of
12991299 its own would give, and a red candidate bisects as above. So a landing that
13001300 moves main does not send every other approved pull off for a check of its
13011301 own first; the batch lands in one gate. A pull's own check runs again only
1302 when its head changes. A person's merge, which no candidate gated, still
1302 when its head changes: an approval checks a head that was never checked,
1303 and a green one only when main moved since and the repository is not
1304 autonomous, whose land gates on today's main anyway (`src/checks/approval.ts`). A person's merge, which no candidate gated, still
13031305 refuses a stale green. Each sweep reads every open pull, oldest first, not
13041306 only the newest page, so an older approved pull is batched with the rest
13051307 rather than joining alone on its own event. The autonomy tick line counts the approved pulls the
src/checks/approval.ts added +35
@@ -0,0 +1,35 @@
1 /**
2 * Whether an approval sends a pull's head through the lane again (#4391).
3 *
4 * An approval is what lets a reader's pull run, so a head that has never
5 * been checked is checked when it is approved. But a crew reviewer approves
6 * nearly every pull minutes after its push check passed, and checking that
7 * head again cost the lane a full run per pull: 21 of 52 heads in one hour
8 * were checked twice, and 27 checks queued behind them.
9 *
10 * A head already green in its environment is checked again only when that
11 * says something new: its base has moved since the green check merged with
12 * it, and nothing else will judge the pull on today's base. An autonomous
13 * repository's land gates the merge on today's base anyway
14 * (`checks/landing.ts`), so there the approval adds nothing.
15 */
16
17 import type { Run } from "../core/runs.ts";
18
19 export function checkOnApproval(
20 /** The runs at this head (`checksForHead`). */
21 runs: readonly Run[],
22 /** The environment the pull's checks run in, `pull-<n>`. */
23 env: string,
24 /** The base branch's tip now; undefined when it could not be read. */
25 baseTip: string | undefined,
26 /** Whether the pull lands through the merge queue's full gate. */
27 landGated: boolean,
28 ): boolean {
29 const green = runs.filter(
30 (r) => r.kind === "check" && r.env === env && r.state === "ok",
31 );
32 if (green.length === 0) return true;
33 if (landGated || baseTip === undefined) return false;
34 return !green.some((r) => r.baseOid === baseTip);
35 }
src/checks/service.ts +11 −9
@@ -45,6 +45,7 @@ import {
4545 import { Gate, runGit } from "../git/env.ts";
4646 import { resolveOid } from "../git/merge.ts";
4747 import { mergeForCheck } from "./merged.ts";
48 import { checkOnApproval } from "./approval.ts";
4849 import { checkKey, provenBy, recordProven } from "./proven.ts";
4950 import { giveHistory } from "./history.ts";
5051 import { readLearned } from "./learned.ts";
@@ -1093,15 +1094,16 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
10931094 const pull = getPull(app.db, repo.id, number);
10941095 if (!pull || pull.state !== "open") return;
10951096 const head = await headOf(repo, pull);
1096 if (head !== undefined)
1097 await request(
1098 repo,
1099 `pull-${pull.number}`,
1100 head,
1101 "approved",
1102 pull.id,
1103 actor,
1104 );
1097 if (head === undefined) return;
1098 // A head already green is not checked again for nothing (#4391).
1099 const env = `pull-${pull.number}`;
1100 const dir = await pullDir(app.db, app.reposRoot, repo, pull);
1101 const tip = await resolveOid(dir, `refs/heads/${pull.baseBranch}`);
1102 // The merge queue gates only an autonomous repository's lands (`crew/autonomy.ts` admit).
1103 const landGated = repo.autonomy === "autonomous" && repo.autonomyBy !== null;
1104 if (!checkOnApproval(checksForHead(app.db, pull.id, head), env, tip, landGated))
1105 return;
1106 await request(repo, env, head, "approved", pull.id, actor);
11051107 return;
11061108 }
11071109 // Landed or closed: nothing left to check, so nothing waits in the lane.
test/check-on-approval.test.ts added +47
@@ -0,0 +1,47 @@
1 import { describe, expect, test } from "bun:test";
2 import { checkOnApproval } from "../src/checks/approval.ts";
3 import type { Run } from "../src/core/runs.ts";
4
5 const run = (over: Partial<Run>): Run =>
6 ({
7 id: "r",
8 kind: "check",
9 env: "pull-7",
10 state: "ok",
11 baseOid: "base1",
12 ...over,
13 }) as Run;
14
15 describe("checkOnApproval: an approval checks a head again only when it says something new (#4391)", () => {
16 test("a head never checked is checked when approved", () => {
17 expect(checkOnApproval([], "pull-7", "base1", true)).toBe(true);
18 expect(checkOnApproval([], "pull-7", "base1", false)).toBe(true);
19 });
20
21 test("a head with only a red, a cancelled or another environment's green is checked", () => {
22 const runs = [
23 run({ state: "failed" }),
24 run({ state: "cancelled" }),
25 run({ env: "pull-8" }),
26 run({ kind: "deploy" }),
27 ];
28 expect(checkOnApproval(runs, "pull-7", "base1", true)).toBe(true);
29 });
30
31 test("a green head on an unmoved base is not checked again", () => {
32 expect(checkOnApproval([run({})], "pull-7", "base1", false)).toBe(false);
33 });
34
35 test("a green head on a moved base is checked again where nothing gates the land", () => {
36 expect(checkOnApproval([run({})], "pull-7", "base2", false)).toBe(true);
37 expect(checkOnApproval([run({ baseOid: "" })], "pull-7", "base2", false)).toBe(true);
38 });
39
40 test("a green head on a moved base is not checked again where the land gates on today's base", () => {
41 expect(checkOnApproval([run({})], "pull-7", "base2", true)).toBe(false);
42 });
43
44 test("an unreadable base is no reason to check a green head again", () => {
45 expect(checkOnApproval([run({})], "pull-7", undefined, false)).toBe(false);
46 });
47 });
test/checks.test.ts +45 −1
@@ -44,7 +44,7 @@ import { repoDir } from "../src/git/repos.ts";
4444 import { STALE_AFTER_MS } from "../src/runs/lease.ts";
4545 import { pullWord, servingHistory } from "../src/web/studio/words.ts";
4646 import { createUser, type User } from "../src/core/users.ts";
47 import type { Repo } from "../src/core/repos.ts";
47 import { getRepo, type Repo } from "../src/core/repos.ts";
4848 import { serve, type Servers } from "../src/http/server.ts";
4949 import { listEvents } from "../src/core/events.ts";
5050 import { readRunLog } from "../src/runs/log.ts";
@@ -608,6 +608,50 @@ describe("checks", () => {
608608 expect(second.state).toBe("ok");
609609 }, 60_000);
610610
611 test("approving a head whose check is already green queues no second check (#4391)", async () => {
612 const based = await commitToBranch(app, alice, repo, {
613 message: "The base's check passes",
614 files: [{ path: "forge.json", content: checkJson("true") }],
615 });
616 if (!based.ok) throw new Error(JSON.stringify(based.error));
617 // Bob's fork, made by the fork test above when it ran first.
618 const fork =
619 getRepo(app.db, "bob", repo.name) ??
620 (await forkRepo(app, bob, repo).then((f) => (f.ok ? f.value : undefined)));
621 if (fork === undefined) throw new Error("no fork");
622 const committed = await commitToBranch(app, bob, fork, {
623 branch: "green-then-approved",
624 message: "Green before the second look",
625 files: [{ path: "GREEN.md", content: "x\n" }],
626 });
627 if (!committed.ok) throw new Error(JSON.stringify(committed.error));
628 const opened = await openPull(app, bob, repo, {
629 title: "Green before the second look",
630 baseBranch: "main",
631 headBranch: "green-then-approved",
632 headRepo: fork,
633 });
634 if (!opened.ok) throw new Error(String(opened.error));
635 const pull = opened.value;
636 const mine = (): Run[] =>
637 listRuns(app.db, repo.id, { kind: "check" }).filter(
638 (r) => r.issueId === pull.id && r.commitOid === committed.value.commitOid,
639 );
640 // The first approval is what lets a reader's head run at all.
641 const approved = await submitReview(app, alice, repo, pull, { state: "approved" });
642 if (!approved.ok) throw new Error(String(approved.error));
643 const first = await settled((r) => r.issueId === pull.id);
644 expect(first.state).toBe("ok");
645 // A second approval of the same green head on the same base is nothing new.
646 const again = await submitReview(app, alice, repo, pull, {
647 state: "approved",
648 body: "Still good.",
649 });
650 if (!again.ok) throw new Error(String(again.error));
651 await Bun.sleep(300);
652 expect(mine().map((r) => r.id)).toEqual([first.id]);
653 }, 60_000);
654
611655 test("a commit already proven green is not run again; the re-request answers with the earlier verdict (#1795)", async () => {
612656 const made = await commitToBranch(app, alice, repo, {
613657 message: "Proven once",

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs