Public: anyone can read this; the crew can change it.

Asks read four rows per open pull, and home runs them twice #2780

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/2772-asks-read-four-rows-per-open-pull-and-ho → main 3 commits fork 1 thread · 1 unresolved

  • Merged. mason/2772-asks-read-four-rows-per-open-pull-and-ho was merged into main as 486d269.

Review

Sign in to review this change.
6 files changed +314 −13
src/core/runs.ts +27
@@ -741,6 +741,33 @@ export function activeRunFor(
741741 return row ? toRun(row) : undefined;
742742 }
743743
744 /**
745 * `activeRunFor` for many issues at once: each issue's earliest queued or
746 * running run of `kind`, one query per chunk. Issues with none are absent.
747 */
748 export function activeRunsFor(
749 db: Database,
750 kind: RunKind,
751 issueIds: readonly string[],
752 ): Map<string, Run> {
753 const out = new Map<string, Run>();
754 const unique = [...new Set(issueIds)];
755 const CHUNK = 400;
756 for (let i = 0; i < unique.length; i += CHUNK) {
757 const batch = unique.slice(i, i + CHUNK);
758 const rows = db
759 .query<RunRow, string[]>(
760 `SELECT * FROM run WHERE state IN ('queued','running') AND kind = ?1
761 AND issue_id IN (${batch.map((_, n) => `?${n + 2}`).join(",")})
762 ORDER BY id`,
763 )
764 .all(kind, ...batch);
765 for (const row of rows)
766 if (row.issue_id && !out.has(row.issue_id)) out.set(row.issue_id, toRun(row));
767 }
768 return out;
769 }
770
744771 /** The variant group an issue's earlier runs were started in, if any. */
745772 export function variantGroupOf(db: Database, issueId: string): string | undefined {
746773 const row = db
src/crew/guard.ts +45
@@ -280,6 +280,51 @@ export function handoverOf(
280280 )
281281 .get(pullId, repo.id)!.n;
282282 if (crewAsked < HANDOVER_ROUNDS) return undefined;
283 return handoverSince(db, repo, pullId);
284 }
285
286 /** Pulls `handoversOf` counts per query: well under SQLite's bound on variables. */
287 const HANDOVER_CHUNK = 500;
288
289 /**
290 * `handoverOf` for many pulls at once: the cheap count is one grouped query
291 * per chunk, and only a pull with `HANDOVER_ROUNDS` crew verdicts — the rare
292 * one — reads further. Only handed-over pulls get an entry.
293 */
294 export function handoversOf(
295 db: Database,
296 pulls: readonly { readonly id: string; readonly repo: Repo }[],
297 ): Map<string, Handover> {
298 const out = new Map<string, Handover>();
299 const repoOf = new Map(pulls.map((p) => [p.id, p.repo]));
300 const ids = [...repoOf.keys()];
301 for (let i = 0; i < ids.length; i += HANDOVER_CHUNK) {
302 const chunk = ids.slice(i, i + HANDOVER_CHUNK);
303 const rows = db
304 .query<{ issueId: string; n: number }, string[]>(
305 `SELECT r.issue_id AS issueId, COUNT(*) AS n FROM review r
306 JOIN issue i ON i.id = r.issue_id
307 JOIN repo_crew c ON c.user_id = r.author_id AND c.repo_id = i.repo_id
308 WHERE r.issue_id IN (${chunk.map((_, n) => `?${n + 1}`).join(",")})
309 AND r.state = 'changes_requested' AND c.role = 'reviewer'
310 GROUP BY r.issue_id`,
311 )
312 .all(...chunk);
313 for (const row of rows) {
314 if (row.n < HANDOVER_ROUNDS) continue;

Minor: the grouped count duplicates the singular count SQL. If the crew-verdict rule changes, both must change. A shared fragment or constant would keep them in step.

315 const over = handoverSince(db, repoOf.get(row.issueId)!, row.issueId);
316 if (over !== undefined) out.set(row.issueId, over);
317 }
318 }
319 return out;
320 }
321
322 /** The hand-over, once a pull has had enough crew verdicts to have one. */
323 function handoverSince(
324 db: Database,
325 repo: Repo,
326 pullId: string,
327 ): Handover | undefined {
283328 const approved =
284329 db
285330 .query<{ at: string | null }, [string]>(
src/feed/feed.ts +3 −1
@@ -55,7 +55,9 @@ export function readFeed(
5555 );
5656 const boosted = new Set(shared.map((c) => c.itemId));
5757 const candidates = [
58 ...candidatesFor(db, viewer, now, chips).filter((c) => !boosted.has(c.id)),
58 ...candidatesFor(db, viewer, now, chips, options.withoutAsks !== true).filter(
59 (c) => !boosted.has(c.id),
60 ),
5961 ...shared,
6062 ]
6163 // The chips: the one filter. An ask was filtered by it in `askLinesFor`.
src/feed/items.ts +3 −1
@@ -201,12 +201,14 @@ export function candidatesFor(
201201 viewer: User,
202202 now: number,
203203 chips?: Chips,
204 /** False when the caller shows asks itself (the home pins them): they are not read twice. */
205 withAsks = true,
204206 ): Candidate[] {
205207 const events = listEventsVisibleTo(db, viewer, { limit: EVENT_WINDOW }).filter(
206208 audienceFilter(db, viewer),
207209 );
208210 const writable = listReposWritableTo(db, viewer, { limit: 200 });
209 const asks = askLinesFor(db, viewer, chips);
211 const asks = withAsks ? askLinesFor(db, viewer, chips) : [];
210212 const decided = listDecidedIn(
211213 db,
212214 writable.map((r) => r.id),
src/web/studio/asks.ts +22 −11
@@ -14,7 +14,7 @@ import { previewEnv } from "../../apps/data.ts";
1414 import { waitingFor, type WaitingFor } from "../../core/waiting_for.ts";
1515 import { crewForRepos } from "../../core/agents.ts";
1616 import { capacityDefault, isCapacityKey, storedCapacity } from "../../core/capacity.ts";
17 import { appsForRepos, getAppByRepo } from "../../core/apps.ts";
17 import { appsForRepos } from "../../core/apps.ts";
1818 import { latestDecision, stopStands } from "../../core/decisions.ts";
1919 import { getHold, holdsFor } from "../../core/holds.ts";
2020 import { issueTitles, proposalsFor, type Proposal } from "../../core/proposals.ts";
@@ -44,7 +44,7 @@ import {
4444 } from "../../core/repos.ts";
4545 import { listReviews } from "../../core/reviews.ts";
4646 import {
47 activeRunFor,
47 activeRunsFor,
4848 automaticRunsSince,
4949 latestRunsForIssues,
5050 type Run,
@@ -57,7 +57,7 @@ import {
5757 } from "../../crew/autonomy.ts";
5858 import { breakerParked, WAITING_NOTES } from "../../crew/breaker.ts";
5959 import { CHECK_FAILED_LIMIT } from "../../crew/dispatch.ts";
60 import { handoverLines, handoverOf, type Handover } from "../../crew/guard.ts";
60 import { handoverLines, handoversOf, type Handover } from "../../crew/guard.ts";
6161 import { conflictsOf, isStalePull, mergesAgainSince, staleGreen } from "../../crew/stale_asks.ts";
6262 import { reviewAttempts, waitsForPerson, waitsForReviewer, type ReviewWait } from "../../crew/review.ts";
6363 import type { Ctx } from "../../http/context.ts";
@@ -134,6 +134,8 @@ export interface Decision {
134134 readonly likely?: { readonly option: string; readonly p: number } | undefined;
135135 /** The preview pictured on a phone, when the forge has taken one. */
136136 readonly picture?: string | undefined;
137 /** Where the pull's preview opens, when its repository is an app. */
138 readonly preview?: string | undefined;
137139 readonly consequence: Consequence;
138140 /** For "setting": the key, its value, its default, and who set it and when. */
139141 readonly setting?: SettingAway | undefined;
@@ -320,6 +322,15 @@ function askCandidatesFor(
320322 db,
321323 pulls.map((p) => p.id),
322324 );
325 const working = activeRunsFor(
326 db,
327 "agent",
328 pulls.map((p) => p.id),
329 );
330 const handedOver = handoversOf(
331 db,
332 pulls.map((p) => ({ id: p.id, repo: byId.get(p.repoId)! })),
333 );
323334 for (const issue of pulls) {
324335 const repo = byId.get(issue.repoId)!;
325336 const pull = joined.get(issue.id);
@@ -330,10 +341,10 @@ function askCandidatesFor(
330341 const check = checks.get(issue.id);
331342 const who = issue.authorName;
332343 // The crew is on it: that is working, not a question for a person.
333 if (activeRunFor(db, "agent", issue.id) !== undefined) continue;
344 if (working.has(issue.id)) continue;
334345 // Three reviews without an approval: the crew stopped, and says so
335346 // before anything about the check (`guard.ts`).
336 const over = handoverOf(db, repo, issue.id);
347 const over = handedOver.get(issue.id);
337348 // An autonomous repository's spent review loop is the planner's
338349 // (`handedOverPulls`), not a question for a person (#1655).
339350 // A person's pull has no planner pass to take it, so it still asks.
@@ -1203,8 +1214,13 @@ export function decisionsFor(
12031214 short: `at its $${dollars(c.repo.autonomyCeilingUsd)} ceiling until 00:00 UTC`,
12041215 };
12051216 };
1217 const previewOf = (c: Scored): string | undefined =>
1218 c.pull === undefined || !records.has(c.repo.id)
1219 ? undefined
1220 : `${repoBase(c.repo)}/app/pull-${c.pull.number}/open`;
12061221 return candidates.map((c) => ({
12071222 ...worded(c),
1223 preview: previewOf(c),
12081224 consequence: c.consequence,
12091225 waiting: c.waiting,
12101226 }));
@@ -1263,12 +1279,7 @@ export function decisionActions(ctx: Ctx, d: Decision): Html {
12631279 export function decisionButtons(ctx: Ctx, d: Decision): { primary: Html; others: Html[] } {
12641280 const base = repoBase(d.repo);
12651281 const pull = d.pull;
1266 const preview =
1267 pull === undefined
1268 ? undefined
1269 : getAppByRepo(ctx.app.db, d.repo.id) === undefined
1270 ? undefined
1271 : `${base}/app/pull-${pull.number}/open`;
1282 const preview = d.preview;
12721283 if ((d.kind === "stopped" || d.kind === "handover") && pull)
12731284 return {
12741285 primary: btn("Tell them", {
test/asks-counts.test.ts added +214
@@ -0,0 +1,214 @@
1 /**
2 * The home's asks without N+1 (#2772). What asks the owner is read from
3 * every open pull they may land: whether the crew is on it, whether its
4 * review loop handed it to a person, its app. Each of those is one grouped
5 * read for the whole set (`test/counts.test.ts`'s rule), and one request
6 * reads the set once — the feed under "Needs you" no longer reads it again.
7 *
8 * Pulls go in as rows, in every state the candidates sort into, so each
9 * branch of `askCandidatesFor` is on the measured path.
10 */
11
12 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
13 import type { Database } from "bun:sqlite";
14 import { mkdtemp, rm } from "node:fs/promises";
15 import { tmpdir } from "node:os";
16 import { join } from "node:path";
17 import { createApp, type App } from "../src/app.ts";
18 import { addCrew, setCrewRole } from "../src/core/agents.ts";
19 import { createApp as createAppRecord } from "../src/core/apps.ts";
20 import { createRepo, type Repo } from "../src/core/repos.ts";
21 import { submitReview } from "../src/core/reviews.ts";
22 import { activeRunFor, activeRunsFor, createRun } from "../src/core/runs.ts";
23 import { createSession } from "../src/core/sessions.ts";
24 import { createUser, type User } from "../src/core/users.ts";
25 import { handoverOf, handoversOf } from "../src/crew/guard.ts";
26 import { serve, type Servers } from "../src/http/server.ts";
27 import { id } from "../src/kernel/id.ts";
28 import { quiet, statements } from "./helpers/statements.ts";
29
30 const ZERO = "0".repeat(40);
31
32 /** The SQL `work` runs, each statement once per execution. */
33 async function seen(db: Database, work: () => Promise<void>): Promise<string[]> {
34 await quiet(db);
35 const door = db as unknown as { query: unknown };
36 const original = db.query.bind(db);
37 const out: string[] = [];
38 door.query = (sql: string) => {
39 out.push(sql.replace(/\s+/g, " ").trim());
40 return original(sql);
41 };
42 try {
43 await work();
44 } finally {
45 door.query = original;
46 }
47 return out;
48 }
49
50 describe("home's asks are read grouped, and once", () => {
51 let dataDir: string;
52 let app: App;
53 let servers: Servers;
54 let owner: User;
55 let builder: User;
56 let reviewer: User;
57 let assisted: Repo;
58 let autonomous: Repo;
59 let cookie = "";
60 let seq = 0;
61
62 const user = async (username: string, agent: boolean) => {
63 const made = await createUser(app.db, {
64 username,
65 email: `${username}@forge.test`,
66 ...(agent
67 ? { agent: true, origin: "agent" as const }
68 : { password: "a-good-passphrase", origin: "seed" as const, trusted: true }),
69 });
70 if (!made.ok) throw new Error(`seed ${username}: ${made.error}`);
71 return made.value;
72 };
73
74 const repo = async (name: string, autonomy: "assisted" | "autonomous") => {
75 const made = await createRepo(app.db, app.reposRoot, {
76 ownerId: owner.id,
77 name,
78 visibility: "public",
79 description: name,
80 });
81 if (!made.ok) throw new Error(`seed repo ${name}: ${made.error}`);
82 for (const agent of [builder, reviewer]) addCrew(app.db, made.value.id, agent.id, owner.id);
83 setCrewRole(app.db, made.value.id, reviewer.id, "reviewer");
84 app.db.query("UPDATE repo SET autonomy = ?2 WHERE id = ?1").run(made.value.id, autonomy);
85 return made.value;
86 };
87
88 const check = (on: Repo, issueId: string, state: "ok" | "failed") => {
89 const run = createRun(app.db, { repoId: on.id, kind: "check", issueId, commitOid: ZERO });
90 app.db.query("UPDATE run SET state = ?2 WHERE id = ?1").run(run.id, state);
91 };
92
93 /** An open pull on `on` by `author`, as rows: its id and number. */
94 const filePull = (on: Repo, author: User, title: string) => {
95 const issueId = id();
96 const number = app.db
97 .query<{ issue_seq: number }, [string]>(
98 "UPDATE repo SET issue_seq = issue_seq + 1 WHERE id = ?1 RETURNING issue_seq",
99 )
100 .get(on.id)!.issue_seq;
101 app.db
102 .query(
103 `INSERT INTO issue (id, repo_id, number, kind, title, body, author_id, state)
104 VALUES (?1, ?2, ?3, 'pull', ?4, '', ?5, 'open')`,
105 )
106 .run(issueId, on.id, number, title, author.id);
107 app.db
108 .query(
109 `INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, head_repo_id)
110 VALUES (?1, 'main', ?2, ?3, ?3, ?4)`,
111 )
112 .run(issueId, `crew/${++seq}`, ZERO, on.id);
113 return { issueId, number };
114 };
115
116 const changesRequested = (issueId: string, rounds: number) => {
117 for (let r = 0; r < rounds; r++)
118 submitReview(app.db, {
119 issueId,
120 authorId: reviewer.id,
121 state: "changes_requested",
122 body: `Round ${r}.`,
123 commitOid: ZERO,
124 });
125 };
126
127 /**
128 * One open pull in each state that reaches the reads grouped here. A
129 * pull with a check result reads more, per pull, on its way to a card
130 * (staleness, the reviewer's wait, the retry count): that is #2772's
131 * follow-up, not measured here.
132 */
133 const seedRound = () => {
134 const states = ["quiet", "working", "reviewed", "autonomous-quiet"] as const;
135 for (const state of states) {
136 const on = state.startsWith("autonomous") ? autonomous : assisted;
137 const { issueId } = filePull(on, builder, `Pull (${state})`);
138 if (state === "working")
139 createRun(app.db, { repoId: on.id, kind: "agent", issueId, actorId: builder.id });
140 if (state === "reviewed") changesRequested(issueId, 1);
141 }
142 };
143
144 const home = async () => {
145 const response = await fetch(`${servers.url}/`, { headers: { cookie } });
146 expect(response.status).toBe(200);
147 return response.text();
148 };
149
150 beforeAll(async () => {
151 dataDir = await mkdtemp(join(tmpdir(), "forge-asks-counts-"));
152 app = createApp({ dataDir, port: 0 });
153 owner = await user("keeper", false);
154 builder = await user("wrench", true);
155 reviewer = await user("plumb", true);
156 assisted = await repo("shop", "assisted");
157 autonomous = await repo("works", "autonomous");
158 cookie = `forge_session=${createSession(app.db, owner.id).token}`;
159 servers = await serve(app);
160 });
161
162 afterAll(async () => {
163 servers?.stop();
164 app?.close();
165 await rm(dataDir, { recursive: true, force: true });
166 });
167
168 test("home costs the same for one round of open pulls as for four", async () => {
169 seedRound();
170 await home();
171 const few = await statements(app.db, async () => void (await home()));
172 seedRound();
173 seedRound();
174 seedRound();
175 const many = await statements(app.db, async () => void (await home()));
176 expect(many).toBe(few);
177 });
178
179 test("one request reads the open pulls' active runs and hand-overs once", async () => {
180 const sql = await seen(app.db, async () => void (await home()));
181 const active = sql.filter((s) => s.includes("state IN ('queued','running')") && s.includes("issue_id IN"));
182 const handovers = sql.filter((s) => s.startsWith("SELECT r.issue_id") && s.includes("changes_requested"));
183 expect(active.length).toBe(1);
184 expect(handovers.length).toBe(1);
185 });
186
187 test("the grouped reads agree with the singular ones", () => {
188 const handed = filePull(assisted, builder, "Handed over");
189 changesRequested(handed.issueId, 3);
190 const pulls = app.db
191 .query<{ id: string; repoId: string }, []>(
192 "SELECT id, repo_id AS repoId FROM issue WHERE kind = 'pull' AND state = 'open'",
193 )
194 .all();
195 const repoOf = (repoId: string) => (repoId === assisted.id ? assisted : autonomous);
196 const active = activeRunsFor(app.db, "agent", pulls.map((p) => p.id));
197 const over = handoversOf(app.db, pulls.map((p) => ({ id: p.id, repo: repoOf(p.repoId) })));
198 for (const p of pulls) {
199 expect(active.get(p.id)).toEqual(activeRunFor(app.db, "agent", p.id));
200 expect(over.get(p.id)).toEqual(handoverOf(app.db, repoOf(p.repoId), p.id));
201 }
202 expect(over.has(handed.issueId)).toBe(true);
203 expect(active.size).toBeGreaterThan(0);
204 });
205
206 test("a green pull on an app still offers its preview", async () => {
207 const green = filePull(assisted, owner, "Green and waiting");
208 check(assisted, green.issueId, "ok");
209 expect(await home()).not.toContain(`/app/pull-${green.number}/open`);
210 const made = createAppRecord(app.db, { repoId: assisted.id, slug: "shop", access: "public" });
211 expect(made.ok).toBe(true);
212 expect(await home()).toContain(`/app/pull-${green.number}/open`);
213 });
214 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs