Public: anyone can read this; the crew can change it.

A repository's home answers under 50 ms on the server: `now` and `following` under 5 ms each for trevato/forge #6188

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/6184-a-repository-s-home-answers-under-50-ms → main 3 commits fork 6 threads · 6 unresolved

  • Merged. scout/6184-a-repository-s-home-answers-under-50-ms was merged into main as 98ecd5b.

Review

Sign in to review this change.
6 files changed +295 −29
src/core/dependencies.ts +25
@@ -278,6 +278,31 @@ export function deferredIn(
278278
279279 const GRAPH_MAX = 400;
280280
281 /**
282 * Which of these issues sit on either end of a dependency edge, in one read
283 * of the edges themselves. An issue on no edge can only be deferred by being
284 * the Later umbrella (`pillars.ts` finds it by a title starting "Later"), so
285 * a page that words a few tells can skip the whole-repository graph for them.
286 */
287 export function linkedIssueIds(db: Database, ids: readonly string[]): Set<string> {
288 const out = new Set<string>();
289 if (ids.length === 0) return out;
290 const rows = db
291 .query<{ a: string; b: string }, [string]>(
292 `SELECT issue_id AS a, blocked_by_id AS b FROM issue_dependency
293 WHERE issue_id IN (SELECT value FROM json_each(?1))
294 UNION
295 SELECT issue_id, blocked_by_id FROM issue_dependency
296 WHERE blocked_by_id IN (SELECT value FROM json_each(?1))`,
297 )
298 .all(JSON.stringify(ids));
299 for (const r of rows) {
300 out.add(r.a);
301 out.add(r.b);
302 }
303 return out;
304 }
305
281306 export function repoGraph(
282307 db: Database,
283308 repoId: string,
src/core/pillars.ts +2
@@ -50,6 +50,8 @@ export interface Pillars {
5050 const PILLAR = /^(P\d+) [^:]+(?=:)/;
5151 const GATE = /^Later gate:/i;
5252 const LATER_UMBRELLA = /^Later:/i;
53 /** A title that may be the Later gate or umbrella: what `TITLED` selects for below. */
54 export const LATER_TITLED = /^later/i;
5355 /** More umbrellas than a direction has pillars. */
5456 const MAX = 64;
5557
src/web/studio/now.ts +26 −27
@@ -30,9 +30,11 @@ import { changeHref, tellHref } from "./nouns.ts";
3030 import { LOG_TAIL_CAP, logBox } from "./log.ts";
3131 import { appPanels } from "./previews.ts";
3232 import { stopForm } from "./run.ts";
33 import { tellWord } from "./tell.ts";
33 import { tellWords } from "./tell.ts";
3434 import { voiceFor, type Actor } from "./voice.ts";
3535 import {
36 knowPulls,
37 knowRunPulls,
3638 pullWord,
3739 runWord,
3840 servingHistory,
@@ -151,6 +153,10 @@ export async function nowCard(
151153 (r, idx, all) => all.findIndex((x) => x.issueId === r.issueId) === idx,
152154 )
153155 .slice(0, 6);
156 // The words below ask per open change and per finished run; read them for
157 // all the rows at once, so the card is a handful of queries, not one apiece.
158 knowPulls(wordCtx, openPulls);
159 knowRunPulls(wordCtx, rowsNow);
154160 // Who signs a "Now" row is the one rule for it, `voiceFor`
155161 // (`studio/voice.ts`): a person is named to anyone who can see the row —
156162 // the same name `/pulls` and the pull request's own page give — and only the
@@ -228,6 +234,24 @@ export async function nowCard(
228234 ];
229235 });
230236
237 const heldWords = await tellWords(wordCtx, repo, held, new Map());
238 const heldRows = held.map((i, n) => {
239 const { word, note } = heldWords[n] ?? { word: "told" as const, note: "held" };
240 const who = voiceFor(
241 { name: i.assigneeName ?? undefined, isAgent: false },
242 ownerFacts,
243 "someone",
244 );
245 return row({
246 href: tellHref(base, i.number),
247 nav: true,
248 lead: avatar(who),
249 title: i.title,
250 meta: html`${who} · ${note} · ${time(i.createdAt)}`,
251 trail: wordBadge(word),
252 });
253 });
254
231255 const nowRows: Html[] = [
232256 ...busyRows,
233257 ...(await Promise.all(
@@ -267,32 +291,7 @@ export async function nowCard(
267291 // note the issue's own page and the issues list already show for it.
268292 // `personHeld` has already established the holder is a person, so
269293 // `voiceFor` names them to every viewer who can see the row.
270 ...(await Promise.all(
271 held.map(async (i) => {
272 const { word, note } = (await tellWord(
273 wordCtx,
274 repo,
275 i,
276 undefined,
277 )) ?? {
278 word: "told" as const,
279 note: "held",
280 };
281 const who = voiceFor(
282 { name: i.assigneeName ?? undefined, isAgent: false },
283 ownerFacts,
284 "someone",
285 );
286 return row({
287 href: tellHref(base, i.number),
288 nav: true,
289 lead: avatar(who),
290 title: i.title,
291 meta: html`${who} · ${note} · ${time(i.createdAt)}`,
292 trail: wordBadge(word),
293 });
294 }),
295 )),
294 ...heldRows,
296295 ...stuck.map((i) => {
297296 return row({
298297 lead: avatar("crew"),
src/web/studio/tell.ts +12 −2
@@ -16,7 +16,8 @@ import { threadNote } from "../../crew/direct.ts";
1616 import { alreadyDone } from "../../core/done_on_main.ts";
1717 import { appsRuntime } from "../../apps/runtime.ts";
1818 import { getAppByRepo } from "../../core/apps.ts";
19 import { repoGraph } from "../../core/dependencies.ts";
19 import { LATER_TITLED } from "../../core/pillars.ts";
20 import { linkedIssueIds, repoGraph } from "../../core/dependencies.ts";
2021 import {
2122 getIssuesByNumbers,
2223 labelsFor,
@@ -155,6 +156,10 @@ const told = new WeakMap<
155156 active: Map<string, Run | undefined>;
156157 last: Map<string, Run | undefined>;
157158 shipped: Map<number, Pull | undefined>;
159 /** The page's issues that sit on a dependency edge; the rest are never deferred. */
160 linked: Set<string>;
161 /** The ids the batch covered: only these may skip the graph. */
162 covered: Set<string>;
158163 }
159164 >();
160165
@@ -175,6 +180,8 @@ export async function tellWords(
175180 active: new Map<string, Run | undefined>(),
176181 last: new Map<string, Run | undefined>(),
177182 shipped: new Map<number, Pull | undefined>(),
183 linked: linkedIssueIds(db, issues.map((i) => i.id)),
184 covered: new Set(issues.map((i) => i.id)),
178185 };
179186 const pulls: Pull[] = [];
180187 const closed = issues.filter((i) => i.state === "closed");
@@ -259,7 +266,10 @@ export async function tellWord(
259266 return { word: "working", note: "working now" };
outdated

Medium: read is stored per wordCtx, but read.linked only holds the ids that tellWords was given. Any other tellWord call on the same wordCtx while that batch is in flight would see read !== undefined. If its issue is not in the batch, it is "not linked" and silently never deferred. Nothing calls it that way today (nowCard awaits heldWords on its own), but the active/last maps already guard against this with .has(issue.id). Do the same here: keep the ids the batch covered and take the shortcut only for those, e.g. read?.covered.has(issue.id) && !read.linked.has(issue.id) && !/^later/i.test(issue.title).

260267 // Deferred work is not waiting to be picked up: the crew will not take
261268 // it, and the page says why, as the map does.
262 const deferred = deferredWhy(wordCtx, repo, issue);
269 const deferred =
270 read?.covered.has(issue.id) === true && !read.linked.has(issue.id) && !LATER_TITLED.test(issue.title)
outdated

This shortcut is the behaviour change, and nothing tests that it gives the same answer as the graph. Add the cases from the review body. Also consider naming the condition (e.g. onNoEdge(read, issue)) so the Later-title rule lives next to the code in pillars.ts that defines it, instead of being a second copy of the regex.

271 ? undefined
272 : deferredWhy(wordCtx, repo, issue);
263273 if (deferred !== undefined)
264274 return { word: "told", note: `deferred: ${deferred}` };
265275 const last = read?.last.has(issue.id)
test/app-home-batched.test.ts added +133
@@ -0,0 +1,133 @@
1 /** An app home's `now` and `following` read in batches: 200 issues, 40 pulls, 5 runs (#6184). */
2 import { afterAll, beforeAll, expect, test } from "bun:test";
3 import { mkdtempSync, rmSync } from "node:fs";
4 import { tmpdir } from "node:os";
5 import { join } from "node:path";
6 import { enableApp } from "../src/actions/apps.ts";
7 import { commitToBranch, createRepo, forkRepo } from "../src/actions/repos.ts";
8 import { createApp, type App } from "../src/app.ts";
9 import { forkRelation } from "../src/core/following.ts";
10 import { createUser } from "../src/core/users.ts";
11 import { serve, type Servers } from "../src/http/server.ts";
12 import { proveAll } from "./helpers/verified.ts";
13 import { baselineMs, expectBudget } from "./helpers/budget.ts";
14 import { quietLogs } from "./helpers/globals.ts";
15
16 quietLogs();
17
18 let root = "";
19 let app: App;
20 let servers: Servers;
21 let cookie = "";
22
23 beforeAll(async () => {
24 root = mkdtempSync(join(tmpdir(), "forge-home-batched-"));
25 app = createApp({ dataDir: root, port: 0, forgeRepo: "alice/forge" });
26 const u = await createUser(app.db, {
27 origin: "seed",
28 trusted: true,
29 username: "alice",
30 email: "[email protected]",
31 password: "a-good-passphrase",
32 adminIfFirst: true,
33 });
34 if (!u.ok) throw new Error(u.error);
35 proveAll(app);
36 const made = await createRepo(app, { id: u.value.id, username: "alice" } as never, { name: "vault" });
37 if (!made.ok) throw new Error(String(made.error));
38 const seeded = await commitToBranch(app, u.value, made.value, {
39 message: "Start",
40 files: [{ path: "README.md", content: "v\n" }],
41 });
42 if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
43 const enabled = await enableApp(app, u.value, made.value, { slug: "vault" });
44 if (!enabled.ok) throw new Error(String(enabled.error));
45 const forked = await forkRepo(app, u.value, made.value, "vault-fork");
outdated

A fixed 500 ms sleep waits for the follow to settle. Poll until the fork reports diverged, with a short cap, so the test is neither slow nor racy.

46 if (!forked.ok) throw new Error(String(forked.error));
47 const enabledFork = await enableApp(app, u.value, forked.value, { slug: "vault-fork" });
48 if (!enabledFork.ok) throw new Error(String(enabledFork.error));
49 // Both sides edit the same line: diverged, and git cannot merge them.
50 for (const [repo, content] of [[forked.value, "fork\n"], [made.value, "parent\n"]] as const) {
51 const moved = await commitToBranch(app, u.value, repo, {
52 message: "Edit",
53 files: [{ path: "README.md", content }],
54 });
55 if (!moved.ok) throw new Error(JSON.stringify(moved.error));
56 }
57 // The parent's push is followed as it lands; wait for the fork to read diverged.
58 for (let i = 0; i < 100; i++) {
59 const relation = await forkRelation(app, forked.value);
60 if (relation?.state === "diverged") break;
61 await new Promise((r) => setTimeout(r, 30));
62 }
63 const db = app.db;
64 const agent = db.query<{ id: string }, []>("SELECT id FROM user WHERE kind = 'agent' LIMIT 1").get();
65 const assignee = agent?.id ?? u.value.id;
66 db.transaction(() => {
67 for (const repoId of [made.value.id, forked.value.id]) {
68 let n = 0;
69 for (let i = 0; i < 200; i++) {
70 db.query(
71 `INSERT INTO issue (id, repo_id, number, kind, title, body, author_id, assignee_id, state)
72 VALUES (?1, ?2, ?3, 'issue', ?4, '', ?5, ?6, 'open')`,
73 ).run(`${repoId}i${i}`, repoId, ++n, `Tell ${i}`, u.value.id, i % 2 === 0 ? assignee : null);
74 }
75 for (let i = 0; i < 40; i++) {
76 db.query(
77 `INSERT INTO issue (id, repo_id, number, kind, title, body, author_id, state)
78 VALUES (?1, ?2, ?3, 'pull', ?4, ?5, ?6, 'open')`,
79 ).run(`${repoId}p${i}`, repoId, ++n, `Pull ${i}`, `Closes #${i + 1}`, u.value.id);
80 db.query(
81 `INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid)
82 VALUES (?1, 'main', ?2, 'x', 'y')`,
83 ).run(`${repoId}p${i}`, `b${i}`);
84 }
85 for (let i = 0; i < 5; i++)
86 db.query(
87 `INSERT INTO run (id, repo_id, kind, state, issue_id) VALUES (?1, ?2, 'agent', 'running', ?3)`,
88 ).run(`${repoId}r${i}`, repoId, `${repoId}i${i * 2 + 1}`);
89 }
90 })();
91 servers = await serve(app);
92 const signin = await fetch(`${servers.url}/signin`, {
93 method: "POST",
94 headers: { origin: servers.url },
95 body: new URLSearchParams({ username: "alice", password: "a-good-passphrase" }),
96 redirect: "manual",
97 });
98 cookie = (signin.headers.get("set-cookie") ?? "").split(";")[0]!;
99 });
100 afterAll(() => {
101 servers.stop();
102 app.close();
103 rmSync(root, { recursive: true, force: true });
104 });
105
106 async function phases(path: string): Promise<Record<string, number>> {
107 const best: Record<string, number> = {};
108 for (let i = 0; i < 6; i++) {
109 const res = await fetch(`${servers.url}${path}`, { headers: { cookie } });
110 expect(res.status).toBe(200);
111 const body = await res.text();
112 if (path.endsWith("-fork")) expect(body.replace(/\s+/g, " ")).toContain("needs a decision");
113 for (const part of (res.headers.get("server-timing") ?? "").split(", ")) {
114 const m = /^([a-z]+);dur=([\d.]+)/.exec(part);
115 if (m) best[m[1]!] = Math.min(best[m[1]!] ?? Infinity, Number(m[2]));
outdated

High: CLAUDE.md and test/flake.test.ts refuse a test title that says "under N ms", and rawBudgets flags this line. Rename it to what the test proves (the card reads in batches), not to a speed.

116 }
117 }
outdated

High: this is a raw 5 ms wall-clock bound on a real request. It will flake in a loaded check cell, and CLAUDE.md forbids it. Use expectBudget(best.now!, { ms: 5, against: await baselineMs(servers.url) }) from test/helpers/budget.ts.

118 return best;
119 }
120
121 test("now reads its rows in batches with 200 open issues, 40 open pulls and 5 active runs", async () => {
122 const against = await baselineMs(servers.url);
123 const best = await phases("/alice/vault");
outdated

High: same as above. Use expectBudget with a same-run baseline. This pull does not change following, so either drop the bound here or name it as a regression guard.

124 expectBudget(best.now!, { ms: 5, against });
125 });
126
127 test("following stays cheap on a diverged fork (regression guard; this pull does not change it)", async () => {
128 const against = await baselineMs(servers.url);
129 const best = await phases("/alice/vault-fork");
130 expect(best.following).toBeDefined();
131 expectBudget(best.following!, { ms: 5, against });
132 expectBudget(best.now!, { ms: 5, against });
133 });
test/tell-deferral.test.ts added +97
@@ -0,0 +1,97 @@
1 /** The batch skips the dependency graph only for issues on no edge (#6184): same words as the graph gives. */
2 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
3 import { mkdtemp, rm } from "node:fs/promises";
4 import { tmpdir } from "node:os";
5 import { join } from "node:path";
6 import { createApp, type App } from "../src/app.ts";
7 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
8 import { addDependency, linkedIssueIds } from "../src/core/dependencies.ts";
9 import { createIssue, type Issue } from "../src/core/issues.ts";
10 import type { Repo } from "../src/core/repos.ts";
11 import { getUserByName } from "../src/core/users.ts";
12 import { repoDir } from "../src/git/repos.ts";
13 import { tellWord, tellWords } from "../src/web/studio/tell.ts";
14 import { servingHistory, type WordCtx } from "../src/web/studio/words.ts";
15 import { adminCookie } from "./helpers/admin.ts";
16 import { quietLogs } from "./helpers/globals.ts";
17
18 quietLogs();
19
20 describe("deferral in a batch of tells", () => {
21 let dataDir = "";
22 let app: App;
23 let repo: Repo;
24 let gate: Issue;
25 let umbrella: Issue;
26 let child: Issue;
27 let loose: Issue;
28
29 const ctx = (): WordCtx => ({
30 db: app.db,
31 dir: repoDir(app.reposRoot, repo.ownerName, repo.name),
32 runtime: undefined,
33 record: undefined,
34 history: servingHistory(),
35 viewer: undefined,
36 });
37
38 beforeAll(async () => {
39 dataDir = await mkdtemp(join(tmpdir(), "forge-tell-deferral-"));
40 app = createApp({ dataDir, port: 0 });
41 await adminCookie(app, { username: "alice", email: "[email protected]", password: "a-long-words-passphrase" });
42 const alice = getUserByName(app.db, "alice")!;
43 const made = await createRepo(app, alice, { name: "garden", visibility: "public" });
44 if (!made.ok) throw new Error(String(made.error));
45 repo = made.value;
46 await commitToBranch(app, alice, repo, { message: "seed", files: [{ path: "README.md", content: "# g\n" }] });
47 const open = (title: string): Issue => {
48 const made = createIssue(app.db, { repoId: repo.id, authorId: alice.id, title });
49 if (!made.ok) throw new Error(String(made.error));
50 return made.value;
51 };
52 gate = open("Later gate: not yet");
53 umbrella = open("Later: someday");
54 child = open("Waits behind the gate");
55 loose = open("Tidy the README");
56 const linked = addDependency(app.db, child, gate, alice.id);
57 if (!linked.ok) throw new Error(linked.error);
58 });
59 afterAll(async () => {
60 app.close();
61 await rm(dataDir, { recursive: true, force: true });
62 });
63
64 test("linkedIssueIds is exactly the issues on either end of an edge", () => {
65 const all = [gate, umbrella, child, loose].map((i) => i.id);
66 expect(linkedIssueIds(app.db, all)).toEqual(new Set([gate.id, child.id]));
67 expect(linkedIssueIds(app.db, [loose.id, umbrella.id])).toEqual(new Set());
68 expect(linkedIssueIds(app.db, [])).toEqual(new Set());
69 });
70
71 test("a batch words an issue on an edge as deferred, as the graph does", async () => {
72 const [word] = await tellWords(ctx(), repo, [child], new Map());
73 expect(word?.note).toStartWith("deferred:");
74 expect(word).toEqual(await tellWord(ctx(), repo, child, undefined));
75 });
76
77 test("an issue on no edge is worded as it is without the shortcut", async () => {
78 const [word] = await tellWords(ctx(), repo, [loose], new Map());
79 expect(word?.note).not.toStartWith("deferred:");
80 expect(word).toEqual(await tellWord(ctx(), repo, loose, undefined));
81 });
82
83 test("a Later umbrella on no edge still goes through the graph", async () => {
84 const [word] = await tellWords(ctx(), repo, [umbrella], new Map());
85 expect(word?.note).toStartWith("deferred:");
86 expect(word).toEqual(await tellWord(ctx(), repo, umbrella, undefined));
87 });
88
89 test("an issue outside the batch, on the same context, still gets the graph path", async () => {
90 const c = ctx();
91 // The batch is in flight (its reads are set) when the other tell is worded.
92 const batch = tellWords(c, repo, [loose], new Map());
93 const outside = await tellWord(c, repo, child, undefined);
94 await batch;
95 expect(outside?.note).toStartWith("deferred:");
96 });
97 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs