Public: anyone can read this; the crew can change it.

GET a repository's branches compares only the branches it returns: `git` under 60 ms for a 30-branch page of trevato/forge #7054

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/6225-get-a-repository-s-branches-compares-onl → main 3 commits fork 4 threads · 4 unresolved

  • Merged. scout/6225-get-a-repository-s-branches-compares-onl was merged into main as adcbbb8.

Review

Sign in to review this change.
4 files changed +192 −28
src/git/branches.ts +47 −9
@@ -151,6 +151,24 @@ export async function listBranches(
151151 defaultBranch: string,
152152 opts: { readonly bulk?: boolean } = {},
153153 ): Promise<Result<BranchListing, GitFailure>> {
154 const rows = await listBranchRows(dir, defaultBranch);
155 if (!rows.ok) return rows;
156 return ok(await compareBranches(dir, defaultBranch, rows.value, rows.value.length, opts));
157 }
158
159 export interface BranchRow extends BranchRef {
160 readonly isDefault: boolean;
161 }
162
163 /**
164 * Names and tips only, default first then alphabetical: one `for-each-ref`
165 * and no comparison, so a caller that shows a page can take the page before
166 * it pays for ahead/behind.
167 */
168 export async function listBranchRows(
169 dir: string,
170 defaultBranch: string,
171 ): Promise<Result<BranchRow[], GitFailure>> {
154172 const refs = await listRefs(dir, "refs/heads/");
155173 if (!refs.ok) return refs;
156174
@@ -166,28 +184,44 @@ export async function listBranches(
166184 ? -1
167185 : 1,
168186 );
187 return ok(rows);
188 }
169189
190 /**
191 * Ahead/behind for just `rows` (a page of the list). `total` is how many
192 * branches the repo has: past `MAX_BRANCH_COMPARE` nothing is compared, as
193 * for the whole list.
194 */
195 export async function compareBranches(
196 dir: string,
197 defaultBranch: string,
198 rows: readonly BranchRow[],
199 total: number,
200 opts: { readonly bulk?: boolean } = {},
201 ): Promise<BranchListing> {
170202 const compare =
171 rows.length <= MAX_BRANCH_COMPARE && isSafeRef(defaultBranch)
203 total <= MAX_BRANCH_COMPARE && isSafeRef(defaultBranch)
172204 ? await resolveOid(dir, `refs/heads/${defaultBranch}`)
173205 : undefined;
174206 if (compare === undefined) {
175 return ok({
207 return {
176208 branches: rows.map((r) => ({ ...r, ahead: null, behind: null })),
177 compared: rows.length <= MAX_BRANCH_COMPARE,
178 });
209 compared: total <= MAX_BRANCH_COMPARE,
210 };
179211 }
180212
181213 const counted =
182 opts.bulk === false ? undefined : await aheadBehindAll(dir, compare);
214 opts.bulk === false
215 ? undefined
216 : await aheadBehindAll(dir, compare, rows.map((r) => r.name));
183217 if (counted !== undefined) {
184 return ok({
218 return {
185219 branches: rows.map((r) => ({
186220 ...r,
187221 ...(counted.get(r.name) ?? { ahead: null, behind: null }),
188222 })),
189223 compared: true,
190 });
224 };
191225 }
192226
193227 const branches: BranchInfo[] = [];
@@ -203,7 +237,7 @@ export async function listBranches(
203237 });
204238 }
205239 }
206 return ok({ branches, compared: true });
240 return { branches, compared: true };
207241 }
208242
209243 /**
@@ -214,14 +248,18 @@ export async function listBranches(
214248 async function aheadBehindAll(
215249 dir: string,
216250 base: string,
251 names: readonly string[],
217252 ): Promise<Map<string, { ahead: number; behind: number }> | undefined> {
218253 if (!isSafeOid(base)) return undefined;
254 // An empty page has nothing to count; asking git would compare every branch.
255 if (names.length === 0) return new Map();
256 if (!names.every(isBranchName)) return undefined;
219257 const listed = await runGit(
220258 [
221259 "for-each-ref",
222260 `--format=%(refname)%00%(ahead-behind:${base})`,
outdated

Minor: with names empty, this falls back to refs/heads/ and counts every branch, only for none of the counts to be used. An empty page (no branches, or a cursor at the last row) should return an empty map without calling git. Today the empty case compares the whole repo, the opposite of what this pull is for.

223261 "--",
224 "refs/heads/",
262 ...names.map((n) => `refs/heads/${n}`),
225263 ],
226264 { cwd: dir },
227265 );
src/http/api.ts +23 −19
@@ -244,7 +244,7 @@ import { runGit } from "../git/env.ts";
244244 import { appLimitWords, appsOwnedBy } from "../actions/app_limit.ts";
245245 import { getQuota } from "../settings/quotas.ts";
246246 import { isSafeRef, isSafeTreePath } from "../git/guard.ts";
247 import { listBranches } from "../git/branches.ts";
247 import { compareBranches, listBranchRows } from "../git/branches.ts";
248248 import {
249249 appView,
250250 appVisitors,
@@ -2475,25 +2475,29 @@ export function apiRoutes(router: Router<Handler>): void {
24752475 const cursor = q.get("cursor");
24762476 if (cursor !== null && (cursor === "" || cursor.length > 255))
24772477 return apiError("invalid_cursor", 422, "cursor is the next_cursor of a page before");
2478 const listed = await timePhaseAsync(ctx.request, "git", () =>
2479 listBranches(dirOf(ctx, repo), repo.defaultBranch),
2480 );
2481 const rows = listed.ok
2482 ? listed.value.branches.map((b) => ({
2483 name: b.name,
2484 oid: b.oid,
2485 isDefault: b.isDefault,
2486 ahead: b.ahead,
2487 behind: b.behind,
2488 protected: b.isDefault && repo.protectDefault,
2489 }))
2490 : [];
2491 // The cursor is the name of the last branch of the page before.
2492 const start = cursor === null ? 0 : rows.findIndex((r) => r.name === cursor) + 1;
2493 if (start === 0 && cursor !== null)
2478 // List the refs (cheap), take the page, and compare only its rows.
2479 const dir = dirOf(ctx, repo);
2480 const page = await timePhaseAsync(ctx.request, "git", async () => {
2481 const listed = await listBranchRows(dir, repo.defaultBranch);
2482 const all = listed.ok ? listed.value : [];
2483 // The cursor is the name of the last branch of the page before.
2484 const start = cursor === null ? 0 : all.findIndex((r) => r.name === cursor) + 1;
2485 if (start === 0 && cursor !== null) return undefined;
2486 const slice = all.slice(start, start + limit);
2487 const compared = await compareBranches(dir, repo.defaultBranch, slice, all.length);
2488 return { branches: compared.branches, more: start + limit < all.length };
2489 });
2490 if (page === undefined)
24942491 return apiError("invalid_cursor", 422, "cursor is the next_cursor of a page before");
2495 const items = rows.slice(start, start + limit);
2496 const more = start + limit < rows.length;
2492 const items = page.branches.map((b) => ({
2493 name: b.name,
2494 oid: b.oid,
2495 isDefault: b.isDefault,
2496 ahead: b.ahead,
2497 behind: b.behind,
2498 protected: b.isDefault && repo.protectDefault,
2499 }));
2500 const more = page.more;
24972501 return timePhase(ctx.request, "render", () =>
24982502 json({ items, next_cursor: more ? items[items.length - 1]!.name : null }),
24992503 );
test/branches-deep.test.ts added +102
@@ -0,0 +1,102 @@
1 /**
2 * The branches list compares only the page it returns (#6225): on a repo with
3 * deep history and 200 branches forked at spread-out points, a 30-row page
4 * costs clearly less git time than a 200-row page. Comparing every branch
5 * for every page would make the two equal.
6 */
7 import { afterAll, beforeAll, expect, test } from "bun:test";
8 import { mkdtemp, rm } from "node:fs/promises";
9 import { tmpdir } from "node:os";
10 import { join } from "node:path";
11 import { createApp, type App } from "../src/app.ts";
12 import { serve, type Servers } from "../src/http/server.ts";
13 import { repoDir } from "../src/git/repos.ts";
14 import { compareBranches } from "../src/git/branches.ts";
15 import { gitClientEnv } from "./helpers/git-client.ts";
16 import { seedAdmin } from "./helpers/admin.ts";
17
18 const COMMITS = 3000;
19 /** Each branch carries its own run of commits, so each comparison costs a walk. */
20 const OWN = 150;
21 const env = gitClientEnv({
22 GIT_AUTHOR_NAME: "A",
23 GIT_AUTHOR_EMAIL: "[email protected]",
24 GIT_COMMITTER_NAME: "A",
25 GIT_COMMITTER_EMAIL: "[email protected]",
26 });
27
28 let dataDir = "";
29 let app: App, servers: Servers, bearer = "", dir = "";
30
31 beforeAll(async () => {
32 dataDir = await mkdtemp(join(tmpdir(), "forge-bdeep-"));
33 app = createApp({ dataDir, port: 0 });
34 servers = await serve(app);
35 await seedAdmin(app, { username: "alice", email: "[email protected]", password: "correct-horse-battery" });
36 const minted = await fetch(`${servers.url}/api/v1/tokens`, {
37 method: "POST",
38 headers: { "content-type": "application/json" },
39 body: JSON.stringify({ adminConfirm: true, username: "alice", password: "correct-horse-battery", name: "cli", scope: "admin" }),
40 });
41 bearer = ((await minted.json()) as { token: string }).token;
42 await fetch(`${servers.url}/api/v1/repos`, {
43 method: "POST",
44 headers: { "content-type": "application/json", authorization: `Bearer ${bearer}` },
45 body: JSON.stringify({ name: "deep", description: "x" }),
46 });
47 dir = repoDir(app.reposRoot, "alice", "deep");
48 let stream = "";
49 for (let i = 0; i < COMMITS; i++)
50 stream += `commit refs/heads/main\nmark :${i + 1}\ncommitter A <[email protected]> ${1700000000 + i} +0000\ndata 1\nc\n\n`;
51 for (let b = 0; b < 199; b++) {
52 const name = `feat/b${String(b).padStart(3, "0")}`;
53 for (let k = 0; k < OWN; k++)
54 stream += `commit refs/heads/${name}\ncommitter A <[email protected]> ${1800000000 + k} +0000\ndata 1\nx\n${k === 0 ? `from :${b * 10 + 5}\n` : ""}\n`;
55 }
56 const fi = Bun.spawn(["git", "fast-import", "--quiet"], { cwd: dir, env, stdin: new TextEncoder().encode(stream), stderr: "pipe" });
57 if ((await fi.exited) !== 0) throw new Error(await new Response(fi.stderr).text());
58 }, 120_000);
59
60 afterAll(async () => {
61 servers.stop();
62 app.close();
63 await rm(dataDir, { recursive: true, force: true });
64 });
65
66 async function gitMs(query: string): Promise<{ ms: number; items: { name: string; ahead: number | null }[] }> {
67 const res = await fetch(`${servers.url}/api/v1/repos/alice/deep/branches?${query}`, {
68 headers: { authorization: `Bearer ${bearer}` },
69 });
70 const body = (await res.json()) as { items: { name: string; ahead: number | null }[] };
71 expect(res.status).toBe(200);
72 const ms = Number(/(?:^|, )git;dur=([0-9.]+)/.exec(res.headers.get("server-timing") ?? "")?.[1]);
73 return { ms, items: body.items };
74 }
75
76 // Four runs, the first dropped as warm-up (page cache, git startup): best of three timed.
outdated

Nit: the loop runs four times and drops the first run as warm-up. The comment should say that, so the next reader doesn't "fix" it to three.

77 async function best(query: string) {
78 let fastest = Infinity;
79 let items: { name: string; ahead: number | null }[] = [];
80 for (let i = 0; i < 4; i++) {
81 const got = await gitMs(query);
82 if (i > 0) fastest = Math.min(fastest, got.ms);
83 items = got.items;
84 }
85 return { fastest, items };
86 }
87
88 test("a 30-row page compares 30 branches, not 200: its git phase is well under the 200-row page's", async () => {
89 const small = await best("limit=30");
90 const full = await best("limit=200");
91 expect(small.items).toHaveLength(30);
92 expect(full.items).toHaveLength(200);
93 expect(small.items.every((i) => i.ahead === OWN || i.name === "main")).toBe(true);
94 expect(small.fastest).toBeLessThan(full.fastest * 0.6);
95 }, 60_000);
96
97 test("an empty page compares nothing", async () => {
outdated

Non-blocking: this test's name says git is never asked, but it only checks the return value. Rename it to say what it checks ("an empty page compares nothing"), or make it prove the claim. For example, with a 200-branch repo, time compareBranches on [] against one row.

Minor: this test only checks the return value. It would still pass if aheadBehindAll went back to comparing the whole repo for an empty page and then dropped the result. The limit=200 call on lines 100–101 checks nothing that the first test doesn't already. To pin the actual behaviour, call aheadBehindAll with an empty list and assert that git is never spawned (or that it returns the empty map without git). That can be a follow-up; it doesn't block.

98 const listed = await compareBranches(dir, "main", [], 200);
99 expect(listed.branches).toEqual([]);
100 const last = await gitMs("limit=200");
101 expect(last.items).toHaveLength(200);
102 });
test/branches-many.test.ts +20
@@ -115,6 +115,26 @@ test("the one-call counts equal the per-branch rev-list counts", async () => {
115115 expect(listed.value.branches[1]).toMatchObject({ name: "feat/b000", ahead: 1, behind: 1 });
116116 }, 60_000);
117117
118 test("the API returns the first 30 of 200 branches with the whole list's counts, and its git phase fits the budget", async () => {
119 const dir = repoDir(app.reposRoot, "alice", "app");
120 const full = await listBranches(dir, "main");
121 if (!full.ok) throw new Error("listBranches failed");
122 const path = "/api/v1/repos/alice/app/branches?limit=30";
123 await get(path);
124 const { text, timing } = await get(path);
125 const dur = Number(/(?:^|, )git;dur=([0-9.]+)/.exec(timing)?.[1]);

This budget holds on main too: in this fixture, comparing all 200 one-commit branches already fits in 60 ms. A test of 'compares only the page' has to fail when the whole list is compared. Make ahead/behind costly (deep history, forks spread out) and assert that limit=30 is cheaper than limit=200 on the same repo, best of three.

126 expectBudget(dur, { ms: 60, against: await baselineMs(servers.url) });
127 const page = JSON.parse(text) as { items: { name: string; ahead: number | null; behind: number | null }[]; next_cursor: string | null };
128 // Rows, order and cursor are the whole list's first 30.
129 expect(page.items.map((i) => [i.name, i.ahead, i.behind])).toEqual(
130 full.value.branches.slice(0, 30).map((b) => [b.name, b.ahead, b.behind]),
131 );
132 expect(page.next_cursor).toBe(page.items[29]!.name);
133 const next = JSON.parse((await get(`${path}&cursor=${encodeURIComponent(page.next_cursor!)}`)).text) as typeof page;
134 expect(next.items[0]!.name).toBe(full.value.branches[30]!.name);
135 expect(next.items[0]).toMatchObject({ ahead: 1, behind: 1 });
136 }, 30_000);
137
118138 test("the per-branch fallback gives the same rows, and the one-call path is faster", async () => {
119139 const dir = repoDir(app.reposRoot, "alice", "app");
120140 // Best of three each, so one stalled run on a loaded machine cannot flip it.

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs