trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRuns4g rEventsFlows

Displays read git once per repository, and the picker reads fresh crew pulls once per pass #10534

Merged scoutcrew opened this in answer to trevatoowner

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

scout/forge:scout/10445-fast-everywhere-displays-first-then-a-ti → main 5 commits fork

  • Merged. scout/10445-fast-everywhere-displays-first-then-a-ti was merged into main as f08d37f.

Review

Sign in to review this change.
10 files changed +541 −22
src/actions/pin_demos.ts +30 −5
@@ -9,6 +9,7 @@ import type { Repo } from "../core/repos.ts";
99 import { exactRefOid } from "../git/branches.ts";
1010 import type { Demo, Launch } from "../git/builds.ts";
1111 import { isSafeOid } from "../git/guard.ts";
12 import { PINS_FRESH_MS } from "../git/pins_kept.ts";
1213 import { readerFor } from "../git/read.ts";
1314 import { repoDir } from "../git/repos.ts";
1415
@@ -26,9 +27,32 @@ export interface DemoFacts {
2627 * branch names now. A page the old build does not serve is simply not
2728 * found when played.
2829 */
29 export async function demoFactsAt(app: App, repo: Repo, oid: string): Promise<DemoFacts> {
30 if (!isSafeOid(oid)) return { pages: [] };
30 export function demoFactsAt(app: App, repo: Repo, oid: string): Promise<DemoFacts> {
31 if (!isSafeOid(oid)) return Promise.resolve({ pages: [] });
3132 const dir = repoDir(app.reposRoot, repo.ownerName, repo.name);
33 const key = `${dir}\0${repo.defaultBranch}\0${oid}`;
34 const seen = factsKept.get(key);
35 if (seen !== undefined && Date.now() <= seen.until) return seen.facts;
36 const facts = readDemoFacts(dir, repo, oid);
37 const entry = { facts, until: Number.POSITIVE_INFINITY };
38 factsKept.delete(key);
39 factsKept.set(key, entry);
40 if (factsKept.size > FACTS_KEPT_MAX) factsKept.delete(factsKept.keys().next().value!);
41 // What a commit holds never changes; facts borrowed from the branch tip are read again after a while.
42 facts.then(
43 (f) => {
44 if (f.borrowed) entry.until = Date.now() + PINS_FRESH_MS;
45 },
46 () => factsKept.delete(key),
47 );
48 return facts.then(({ pages, launch }) => (launch === undefined ? { pages } : { pages, launch }));
49 }
50
51 /** The most commits' facts kept at once (#10445): a display reads them once per build, not once per request. */
52 const FACTS_KEPT_MAX = 4_096;
53 const factsKept = new Map<string, { facts: Promise<DemoFacts & { borrowed?: true }>; until: number }>();
54
55 async function readDemoFacts(dir: string, repo: Repo, oid: string): Promise<DemoFacts & { borrowed?: true }> {
3256 const read = async (at: string): Promise<DemoFacts> => {
3357 const [forgeJson, packageJson] = await Promise.all([
3458 readFileAt(dir, at, "forge.json"),
@@ -45,11 +69,12 @@ export async function demoFactsAt(app: App, repo: Repo, oid: string): Promise<De
4569 if (own.pages.length > 0) return own;
4670 // Only a commit this repository holds borrows anything.
4771 const stat = await readerFor(dir).stat(oid);
48 if (!stat.ok || stat.value.type !== "commit") return own;
72 // No pages of its own: whatever is answered may change with the branch, so it is kept a while only.
73 if (!stat.ok || stat.value.type !== "commit") return { ...own, borrowed: true };
4974 const tip = await exactRefOid(dir, `refs/heads/${repo.defaultBranch}`);
50 if (tip === undefined || tip === oid) return own;
75 if (tip === undefined || tip === oid) return { ...own, borrowed: true };
5176 const now = await read(tip);
52 return { pages: now.pages, ...((own.launch ?? now.launch) === undefined ? {} : { launch: own.launch ?? now.launch }) };
77 return { pages: now.pages, borrowed: true, ...((own.launch ?? now.launch) === undefined ? {} : { launch: own.launch ?? now.launch }) };
5378 }
5479
5580 /** The facts a pin's demo starts from, before anyone edits them. */
src/app.ts +6
@@ -31,6 +31,7 @@ import {
3131 } from "./kernel/log.ts";
3232 import { RefusedLedger } from "./kernel/internal_refused.ts";
3333 import { closeReadersUnder } from "./git/read.ts";
34 import { dropAllPins } from "./git/pins_kept.ts";
3435
3536 export interface ForgeConfig {
3637 /** Everything lives here: index db, bare repos, the internal socket. */
@@ -255,6 +256,11 @@ export function createApp(config: ForgeConfig): App {
255256 // sends them. Listener throws are swallowed by the bus, so this can never
256257 // break the action that produced the event.
257258 bus.subscribe((event) => enqueueForEvent(db, event));
259 // A push, a merge, a repository change or a build event may move pins: the kept listings go (`git/pins_kept.ts`).
260 bus.subscribe((event) => {
261 const k = event.kind;
262 if (k === "push" || k === "pull.merged" || k.startsWith("repo.") || k.startsWith("build.")) dropAllPins();
263 });
258264 // What a push moves besides its own branch: the pulls it heads elsewhere,
259265 // and the forks whose default branch is still a copy of this one's.
260266 follow({
src/crew/fresh_pull.ts +51 −15
@@ -1,7 +1,13 @@
11 /**
22 * A pull a crew member opened for an issue a moment ago (#8473). The
33 * dispatcher refuses a second build run on such an issue, and the picker
4 * holds it as `open_pull`: both ask this one function.
4 * holds it as `open_pull`: both ask this one module.
5 *
6 * The picker asks for every ready issue on every tick, so the database is
7 * read once per repository (`freshPulls`, a range on `issue_created` that
8 * holds only the last ten minutes) and each issue is matched in memory
9 * (`pullFor`): it used to be one query per issue per builder, about 1,300 a
10 * minute and 11 s of every minute of SQL time (#10445).
511 */
612
713 import type { Database } from "bun:sqlite";
@@ -9,6 +15,49 @@ import type { Database } from "bun:sqlite";
915 /** How long after a crew pull opens its issue takes no second build run. */
1016 export const BUILD_SETTLE_MS = 10 * 60_000;
1117
18 /** A pull opened within `BUILD_SETTLE_MS`: its number and its head branch. */
19 export interface FreshPull {
20 number: number;
21 headBranch: string;
22 }
23
24 /**
25 * The repository's pulls opened since `?2` (`?1` is the repository).
26 * `+i.repo_id` keeps the planner on issue_created: the window is ten
27 * minutes wide, the repository's issues are thousands.
28 */
29 export const FRESH_PULLS_SQL = `SELECT i.number AS number, p.head_branch AS headBranch
30 FROM issue i JOIN pull p ON p.issue_id = i.id
31 WHERE i.created_at > ?2 AND +i.repo_id = ?1`;
32
33 /** Every pull in the repository opened within `BUILD_SETTLE_MS` of `now`. */
34 export function freshPulls(db: Database, repoId: string, now = Date.now()): FreshPull[] {
35 const since = new Date(now - BUILD_SETTLE_MS).toISOString();
36 return db.query<FreshPull, [string, string]>(FRESH_PULLS_SQL).all(repoId, since);
37 }
38
39 /** ASCII letters folded to lower case, as SQLite's LIKE folds them; every other character as is. */
40 const asciiLower = (s: string): string => s.replace(/[A-Z]/g, (c) => c.toLowerCase());
41
42 /**
43 * The fresh pull on a branch of one of `usernames` for the issue, if any:
44 * `<user>/<n>` exactly, or `<user>/<n>-…` with ASCII case ignored — the
45 * matching the old query's `=` and `LIKE` did.
46 */
47 export function pullFor(
48 fresh: readonly FreshPull[],
49 issueNumber: number,
50 usernames: readonly string[],
51 ): { number: number } | undefined {
52 for (const username of usernames) {
53 const exact = `${username}/${issueNumber}`;
54 const prefix = asciiLower(`${exact}-`);
55 const found = fresh.find((p) => p.headBranch === exact || asciiLower(p.headBranch).startsWith(prefix));
56 if (found) return { number: found.number };
57 }
58 return undefined;
59 }
60
1261 /** The pull opened within `BUILD_SETTLE_MS` on a branch of one of `usernames` for the issue, if any. */
1362 export function freshPullFor(
1463 db: Database,
@@ -16,18 +65,5 @@ export function freshPullFor(
1665 issueNumber: number,
1766 usernames: readonly string[],
1867 ): { number: number } | undefined {
19 const since = new Date(Date.now() - BUILD_SETTLE_MS).toISOString();
20 for (const username of usernames) {
21 const name = username.replace(/[\\%_]/g, "\\$&");
22 const found = db
23 .query<{ number: number }, [string, string, string, string]>(
24 `SELECT i.number FROM pull p JOIN issue i ON i.id = p.issue_id
25 WHERE i.repo_id = ?1 AND i.created_at > ?2
26 AND (p.head_branch = ?3 OR p.head_branch LIKE ?4 ESCAPE '\\')
27 LIMIT 1`,
28 )
29 .get(repoId, since, `${username}/${issueNumber}`, `${name}/${issueNumber}-%`);
30 if (found) return found;
31 }
32 return undefined;
68 return pullFor(freshPulls(db, repoId), issueNumber, usernames);
3369 }
src/crew/held.ts +4 −2
@@ -44,7 +44,7 @@ import { listActive, type Run } from "../core/runs.ts";
4444 import { getUserById } from "../core/users.ts";
4545 import type { Stuck } from "./brief.ts";
4646 import { proposedWork, speaksForCrew } from "./direct.ts";
47 import { freshPullFor } from "./fresh_pull.ts";
47 import { freshPulls, pullFor } from "./fresh_pull.ts";
4848 import { capReached, runCapOf } from "./guard.ts";
4949 import { interruptionOf } from "./interrupted.ts";
5050 import { landedUnanswered } from "./landed.ts";
@@ -143,6 +143,8 @@ export function heldBy(
143143 crew.filter((c) => c.role !== "builder").map((c) => c.agent.user.id),
144144 );
145145 const builders = crew.filter((c) => c.role === "builder").map((c) => c.agent.user.username);
146 // Read once for the whole pass, matched per issue in memory (#10445).
147 const fresh = builders.length === 0 ? [] : freshPulls(app.db, repo.id);
146148 const handed = withPlanner(app.db, repo);
147149 const hot = hotFiles(repo.id, admit);
148150 const pins = new Map(heldFiles(app.db, repo.id).map((h) => [h.path, h]));
@@ -199,7 +201,7 @@ export function heldBy(
199201 if (hotOverlap(issue, hot).length > 0) return "hot_file";
200202 if (waitsOnRelief(issue)) return "hot_file";
201203 // The dispatcher refuses a build on an issue whose pull just opened (#8473): held, not picked.
202 if (freshPullFor(app.db, repo.id, issue.number, builders) !== undefined) return "open_pull";
204 if (pullFor(fresh, issue.number, builders) !== undefined) return "open_pull";
203205 return null;
204206 };
205207 const by = Object.fromEntries(HOLDS.map((h) => [h, [] as Issue[]])) as Record<
src/git/builds.ts +11
@@ -14,6 +14,7 @@ import { join } from "node:path";
1414 import { err, ok, type Result } from "../kernel/result.ts";
1515 import { runGit, runGitCapture, runGitRaw } from "./env.ts";
1616 import { isSafeOid, isSafeRef } from "./guard.ts";
17 import { keptPins, pinsMoved } from "./pins_kept.ts";
1718
1819 export const BUILDS_PREFIX = "refs/builds/";
1920
@@ -173,6 +174,7 @@ export async function pinBuild(
173174 ["update-ref", `${BUILDS_PREFIX}${input.label}`, made.value.trim(), ""],
174175 { cwd: dir },
175176 );
177 pinsMoved(dir);
176178 if (created.code !== 0)
177179 return err(/already exists|reference already/i.test(created.stderr) ? "exists" : "git_failed");
178180 return ok({
@@ -305,6 +307,7 @@ async function rewritePin(
305307 const made = await runGit(["mktag"], { cwd: dir, stdin: tag });
306308 if (!made.ok) return err("git_failed");
307309 const moved = await runGitCapture(["update-ref", ref, made.value.trim(), oldTag], { cwd: dir });
310 pinsMoved(dir);
308311 if (moved.code !== 0) return err("moved");
309312 return ok({
310313 label,
@@ -561,6 +564,7 @@ export async function appendNote(
561564 if (!commit.ok) return err("git_failed");
562565 const oid = commit.value.trim();
563566 const moved = await runGitCapture(["update-ref", NOTES_REF, oid, tip ?? ""], { cwd: dir });
567 pinsMoved(dir);
564568 if (moved.code === 0) return ok(oid);
565569 }
566570 return err("moved");
@@ -624,6 +628,7 @@ export async function deletePin(dir: string, label: string): Promise<Result<{ re
624628 if (at !== undefined) lines.push(`delete ${other} ${at}`);
625629 }
626630 const done = await runGitCapture(["update-ref", "--stdin"], { cwd: dir, stdin: `start\n${lines.join("\n")}\nprepare\ncommit\n` });
631 pinsMoved(dir);
627632 if (done.code !== 0) return err(/cannot lock|is at|expected/i.test(done.stderr) ? "moved" : "git_failed");
628633 return ok({ oid: commit.stdout.trim(), tag });
629634 }
@@ -662,6 +667,12 @@ export async function readCover(dir: string, oid: string): Promise<Uint8Array |
662667 * else its tag message's.
663668 */
664669 export async function listBuildPins(dir: string, label?: string): Promise<BuildPin[]> {
670 // The whole listing is kept per repository (`pins_kept.ts`); one label is read fresh.
671 if (label === undefined) return [...(await keptPins(dir, () => readBuildPins(dir)))];
672 return readBuildPins(dir, label);
673 }
674
675 async function readBuildPins(dir: string, label?: string): Promise<BuildPin[]> {
665676 const pins = await listTagPins(dir, label);
666677 const notes = await readBuildNotes(dir, pins.map((p) => p.label));
667678 if (notes.size === 0) return pins;
src/git/pins_kept.ts added +58
@@ -0,0 +1,58 @@
1 /**
2 * Each repository's pins as last listed (`listBuildPins` with no label),
3 * kept in memory so a page that reads many repositories' pins — a display
4 * and its chooser, the Arcade's shelf — spawns git once per repository, not
5 * once per request (#10445).
6 *
7 * A listing is dropped when this process writes the repository's pins or
8 * notes (`git/builds.ts` calls `pinsMoved`), when an event may have moved
9 * them (`dropAllPins`, from the display cache's bus subscription), and after
10 * `PINS_FRESH_MS` in any case, so a ref moved by nothing the forge saw is
11 * read again within that. Git stays the record: this is a cache, empty on
12 * boot, bounded by `PINS_KEPT_MAX` repositories.
13 */
14
15 /** How long a listing is answered before it is read again. */
16 export const PINS_FRESH_MS = 30_000;
17 /** The most repositories' listings kept at once; the oldest goes first. */
18 export const PINS_KEPT_MAX = 2_000;
19
20 interface Entry<T> {
21 readonly value: Promise<T>;
22 readonly at: number;
23 readonly epoch: number;
24 }
25
26 const kept = new Map<string, Entry<unknown>>();
27 const epochs = new Map<string, number>();
28 let allEpoch = 0;
29
30 const epochOf = (dir: string): number => (epochs.get(dir) ?? 0) + allEpoch * 1e9;
31
32 /** This process wrote `dir`'s pins or notes: its listing is read again. */
33 export function pinsMoved(dir: string): void {
34 epochs.set(dir, (epochs.get(dir) ?? 0) + 1);
35 kept.delete(dir);
36 }
37
38 /** An event may have moved any repository's pins: every listing is read again. */
39 export function dropAllPins(): void {
40 allEpoch++;
41 kept.clear();
42 }
43
44 /** `dir`'s listing as kept, or `read` once, shared by concurrent callers; a failed read is not kept. */
45 export function keptPins<T>(dir: string, read: () => Promise<T>): Promise<T> {
46 const seen = kept.get(dir) as Entry<T> | undefined;
47 const epoch = epochOf(dir);
48 if (seen !== undefined && seen.epoch === epoch && Date.now() - seen.at <= PINS_FRESH_MS) return seen.value;
49 const value = read();
50 const entry: Entry<T> = { value, at: Date.now(), epoch };
51 kept.delete(dir);
52 kept.set(dir, entry);
53 if (kept.size > PINS_KEPT_MAX) kept.delete(kept.keys().next().value!);
54 value.catch(() => {
55 if (kept.get(dir) === entry) kept.delete(dir);
56 });
57 return value;
58 }
test/crew-sandbox.test.ts +7
@@ -606,10 +606,17 @@ test.skipIf(process.platform !== "linux")("a child forge's run reads and writes
606606 mkdirSync(join(data, "crew", "scottishbot"), { recursive: true });
607607 const neighbour = join(tmpdir(), `fc-neighbour-${Date.now()}`);
608608 mkdirSync(neighbour, { recursive: true });
609 const otherRun = join(tmpdir(), `fr-other${Date.now()}`);
610 mkdirSync(otherRun, { recursive: true });
609611 const p = sandboxPolicy({ dataDir: data, agentName: "scottishbot", urls: [], runId: "01m71kw4dfp4z8arr359fkygy5" });
610612 expect(p.tmpDir).toBe(join(tmpdir(), "fr-arr359fkygy5"));
611613 expect(p.denyRead).not.toContain(p.tmpDir);
612614 expect(p.denyRead).toContain(neighbour);
615 expect(p.denyRead).toContain(otherRun);
616 // The data directory is in the temp directory too: it is not denied whole,
617 // so the agent still reaches its own working directory.
618 const under = (path: string) => p.denyRead.some((d) => path === d || path.startsWith(`${d}/`));
619 expect(under(join(data, "crew", "scottishbot", "01repo", "7"))).toBe(false);
613620 expect(p.denyRead).toContain(join(data, "forge.db"));
614621 });
615622
test/display-cache.test.ts added +141
@@ -0,0 +1,141 @@
1 /**
2 * What a display keeps in memory is dropped when it should be (#10445): a
3 * repository's pin listing after `PINS_FRESH_MS`, and on every push, merge,
4 * repository or build event through the bus; a commit's demo facts are kept
5 * by oid, and facts borrowed from the branch tip are read again after
6 * `PINS_FRESH_MS`.
7 */
8 import { afterAll, afterEach, beforeAll, describe, expect, spyOn, test } from "bun:test";
9 import { mkdtemp, rename, rm } from "node:fs/promises";
10 import { tmpdir } from "node:os";
11 import { join } from "node:path";
12 import { createApp, type App } from "../src/app.ts";
13 import { demoFactsAt } from "../src/actions/pin_demos.ts";
14 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
15 import type { Repo } from "../src/core/repos.ts";
16 import { createUser, type User } from "../src/core/users.ts";
17 import { keptPins, PINS_FRESH_MS } from "../src/git/pins_kept.ts";
18 import { closeReadersUnder } from "../src/git/read.ts";
19 import { repoDir } from "../src/git/repos.ts";
20
21 let dataDir: string;
22 let app: App;
23 let alice: User;
24 let clock: ReturnType<typeof spyOn> | undefined;
25
26 /** Date.now moved `ms` ahead of the real clock until the test ends. */
27 function later(ms: number): void {
28 const real = Date.now.bind(Date);
29 clock?.mockRestore();
30 clock = spyOn(Date, "now").mockImplementation(() => real() + ms);
31 }
32
33 afterEach(() => {
34 clock?.mockRestore();
35 clock = undefined;
36 });
37
38 beforeAll(async () => {
39 dataDir = await mkdtemp(join(tmpdir(), "forge-display-cache-"));
40 app = createApp({ dataDir, port: 0 });
41 const made = await createUser(app.db, {
42 origin: "seed",
43 trusted: true,
44 username: "alice",
45 email: "[email protected]",
46 password: "a-good-passphrase",
47 adminIfFirst: true,
48 });
49 if (!made.ok) throw new Error(made.error);
50 alice = made.value;
51 });
52
53 afterAll(async () => {
54 app.close();
55 await rm(dataDir, { recursive: true, force: true });
56 });
57
58 /** A listing read through `keptPins` that counts its reads. */
59 function counted(dir: string) {
60 let reads = 0;
61 const list = () => keptPins(dir, async () => ++reads);
62 return { list, reads: () => reads };
63 }
64
65 describe("a kept pin listing", () => {
66 test("is answered for PINS_FRESH_MS, then read again", async () => {
67 const pins = counted(join(dataDir, "ttl"));
68 expect(await pins.list()).toBe(1);
69 expect(await pins.list()).toBe(1);
70 later(PINS_FRESH_MS - 1_000);
71 expect(await pins.list()).toBe(1);
72 later(PINS_FRESH_MS + 1_000);
73 expect(await pins.list()).toBe(2);
74 });
75
76 for (const kind of ["push", "pull.merged", "repo.created", "build.pinned"]) {
77 test(`is dropped by a ${kind} event on the bus`, async () => {
78 const pins = counted(join(dataDir, `event-${kind}`));
79 expect(await pins.list()).toBe(1);
80 expect(await pins.list()).toBe(1);
81 app.bus.append({ kind, payload: {} });
82 expect(await pins.list()).toBe(2);
83 });
84 }
85
86 test("is kept through an event that moves no pins", async () => {
87 const pins = counted(join(dataDir, "event-other"));
88 expect(await pins.list()).toBe(1);
89 app.bus.append({ kind: "issue.opened", payload: {} });
90 expect(await pins.list()).toBe(1);
91 });
92 });
93
94 describe("a commit's demo facts", () => {
95 const manifest = (pages: string[]) =>
96 JSON.stringify({ app: { run: "bun run index.ts", pages } });
97 let repo: Repo;
98 let dir: string;
99 const commit = async (path: string, content: string) => {
100 const made = await commitToBranch(app, alice, repo, { message: path, files: [{ path, content }] });
101 if (!made.ok) throw new Error(JSON.stringify(made.error));
102 return made.value.commitOid;
103 };
104 /** `run` with the repository out of reach: whatever it answers was kept. */
105 const unreachable = async <T>(run: () => Promise<T>): Promise<T> => {
106 const away = `${dir}.away`;
107 closeReadersUnder(dir);
108 await rename(dir, away);
109 try {
110 return await run();
111 } finally {
112 await rename(away, dir);
113 closeReadersUnder(dir);
114 }
115 };
116
117 beforeAll(async () => {
118 const made = await createRepo(app, alice, { name: "game", visibility: "public" });
119 if (!made.ok) throw new Error(String(made.error));
120 repo = made.value;
121 dir = repoDir(app.reposRoot, "alice", "game");
122 });
123
124 test("are kept by oid: a commit with pages is not read again", async () => {
125 const oid = await commit("forge.json", manifest(["/own"]));
126 expect((await demoFactsAt(app, repo, oid)).pages).toEqual(["/own"]);
127 later(PINS_FRESH_MS + 1_000);
128 expect((await unreachable(() => demoFactsAt(app, repo, oid))).pages).toEqual(["/own"]);
129 });
130
131 test("borrowed from the tip are kept a while, then read again", async () => {
132 const old = await commit("forge.json", manifest([]));
133 await commit("forge.json", manifest(["/first"]));
134 expect((await demoFactsAt(app, repo, old)).pages).toEqual(["/first"]);
135 await commit("forge.json", manifest(["/second"]));
136 // Within PINS_FRESH_MS the kept answer stands, git or no git.
137 expect((await unreachable(() => demoFactsAt(app, repo, old))).pages).toEqual(["/first"]);
138 later(PINS_FRESH_MS + 1_000);
139 expect((await demoFactsAt(app, repo, old)).pages).toEqual(["/second"]);
140 });
141 });
test/display-warm-reads.test.ts added +136
@@ -0,0 +1,136 @@
1 /**
2 * A display over many builds reads git once per repository, not once per
3 * build per request (#10445): a warm `/:owner/:name/display` and a warm
4 * `/displays` spawn no git, for a reader and for a writer. Counted, never
5 * timed.
6 */
7 import { afterAll, beforeAll, describe, expect, spyOn, test } from "bun:test";
8 import { mkdtemp, rm } from "node:fs/promises";
9 import { tmpdir } from "node:os";
10 import { join } from "node:path";
11 import { createApp, type App } from "../src/app.ts";
12 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
13 import { getRepo } from "../src/core/repos.ts";
14 import { createSession } from "../src/core/sessions.ts";
15 import { createUser, type User } from "../src/core/users.ts";
16 import { pinBuild } from "../src/git/builds.ts";
17 import { repoDir } from "../src/git/repos.ts";
18 import { serve, type Servers } from "../src/http/server.ts";
19 import { proveAll } from "./helpers/verified.ts";
20
21 const REPOS = 10;
22 const PINS = 3;
23
24 /** The calls in a spawn spy that ran git, whatever the argv shape. */
25 function gitCalls(calls: unknown[][]): number {
26 return calls.filter((c) => {
27 const first = c[0];
28 const argv = Array.isArray(first) ? first : (first as { cmd?: unknown } | undefined)?.cmd;
29 return Array.isArray(argv) && /(^|\/)git$/.test(String(argv[0]));
30 }).length;
31 }
32
33 describe("a warm display reads no git", () => {
34 let dataDir: string;
35 let app: App;
36 let servers: Servers;
37 let alice: User;
38 let bob: User;
39
40 const get = (path: string, u: User) =>
41 fetch(`${servers.url}${path}`, { headers: { cookie: `forge_session=${createSession(app.db, u.id).token}` }, redirect: "manual" });
42 const commit = async (name: string, path: string, content: string) => {
43 const made = await commitToBranch(app, alice, getRepo(app.db, "alice", name)!, { message: path, files: [{ path, content }] });
44 if (!made.ok) throw new Error(JSON.stringify(made.error));
45 return made.value.commitOid;
46 };
47 /** Git spawned by one warm request for `path`. */
48 const warm = async (path: string, u: User) => {
49 const first = await get(path, u);
50 expect(first.status).toBe(200);
51 await first.text();
52 const spawn = spyOn(Bun, "spawn");
53 const spawnSync = spyOn(Bun, "spawnSync");
54 try {
55 const res = await get(path, u);
56 const page = await res.text();
57 expect(res.status).toBe(200);
58 return { page, git: gitCalls(spawn.mock.calls) + gitCalls(spawnSync.mock.calls) };
59 } finally {
60 spawn.mockRestore();
61 spawnSync.mockRestore();
62 }
63 };
64
65 beforeAll(async () => {
66 dataDir = await mkdtemp(join(tmpdir(), "forge-display-warm-"));
67 app = createApp({ dataDir, port: 0 });
68 servers = await serve(app);
69 for (const name of ["alice", "bob"]) {
70 const made = await createUser(app.db, { origin: "seed", trusted: true, username: name, email: `${name}@forge.local`, password: "a-good-passphrase", adminIfFirst: true });
71 if (!made.ok) throw new Error(made.error);
72 if (name === "alice") alice = made.value;
73 else bob = made.value;
74 }
75 proveAll(app);
76 for (let r = 0; r < REPOS; r++) {
77 const name = `game${r}`;
78 const made = await createRepo(app, alice, { name, visibility: "public" });
79 if (!made.ok) throw new Error(String(made.error));
80 // No pages and no launch on the demo: each build's facts come from git (`demoFactsAt`).
81 const oid = await commit(name, "a.txt", String(r));
82 for (let p = 0; p < PINS; p++) {
83 const pinned = await pinBuild(repoDir(app.reposRoot, "alice", name), {
84 label: `v${p}`,
85 oid,
86 note: "",
87 source: "commit",
88 tagger: { name: "alice", email: "[email protected]" },
89 at: 1_800_000_000 + p,
90 demo: { name: `${name}-v${p}`, version: `${p}.0`, page: "/", showcase: true },
91 });
92 if (!pinned.ok) throw new Error(pinned.error);
93 }
94 }
95 await createRepo(app, alice, { name: "shelf", visibility: "public" });
96 await commit("shelf", "display.json", JSON.stringify({ title: "Game night", layout: "grid", sources: ["alice"] }));
97 }, 120_000);
98
99 afterAll(async () => {
100 servers.stop();
101 app.close();
102 await rm(dataDir, { recursive: true, force: true });
103 });
104
105 test("a reader's display page", async () => {
106 const { page, git } = await warm("/alice/shelf/display", bob);
107 expect(page).toContain(`${REPOS * PINS} builds`);
108 expect(git).toBe(0);
109 });
110
111 test("a writer's display page, with its chooser", async () => {
112 const { page, git } = await warm("/alice/shelf/display", alice);
113 expect(page).toContain("Choose builds");
114 // The form's tip is read fresh: it guards the save against a moved branch.
115 expect(git).toBeLessThanOrEqual(1);
116 });
117
118 test("/displays", async () => {
119 const { page, git } = await warm("/displays", bob);
120 expect(page).toContain("Game night");
121 expect(git).toBe(0);
122 });
123
124 test("a new pin shows at once", async () => {
125 const dir = repoDir(app.reposRoot, "alice", "game0");
126 const oid = (await commit("game0", "b.txt", "b"));
127 const pinned = await pinBuild(dir, {
128 label: "v9", oid, note: "", source: "commit",
129 tagger: { name: "alice", email: "[email protected]" },
130 demo: { name: "game0-v9", version: "9.0", page: "/", showcase: true },
131 });
132 expect(pinned.ok).toBe(true);
133 const page = await (await get("/alice/shelf/display", bob)).text();
134 expect(page).toContain("game0-v9");
135 });
136 });
test/fresh-pull.test.ts added +97
@@ -0,0 +1,97 @@
1 /**
2 * A crew pull opened a moment ago (#8473), read once per repository and
3 * matched in memory (#10445): the picker used to ask SQL once per ready
4 * issue per builder.
5 */
6 import { afterAll, beforeAll, expect, test } from "bun:test";
7 import { mkdtemp, rm } from "node:fs/promises";
8 import { tmpdir } from "node:os";
9 import { join } from "node:path";
10 import { createApp, type App } from "../src/app.ts";
11 import { openPull } from "../src/actions/pulls.ts";
12 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
13 import { createUser } from "../src/core/users.ts";
14 import { BUILD_SETTLE_MS, FRESH_PULLS_SQL, freshPullFor, freshPulls, pullFor } from "../src/crew/fresh_pull.ts";
15 import { quietWeek } from "./helpers/week-quiet.ts";
16
17 let dataDir: string;
18 let app: App;
19
20 beforeAll(async () => {
21 dataDir = await mkdtemp(join(tmpdir(), "forge-fresh-"));
22 app = createApp({ dataDir, port: 0 });
23 quietWeek(app.db);
24 });
25
26 afterAll(async () => {
27 app.close();
28 await rm(dataDir, { recursive: true, force: true });
29 });
30
31 test("a branch of a builder for the issue matches, a lookalike does not", () => {
32 const fresh = [
33 { number: 7, headBranch: "mason/12-fix-the-thing" },
34 { number: 8, headBranch: "mason/120" },
35 { number: 9, headBranch: "atlas/13" },
36 ];
37 expect(pullFor(fresh, 12, ["mason"])).toEqual({ number: 7 });
38 expect(pullFor(fresh, 120, ["mason"])).toEqual({ number: 8 });
39 expect(pullFor(fresh, 1, ["mason"])).toBeUndefined();
40 expect(pullFor(fresh, 13, ["mason"])).toBeUndefined();
41 expect(pullFor(fresh, 13, ["mason", "atlas"])).toEqual({ number: 9 });
42 expect(pullFor(fresh, 12, [])).toBeUndefined();
43 // As the old LIKE did: the `-…` form ignores ASCII case, the exact form does not.
44 expect(pullFor([{ number: 4, headBranch: "Mason/12-Fix" }], 12, ["mason"])).toEqual({ number: 4 });
45 expect(pullFor([{ number: 4, headBranch: "mason/12-fix" }], 12, ["MASON"])).toEqual({ number: 4 });
46 expect(pullFor([{ number: 5, headBranch: "Mason/12" }], 12, ["mason"])).toBeUndefined();
47 expect(pullFor([{ number: 6, headBranch: "mÅson/12-x" }], 12, ["måson"])).toBeUndefined();
48 // A username with LIKE's wildcards is matched literally.
49 expect(pullFor([{ number: 3, headBranch: "ma_on/12" }], 12, ["mason"])).toBeUndefined();
50 });
51
52 test("the repository's fresh pulls are read once, inside the window only, through issue_created", async () => {
53 const u = await createUser(app.db, {
54 origin: "seed",
55 trusted: true,
56 username: "mason",
57 email: "[email protected]",
58 password: "a-good-passphrase",
59 adminIfFirst: true,
60 });
61 if (!u.ok) throw new Error(u.error);
62 const made = await createRepo(app, u.value, { name: "engine", visibility: "private" });
63 if (!made.ok) throw new Error(String(made.error));
64 const repo = made.value;
65 const seeded = await commitToBranch(app, u.value, repo, {
66 message: "Start",
67 files: [{ path: "README.md", content: "# engine\n" }],
68 });
69 if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
70 const branch = await commitToBranch(app, u.value, repo, {
71 branch: "mason/4-start",
72 base: "main",
73 message: "Work",
74 files: [{ path: "a.ts", content: "export const a = 1;\n" }],
75 });
76 if (!branch.ok) throw new Error(JSON.stringify(branch.error));
77 const pull = await openPull(app, u.value, repo, {
78 title: "Work",
79 body: "Closes #4",
80 baseBranch: "main",
81 headBranch: "mason/4-start",
82 });
83 if (!pull.ok) throw new Error(String(pull.error));
84 const number = pull.value.number;
85
86 expect(freshPulls(app.db, repo.id)).toEqual([{ number, headBranch: "mason/4-start" }]);
87 expect(freshPullFor(app.db, repo.id, 4, ["mason"])).toEqual({ number });
88 expect(freshPulls(app.db, repo.id, Date.now() + BUILD_SETTLE_MS + 1000)).toEqual([]);
89 expect(freshPulls(app.db, "no-such-repo")).toEqual([]);
90
91 const plan = app.db
92 .query<{ detail: string }, [string, string]>(`EXPLAIN QUERY PLAN ${FRESH_PULLS_SQL}`)
93 .all(repo.id, "2000")
94 .map((r) => r.detail)
95 .join("\n");
96 expect(plan).toContain("issue_created");
97 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs