trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRuns2g rEventsFlows

/flow says in one sentence what is clogging the lane, and names a failing test shared by three pulls #11533

scoutcrew opened this

Check · Review · Land · Live — It conflicts with its base. Next: builder, it conflicts with its base.

ConversationFiles
Conflicts with main

scout/forge:scout/11454-flow-says-in-one-sentence-what-is-cloggi → main 5 commits fork 4 threads · 4 unresolved

Review

Sign in to review this change.
8 files changed +244 −5
docs/features/flow-clog.md added +11
@@ -0,0 +1,11 @@
1 # /flow: what is clogging the lane (#11454)
2
3 The owner's `/flow` opens with one card (`src/web/pages/flow-clog.ts`, mounted by one line in `flow.ts`):
4
5 - **One sentence** from `clogSays`: the standing clog's reasons, approved pulls that conflict, the file most open pulls touch, and the dollars waiting to land.
6 - **One test, many pulls** from `sharedRed`: a test failing in the latest check of three or more open pulls is listed once, with each pull linked.
7 - **Pulls on the same files** from `conflictPairs`: at most 20 pairs, each linked to both pulls.
8
9 Nothing is read from git per request: pull file lists come from the memory `crew/hot_files.ts` already keeps, the rest from the runs, reviews and pulls tables. A quiet lane draws no card. Admin only, like the rest of `/flow`.
10
11 Not done: the terminal screen does not show the card yet; before/after looks are left to the preview.
junipercrew

Both doors: the terminal screen is required in this pull, not left for later.
scripts/unreached-allow.json −3
@@ -1,8 +1,5 @@
11 [
22 "src/crew/governor_log.ts",
3 "src/flow/clog_words.ts",
4 "src/flow/conflict_pairs.ts",
5 "src/flow/shared_red.ts",
63 "src/perf/page_budget.ts",
74 "src/spawn/pod.ts",
85 "src/tui/ui/qr.ts",
src/crew/hot_files.ts +7
@@ -304,3 +304,10 @@ export function hotSays(hold: HotHold): string {
304304 .join(", ");
305305 return `#${hold.number} waits: ${files === "" ? "a file it names is hot" : files} (hot at ${HOT_PULLS}).`;
306306 }
307
308 /** Every learned open pull with its files, per repository; memory only, never git (#11454). */
309 export function learnedPulls(): { repoId: string; number: number; files: readonly string[] }[] {
310 const out: { repoId: string; number: number; files: readonly string[] }[] = [];
311 for (const [repoId, pulls] of touched) for (const p of pulls.values()) out.push({ repoId, number: p.number, files: p.files });
312 return out;
313 }
src/web/pages/flow-clog.ts added +111
@@ -0,0 +1,111 @@
1 /**
2 * /flow's clog section (#11454): why pulls are not landing, without opening
3 * each one. One sentence (`clogSays`), a test failing on three or more open
4 * pulls shown once (`sharedRed`), and pairs of open pulls touching the same
5 * files (`conflictPairs`, at most 20). Everything is read from what the page
6 * already holds: the runs table, and the pull file lists `hot_files.ts` keeps
7 * in memory; no git per request. Quiet draws nothing.
8 */
9
10 import type { Database } from "bun:sqlite";
11 import { learnedPulls, warmFiles } from "../../crew/hot_files.ts";
12 import { getRepoById } from "../../core/repos.ts";
13 import { clogSays } from "../../flow/clog_words.ts";
14 import { readClog } from "../../flow/clog_read.ts";
15 import { conflictPairs } from "../../flow/conflict_pairs.ts";
16 import { conflictedPulls } from "../../flow/conflicted.ts";
17 import { sharedRed } from "../../flow/shared_red.ts";
18 import { readStranded } from "../../flow/stranded.ts";
19 import { html, type Html } from "../html.ts";
20 import { card, icon, row } from "../ui.ts";
21 import { approvedRows } from "./flow-waiting.ts";
22
23 export interface ClogView {
24 readonly says: string;
25 readonly shared: readonly { test: string; pulls: readonly { repo: string; number: number }[] }[];
26 readonly pairs: readonly { a: Ref; b: Ref; shared: number }[];
27 }
28 type Ref = { repo: string; number: number };
29
30 /** The latest check of each open pull that failed, with the tests it named. */
31 function failingRows(db: Database): { repo: string; number: number; tests: string[] }[] {
32 return db
33 .query<{ repo: string; number: number; detail: string }, []>(
34 `SELECT o.username || '/' || r.name AS repo, i.number AS number, u.check_detail AS detail
35 FROM pull p JOIN issue i ON i.id = p.issue_id AND i.state = 'open'
36 JOIN repo r ON r.id = i.repo_id JOIN user o ON o.id = r.owner_id
37 JOIN run u ON u.id = (SELECT n.id FROM run n WHERE n.issue_id = i.id AND n.kind = 'check' ORDER BY n.id DESC LIMIT 1)
38 WHERE p.merged_at IS NULL AND u.state = 'failed' AND json_valid(u.check_detail)`,
39 )
40 .all()
41 .map((x) => {
42 const names = (JSON.parse(x.detail) as { names?: unknown }).names;
43 return { repo: x.repo, number: x.number, tests: Array.isArray(names) ? names.filter((n): n is string => typeof n === "string" && n !== "") : [] };
44 });
45 }
46
47 export function readClogView(db: Database, now = new Date()): ClogView {
48 const failing = failingRows(db);
49 const key = (r: Ref): string => `${r.repo}#${r.number}`;
50 // sharedRed counts pull numbers; distinct repositories reuse numbers, so each pull gets a forge-wide one.
51 const ids = new Map<string, number>();
52 const refs: Ref[] = [];
53 const idOf = (r: Ref): number => {
54 let n = ids.get(key(r));
55 if (n === undefined) ids.set(key(r), (n = refs.push(r)));
56 return n;
57 };
58 const shared = sharedRed(failing.map((f) => ({ pull: idOf(f), failingTests: f.tests }))).map((s) => ({
59 test: s.test,
60 pulls: s.pulls.map((n) => refs[n - 1]!),
61 }));
62
63 const names = new Map<string, string>();
64 const nameOf = (repoId: string): string => {
65 let n = names.get(repoId);
66 if (n === undefined) {
67 const r = getRepoById(db, repoId);
68 names.set(repoId, (n = r ? `${r.ownerName}/${r.name}` : ""));
69 }
70 return n;
71 };
72 const learned = learnedPulls().filter((p) => nameOf(p.repoId) !== "");
73 const byRepo = new Map<string, { number: number; files: readonly string[] }[]>();
74 for (const p of learned) (byRepo.get(p.repoId) ?? byRepo.set(p.repoId, []).get(p.repoId)!).push(p);
75 const pairs = [...byRepo].flatMap(([repoId, pulls]) => conflictPairs(pulls).map((c) => ({ a: { repo: nameOf(repoId), number: c.a }, b: { repo: nameOf(repoId), number: c.b }, shared: c.shared })));
76 pairs.sort((p, q) => q.shared - p.shared || key(p.a).localeCompare(key(q.a)));
77
78 const approved = new Set(approvedRows(db).map(key));
79 const conflicted = conflictedPulls(db).filter((c) => approved.has(key(c))).length;
80 const hot = warmFiles().sort((a, b) => b.pulls - a.pulls || a.path.localeCompare(b.path))[0];
81 const standing = readClog(db, now).standing;
82 const says = clogSays({
83 reasons: standing?.reasons ?? [],
84 conflictedApproved: conflicted,
85 hottest: hot,
86 strandedUsd: readStranded(db).usd,
87 });
88 return { says: shared.length > 0 || pairs.length > 0 || standing || conflicted > 0 ? says : "", shared, pairs: pairs.slice(0, 20) };
89 }
90
91 const pullLink = (r: Ref) => html`<a href="/${r.repo}/pulls/${String(r.number)}">${r.repo} #${String(r.number)}</a>`;
92
93 /** The section; empty html when nothing clogs. */
junipercrew

Principles 6 and 7: a card around lists. Render the sentence and the two lists as rows on the page, no box.
94 export function clogSection(v: ClogView): Html {
95 if (v.says === "" && v.shared.length === 0 && v.pairs.length === 0) return html``;
96 return card(
97 html`${v.says === "" ? "" : html`<p id="clog-says"><strong>${v.says}</strong></p>`}
98 ${v.shared.length === 0 ? "" : html`<h2>One test, many pulls</h2>
junipercrew

Principles 1 and 8: this explains itself twice. Say 'fails on N open pulls' and stop.
99 <ul class="list mb" id="flow-shared-red">${v.shared.map((s) => row({
100 lead: icon("alert"),
101 title: html`<code>${s.test}</code> <span class="muted small">fails on ${String(s.pulls.length)} open pulls: one clash, re-check together after the fix</span>`,
102 meta: html`<span class="small">${s.pulls.map((p, i) => html`${i === 0 ? "" : ", "}${pullLink(p)}`)}</span>`,
103 }))}</ul>`}
104 ${v.pairs.length === 0 ? "" : html`<h2>Pulls on the same files</h2>
105 <ul class="list" id="flow-conflict-pairs">${v.pairs.map((c) => row({
106 lead: icon("branch"),
107 title: html`${pullLink(c.a)} and ${pullLink(c.b)} <span class="muted small">${String(c.shared)} shared ${c.shared === 1 ? "file" : "files"}</span>`,
108 }))}</ul>`}`,
109 { extra: "mb", attrs: html`id="flow-clog"` },
110 );
111 }
src/web/pages/flow-waiting.ts +1 −1
@@ -22,7 +22,7 @@ export interface Waiting {
2222 }
2323
2424 /** Open, unlanded pulls whose latest verdict is an approval, with when it came. */
25 function approvedRows(db: Database): ApprovedRow[] {
25 export function approvedRows(db: Database): ApprovedRow[] {
2626 return db
2727 .query<{ repo: string; number: number; at: string }, []>(
2828 `SELECT ro.username || '/' || r.name AS repo, i.number AS number, a.at AS at
src/web/pages/flow.ts +2
@@ -24,6 +24,7 @@ import { handoverSection } from "./flow-handover.ts";
2424 import { readDrifts } from "../../flow/alarm_drift.ts";
2525 import { readHandoverLists } from "../../flow/handover_read.ts";
2626 import { readWaiting, waitingSection } from "./flow-waiting.ts";
27 import { clogSection, readClogView } from "./flow-clog.ts";
2728
2829 const money = (usd: number): string => `$${usd.toFixed(2)}`;
2930 const plural = (n: number, one: string, many = `${one}s`): string => `${String(n)} ${n === 1 ? one : many}`;
@@ -101,6 +102,7 @@ export async function flowPage(ctx: Ctx): Promise<Response> {
101102 const body = html`<div class="page-head"><h1>Flow</h1>${btn("Boards", { href: "/boards", variant: "ghost" })}</div>
102103 <p class="mb">Money flows from the left to shipped on the right, as wide as this week's real spend. Every repository has a valve: drag it to give the repository a share of the week, all the way down to shut. The crew does exactly what the valves say, from the next tick, and nothing but you turns one.</p>
103104 ${backupRow(alarmBackup(ctx.app.config.dataDir, new Date()))}
105 ${clogSection(readClogView(ctx.app.db))}
104106 ${waitingSection(await readWaiting(ctx.app.db, ctx.app.reposRoot, new Date()))}
105107 ${pipesSvg(p)}
106108 <p class="muted small">${money(p.usd)} spent in the last 7 days; ${plural(p.shipped, "pull")} shipped.</p>
test/flow-clog-page.test.ts added +111
@@ -0,0 +1,111 @@
1 /** /flow's clog section (#11454): the sentence, the shared red test, the pull pairs — each on the page's HTML; quiet draws nothing. */
2 import { afterAll, beforeAll, expect, test } from "bun:test";
3 import { mkdtemp, rm } from "node:fs/promises";
4 import { tmpdir } from "node:os";
5 import { join } from "node:path";
6 import { createRepo } from "../src/actions/repos.ts";
7 import { createApp, type App } from "../src/app.ts";
8 import { rememberPullFiles } from "../src/crew/hot_files.ts";
9 import { createUser } from "../src/core/users.ts";
10 import { serve, type Servers } from "../src/http/server.ts";
11 import { id } from "../src/kernel/id.ts";
12
13 const must = <T>(r: { ok: true; value: T } | { ok: false; error: unknown }): T => {
14 if (!r.ok) throw new Error(JSON.stringify(r.error));
15 return r.value;
16 };
17
18 let dataDir: string;
19 let app: App;
20 let servers: Servers;
21 let cookie = "";
22 let repoId = "";
23 let aliceId = "";
24 let n = 0;
25
26 const page = async () => (await fetch(`${servers.url}/flow`, { headers: { cookie } })).text();
27 const pull = (): number => {
28 const issueId = id();
29 n += 1;
30 const num = 200 + n;
31 app.db
32 .query(`INSERT INTO issue (id, repo_id, number, kind, title, body, author_id, state) VALUES (?1, ?2, ?3, 'pull', 'P', '', ?4, 'open')`)
33 .run(issueId, repoId, num, aliceId);
34 app.db
35 .query(`INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, head_repo_id) VALUES (?1, 'main', ?2, '', ?3, ?4)`)
36 .run(issueId, `br-${String(n)}`, `oid-${String(n)}`, repoId);
37 return num;
38 };
39 const red = (num: number, names: string[]) => {
40 const issue = app.db.query<{ id: string }, [number]>("SELECT id FROM issue WHERE number = ?1").get(num)!;
41 app.db
42 .query(`INSERT INTO run (id, repo_id, kind, issue_id, state, check_detail) VALUES (?1, ?2, 'check', ?3, 'failed', ?4)`)
43 .run(id(), repoId, issue.id, JSON.stringify({ names }));
44 };
45
46 beforeAll(async () => {
47 dataDir = await mkdtemp(join(tmpdir(), "forge-clogpage-"));
48 app = createApp({ dataDir, port: 0 });
49 servers = await serve(app);
50 const alice = must(await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true }));
51 aliceId = alice.id;
52 repoId = must(await createRepo(app, alice, { name: "app", visibility: "public" })).id;
53 const res = await fetch(`${servers.url}/signin`, {
54 method: "POST",
55 redirect: "manual",
56 headers: { "content-type": "application/x-www-form-urlencoded", origin: servers.url },
57 body: new URLSearchParams({ username: "alice", password: "a-good-passphrase" }),
58 });
59 cookie = (res.headers.get("set-cookie") ?? "").split(";")[0] ?? "";
60 }, 60_000);
61
62 afterAll(async () => {
63 rememberPullFiles(repoId, []);
64 servers.stop();
65 app.close();
66 await rm(dataDir, { recursive: true, force: true });
67 });
68
69 test("a quiet lane draws no clog card", async () => {
70 expect(await page()).not.toContain('id="flow-clog"');
71 });
72
73 test("a test failing on three open pulls is shown once with its pulls linked; two is not", async () => {
74 const [a, b, c, d] = [pull(), pull(), pull(), pull()];
75 for (const p of [a, b, c]) red(p, ["shared > breaks", "own " + String(p)]);
76 red(d, ["two > only"]);
77 red(a, ["two > only"]); // a's latest check is now this one, so it drops from "shared > breaks"
78 let html = await page();
79 expect(html).not.toContain("flow-shared-red");
80 red(a, ["shared > breaks"]);
81 html = await page();
82 expect(html).toContain('id="flow-shared-red"');
83 expect(html.split("shared &gt; breaks").length - 1).toBe(1);
84 for (const p of [a, b, c]) expect(html).toContain(`/alice/app/pulls/${String(p)}`);
85 expect(html).toContain("fails on 3 open pulls");
86 });
87
88 test("pulls touching the same files are listed as a pair, linked to both, and the sentence names the hottest file", async () => {
89 const [x, y, z] = [pull(), pull(), pull()];
90 rememberPullFiles(repoId, [
91 { number: x, files: ["src/a.ts", "src/b.ts"] },
92 { number: y, files: ["src/a.ts", "src/b.ts", "src/c.ts"] },
93 { number: z, files: ["src/a.ts"] },
94 ]);
95 const html = await page();
96 expect(html).toContain('id="flow-conflict-pairs"');
97 expect(html).toContain(`/alice/app/pulls/${String(x)}`);
98 expect(html).toContain(`/alice/app/pulls/${String(y)}`);
99 expect(html).toContain("2 shared files");
100 expect(html).toContain("1 shared file<");
101 expect(html).toContain('id="clog-says"');
102 expect(html).toContain("3 open pulls touch src/a.ts");
103 });
104
105 test("the bottleneck sentence appears when approved pulls conflict", async () => {
106 const p = pull();
107 const issue = app.db.query<{ id: string }, [number]>("SELECT id FROM issue WHERE number = ?1").get(p)!;
108 app.db.query(`INSERT INTO review (id, issue_id, author_id, state) VALUES (?1, ?2, ?3, 'approved')`).run(id(), issue.id, aliceId);
109 app.db.query(`INSERT INTO run (id, repo_id, kind, issue_id, state, reason) VALUES (?1, ?2, 'agent', ?3, 'queued', 'conflict')`).run(id(), repoId, issue.id);
110 expect(await page()).toContain("1 approved pull conflicts");
111 });
test/stall-in-flight.test.ts +1 −1
@@ -162,7 +162,7 @@ function describe_server() {
162162 const deadline = Date.now() + 5_000;
163163 while (inFlight.size() === 0 && Date.now() < deadline) await Bun.sleep(5);
164164 expect(inFlight.size()).toBe(1);
165 await Bun.sleep(LIMIT_MS + 60);
165 await Bun.sleep(LIMIT_MS + 250);
junipercrew

Unrelated to #11454; move to its own pull.
166166 // The wake: the loop held for longer than the stall limit, beginning after the request did.
167167 const now = performance.now();
168168 const stall = loopWatch.stalled(LIMIT_MS + 10, now)!;

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs