ff46372 · 26a5d98
@@ -0,0 +1,173 @@
/** The crew scorecard in the terminal (#2677): a forge admin's only, the figures Settings → Crew shows. */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { createRepo } from "../src/actions/repos.ts";
import { createAgent } from "../src/actions/crew.ts";
import { createIssue } from "../src/core/issues.ts";
import { claimRun, createRun, finishRun, noteRunProgress } from "../src/core/runs.ts";
import { crewScorecards, crewScorecardSplits, recentWindow, spanText } from "../src/core/scorecard.ts";
import { createUser, getUserByName } from "../src/core/users.ts";
import { createForgeApp } from "../src/tui/forge.ts";
import { headlessTerminal } from "../src/tui/term/terminal.ts";
import { quietLogs } from "./helpers/globals.ts";
import { settleApp } from "./helpers/tui.ts";
quietLogs();
const TITLE = "Crew scorecard per agent";
/** The palette entry's hint: on screen only when the command is listed (the query echoes the title). */
const HINT = "each agent's pulls landed";
const DAY = 86_400_000;
let dirs: string[] = [];
let apps: App[] = [];
async function forge(): Promise<App> {
  const dir = await mkdtemp(join(tmpdir(), "forge-tui-scorecard-"));
  const app = createApp({ dataDir: dir, port: 0 });
  dirs.push(dir);
  apps.push(app);
  for (const [username, adminIfFirst] of [["ada", true], ["bob", false]] as const) {
    const made = await createUser(app.db, {
      origin: "seed",
      trusted: true,
      username,
      email: `${username}@forge.local`,
      password: "a-good-passphrase",
      adminIfFirst,
    });
    if (!made.ok) throw new Error(made.error);
  }
  return app;
}
async function agent(app: App, username: string): Promise<string> {
  const made = await createAgent(app, getUserByName(app.db, "ada"), { username });
  if (!made.ok) throw new Error(String(made.error));
  return made.value.user.id;
}
afterAll(async () => {
  for (const a of apps) a.close();
  for (const d of dirs) await rm(d, { recursive: true, force: true });
});
function terminal(app: App, username: string) {
  const user = getUserByName(app.db, username)!;
  const { app: tui } = createForgeApp(
    app,
    { sshUsername: "t", remoteAddress: "127.0.0.1", user },
    { terminal: headlessTerminal(200, 40), exitHooks: false },
  );
  tui.start();
  const key = (k: string, ctrl = false) =>
    tui.dispatch({ type: "key", key: k, ctrl, alt: false, shift: false });
  const t = {
    text: () => tui.screen.toText(),
    search: async () => {
      key(":");
      for (const ch of TITLE) key(ch);
      await settleApp(tui);
    },
    open: async () => {
      await t.search();
      key("enter");
      await settleApp(tui);
    },
    stop: () => tui.stop(),
  };
  return t;
}
let app: App;
let repoId = "";
const agentIds: Record<string, string> = {};
function pull(authorId: string, title: string, model: string, ageDays: number, merged: boolean): void {
  const issue = createIssue(app.db, { repoId, authorId, title, kind: "pull" });
  if (!issue.ok) throw new Error(issue.error);
  const opened = new Date(Date.now() - ageDays * DAY).toISOString();
  app.db.query("UPDATE issue SET created_at = ?2 WHERE id = ?1").run(issue.value.id, opened);
  app.db
    .query(
      `INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, merged_at)
       VALUES (?1, 'main', ?2, 'a', 'b', ?3)`,
    )
    .run(
      issue.value.id,
      `h${issue.value.number}`,
      merged ? new Date(Date.parse(opened) + 3 * 3_600_000).toISOString() : null,
    );
  const run = createRun(app.db, { repoId, kind: "agent", actorId: authorId, issueId: issue.value.id });
  claimRun(app.db, run.id);
  noteRunProgress(app.db, run.id, { model, promptVersion: "v1" });
  finishRun(app.db, run.id, { state: "ok", summary: "" });
  const check = createRun(app.db, { repoId, kind: "check", issueId: issue.value.id });
  claimRun(app.db, check.id);
  finishRun(app.db, check.id, { state: merged ? "ok" : "failed", summary: "" });
}
beforeAll(async () => {
  app = await forge();
  const repo = await createRepo(app, getUserByName(app.db, "ada"), { name: "hello", visibility: "public" });
  if (!repo.ok) throw new Error(String(repo.error));
  repoId = repo.value.id;
  const smith = await agent(app, "smith");
  const jones = await agent(app, "jones");
  agentIds.smith = smith;
  agentIds.jones = jones;
  pull(smith, "s1", "opus", 40, true);
  pull(smith, "s2", "opus", 2, true);
  pull(smith, "s3", "haiku", 1, false);
  pull(jones, "j1", "opus", 3, true);
  pull(jones, "j2", "opus", 60, false);
});
test("an admin's terminal shows the figures crewScorecards gives, recent and all time, with a row per model", async () => {
  const all = crewScorecards(app.db);
  const recent = crewScorecards(app.db, recentWindow());
  const splits = crewScorecardSplits(app.db);
  expect(all.size).toBe(2);
  const t = terminal(app, "ada");
  await t.open();
  const text = t.text();
  for (const name of ["smith", "jones"]) {
    const id = agentIds[name]!;
    const a = all.get(id)!;
    const r = recent.get(id)!;
    expect(text).toContain(`landed 30d ${r.landed} of ${r.opened} · all ${a.landed} of ${a.opened}`);
    expect(text).toContain(`reverts ${a.reverts}`);
    expect(text).toContain(`time to land ${spanText(a.meanLandMs!)}`);
    expect(text).toContain(`red checks 30d ${Math.round(100 * r.redCheckRate!)}% · all ${Math.round(100 * a.redCheckRate!)}%`);
    expect(text).toContain(`all ${a.reviewRounds!.toFixed(1)}`);
    for (const p of splits.get(id)!) expect(text).toContain(`↳ ${p.model} · ${p.promptVersion}`);
  }
  expect(splits.get(agentIds.smith!)!.map((p) => p.model).sort()).toEqual(["haiku", "opus"]);
      
        junipercrew
        18d ago
        
      
      bob.open() presses enter on an empty or unrelated palette result. The not.toContain checks hold, but they would also pass if the screen broke for another reason. Add one positive assertion, for example that the palette shows no match for the title, so the test only passes for the right reason.
    
  t.stop();
      
        junipercrew
        18d ago
        outdated
      
      Pass-either-way: without the isAdmin gate this still passes. Assert the palette lacks the command for bob and has it for ada. " of 3" is an arbitrary string.
    
});
test("the palette lists the scorecard for a forge admin and not for anyone else", async () => {
  const ada = terminal(app, "ada");
  await ada.search();
  expect(ada.text()).toContain(HINT);
  ada.stop();
  const bob = terminal(app, "bob");
  await bob.search();
      
        junipercrew
        18d ago
        outdated
      
      Use a local const empty = await forge() instead of reassigning the shared app.
    
  expect(bob.text()).not.toContain(HINT);
  await bob.open();
  expect(bob.text()).not.toContain("smith");
  expect(bob.text()).not.toContain("landed 30d");
  bob.stop();
});
test("with no crew at all it says so in one line, with no table", async () => {
  const empty = await forge();
  const t = terminal(empty, "ada");
  await t.open();
  const text = t.text();
  expect(text).toContain("No agents yet.");
  expect(text).not.toContain("landed 30d");
  expect(text).not.toContain("no pull yet");
  t.stop();
});
		@@ -0,0 +1,173 @@
	1	/** The crew scorecard in the terminal (#2677): a forge admin's only, the figures Settings → Crew shows. */
	2	import { afterAll, beforeAll, expect, test } from "bun:test";
	3	import { mkdtemp, rm } from "node:fs/promises";
	4	import { tmpdir } from "node:os";
	5	import { join } from "node:path";
	6	import { createApp, type App } from "../src/app.ts";
	7	import { createRepo } from "../src/actions/repos.ts";
	8	import { createAgent } from "../src/actions/crew.ts";
	9	import { createIssue } from "../src/core/issues.ts";
	10	import { claimRun, createRun, finishRun, noteRunProgress } from "../src/core/runs.ts";
	11	import { crewScorecards, crewScorecardSplits, recentWindow, spanText } from "../src/core/scorecard.ts";
	12	import { createUser, getUserByName } from "../src/core/users.ts";
	13	import { createForgeApp } from "../src/tui/forge.ts";
	14	import { headlessTerminal } from "../src/tui/term/terminal.ts";
	15	import { quietLogs } from "./helpers/globals.ts";
	16	import { settleApp } from "./helpers/tui.ts";
	17
	18	quietLogs();
	19
	20	const TITLE = "Crew scorecard per agent";
	21	/** The palette entry's hint: on screen only when the command is listed (the query echoes the title). */
	22	const HINT = "each agent's pulls landed";
	23	const DAY = 86_400_000;
	24	let dirs: string[] = [];
	25	let apps: App[] = [];
	26
	27	async function forge(): Promise<App> {
	28	const dir = await mkdtemp(join(tmpdir(), "forge-tui-scorecard-"));
	29	const app = createApp({ dataDir: dir, port: 0 });
	30	dirs.push(dir);
	31	apps.push(app);
	32	for (const [username, adminIfFirst] of [["ada", true], ["bob", false]] as const) {
	33	const made = await createUser(app.db, {
	34	origin: "seed",
	35	trusted: true,
	36	username,
	37	email: `${username}@forge.local`,
	38	password: "a-good-passphrase",
	39	adminIfFirst,
	40	});
	41	if (!made.ok) throw new Error(made.error);
	42	}
	43	return app;
	44	}
	45
	46	async function agent(app: App, username: string): Promise<string> {
	47	const made = await createAgent(app, getUserByName(app.db, "ada"), { username });
	48	if (!made.ok) throw new Error(String(made.error));
	49	return made.value.user.id;
	50	}
	51	afterAll(async () => {
	52	for (const a of apps) a.close();
	53	for (const d of dirs) await rm(d, { recursive: true, force: true });
	54	});
	55
	56	function terminal(app: App, username: string) {
	57	const user = getUserByName(app.db, username)!;
	58	const { app: tui } = createForgeApp(
	59	app,
	60	{ sshUsername: "t", remoteAddress: "127.0.0.1", user },
	61	{ terminal: headlessTerminal(200, 40), exitHooks: false },
	62	);
	63	tui.start();
	64	const key = (k: string, ctrl = false) =>
	65	tui.dispatch({ type: "key", key: k, ctrl, alt: false, shift: false });
	66	const t = {
	67	text: () => tui.screen.toText(),
	68	search: async () => {
	69	key(":");
	70	for (const ch of TITLE) key(ch);
	71	await settleApp(tui);
	72	},
	73	open: async () => {
	74	await t.search();
	75	key("enter");
	76	await settleApp(tui);
	77	},
	78	stop: () => tui.stop(),
	79	};
	80	return t;
	81	}
	82
	83	let app: App;
	84	let repoId = "";
	85	const agentIds: Record<string, string> = {};
	86
	87	function pull(authorId: string, title: string, model: string, ageDays: number, merged: boolean): void {
	88	const issue = createIssue(app.db, { repoId, authorId, title, kind: "pull" });
	89	if (!issue.ok) throw new Error(issue.error);
	90	const opened = new Date(Date.now() - ageDays * DAY).toISOString();
	91	app.db.query("UPDATE issue SET created_at = ?2 WHERE id = ?1").run(issue.value.id, opened);
	92	app.db
	93	.query(
	94	`INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, merged_at)
	95	VALUES (?1, 'main', ?2, 'a', 'b', ?3)`,
	96	)
	97	.run(
	98	issue.value.id,
	99	`h${issue.value.number}`,
	100	merged ? new Date(Date.parse(opened) + 3 * 3_600_000).toISOString() : null,
	101	);
	102	const run = createRun(app.db, { repoId, kind: "agent", actorId: authorId, issueId: issue.value.id });
	103	claimRun(app.db, run.id);
	104	noteRunProgress(app.db, run.id, { model, promptVersion: "v1" });
	105	finishRun(app.db, run.id, { state: "ok", summary: "" });
	106	const check = createRun(app.db, { repoId, kind: "check", issueId: issue.value.id });
	107	claimRun(app.db, check.id);
	108	finishRun(app.db, check.id, { state: merged ? "ok" : "failed", summary: "" });
	109	}
	110
	111	beforeAll(async () => {
	112	app = await forge();
	113	const repo = await createRepo(app, getUserByName(app.db, "ada"), { name: "hello", visibility: "public" });
	114	if (!repo.ok) throw new Error(String(repo.error));
	115	repoId = repo.value.id;
	116	const smith = await agent(app, "smith");
	117	const jones = await agent(app, "jones");
	118	agentIds.smith = smith;
	119	agentIds.jones = jones;
	120	pull(smith, "s1", "opus", 40, true);
	121	pull(smith, "s2", "opus", 2, true);
	122	pull(smith, "s3", "haiku", 1, false);
	123	pull(jones, "j1", "opus", 3, true);
	124	pull(jones, "j2", "opus", 60, false);
	125	});
	126
	127	test("an admin's terminal shows the figures crewScorecards gives, recent and all time, with a row per model", async () => {
	128	const all = crewScorecards(app.db);
	129	const recent = crewScorecards(app.db, recentWindow());
	130	const splits = crewScorecardSplits(app.db);
	131	expect(all.size).toBe(2);
	132	const t = terminal(app, "ada");
	133	await t.open();
	134	const text = t.text();
	135	for (const name of ["smith", "jones"]) {
	136	const id = agentIds[name]!;
	137	const a = all.get(id)!;
	138	const r = recent.get(id)!;
	139	expect(text).toContain(`landed 30d ${r.landed} of ${r.opened} · all ${a.landed} of ${a.opened}`);
	140	expect(text).toContain(`reverts ${a.reverts}`);
	141	expect(text).toContain(`time to land ${spanText(a.meanLandMs!)}`);
	142	expect(text).toContain(`red checks 30d ${Math.round(100 * r.redCheckRate!)}% · all ${Math.round(100 * a.redCheckRate!)}%`);
	143	expect(text).toContain(`all ${a.reviewRounds!.toFixed(1)}`);
	144	for (const p of splits.get(id)!) expect(text).toContain(`↳ ${p.model} · ${p.promptVersion}`);
	145	}
	146	expect(splits.get(agentIds.smith!)!.map((p) => p.model).sort()).toEqual(["haiku", "opus"]);
		junipercrew 18d ago bob.open() presses enter on an empty or unrelated palette result. The not.toContain checks hold, but they would also pass if the screen broke for another reason. Add one positive assertion, for example that the palette shows no match for the title, so the test only passes for the right reason.
	147	t.stop();
		junipercrew 18d ago outdated Pass-either-way: without the isAdmin gate this still passes. Assert the palette lacks the command for bob and has it for ada. " of 3" is an arbitrary string.
	148	});
	149
	150	test("the palette lists the scorecard for a forge admin and not for anyone else", async () => {
	151	const ada = terminal(app, "ada");
	152	await ada.search();
	153	expect(ada.text()).toContain(HINT);
	154	ada.stop();
	155	const bob = terminal(app, "bob");
	156	await bob.search();
		junipercrew 18d ago outdated Use a local `const empty = await forge()` instead of reassigning the shared `app`.
	157	expect(bob.text()).not.toContain(HINT);
	158	await bob.open();
	159	expect(bob.text()).not.toContain("smith");
	160	expect(bob.text()).not.toContain("landed 30d");
	161	bob.stop();
	162	});
	163
	164	test("with no crew at all it says so in one line, with no table", async () => {
	165	const empty = await forge();
	166	const t = terminal(empty, "ada");
	167	await t.open();
	168	const text = t.text();
	169	expect(text).toContain("No agents yet.");
	170	expect(text).not.toContain("landed 30d");
	171	expect(text).not.toContain("no pull yet");
	172	t.stop();
	173	});