Public: anyone can read this; the crew can change it.

The model-review proposal can be applied or declined through the existing scorecard MCP tool, in the words of the page's button #15282

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/15265-the-model-review-proposal-can-be-applied → main 6 commits fork 2 threads · 2 unresolved

  • Merged. mason/15265-the-model-review-proposal-can-be-applied was merged into main as 19677ab.

Review

Sign in to review this change.
7 files changed +165 −13
docs/features/model-review.md +3 −1
@@ -13,4 +13,6 @@ Once a week (`runModelReview`, ticked hourly from `src/main.ts`, runs at most on
1313 - **On the trail**: `crew.model_asked`, `crew.model_changed` (with the before numbers), `crew.model_declined`, `crew.model_result`.
1414 - **A week after** a change (kept waiting up to two weeks for enough landed pulls) the result is reported; the change is undone when reverts rose, the red rate rose by more than `model_review.slack_pct` points (default 10), or rework rose by more than half a round. A model changed by hand meanwhile is left alone.
1515
16 Not done: an MCP tool for the button (the decision is on the page and in the terminal), and the review does not look at an agent running on the runner's default model.
16 The same button is `action: apply|decline` with `change` on the `scorecard` MCP tool: a person who administers only, an agent is refused, and it answers the page's sentence.
17
18 Not done: the review does not look at an agent running on the runner's default model.
src/actions/model_review.ts +26
@@ -192,3 +192,29 @@ export function runModelReview(app: App, now = new Date()): ModelChange[] {
192192 }
193193 return made;
194194 }
195
196 /** The words after the button, the page's flash and the MCP receipt alike. */
197 export const CHANGE_WORDS = {
198 taken: (toModel: string) => `${toModel} it is.`,
199 left: "Left as it was.",
200 gone: "That change is no longer waiting.",
201 /**
202 * An agent never moves a model, its own or another's: only a person who administers the forge.
203 */
204 person: "the model review's button is a person's who administers the forge; an agent cannot press it",
205 } as const;
206
207 /** The button as one answer: who, what the change was, and the sentence the page shows. */
208 export function answerChange(app: App, actor: User, changeId: string, take: boolean) {
209 const done = decideChange(app, actor, changeId, take);
210 if (!done.ok) return err(CHANGE_WORDS.gone);
211 const c = done.value;
212 return ok({
213 change: c.id,
214 agent: nameOf(app, c.agentId),
215 from: c.fromModel,
216 to: c.toModel,
217 state: c.state,
218 says: take ? CHANGE_WORDS.taken(c.toModel) : CHANGE_WORDS.left,
219 });
220 }
src/http/mcp/tools/scorecard.ts +24 −7
@@ -1,17 +1,34 @@
11 import { readScorecards } from "../../../actions/scorecard.ts";
22 import { ADMIN_WORDS } from "../../../actions/admin_words.ts";
3 import { answerChange, CHANGE_WORDS } from "../../../actions/model_review.ts";
34 import { good, bad, type Tool } from "../shared.ts";
45
56 export const tool: Tool = {
67 name: "scorecard",
78 description:
8 "Crew scorecard (forge admin): per agent, all time and 30 days.",
9 inputSchema: { type: "object", properties: {} },
10 writes: false,
11 doc: "(forge admin) no input: the crew's scorecard per agent, all time and 30 days: landed, review rounds, red rate, reverts; week {merged, returned, cost}",
9 "Crew scorecard (forge admin): per agent, all time and 30 days; `action` answers a model proposal.",
10 inputSchema: {
11 type: "object",
12 properties: {
13 action: { type: "string", enum: ["apply", "decline"] },
14 change: { type: "string", maxLength: 40 },
15 },
16 },
17 writes: true,
18 readsWithout: "action",
19 doc: "(forge admin) no input: the crew's scorecard per agent, all time and 30 days: landed, review rounds, red rate, reverts; week {merged, returned, cost}. `{ action: apply|decline, change }` answers one waiting model-review proposal (`modelReview.changes[].id`), the same as the page's **Move to …** / **Leave it**, and answers `{ change, agent, from, to, state, says }`; a person who administers only, an agent is refused",
1220 adminOnly: true,
13 run(ctx, user) {
14 const card = readScorecards(ctx.app, user);
15 return card ? good(card) : bad("forbidden", ADMIN_WORDS.scorecard);
21 run(ctx, user, args) {
22 const action = args["action"];
23 if (action === undefined) {
24 const card = readScorecards(ctx.app, user);
25 return card ? good(card) : bad("forbidden", ADMIN_WORDS.scorecard);
26 }
27 if (action !== "apply" && action !== "decline") return bad("invalid", "action is apply or decline");
28 if (user.kind !== "user" || !user.isAdmin) return bad("forbidden", CHANGE_WORDS.person);

Reachable only by an admin agent; the test never gets here. Cover it.

29 const change = args["change"];
30 if (typeof change !== "string" || change === "" || change.length > 40) return bad("invalid", "change is the id of a waiting proposal");
31 const done = answerChange(ctx.app, user, change, action === "apply");
32 return done.ok ? good(done.value) : bad("not_found", done.error);
1633 },
1734 };
src/web/pages/crew.ts +3 −3
@@ -45,7 +45,7 @@ import { keeperDays, type KeeperDay } from "../../flow/keeper_rate.ts";
4545 import { weekCardRows } from "../../crew/week.ts";
4646 import { weekCard, weekReturns, WEEK_DAYS, type ReturnedPull, type WeekLine } from "../../crew/week_card.ts";
4747 import { knobsCard } from "./knobs.ts";
48 import { decideChange, readReview } from "../../actions/model_review.ts";
48 import { CHANGE_WORDS, decideChange, readReview } from "../../actions/model_review.ts";
4949 import { wastedDays, type WastedDay } from "../../core/wasted_days.ts";
5050 import { modelDays, type ModelDay } from "../../core/model_days.ts";
5151 import { notePhase, takeSessionTime } from "../../core/route_timing.ts";
@@ -1194,8 +1194,8 @@ export async function modelReviewDecide(ctx: Ctx, take: boolean): Promise<Respon
11941194 const done = decideChange(ctx.app, ctx.auth.user, ctx.params.id ?? "", take);
11951195 return flashTo(ctx, "/settings/crew", {
11961196 ...(done.ok
1197 ? { message: take ? `${done.value.toModel} it is.` : "Left as it was." }
1198 : { tone: "danger" as const, message: "That change is no longer waiting." }),
1197 ? { message: take ? CHANGE_WORDS.taken(done.value.toModel) : CHANGE_WORDS.left }
1198 : { tone: "danger" as const, message: CHANGE_WORDS.gone }),
11991199 });
12001200 }
12011201 export const modelReviewTake = (ctx: Ctx) => modelReviewDecide(ctx, true);
test/conversation-confirm.test.ts +1 −1
@@ -126,7 +126,7 @@ describe("the one list of what asks first", () => {
126126 "crew", "decisions_by", "desk", "detach", "economics", "edit", "email_verify", "estimate", "feed_chips", "feed_dismiss", "feed_like", "flag_set", "flag_winner",
127127 "follow_mode", "forge", "fork", "front_page", "hold", "issue_teams", "keep_plan", "knobs", "labels", "lift", "limb", "maintenance", "make_it_yours", "map", "merge", "new_app",
128128 "next_up", "week", "notify", "open_pull", "org_create", "origin_apps", "pins", "plan", "profile", "propose_to_parent_forge", "repo_labels", "reports",
129 "repo", "restore_drill", "review", "say", "signup", "snapshot_now", "spend", "steer", "sync", "system", "take_back", "take_updates",
129 "repo", "restore_drill", "review", "say", "scorecard", "signup", "snapshot_now", "spend", "steer", "sync", "system", "take_back", "take_updates",
130130 "teams", "tell", "threads", "trail", "upstream", "variant", "wait_for", "watch", "whoami_can", "write_file",
131131 ]);
132132
test/mcp-scorecard-decide.test.ts added +107
@@ -0,0 +1,107 @@
1 /**
2 * The scorecard MCP tool answers a model-review proposal in the page's words, for a person who
3 * administers only.
4 */
5 import { afterAll, beforeAll, expect, test } from "bun:test";
6 import { mkdtemp, rm } from "node:fs/promises";
7 import { tmpdir } from "node:os";
8 import { join } from "node:path";
9 import { createApp, type App } from "../src/app.ts";
10 import { createAgentRow, getAgent } from "../src/core/agents.ts";
11 import { getChange, recordChange, type ModelStats } from "../src/core/model_review.ts";
12 import { CHANGE_WORDS } from "../src/actions/model_review.ts";
13 import { createToken } from "../src/core/tokens.ts";
14 import { createUser } from "../src/core/users.ts";
15 import { serve, type Servers } from "../src/http/server.ts";
16 import { quietLogs } from "./helpers/globals.ts";
17
18 quietLogs();
19
20 let dataDir: string;
21 let app: App;
22 let servers: Servers;
23 let adminToken = "";
24 let agentToken = "";
25 let botId = "";
26 let n = 1;
27
28 const call = async (token: string, args: unknown) => {
29 const res = await fetch(`${servers.url}/mcp`, {
30 method: "POST",
31 headers: { "content-type": "application/json", accept: "application/json, text/event-stream", authorization: `Bearer ${token}` },
32 body: JSON.stringify({ jsonrpc: "2.0", id: n++, method: "tools/call", params: { name: "scorecard", arguments: args } }),
33 });
34 const body = (await res.json()) as any;
35 return { error: body.result?.isError === true || body.error !== undefined, text: String(body.result?.content?.[0]?.text ?? body.error?.message ?? "") };
36 };
37
38 const before: ModelStats = { model: "opus", opened: 9, landed: 9, costPerLanded: 20, redRate: 0.1, reworkRounds: 0, reverts: 0, hardShare: 0 };
39 const ask = () => recordChange(app.db, { agentId: botId, fromModel: "opus", toModel: "sonnet", state: "asked", why: "healthy", before, at: new Date() });
40
41 beforeAll(async () => {
42 dataDir = await mkdtemp(join(tmpdir(), "forge-mcp-scorecard-decide-"));
43 app = createApp({ dataDir, port: 0 });
44 servers = await serve(app);
45 const a = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
46 if (!a.ok) throw new Error("users");
47 adminToken = createToken(app.db, a.value.id, "admin", "admin").token;
48 const bot = await createUser(app.db, { origin: "seed", trusted: true, username: "smith", email: "[email protected]", password: "a-good-passphrase" });
49 if (!bot.ok) throw new Error("bot");
50 app.db.query("UPDATE user SET is_agent = 1 WHERE id = ?1").run(bot.value.id);
51 createAgentRow(app.db, { userId: bot.value.id, model: "opus" });
52 botId = bot.value.id;
53 agentToken = createToken(app.db, bot.value.id, "admin", "admin").token;
54 }, 30_000);
55
56 afterAll(async () => {
57 servers.stop();
58 app.close();
59 await rm(dataDir, { recursive: true, force: true });
60 });
61
62 test("apply moves the model and answers the page's receipt", async () => {
63 const id = ask();
64 const got = await call(adminToken, { action: "apply", change: id });
65 expect(got.error).toBe(false);
66 expect(JSON.parse(got.text)).toMatchObject({ change: id, agent: "smith", from: "opus", to: "sonnet", state: "applied", says: "sonnet it is." });
67 expect(getAgent(app.db, botId)?.model).toBe("sonnet");
68 // once answered it is no longer waiting
69 const again = await call(adminToken, { action: "apply", change: id });
70 expect(again.error).toBe(true);
71 expect(again.text).toContain("no longer waiting");
72 });
73
74 test("decline leaves the model", async () => {
75 app.db.query("UPDATE agent SET model = 'opus' WHERE user_id = ?1").run(botId);
76 const id = ask();
outdated

This agent is not an admin, so visible() refuses it with "no tool named…" and run never executes. Assert CHANGE_WORDS.person with an agent that is_admin=1, so the new refusal is covered and would fail without the change.

77 const got = await call(adminToken, { action: "decline", change: id });
78 expect(JSON.parse(got.text)).toMatchObject({ state: "declined", says: "Left as it was." });
79 expect(getAgent(app.db, botId)?.model).toBe("opus");
80 });
81
82 test("an agent is refused and nothing changes", async () => {
83 const id = ask();
84 const got = await call(agentToken, { action: "apply", change: id });
85 expect(got.error).toBe(true);
86 expect(getChange(app.db, id)?.state).toBe("asked");
87 expect(getAgent(app.db, botId)?.model).toBe("opus");
88 });
89
90 test("an agent that administers is refused in the page's sentence for an agent, and nothing changes", async () => {
91 app.db.query("UPDATE user SET is_admin = 1 WHERE id = ?1").run(botId);
92 try {
93 const id = ask();
94 const got = await call(agentToken, { action: "apply", change: id });
95 expect(got.error).toBe(true);
96 expect(got.text).toContain(CHANGE_WORDS.person);
97 expect(getChange(app.db, id)?.state).toBe("asked");
98 expect(getAgent(app.db, botId)?.model).toBe("opus");
99 } finally {
100 app.db.query("UPDATE user SET is_admin = 0 WHERE id = ?1").run(botId);
101 }
102 });
103
104 test("a bad action or a missing change is named", async () => {
105 expect((await call(adminToken, { action: "yes", change: "x" })).error).toBe(true);
106 expect((await call(adminToken, { action: "apply" })).error).toBe(true);
107 });
test/route-marks.test.ts +1 −1
@@ -233,7 +233,7 @@ describe("every admin-only action refuses the everyday account", () => {
233233 });
234234
235235 test("the admin calls each read-only admin MCP tool and it answers; the write ones stay listed only", async () => {
236 const reads = TOOLS.filter((t) => t.adminOnly === true && !t.writes);
236 const reads = TOOLS.filter((t) => t.adminOnly === true && (!t.writes || t.readsWithout !== undefined));
237237 expect(reads.length).toBeGreaterThan(3);
238238 const wrong: string[] = [];
239239 for (const t of reads) {

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs