Public: anyone can read this; the crew can change it.
The model-review proposal can be applied or declined through the existing scorecard MCP tool, in the words of the page's button #15282
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/15265-the-model-review-proposal-can-be-applied → main 6 commits fork 2 threads · 2 unresolved
7 files
changed +165
−13
docs/features/model-review.md +3 −1
| @@ -13,4 +13,6 @@ Once a week (`runModelReview`, ticked hourly from `src/main.ts`, runs at most on | ||
| 13 | 13 | - **On the trail**: `crew.model_asked`, `crew.model_changed` (with the before numbers), `crew.model_declined`, `crew.model_result`. |
| 14 | 14 | - **A week after** a change (kept waiting up to two weeks for enough landed pulls) the result is reported; the change is undone when reverts rose, the red rate rose by more than `model_review.slack_pct` points (default 10), or rework rose by more than half a round. A model changed by hand meanwhile is left alone. |
| 15 | 15 | |
| 16 | Not done: an MCP tool for the button (the decision is on the page and in the terminal), and the review does not look at an agent running on the runner's default model. | |
| 16 | The same button is `action: apply|decline` with `change` on the `scorecard` MCP tool: a person who administers only, an agent is refused, and it answers the page's sentence. | |
| 17 | ||
| 18 | Not done: the review does not look at an agent running on the runner's default model. |
src/actions/model_review.ts +26
| @@ -192,3 +192,29 @@ export function runModelReview(app: App, now = new Date()): ModelChange[] { | ||
| 192 | 192 | } |
| 193 | 193 | return made; |
| 194 | 194 | } |
| 195 | ||
| 196 | /** The words after the button, the page's flash and the MCP receipt alike. */ | |
| 197 | export const CHANGE_WORDS = { | |
| 198 | taken: (toModel: string) => `${toModel} it is.`, | |
| 199 | left: "Left as it was.", | |
| 200 | gone: "That change is no longer waiting.", | |
| 201 | /** | |
| 202 | * An agent never moves a model, its own or another's: only a person who administers the forge. | |
| 203 | */ | |
| 204 | person: "the model review's button is a person's who administers the forge; an agent cannot press it", | |
| 205 | } as const; | |
| 206 | ||
| 207 | /** The button as one answer: who, what the change was, and the sentence the page shows. */ | |
| 208 | export function answerChange(app: App, actor: User, changeId: string, take: boolean) { | |
| 209 | const done = decideChange(app, actor, changeId, take); | |
| 210 | if (!done.ok) return err(CHANGE_WORDS.gone); | |
| 211 | const c = done.value; | |
| 212 | return ok({ | |
| 213 | change: c.id, | |
| 214 | agent: nameOf(app, c.agentId), | |
| 215 | from: c.fromModel, | |
| 216 | to: c.toModel, | |
| 217 | state: c.state, | |
| 218 | says: take ? CHANGE_WORDS.taken(c.toModel) : CHANGE_WORDS.left, | |
| 219 | }); | |
| 220 | } |
src/http/mcp/tools/scorecard.ts +24 −7
| @@ -1,17 +1,34 @@ | ||
| 1 | 1 | import { readScorecards } from "../../../actions/scorecard.ts"; |
| 2 | 2 | import { ADMIN_WORDS } from "../../../actions/admin_words.ts"; |
| 3 | import { answerChange, CHANGE_WORDS } from "../../../actions/model_review.ts"; | |
| 3 | 4 | import { good, bad, type Tool } from "../shared.ts"; |
| 4 | 5 | |
| 5 | 6 | export const tool: Tool = { |
| 6 | 7 | name: "scorecard", |
| 7 | 8 | description: |
| 8 | "Crew scorecard (forge admin): per agent, all time and 30 days.", | |
| 9 | inputSchema: { type: "object", properties: {} }, | |
| 10 | writes: false, | |
| 11 | doc: "(forge admin) no input: the crew's scorecard per agent, all time and 30 days: landed, review rounds, red rate, reverts; week {merged, returned, cost}", | |
| 9 | "Crew scorecard (forge admin): per agent, all time and 30 days; `action` answers a model proposal.", | |
| 10 | inputSchema: { | |
| 11 | type: "object", | |
| 12 | properties: { | |
| 13 | action: { type: "string", enum: ["apply", "decline"] }, | |
| 14 | change: { type: "string", maxLength: 40 }, | |
| 15 | }, | |
| 16 | }, | |
| 17 | writes: true, | |
| 18 | readsWithout: "action", | |
| 19 | doc: "(forge admin) no input: the crew's scorecard per agent, all time and 30 days: landed, review rounds, red rate, reverts; week {merged, returned, cost}. `{ action: apply|decline, change }` answers one waiting model-review proposal (`modelReview.changes[].id`), the same as the page's **Move to …** / **Leave it**, and answers `{ change, agent, from, to, state, says }`; a person who administers only, an agent is refused", | |
| 12 | 20 | adminOnly: true, |
| 13 | run(ctx, user) { | |
| 14 | const card = readScorecards(ctx.app, user); | |
| 15 | return card ? good(card) : bad("forbidden", ADMIN_WORDS.scorecard); | |
| 21 | run(ctx, user, args) { | |
| 22 | const action = args["action"]; | |
| 23 | if (action === undefined) { | |
| 24 | const card = readScorecards(ctx.app, user); | |
| 25 | return card ? good(card) : bad("forbidden", ADMIN_WORDS.scorecard); | |
| 26 | } | |
| 27 | if (action !== "apply" && action !== "decline") return bad("invalid", "action is apply or decline"); | |
| 28 | if (user.kind !== "user" || !user.isAdmin) return bad("forbidden", CHANGE_WORDS.person); | |
| 29 | const change = args["change"]; | |
| 30 | if (typeof change !== "string" || change === "" || change.length > 40) return bad("invalid", "change is the id of a waiting proposal"); | |
| 31 | const done = answerChange(ctx.app, user, change, action === "apply"); | |
| 32 | return done.ok ? good(done.value) : bad("not_found", done.error); | |
| 16 | 33 | }, |
| 17 | 34 | }; |
src/web/pages/crew.ts +3 −3
| @@ -45,7 +45,7 @@ import { keeperDays, type KeeperDay } from "../../flow/keeper_rate.ts"; | ||
| 45 | 45 | import { weekCardRows } from "../../crew/week.ts"; |
| 46 | 46 | import { weekCard, weekReturns, WEEK_DAYS, type ReturnedPull, type WeekLine } from "../../crew/week_card.ts"; |
| 47 | 47 | import { knobsCard } from "./knobs.ts"; |
| 48 | import { decideChange, readReview } from "../../actions/model_review.ts"; | |
| 48 | import { CHANGE_WORDS, decideChange, readReview } from "../../actions/model_review.ts"; | |
| 49 | 49 | import { wastedDays, type WastedDay } from "../../core/wasted_days.ts"; |
| 50 | 50 | import { modelDays, type ModelDay } from "../../core/model_days.ts"; |
| 51 | 51 | import { notePhase, takeSessionTime } from "../../core/route_timing.ts"; |
| @@ -1194,8 +1194,8 @@ export async function modelReviewDecide(ctx: Ctx, take: boolean): Promise<Respon | ||
| 1194 | 1194 | const done = decideChange(ctx.app, ctx.auth.user, ctx.params.id ?? "", take); |
| 1195 | 1195 | return flashTo(ctx, "/settings/crew", { |
| 1196 | 1196 | ...(done.ok |
| 1197 | ? { message: take ? `${done.value.toModel} it is.` : "Left as it was." } | |
| 1198 | : { tone: "danger" as const, message: "That change is no longer waiting." }), | |
| 1197 | ? { message: take ? CHANGE_WORDS.taken(done.value.toModel) : CHANGE_WORDS.left } | |
| 1198 | : { tone: "danger" as const, message: CHANGE_WORDS.gone }), | |
| 1199 | 1199 | }); |
| 1200 | 1200 | } |
| 1201 | 1201 | export const modelReviewTake = (ctx: Ctx) => modelReviewDecide(ctx, true); |
test/conversation-confirm.test.ts +1 −1
| @@ -126,7 +126,7 @@ describe("the one list of what asks first", () => { | ||
| 126 | 126 | "crew", "decisions_by", "desk", "detach", "economics", "edit", "email_verify", "estimate", "feed_chips", "feed_dismiss", "feed_like", "flag_set", "flag_winner", |
| 127 | 127 | "follow_mode", "forge", "fork", "front_page", "hold", "issue_teams", "keep_plan", "knobs", "labels", "lift", "limb", "maintenance", "make_it_yours", "map", "merge", "new_app", |
| 128 | 128 | "next_up", "week", "notify", "open_pull", "org_create", "origin_apps", "pins", "plan", "profile", "propose_to_parent_forge", "repo_labels", "reports", |
| 129 | "repo", "restore_drill", "review", "say", "signup", "snapshot_now", "spend", "steer", "sync", "system", "take_back", "take_updates", | |
| 129 | "repo", "restore_drill", "review", "say", "scorecard", "signup", "snapshot_now", "spend", "steer", "sync", "system", "take_back", "take_updates", | |
| 130 | 130 | "teams", "tell", "threads", "trail", "upstream", "variant", "wait_for", "watch", "whoami_can", "write_file", |
| 131 | 131 | ]); |
| 132 | 132 |
test/mcp-scorecard-decide.test.ts added +107
| @@ -0,0 +1,107 @@ | ||
| 1 | /** | |
| 2 | * The scorecard MCP tool answers a model-review proposal in the page's words, for a person who | |
| 3 | * administers only. | |
| 4 | */ | |
| 5 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 6 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 7 | import { tmpdir } from "node:os"; | |
| 8 | import { join } from "node:path"; | |
| 9 | import { createApp, type App } from "../src/app.ts"; | |
| 10 | import { createAgentRow, getAgent } from "../src/core/agents.ts"; | |
| 11 | import { getChange, recordChange, type ModelStats } from "../src/core/model_review.ts"; | |
| 12 | import { CHANGE_WORDS } from "../src/actions/model_review.ts"; | |
| 13 | import { createToken } from "../src/core/tokens.ts"; | |
| 14 | import { createUser } from "../src/core/users.ts"; | |
| 15 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 16 | import { quietLogs } from "./helpers/globals.ts"; | |
| 17 | ||
| 18 | quietLogs(); | |
| 19 | ||
| 20 | let dataDir: string; | |
| 21 | let app: App; | |
| 22 | let servers: Servers; | |
| 23 | let adminToken = ""; | |
| 24 | let agentToken = ""; | |
| 25 | let botId = ""; | |
| 26 | let n = 1; | |
| 27 | ||
| 28 | const call = async (token: string, args: unknown) => { | |
| 29 | const res = await fetch(`${servers.url}/mcp`, { | |
| 30 | method: "POST", | |
| 31 | headers: { "content-type": "application/json", accept: "application/json, text/event-stream", authorization: `Bearer ${token}` }, | |
| 32 | body: JSON.stringify({ jsonrpc: "2.0", id: n++, method: "tools/call", params: { name: "scorecard", arguments: args } }), | |
| 33 | }); | |
| 34 | const body = (await res.json()) as any; | |
| 35 | return { error: body.result?.isError === true || body.error !== undefined, text: String(body.result?.content?.[0]?.text ?? body.error?.message ?? "") }; | |
| 36 | }; | |
| 37 | ||
| 38 | const before: ModelStats = { model: "opus", opened: 9, landed: 9, costPerLanded: 20, redRate: 0.1, reworkRounds: 0, reverts: 0, hardShare: 0 }; | |
| 39 | const ask = () => recordChange(app.db, { agentId: botId, fromModel: "opus", toModel: "sonnet", state: "asked", why: "healthy", before, at: new Date() }); | |
| 40 | ||
| 41 | beforeAll(async () => { | |
| 42 | dataDir = await mkdtemp(join(tmpdir(), "forge-mcp-scorecard-decide-")); | |
| 43 | app = createApp({ dataDir, port: 0 }); | |
| 44 | servers = await serve(app); | |
| 45 | const a = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true }); | |
| 46 | if (!a.ok) throw new Error("users"); | |
| 47 | adminToken = createToken(app.db, a.value.id, "admin", "admin").token; | |
| 48 | const bot = await createUser(app.db, { origin: "seed", trusted: true, username: "smith", email: "[email protected]", password: "a-good-passphrase" }); | |
| 49 | if (!bot.ok) throw new Error("bot"); | |
| 50 | app.db.query("UPDATE user SET is_agent = 1 WHERE id = ?1").run(bot.value.id); | |
| 51 | createAgentRow(app.db, { userId: bot.value.id, model: "opus" }); | |
| 52 | botId = bot.value.id; | |
| 53 | agentToken = createToken(app.db, bot.value.id, "admin", "admin").token; | |
| 54 | }, 30_000); | |
| 55 | ||
| 56 | afterAll(async () => { | |
| 57 | servers.stop(); | |
| 58 | app.close(); | |
| 59 | await rm(dataDir, { recursive: true, force: true }); | |
| 60 | }); | |
| 61 | ||
| 62 | test("apply moves the model and answers the page's receipt", async () => { | |
| 63 | const id = ask(); | |
| 64 | const got = await call(adminToken, { action: "apply", change: id }); | |
| 65 | expect(got.error).toBe(false); | |
| 66 | expect(JSON.parse(got.text)).toMatchObject({ change: id, agent: "smith", from: "opus", to: "sonnet", state: "applied", says: "sonnet it is." }); | |
| 67 | expect(getAgent(app.db, botId)?.model).toBe("sonnet"); | |
| 68 | // once answered it is no longer waiting | |
| 69 | const again = await call(adminToken, { action: "apply", change: id }); | |
| 70 | expect(again.error).toBe(true); | |
| 71 | expect(again.text).toContain("no longer waiting"); | |
| 72 | }); | |
| 73 | ||
| 74 | test("decline leaves the model", async () => { | |
| 75 | app.db.query("UPDATE agent SET model = 'opus' WHERE user_id = ?1").run(botId); | |
| 76 | const id = ask(); | |
outdated
This agent is not an admin, so |
||
| 77 | const got = await call(adminToken, { action: "decline", change: id }); | |
| 78 | expect(JSON.parse(got.text)).toMatchObject({ state: "declined", says: "Left as it was." }); | |
| 79 | expect(getAgent(app.db, botId)?.model).toBe("opus"); | |
| 80 | }); | |
| 81 | ||
| 82 | test("an agent is refused and nothing changes", async () => { | |
| 83 | const id = ask(); | |
| 84 | const got = await call(agentToken, { action: "apply", change: id }); | |
| 85 | expect(got.error).toBe(true); | |
| 86 | expect(getChange(app.db, id)?.state).toBe("asked"); | |
| 87 | expect(getAgent(app.db, botId)?.model).toBe("opus"); | |
| 88 | }); | |
| 89 | ||
| 90 | test("an agent that administers is refused in the page's sentence for an agent, and nothing changes", async () => { | |
| 91 | app.db.query("UPDATE user SET is_admin = 1 WHERE id = ?1").run(botId); | |
| 92 | try { | |
| 93 | const id = ask(); | |
| 94 | const got = await call(agentToken, { action: "apply", change: id }); | |
| 95 | expect(got.error).toBe(true); | |
| 96 | expect(got.text).toContain(CHANGE_WORDS.person); | |
| 97 | expect(getChange(app.db, id)?.state).toBe("asked"); | |
| 98 | expect(getAgent(app.db, botId)?.model).toBe("opus"); | |
| 99 | } finally { | |
| 100 | app.db.query("UPDATE user SET is_admin = 0 WHERE id = ?1").run(botId); | |
| 101 | } | |
| 102 | }); | |
| 103 | ||
| 104 | test("a bad action or a missing change is named", async () => { | |
| 105 | expect((await call(adminToken, { action: "yes", change: "x" })).error).toBe(true); | |
| 106 | expect((await call(adminToken, { action: "apply" })).error).toBe(true); | |
| 107 | }); |
test/route-marks.test.ts +1 −1
| @@ -233,7 +233,7 @@ describe("every admin-only action refuses the everyday account", () => { | ||
| 233 | 233 | }); |
| 234 | 234 | |
| 235 | 235 | test("the admin calls each read-only admin MCP tool and it answers; the write ones stay listed only", async () => { |
| 236 | const reads = TOOLS.filter((t) => t.adminOnly === true && !t.writes); | |
| 236 | const reads = TOOLS.filter((t) => t.adminOnly === true && (!t.writes || t.readsWithout !== undefined)); | |
| 237 | 237 | expect(reads.length).toBeGreaterThan(3); |
| 238 | 238 | const wrong: string[] = []; |
| 239 | 239 | for (const t of reads) { |
new above
offline
Reachable only by an admin agent; the test never gets here. Cover it.