Public: anyone can read this; the crew can change it.

After each release, the forge counts /talk turns that resumed their session and those that fell back, and a fallback shows on the vitals #8804

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/8795-after-each-release-the-forge-counts-talk → main 1 commit fork

  • Merged. mason/8795-after-each-release-the-forge-counts-talk was merged into main as e287d33.

Review

Sign in to review this change.
6 files changed +147 −1
src/core/migration-pins/20261103110000-conversation-run-resume.pin added +1
@@ -0,0 +1,1 @@
1 e17306d61c6678f21d103d76facc86541db5f598399acbc9175315628e889b15
src/core/migrations/20261103110000-conversation-run-resume.ts added +7
@@ -0,0 +1,7 @@
1 // How a /talk turn that had a stored session went (#8795): `resumed` took it,
2 // `fallback` found it gone and started fresh, empty when there was none to
3 // resume. With the running version, never the text or the session.
4 export default `
5 ALTER TABLE conversation_run ADD COLUMN resume_outcome TEXT NOT NULL DEFAULT '';
6 ALTER TABLE conversation_run ADD COLUMN version TEXT NOT NULL DEFAULT '';
7 `;
src/crew/conversation.ts +6
@@ -26,6 +26,7 @@ import { PLATFORM, pickSubscription, type Pick } from "../core/subscriptions.ts"
2626 import { getUserById } from "../core/users.ts";
2727 import { mintViaToken, VIA_TOKEN_MAX_MS } from "../core/via_token.ts";
2828 import { id } from "../kernel/id.ts";
29 import { running } from "../kernel/version.ts";
2930 import { err, ok, type Result } from "../kernel/result.ts";
3031 import { appendRunLog } from "../runs/log.ts";
3132 import { runClaude, type ClaudeOutcome } from "./claude.ts";
@@ -334,6 +335,11 @@ function begin(
334335 },
335336 });
336337 const lost = !outcome.ok && lostSession(outcome.result);
338 // Only the outcome and the version are kept, never the text or the session (#8795).
339 if (session !== undefined && !options.signal.aborted)
340 app.db
341 .query("UPDATE conversation_run SET resume_outcome = ?2, version = ?3 WHERE id = ?1")
342 .run(runId, named && !lost ? "resumed" : "fallback", running().version);
337343 if (session === undefined || (named && !lost) || options.signal.aborted) break;
338344 appendRunLog(app.config.dataDir, runId, "no session to resume; starting this message fresh");
339345 // What the failed attempt showed (its error may be printed as text) is taken back.
src/flow/talk_resumes.ts added +30
@@ -0,0 +1,30 @@
1 /**
2 * After a release, did /talk conversations pick up their sessions (#8795)?
3 * The turns of the running version that had a stored session, counted by
4 * outcome: resumed it, or fell back to a fresh start. Counts only; no
5 * conversation, person or session is named, so every reader may see it.
6 */
7
8 import type { Database } from "bun:sqlite";
9
10 export interface TalkResumes {
11 readonly version: string;
12 readonly resumed: number;
13 readonly fellBack: number;
14 }
15
16 export function readTalkResumes(db: Database, version: string): TalkResumes {
17 const row = db
18 .query<{ resumed: number | null; fell_back: number | null }, [string]>(
19 `SELECT SUM(resume_outcome = 'resumed') AS resumed, SUM(resume_outcome = 'fallback') AS fell_back
20 FROM conversation_run WHERE version = ?1`,
21 )
22 .get(version);
23 return { version, resumed: row?.resumed ?? 0, fellBack: row?.fell_back ?? 0 };
24 }
25
26 /** `3 conversations resumed after release 0.7.9, 1 fell back`; empty when no turn has had a session to resume. */
27 export function talkResumesWords(t: TalkResumes): string {
28 if (t.resumed + t.fellBack === 0) return "";
29 return `${t.resumed} ${t.resumed === 1 ? "conversation" : "conversations"} resumed after release ${t.version}, ${t.fellBack} fell back`;
30 }
src/system/telemetry.ts +5 −1
@@ -11,6 +11,8 @@ import type { App } from "../app.ts";
1111 import { SOURCES } from "../boards/sources.ts";
1212 import { conflictedPulls, conflictedSays, rebaseLane, rebaseLaneSays } from "../flow/conflicted.ts";
1313 import { firedAlarms } from "../flow/alarms.ts";
14 import { running } from "../kernel/version.ts";
15 import { readTalkResumes, talkResumesWords } from "../flow/talk_resumes.ts";
1416 import { readVitals, vitalsFor, vitalsLine, vitalsMisses } from "../flow/vitals.ts";
1517 import { parkedSays, readParked } from "../flow/parked.ts";
1618 import { readKeeper } from "../flow/keeper.ts";
@@ -39,6 +41,7 @@ export function flowFor(app: App, reader: Reader, now = new Date()) {
3941 const busy = reviewersBusy(app.db, reader.kind === "agent");
4042 // Dollars only for the owner and an agent holding telemetry:read (#6424).
4143 const vitals = vitalsFor(readVitals(app.db, now, reader.kind === "agent"), reader.kind === "owner" || reader.spend === true);
44 const talk = readTalkResumes(app.db, running().version);
4245 const parked = readParked(app.db, now, reader.kind === "agent");
4346 return {
4447 says: verdict(stages, now.getTime()),
@@ -51,7 +54,8 @@ export function flowFor(app: App, reader: Reader, now = new Date()) {
5154 // Issues the crew parked in the last 24 h and 7 d, the newest three named (#6471).
5255 parked: { says: parkedSays(parked), ...parked },
5356 // Open pulls, opened and landed in 1 h and 6 h, and conflicted, against #5924's targets (#5935).
54 vitals: { says: vitalsLine(vitals), misses: vitalsMisses(vitals), ...vitals },
57 // A fallback after a release is a line here, not a banner (#8795).
58 vitals: { says: [vitalsLine(vitals), talkResumesWords(talk)].filter((w) => w !== "").join(" · "), talk, misses: vitalsMisses(vitals), ...vitals },
5559 // What the last red batches were: the pull named and returned with its failing tests, or a load red (#7639).
5660 redBatches: readVerdicts(app.db, now, reader.kind === "agent"),
5761 // Findings reviewers repeated, proposed as checks and retired, beside review comments per pull (#8716).
test/conversation-resume-count.test.ts added +98
@@ -0,0 +1,98 @@
1 /**
2 * Each /talk turn records whether it resumed its stored session or took the
3 * fresh-start fallback, with the running version and nothing else; the flow
4 * vitals say it in words (#8795).
5 */
6
7 import { afterAll, beforeAll, expect, test } from "bun:test";
8 import { chmodSync, writeFileSync } from "node:fs";
9 import { mkdtemp, rm } from "node:fs/promises";
10 import { tmpdir } from "node:os";
11 import { join } from "node:path";
12 import { createApp, type App } from "../src/app.ts";
13 import { createUser } from "../src/core/users.ts";
14 import { say, type ConversationOptions } from "../src/crew/conversation.ts";
15 import { readTalkResumes, talkResumesWords } from "../src/flow/talk_resumes.ts";
16 import { running } from "../src/kernel/version.ts";
17 import { flowFor } from "../src/system/telemetry.ts";
18
19 let dir: string;
20 let app: App;
21 let userId: string;
22
23 const options = (binary: string): ConversationOptions => ({
24 binary,
25 localUrl: "http://127.0.0.1:9",
26 budgetUsd: 1,
27 timeoutMs: 30_000,
28 signal: new AbortController().signal,
29 });
30
31 /** A fake `claude` that answers; with `lost`, a resume is refused as Claude Code refuses a session whose file is gone. */
32 function fake(name: string, lost: boolean): string {
33 const path = join(dir, name);
34 writeFileSync(
35 path,
36 `#!/bin/sh
37 if [ "$1" = "--version" ]; then echo "2.1.0 (fake)"; exit 0; fi
38 IFS= read -r first
39 echo '{"type":"system","subtype":"init","session_id":"sess-count","model":"fake"}'
40 ${lost ? `case " $* " in *" --resume "*)
41 echo '{"type":"result","subtype":"error_during_execution","is_error":true,"result":"","errors":["No conversation found with session ID: sess-count"],"total_cost_usd":0,"num_turns":0,"session_id":"sess-count"}'
42 exit 1 ;; esac` : ""}
43 echo '{"type":"assistant","session_id":"sess-count","message":{"id":"m","content":[{"type":"text","text":"All quiet."}],"usage":{"input_tokens":1,"output_tokens":1}}}'
44 echo '{"type":"result","subtype":"success","is_error":false,"result":"All quiet.","total_cost_usd":0.1,"num_turns":1,"session_id":"sess-count","terminal_reason":"completed","usage":{"input_tokens":1,"output_tokens":1}}'
45 `,
46 );
47 chmodSync(path, 0o755);
48 return path;
49 }
50
51 const turn = async (binary: string, text: string): Promise<void> => {
52 const said = say(app, options(binary), userId, text);
53 if (said.ok) await said.value.done;
54 };
55
56 beforeAll(async () => {
57 dir = await mkdtemp(join(tmpdir(), "forge-resume-count-"));
58 app = createApp({ dataDir: dir, port: 0 });
59 const a = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
60 if (!a.ok) throw new Error("user");
61 userId = a.value.id;
62 });
63 afterAll(async () => {
64 app.db.close();
65 await rm(dir, { recursive: true, force: true });
66 });
67
68 test("a first message has no session to resume and counts for neither", async () => {
69 await turn(fake("a", false), "hello");
70 const t = readTalkResumes(app.db, running().version);
71 expect([t.resumed, t.fellBack]).toEqual([0, 0]);
72 expect(talkResumesWords(t)).toBe("");
73 });
74
75 test("a turn that resumes its session counts as resumed", async () => {
76 await turn(fake("b", false), "again");
77 const t = readTalkResumes(app.db, running().version);
78 expect([t.resumed, t.fellBack]).toEqual([1, 0]);
79 expect(talkResumesWords(t)).toBe(`1 conversation resumed after release ${running().version}, 0 fell back`);
80 });
81
82 test("a turn fed “No conversation found” counts as a fallback and shows in the vitals words", async () => {
83 await turn(fake("c", true), "still there?");
84 const t = readTalkResumes(app.db, running().version);
85 expect([t.resumed, t.fellBack]).toEqual([1, 1]);
86 for (const reader of [{ kind: "owner" }, { kind: "agent" }] as const) {
87 const vitals = flowFor(app, reader).vitals;
88 expect(vitals.says).toContain(`1 conversation resumed after release ${running().version}, 1 fell back`);
89 expect(vitals.talk).toEqual(t);
90 }
91 });
92
93 test("another version's turns are not this version's, and nothing but outcome and version is kept", () => {
94 expect(readTalkResumes(app.db, "0.0.0-other")).toEqual({ version: "0.0.0-other", resumed: 0, fellBack: 0 });
95 const rows = app.db.query<{ resume_outcome: string; version: string }, []>("SELECT resume_outcome, version FROM conversation_run WHERE resume_outcome != ''").all();
96 expect(rows.map((r) => r.resume_outcome).sort()).toEqual(["fallback", "resumed"]);
97 expect(JSON.stringify(rows)).not.toContain("sess-count");
98 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs