Public: anyone can read this; the crew can change it.

A stopped Talk turn is stored as stopped, and run vitals do not count it as a failure #8880

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/8860-a-stopped-talk-turn-is-stored-as-stopped → main 5 commits fork 2 threads · 2 unresolved

  • Merged. scout/8860-a-stopped-talk-turn-is-stored-as-stopped was merged into main as 7d60106.

Review

Sign in to review this change.
7 files changed +67 −2
docs/features/talk.md +2
@@ -67,6 +67,8 @@ Prior art read: Vercel AI Elements (Apache-2.0 React on shadcn; anatomy and beha
6767
6868 **Cost and context.** Under a finished reply a quiet line gives the turn's dollars (`$0.12 this turn`); above the log, "Context 85% full (170k of 200k tokens)". Both are read from the run's record (`conversation_run`: `cost_usd`, and `context_tokens`/`context_window` from the last message's usage and the result's `modelUsage`), shown by `core/conversation_cost.ts`. When a run names no window, 200k is assumed and the line says so by showing it. From 80 % the line adds that a new conversation answers faster and costs less (information only: there is no way to start another yet, see "Many conversations"). A reply that streams in gets both when the stream ends; the terminal prints the last turn's cost and the share after its confirmation line.
6969
70 **Stopped is not failed.** A turn the person stopped is stored with `conversation_run.stopped = 1` (its `state` stays `failed`, so a release still in play reads it). The flow vitals line says it apart: `Talk: 2 turns failed, 1 stopped by the person` (`flow/talk_turns.ts`), and the failure rate leaves stopped turns out. The line is in the flow telemetry (`GET /api/v1/telemetry/flow` and the MCP flow tool) as `vitals.says`, with the counts at `vitals.talkTurns`.
71
7072 **The terminal** (`ssh <forge> talk <words>`, `forge talk`) sends one line and prints the answer; it has no log, so who/when, copy, tool cards, starters and the jump button do not apply to it. `forge talk` prints a thought as one `thinking:` line, then the reply, the steps line and the links as paths (the API's agent turns carry `work: {steps, links}`). The steps line says "opened" for a read issue or pull and "changed" for a tool that writes one (`CHANGES` in `core/talk_work.ts`, held to the MCP tools' `writes` by a test); numbers carry their repo when a turn touched more than one. A reply first on an older page, or first in a stream batch, reads its calls from the turns before the page (`worksOf`), so it says the same on every door. The conversation's whole history is read through `GET /api/v1/conversation/turns`, where every turn carries its role and time.
7173
7274 ## Code, tables and Copy in a reply (#8816)
src/core/migration-pins/20261103130000-conversation-run-stopped.pin added +1
@@ -0,0 +1,1 @@
1 41e05fcff4d6c1c737a1c4b1d7f2a3742d3ce18561f5dae30d599ce7fd06467a
src/core/migrations/20261103130000-conversation-run-stopped.ts added +8
@@ -0,0 +1,8 @@
1 // A /talk turn the person stopped is marked `stopped` (#8860), so the failure
2 // count stays what went wrong. Additive across a roll: the state check cannot
3 // be widened without rebuilding the table, and a release still in play reads
4 // `state`, so a stopped turn keeps state `failed` and this flag says it was a
5 // choice. Counts of failures leave `stopped = 1` out.
6 export default `
7 ALTER TABLE conversation_run ADD COLUMN stopped INTEGER NOT NULL DEFAULT 0;
8 `;
src/crew/conversation.ts +1 −1
@@ -454,7 +454,7 @@ function begin(
454454 cancelHeld(app.db, via);
455455 put("agent", STOPPED_LINE);
456456 app.db
457 .query("UPDATE conversation_run SET state = 'failed', tokens_in = ?2, tokens_out = ?3, cost_usd = ?4, finished_at = ?5, context_tokens = ?6, context_window = ?7 WHERE id = ?1")
457 .query("UPDATE conversation_run SET state = 'failed', stopped = 1, tokens_in = ?2, tokens_out = ?3, cost_usd = ?4, finished_at = ?5, context_tokens = ?6, context_window = ?7 WHERE id = ?1")
458458 .run(runId, outcome?.tokensIn ?? 0, outcome?.tokensOut ?? 0, cost, new Date().toISOString(), outcome?.contextTokens ?? 0, outcome?.contextWindow ?? 0);
459459 app.db.query("UPDATE subscription SET spent_usd = spent_usd + ?2 WHERE id = ?1").run(subscriptionId, cost);
460460 return;
src/flow/talk_turns.ts added +29
@@ -0,0 +1,29 @@
1 /** How /talk's turns ended (#8860): a stop is a choice, so it is counted apart from failures. */
2
outdated

This header line is over-long and repeats the migration comment. Make it one line.

3 import type { Database } from "bun:sqlite";
4
5 export interface TalkTurns {
6 readonly ok: number;
7 readonly failed: number;
8 readonly stopped: number;
9 /** Failed over answered plus failed; stopped turns are in neither. 0 with none. */
10 readonly failureRate: number;
11 }
12
13 export function readTalkTurns(db: Database): TalkTurns {
14 const row = db
15 .query<{ ok: number | null; failed: number | null; stopped: number | null }, []>(
16 `SELECT SUM(state = 'ok') AS ok, SUM(state = 'failed' AND stopped = 0) AS failed, SUM(stopped = 1) AS stopped
17 FROM conversation_run`,
18 )
19 .get();
20 const ok = row?.ok ?? 0;
21 const failed = row?.failed ?? 0;
22 return { ok, failed, stopped: row?.stopped ?? 0, failureRate: ok + failed === 0 ? 0 : failed / (ok + failed) };
23 }
24
25 /** `Talk: 2 turns failed, 1 stopped by the person`; empty when none failed or was stopped. */
26 export function talkTurnsWords(t: TalkTurns): string {
27 if (t.failed + t.stopped === 0) return "";
28 return `Talk: ${t.failed} ${t.failed === 1 ? "turn" : "turns"} failed, ${t.stopped} stopped by the person`;
29 }
src/system/telemetry.ts +3 −1
@@ -12,6 +12,7 @@ import { SOURCES } from "../boards/sources.ts";
1212 import { conflictedPulls, conflictedSays, rebaseLane, rebaseLaneSays } from "../flow/conflicted.ts";
1313 import { firedAlarms } from "../flow/alarms.ts";
1414 import { running } from "../kernel/version.ts";
15 import { readTalkTurns, talkTurnsWords } from "../flow/talk_turns.ts";
1516 import { readTalkResumes, talkResumesWords } from "../flow/talk_resumes.ts";
1617 import { readVitals, vitalsFor, vitalsLine, vitalsMisses } from "../flow/vitals.ts";
1718 import { parkedSays, readParked } from "../flow/parked.ts";
@@ -42,6 +43,7 @@ export function flowFor(app: App, reader: Reader, now = new Date()) {
4243 // Dollars only for the owner and an agent holding telemetry:read (#6424).
4344 const vitals = vitalsFor(readVitals(app.db, now, reader.kind === "agent"), reader.kind === "owner" || reader.spend === true);
4445 const talk = readTalkResumes(app.db, running().version);
46 const talkTurns = readTalkTurns(app.db);
4547 const parked = readParked(app.db, now, reader.kind === "agent");
4648 return {
4749 says: verdict(stages, now.getTime()),
@@ -55,7 +57,7 @@ export function flowFor(app: App, reader: Reader, now = new Date()) {
5557 parked: { says: parkedSays(parked), ...parked },
5658 // Open pulls, opened and landed in 1 h and 6 h, and conflicted, against #5924's targets (#5935).
5759 // A fallback after a release is a line here, not a banner (#8795).
outdated

talkTurns is in the data only. Put stopped and failed counts in the says-line so a person sees them, with a test on the line.

58 vitals: { says: [vitalsLine(vitals), talkResumesWords(talk)].filter((w) => w !== "").join(" · "), talk, misses: vitalsMisses(vitals), ...vitals },
60 vitals: { says: [vitalsLine(vitals), talkResumesWords(talk), talkTurnsWords(talkTurns)].filter((w) => w !== "").join(" · "), talk, talkTurns, misses: vitalsMisses(vitals), ...vitals },
5961 // What the last red batches were: the pull named and returned with its failing tests, or a load red (#7639).
6062 redBatches: readVerdicts(app.db, now, reader.kind === "agent"),
6163 // Findings reviewers repeated, proposed as checks and retired, beside review comments per pull (#8716).
test/conversation-turn.test.ts +23
@@ -22,6 +22,8 @@ import { createSession } from "../src/core/sessions.ts";
2222 import { serve } from "../src/http/server.ts";
2323 import { TOOLS } from "../src/crew/claude.ts";
2424 import { answerHeld, hold, waitingFor } from "../src/core/fence.ts";
25 import { flowFor } from "../src/system/telemetry.ts";
26 import { readTalkTurns, talkTurnsWords } from "../src/flow/talk_turns.ts";
2527 import { MCP_SERVER, mcpConfig, RESUMED_LINE, resumeInterrupted, say, STOPPED_LINE, stopTurn, turnRunning, unendedLine, TURNS_LIVE_MAX, type ConversationOptions } from "../src/crew/conversation.ts";
2628
2729 let dir: string;
@@ -502,12 +504,19 @@ sleep 300
502504 const via = `conversation:${openConversation(app.db, u.value.id).id}`;
503505 const held = hold(app.db, via, u.value.id, "create_issue", { title: "x" });
504506 expect(held).not.toBeNull();
507 const failedBefore = readTalkTurns(app.db).failed;
505508 expect(stopTurn(app, u.value.id)).toBe(true);
506509 await said.value.done;
507510 expect(turnRunning(app, u.value.id)).toBe(false);
508511 expect(turns(u.value).slice(-3)).toEqual(["person:do a slow thing", "agent:Looking into it.", `agent:${STOPPED_LINE}`]);
509512 expect(waitingFor(app.db, u.value.id)).toEqual([]);
510513 expect(answerHeld(app.db, held!, u.value.id).ok).toBe(false);
514 // Stored as stopped, and not a failure (#8860).
515 expect(app.db.query("SELECT state FROM conversation_run WHERE user_id = ?1 ORDER BY started_at DESC LIMIT 1").get(u.value.id)).toEqual({ state: "failed" });
516 expect(app.db.query("SELECT stopped FROM conversation_run WHERE user_id = ?1 ORDER BY started_at DESC LIMIT 1").get(u.value.id)).toEqual({ stopped: 1 });
517 const turnsNow = readTalkTurns(app.db);
518 expect(turnsNow.stopped).toBeGreaterThanOrEqual(1);
519 expect(turnsNow.failed).toBe(failedBefore);
511520 let alive = true;
512521 for (let i = 0; i < 60 && alive; i++) {
513522 try { process.kill(pid, 0); await Bun.sleep(50); } catch { alive = false; }
@@ -516,3 +525,17 @@ sleep 300
516525 expect(stopTurn(app, u.value.id)).toBe(false);
517526 }, 20_000);
518527 });
528
529 describe("talk turn counts (#8860)", () => {
530 test("stopped turns are counted apart and leave the failure rate alone", () => {
531 const base = readTalkTurns(app.db);
532 const c = openConversation(app.db, alice.id);
533 for (const [state, stopped] of [["ok", 0], ["failed", 0], ["failed", 1], ["failed", 1]] as const)
534 app.db.query("INSERT INTO conversation_run (id, conversation_id, user_id, state, stopped, prompt, started_at) VALUES (?1,?2,?3,?4,?5,'x','2026-11-03')").run(crypto.randomUUID(), c.id, alice.id, state, stopped);
535 const now = readTalkTurns(app.db);
536 expect([now.ok - base.ok, now.failed - base.failed, now.stopped - base.stopped]).toEqual([1, 1, 2]);
537 expect(talkTurnsWords({ ok: 0, failed: 0, stopped: 0, failureRate: 0 })).toBe("");
538 expect(talkTurnsWords({ ok: 1, failed: 1, stopped: 2, failureRate: 0.5 })).toBe("Talk: 1 turn failed, 2 stopped by the person");
539 expect(flowFor(app, { kind: "owner" }).vitals.says).toContain(`Talk: ${now.failed} ${now.failed === 1 ? "turn" : "turns"} failed, ${now.stopped} stopped by the person`);
540 });
541 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs