trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRuns12g rEventsFlows

An interrupted run is not a park: provider limits and outages must never strike an issue off the map #463

Merged masoncrew opened this in answer to trevatoowner

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

mason/forge:mason/443-an-interrupted-run-is-not-a-park-provide → main 5 commits fork

  • Merged. mason/443-an-interrupted-run-is-not-a-park-provide was merged into main as ce31981.

Review

Sign in to review this change.
12 files changed +498 −45
forge.json +1 −1
@@ -6,7 +6,7 @@
66 "ready": 90
77 },
88 "check": {
9 "run": "bun x tsc --noEmit && bun scripts/test-shards.ts --timeout 60000 -- crew crew-wrapup crew-limits crew-sandbox steer session-settle landing decide ledger checks dependencies autonomy studio studio-words forks fork-flow cross-repo-pull pulls apps apps-adversarial apps-busy apps-token web git-trust helpers shards migrate vitals guard asks feed feed-page offline share cloud run-secrets forge-cloudflare usage credentials standing board-windows spawn-plan spawn web-apps fleet-ops meter",
9 "run": "bun x tsc --noEmit && bun scripts/test-shards.ts --timeout 60000 -- crew crew-wrapup crew-limits crew-parks crew-sandbox steer session-settle landing decide ledger checks dependencies autonomy studio studio-words forks fork-flow cross-repo-pull pulls apps apps-adversarial apps-busy apps-token web git-trust helpers shards migrate vitals guard asks feed feed-page offline share cloud run-secrets forge-cloudflare usage credentials standing board-windows spawn-plan spawn web-apps fleet-ops meter",
1010 "timeout": 1800
1111 }
1212 }
src/core/schema.ts +21
@@ -1151,4 +1151,25 @@ export const migrations: readonly string[] = [
11511151 UPDATE run SET subscription_id = 'platform'
11521152 WHERE kind = 'agent' AND subscription_id IS NULL;
11531153 `,
1154 // 43 — an interrupted run is not a park (`crew/interrupted.ts`). Park
1155 // notes written for runs the provider's limit, an unreachable API or a
1156 // restart ended were strikes against work nobody attempted; they are
1157 // reworded so they no longer start with the park note and no longer count.
1158 `
1159 UPDATE comment
1160 SET body = 'Interrupted, not parked: ' || substr(body, 21)
1161 WHERE body LIKE 'Parked by the crew: %'
1162 AND (body LIKE '%rate limit%' OR body LIKE '%rate-limit%'
1163 OR body LIKE '%rate_limit%' OR body LIKE '%session limit%'
1164 OR body LIKE '%usage limit%' OR body LIKE '%weekly limit%'
1165 OR body LIKE '%hit your % limit%' OR body LIKE '%overloaded%'
1166 OR body LIKE '%too many requests%' OR body LIKE '%(api_error)%'
1167 OR body LIKE '%may use is resting%' OR body LIKE '%resting —%'
1168 OR body LIKE '%ECONNREFUSED%' OR body LIKE '%ECONNRESET%'
1169 OR body LIKE '%ENOTFOUND%' OR body LIKE '%ETIMEDOUT%'
1170 OR body LIKE '%fetch failed%' OR body LIKE '%unable to connect%'
1171 OR body LIKE '%connection error%' OR body LIKE '%unreachable%'
1172 OR body LIKE '%without a pull: cancelled'
1173 OR body LIKE '%interrupted by a restart%');
1174 `,
11541175 ];
src/crew/autonomy.ts +13 −10
@@ -81,6 +81,8 @@ import { reviewSummary } from "../core/reviews.ts";
8181 import { standing } from "../core/standing.ts";
8282 import { pace, type Pace } from "./pacing.ts";
8383 import { BREAKER_NOTE, breakerTripped } from "./breaker.ts";
84 import { interruptionOf } from "./interrupted.ts";
85 import { parkedCount as countParks, PARK_STRIKES } from "./parks.ts";
8486 import { createLog } from "../kernel/log.ts";
8587 import { crewOffInPreview } from "./preview.ts";
8688
@@ -343,7 +345,7 @@ export async function driveRepo(
343345 !proposed.has(r.issue.number) &&
344346 (r.issue.assigneeId === null || !planners.has(r.issue.assigneeId)) &&
345347 !stalled(app, r.issue) &&
346 parkedCount(app, r.issue.id) < 2,
348 parkedCount(app, r.issue.id) < PARK_STRIKES,
347349 );
348350 const next = await nextOf(app, repo, ready);
349351 if (next !== undefined) {
@@ -479,6 +481,10 @@ function stalled(app: App, issue: Issue): boolean {
479481 if (last === null || last === undefined) return false;
480482 if (last.state === "queued" || last.state === "running") return true;
481483 if (/opened pull #|updated pull #/.test(last.summary)) return false;
484 // Cut short by the provider or a restart: nobody tried it, so the clock
485 // tries again (`interrupted.ts`).
486 if (last.state === "failed" && interruptionOf(last.summary) !== undefined)
487 return false;
482488 // A word since then — a person's, or the planner's when it re-planned
483489 // a parked issue — is what un-stalls it.
484490 const spoken = app.db
@@ -532,15 +538,9 @@ export function parkForBreaker(
532538 });
533539 }
534540
535 /** How many times the crew has parked work on this issue. */
541 /** How many times the crew has parked this issue since a person or the planner last spoke (`parks.ts`). */
536542 export function parkedCount(app: App, issueId: string): number {
537 const row = app.db
538 .query<
539 { n: number },
540 [string]
541 >("SELECT COUNT(*) AS n FROM comment WHERE issue_id = ?1 AND body LIKE 'Parked by the crew%'")
542 .get(issueId);
543 return row?.n ?? 0;
543 return countParks(app.db, issueId);
544544 }
545545
546546 /**
@@ -569,7 +569,9 @@ export function parkIssue(
569569 /**
570570 * After a builder's run on an issue ends without a pull to show for it —
571571 * declined, out of budget, or simply nothing pushed — the issue is parked
572 * rather than left stalled: one fresh try later, then the planner's.
572 * rather than left stalled: one fresh try later, then the planner's. A run
573 * the provider or a restart cut short is interrupted, not parked: no note,
574 * no strike.
573575 */
574576 export function settleAgentRun(
575577 app: App,
@@ -594,6 +596,7 @@ export function settleAgentRun(
594596 }
595597 if (run.reason === "review") return false;
596598 if (/opened pull #|updated pull #/.test(run.summary)) return false;
599 if (interruptionOf(run.summary) !== undefined) return false;
597600 const issue = getIssueById(app.db, run.issueId);
598601 if (!issue || issue.kind !== "issue" || issue.state !== "open") return false;
599602 if (issue.id === repo.directionIssueId) return false;
src/crew/breaker.ts +6 −1
@@ -14,7 +14,11 @@ export const BREAKER_WINDOW_MS = 60 * 60_000;
1414 export const BREAKER_NOTE =
1515 "Parked by the crew: the last three runs here failed or were declined within an hour, so nothing more starts until a person says what to try.";
1616
17 /** Whether the issue's newest runs are all failures inside the window and no person has spoken since. */
17 /**
18 * Whether the issue's newest runs are all failures inside the window and no
19 * person has spoken since. An interrupted run (`interrupted.ts`) is not one
20 * of them: the provider's outage is not the issue's failure.
21 */
1822 export function breakerTripped(
1923 db: Database,
2024 issueId: string,
@@ -23,6 +27,7 @@ export function breakerTripped(
2327 const runs = db
2428 .query<{ state: string; created_at: string }, [string, number]>(
2529 `SELECT state, created_at FROM run WHERE issue_id = ?1 AND kind = 'agent'
30 AND summary NOT LIKE 'interrupted: %'
2631 ORDER BY created_at DESC LIMIT ?2`,
2732 )
2833 .all(issueId, BREAKER_RUNS);
src/crew/converge.ts +2 −1
@@ -30,6 +30,7 @@ import { activeRunFor, listRuns } from "../core/runs.ts";
3030 import { getUserById, type User } from "../core/users.ts";
3131 import { createLog } from "../kernel/log.ts";
3232 import { parkedCount } from "./autonomy.ts";
33 import { PARK_STRIKES } from "./parks.ts";
3334 import { queueAgentRun } from "./dispatch.ts";
3435 import { pickBuilder } from "./route.ts";
3536
@@ -97,7 +98,7 @@ export function convergeState(
9798 )?.agent;
9899 return { kind: "working", issue, agent };
99100 }
100 if (parkedCount(app, issue.id) >= 2) return { kind: "parked", issue };
101 if (parkedCount(app, issue.id) >= PARK_STRIKES) return { kind: "parked", issue };
101102 return { kind: "working", issue, agent: undefined };
102103 }
103104
src/crew/interrupted.ts added +74
@@ -0,0 +1,74 @@
1 /**
2 * An interrupted run is not a park. A builder's run the provider ended —
3 * a usage window full, the API unreachable — or the forge stopped by its
4 * own restart says nothing about the work: nobody attempted it. Such a run
5 * is recorded as `interrupted: <cause>, resumes <when>`, writes no park
6 * note and is no strike against the issue; the clock picks it up again.
7 */
8
9 import type { Database } from "bun:sqlite";
10 import { whenText } from "./limits.ts";
11
12 export type Interruption = "provider limit" | "network error" | "restart";
13
14 /** What every interrupted run's summary starts with. */
15 export const INTERRUPTED = "interrupted: ";
16
17 const PROVIDER =
18 /rate[ _-]?limit|session limit|usage limit|weekly limit|hit your \w+ limit|overloaded|too many requests|\b(?:429|529)\b|\(api_error\)|may use is resting|resting —/i;
19 const NETWORK =
20 /ECONNREFUSED|ECONNRESET|ENOTFOUND|ETIMEDOUT|EAI_AGAIN|fetch failed|unable to connect|connection error|could not reach|unreachable|socket hang up/i;
21 const RESTART =
22 /^cancelled$|interrupted by a restart|stopped for a restart|the forge stopped/i;
23
24 /**
25 * Why a run that ended without a pull was cut short, read from its summary
26 * — one written by `interruptedSummary`, or an older one that names a
27 * provider limit, a network error or a restart — or undefined when the run
28 * really ended on the work. A decline is the agent's own word, never this.
29 */
30 export function interruptionOf(summary: string): Interruption | undefined {
31 const s = summary.trim();
32 if (s.startsWith(INTERRUPTED)) {
33 const cause = s.slice(INTERRUPTED.length);
34 if (cause.startsWith("provider limit")) return "provider limit";
35 if (cause.startsWith("network error")) return "network error";
36 return "restart";
37 }
38 if (/^declined\b/i.test(s)) return undefined;
39 if (RESTART.test(s)) return "restart";
40 if (PROVIDER.test(s)) return "provider limit";
41 if (NETWORK.test(s)) return "network error";
42 return undefined;
43 }
44
45 /**
46 * The summary a failed run is recorded with: `interrupted: provider limit,
47 * resumes 02:00 UTC — <what happened>` when it was cut short, the summary
48 * as it was otherwise.
49 */
50 export function interruptedSummary(
51 summary: string,
52 resumesAt?: Date,
53 ): string {
54 if (summary.startsWith(INTERRUPTED)) return summary;
55 const cause = interruptionOf(summary);
56 if (cause === undefined) return summary;
57 const when =
58 resumesAt === undefined ? "resumes on the next pass" : `resumes ${whenText(resumesAt)}`;
59 return `${INTERRUPTED}${cause}, ${when} — ${summary}`;
60 }
61
62 /** The agent runs on an issue that were interrupted, newest first, for its page. */
63 export function interruptionsOn(
64 db: Database,
65 issueId: string,
66 ): { summary: string; at: string }[] {
67 return db
68 .query<{ summary: string; at: string }, [string]>(
69 `SELECT summary, COALESCE(finished_at, created_at) AS at FROM run
70 WHERE issue_id = ?1 AND kind = 'agent' AND summary LIKE 'interrupted: %'
71 ORDER BY created_at DESC LIMIT 20`,
72 )
73 .all(issueId);
74 }
src/crew/parks.ts added +62
@@ -0,0 +1,62 @@
1 /**
2 * Strikes against an issue: the crew's park notes since the last word that
3 * re-planned it. A person's comment, or the planner's, starts the count
4 * from zero — the issue as written now is not the one that was parked.
5 * Two strikes and the clock stops dispatching it; the map's `parked` word
6 * reads the same rule, so what blocks dispatch is what the map shows.
7 */
8
9 import type { Database } from "bun:sqlite";
10
11 /** Parks since the last word at which the clock no longer dispatches an issue. */
12 export const PARK_STRIKES = 2;
13
14 /** What every park note starts with, for `LIKE`. */
15 const PARK = "Parked by the crew%";
16
17 /**
18 * The parks on `p.issue_id` that count: newer than the newest comment that
19 * is not itself a park, by a person or by the repository's planner.
20 */
21 const COUNTED = `p.body LIKE '${PARK}' AND p.created_at > COALESCE((
22 SELECT MAX(c.created_at) FROM comment c
23 JOIN user u ON u.id = c.author_id
24 JOIN issue i ON i.id = c.issue_id
25 WHERE c.issue_id = p.issue_id AND c.body NOT LIKE '${PARK}'
26 AND (u.is_agent = 0 OR EXISTS (
27 SELECT 1 FROM repo_crew rc
28 WHERE rc.repo_id = i.repo_id AND rc.user_id = c.author_id
29 AND rc.role = 'planner'))
30 ), '')`;
31
32 /** How many times the crew has parked this issue since it was last re-planned. */
33 export function parkedCount(db: Database, issueId: string): number {
34 const row = db
35 .query<{ n: number }, [string]>(
36 `SELECT COUNT(*) AS n FROM comment p WHERE p.issue_id = ?1 AND ${COUNTED}`,
37 )
38 .get(issueId);
39 return row?.n ?? 0;
40 }
41
42 /** Of these issues, the ones struck out: parked `PARK_STRIKES` times since the last word. */
43 export function struckOut(
44 db: Database,
45 issueIds: readonly string[],
46 ): Set<string> {
47 const out = new Set<string>();
48 const CHUNK = 400;
49 for (let i = 0; i < issueIds.length; i += CHUNK) {
50 const batch = issueIds.slice(i, i + CHUNK);
51 const marks = batch.map((_, n) => `?${n + 1}`).join(",");
52 const rows = db
53 .query<{ id: string }, string[]>(
54 `SELECT p.issue_id AS id FROM comment p
55 WHERE p.issue_id IN (${marks}) AND ${COUNTED}
56 GROUP BY p.issue_id HAVING COUNT(*) >= ${PARK_STRIKES}`,
57 )
58 .all(...batch);
59 for (const row of rows) out.add(row.id);
60 }
61 return out;
62 }
src/crew/runner.ts +15 −4
@@ -74,6 +74,7 @@ import {
7474 resumeAt,
7575 whenText,
7676 } from "./limits.ts";
77 import { interruptedSummary } from "./interrupted.ts";
7778 import { noteLimit } from "../core/standing.ts";
7879 import { pauseReason } from "./pacing.ts";
7980 import {
@@ -255,14 +256,18 @@ export async function executeAgentRun(
255256 if (paused) announceRun(app, paused);
256257 return { resumeAt: outcome.resumeAt, attempt: run.attempt + 1 };
257258 }
259 // A run the provider or a restart cut short says so, and when it goes
260 // on: interrupted, never a park (`interrupted.ts`).
261 const summary =
262 outcome.state === "failed"
263 ? interruptedSummary(outcome.summary, outcome.resumesAt)
264 : outcome.summary;
258265 write(
259 outcome.state === "ok"
260 ? `done: ${outcome.summary}`
261 : `failed: ${outcome.summary}`,
266 outcome.state === "ok" ? `done: ${summary}` : `failed: ${summary}`,
262267 );
263268 const done = finishRun(app.db, run.id, {
264269 state: outcome.state,
265 summary: outcome.summary,
270 summary,
266271 sessionId: outcome.sessionId,
267272 costUsd: outcome.costUsd,
268273 turns: outcome.turns,
@@ -349,6 +354,8 @@ type Outcome =
349354 | {
350355 readonly state: "ok" | "failed";
351356 readonly summary: string;
357 /** When a run the provider cut short can go on (`interrupted.ts`). */
358 readonly resumesAt?: Date | undefined;
352359 readonly sessionId?: string | undefined;
353360 readonly costUsd?: number | undefined;
354361 readonly turns?: number | undefined;
@@ -737,6 +744,7 @@ async function work(
737744 return {
738745 state: "failed",
739746 summary: `every subscription this repository may use is resting (${picked.said})`,
747 resumesAt: new Date(picked.until),
740748 };
741749 }
742750 if (picked.kind === "ready") {
@@ -857,6 +865,7 @@ async function work(
857865 // The provider's refusal, not the agent's failure: the run pauses with
858866 // its clone and session as they are, and goes on when the limit lifts.
859867 // A limit that never lifts is not waited on forever.
868 let limitLifts: Date | undefined;
860869 if (isRateLimited(result) && !signal.aborted) {
861870 const said = (
862871 result.result.split("\n").find((l) => l.trim() !== "") ?? result.reason
@@ -927,6 +936,7 @@ async function work(
927936 };
928937 }
929938 write(`paused ${MAX_PAUSES} times already; not waiting again`);
939 limitLifts = resumeAt(result.result, new Date());
930940 }
931941
932942 // The reserve: the same session gets one more turn, to land what exists
@@ -1004,6 +1014,7 @@ async function work(
10041014 return {
10051015 state: result.ok || landed.pushed ? "ok" : "failed",
10061016 summary: landed.summary + (result.ok ? "" : ` (${result.reason})`),
1017 resumesAt: limitLifts,
10071018 sessionId: result.sessionId,
10081019 costUsd: result.costUsd,
10091020 turns: result.turns,
src/web/pages/map.ts +2 −22
@@ -54,6 +54,7 @@ import { html, type Html } from "../html.ts";
5454 import { page } from "../layout.ts";
5555 import { avatar, hueOf, icon } from "../ui.ts";
5656 import { autonomyStatus, type AutonomyStatus } from "../../crew/autonomy.ts";
57 import { struckOut } from "../../crew/parks.ts";
5758 import { studioRecord, tellHref } from "../studio/nouns.ts";
5859
5960 /** Where a card's own link goes: dev's issue page, by default. */
@@ -432,27 +433,6 @@ export function crewMarks(runs: readonly Run[]): Map<string, RunState> {
432433 return marks;
433434 }
434435
435 /** Issues whose latest comment is the crew parking them. */
436 function parkedIds(db: Database, issueIds: readonly string[]): Set<string> {
437 const out = new Set<string>();
438 const CHUNK = 400;
439 for (let i = 0; i < issueIds.length; i += CHUNK) {
440 const batch = issueIds.slice(i, i + CHUNK);
441 const marks = batch.map((_, n) => `?${n + 1}`).join(",");
442 const rows = db
443 .query<{ id: string }, string[]>(
444 `SELECT c.issue_id AS id FROM comment c
445 JOIN (SELECT issue_id, MAX(created_at) AS at FROM comment
446 WHERE issue_id IN (${marks}) GROUP BY issue_id) last
447 ON last.issue_id = c.issue_id AND last.at = c.created_at
448 WHERE c.body LIKE 'Parked by the crew%'`,
449 )
450 .all(...batch);
451 for (const row of rows) out.add(row.id);
452 }
453 return out;
454 }
455
456436 /** The marks for one repository's graph: three reads, all bounded. */
457437 export function marksFor(
458438 db: Database,
@@ -467,7 +447,7 @@ export function marksFor(
467447 if (id !== undefined) waiting.add(id);
468448 }
469449 const open = graph.issues.filter((i) => i.state === "open").map((i) => i.id);
470 return { crew: crewMarks(runs), waiting, parked: parkedIds(db, open) };
450 return { crew: crewMarks(runs), waiting, parked: struckOut(db, open) };
471451 }
472452
473453 export const NO_MARKS: Marks = {
src/web/pages/thread.ts +18
@@ -48,6 +48,7 @@ import {
4848 } from "../ui.ts";
4949 import { loadRepo, notFoundPage, repoBase, signInFirst } from "../chrome.ts";
5050 import { studioRecord } from "../studio/nouns.ts";
51 import { interruptionsOn } from "../../crew/interrupted.ts";
5152
5253 /**
5354 * The four ways a write can fail *because of the origin*, in words.
@@ -166,6 +167,22 @@ function stateEvents(ctx: Ctx, repo: Repo, issue: Issue): Entry[] {
166167 return entries;
167168 }
168169
170 /**
171 * The crew's runs here that the provider or a restart cut short, each as
172 * its own line: `interrupted: provider limit, resumes 02:00 UTC`. Not a
173 * park and not a comment — a comment on an assigned issue starts a run.
174 */
175 function interruptionEntries(ctx: Ctx, issue: Issue): Entry[] {
176 return interruptionsOn(ctx.app.db, issue.id).map((run) => ({
177 at: run.at,
178 node: tlEvent({
179 icon: "clock",
180 tone: "attn",
181 body: html`<span>the crew's run was ${run.summary}</span> ${time(run.at)}`,
182 }),
183 }));
184 }
185
169186 export interface ConversationOptions {
170187 /** The verb on the opening card ("opened this", "wants to merge …"). */
171188 openVerb: Html | string;
@@ -201,6 +218,7 @@ export function conversation(
201218 }),
202219 }));
203220 entries.push(...stateEvents(ctx, repo, issue));
221 entries.push(...interruptionEntries(ctx, issue));
204222 entries.push(...(options.extra ?? []));
205223 entries.sort((a, b) => (a.at < b.at ? -1 : a.at > b.at ? 1 : 0));
206224
test/crew-parks.test.ts added +274
@@ -0,0 +1,274 @@
1 /**
2 * An interrupted run is not a park. A builder's run the provider's limit, an
3 * unreachable API or a restart ended writes no park note and is no strike;
4 * strikes count only since the last word from a person or the planner, and
5 * the map's `parked` is the same rule that stops dispatch.
6 */
7 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
8 import { Database } from "bun:sqlite";
9 import { mkdtemp, rm } from "node:fs/promises";
10 import { tmpdir } from "node:os";
11 import { join } from "node:path";
12 import { createApp, type App } from "../src/app.ts";
13 import { addToCrew, createAgent } from "../src/actions/crew.ts";
14 import { addComment, openIssue } from "../src/actions/issues.ts";
15 import { createRepo, updateRepoSettings } from "../src/actions/repos.ts";
16 import { setCrewRole } from "../src/core/agents.ts";
17 import { getIssueById, listComments, type Issue } from "../src/core/issues.ts";
18 import { getRepoById, type Repo } from "../src/core/repos.ts";
19 import { claimRun, createRun, finishRun } from "../src/core/runs.ts";
20 import { migrations } from "../src/core/schema.ts";
21 import { createUser, getUserByName, type User } from "../src/core/users.ts";
22 import { serve, type Servers } from "../src/http/server.ts";
23 import { driveRepo, parkedCount, settleAgentRun } from "../src/crew/autonomy.ts";
24 import { breakerTripped } from "../src/crew/breaker.ts";
25 import {
26 interruptedSummary,
27 interruptionOf,
28 } from "../src/crew/interrupted.ts";
29 import { struckOut } from "../src/crew/parks.ts";
30
31 describe("which runs were interrupted", () => {
32 test("the provider, the network and a restart; never the work itself", () => {
33 expect(
34 interruptionOf(
35 "every subscription this repository may use is resting (You've hit your weekly limit)",
36 ),
37 ).toBe("provider limit");
38 expect(interruptionOf("no changes pushed (api_error)")).toBe("provider limit");
39 expect(interruptionOf("the run failed: TypeError: fetch failed")).toBe(
40 "network error",
41 );
42 expect(interruptionOf("cancelled")).toBe("restart");
43 expect(interruptionOf("no changes pushed: DECLINED: out of scope")).toBe(
44 undefined,
45 );
46 expect(interruptionOf("declined: a rate limit is not mine to change")).toBe(
47 undefined,
48 );
49 expect(interruptionOf("the check stayed red")).toBe(undefined);
50 });
51
52 test("the summary says the cause and when it resumes", () => {
53 const said = interruptedSummary(
54 "every subscription this repository may use is resting (weekly limit)",
55 new Date("2026-10-03T02:00:00Z"),
56 );
57 expect(said).toStartWith("interrupted: provider limit, resumes 02:00 UTC — ");
58 expect(interruptionOf(said)).toBe("provider limit");
59 expect(interruptedSummary(said)).toBe(said);
60 expect(interruptedSummary("nothing to propose")).toBe("nothing to propose");
61 });
62 });
63
64 describe("an interrupted run is not a park", () => {
65 let dataDir: string;
66 let app: App;
67 let servers: Servers;
68 let alice: User;
69 let atlas: User;
70 let mason: User;
71 let repo: Repo;
72 const fresh = () => getRepoById(app.db, repo.id)!;
73 const tick = () => Bun.sleep(5);
74 const issue = (title: string): Issue => {
75 const made = openIssue(app, alice, fresh(), { title });
76 if (!made.ok) throw new Error(String(made.error));
77 return made.value;
78 };
79 /** A builder's run on the issue that ends without a pull, settled as the crew would. */
80 const ended = (on: Issue, summary: string): boolean => {
81 const run = createRun(app.db, {
82 repoId: repo.id,
83 kind: "agent",
84 issueId: on.id,
85 actorId: mason.id,
86 reason: "assigned",
87 });
88 claimRun(app.db, run.id);
89 const recorded = interruptedSummary(summary);
90 finishRun(app.db, run.id, { state: "failed", summary: recorded });
91 return settleAgentRun(app, fresh(), {
92 issueId: on.id,
93 state: "failed",
94 summary: recorded,
95 reason: "assigned",
96 actorId: mason.id,
97 });
98 };
99 const parks = (on: Issue) =>
100 listComments(app.db, on.id).filter((c) =>
101 c.body.startsWith("Parked by the crew"),
102 );
103
104 beforeAll(async () => {
105 dataDir = await mkdtemp(join(tmpdir(), "forge-parks-"));
106 app = createApp({ dataDir, port: 0 });
107 servers = await serve(app);
108 const a = await createUser(app.db, {
109 username: "alice",
110 email: "[email protected]",
111 password: "a-good-passphrase",
112 adminIfFirst: true,
113 });
114 if (!a.ok) throw new Error(a.error);
115 alice = a.value;
116 const made = await createRepo(app, alice, {
117 name: "engine",
118 visibility: "private",
119 });
120 if (!made.ok) throw new Error(String(made.error));
121 repo = made.value;
122 for (const name of ["atlas", "mason"]) {
123 const agent = await createAgent(app, alice, { username: name });
124 if (!agent.ok) throw new Error(String(agent.error));
125 expect(addToCrew(app, alice, repo, name).ok).toBe(true);
126 }
127 atlas = getUserByName(app.db, "atlas")!;
128 mason = getUserByName(app.db, "mason")!;
129 setCrewRole(app.db, repo.id, atlas.id, "planner");
130 const set = await updateRepoSettings(app, alice, repo, {
131 autonomy: "autonomous",
132 autonomyDirection: "Keep the engine small.",
133 autonomyCeilingUsd: 100,
134 autonomyBy: alice.id,
135 });
136 if (!set.ok) throw new Error(JSON.stringify(set.error));
137 });
138
139 afterAll(async () => {
140 servers.stop();
141 app.close();
142 await rm(dataDir, { recursive: true, force: true });
143 });
144
145 test("an outage run writes no park note, no strike, and the clock tries it again", async () => {
146 const work = issue("Shader hot reload");
147 for (const summary of [
148 "every subscription this repository may use is resting (You've hit your weekly limit)",
149 "the run failed: TypeError: fetch failed",
150 "cancelled",
151 ]) {
152 expect(ended(work, summary)).toBe(false);
153 await tick();
154 }
155 expect(parks(work)).toEqual([]);
156 expect(parkedCount(app, work.id)).toBe(0);
157 // Three outages inside the hour are not three failures.
158 expect(breakerTripped(app.db, work.id)).toBe(false);
159 // Not stalled: nobody needs to speak for it to be picked again.
160 expect(await driveRepo(app, fresh())).toBe("picked");
161 expect(getIssueById(app.db, work.id)!.assigneeId).toBe(mason.id);
162 });
163
164 test("the issue page says the run was interrupted, and when it resumes", async () => {
165 const work = issue("Input mapping");
166 const run = createRun(app.db, {
167 repoId: repo.id,
168 kind: "agent",
169 issueId: work.id,
170 actorId: mason.id,
171 reason: "assigned",
172 });
173 claimRun(app.db, run.id);
174 finishRun(app.db, run.id, {
175 state: "failed",
176 summary: interruptedSummary(
177 "every subscription this repository may use is resting (weekly limit)",
178 new Date("2026-10-03T02:00:00Z"),
179 ),
180 });
181 const cookie = (
182 (
183 await fetch(`${servers.url}/signin`, {
184 method: "POST",
185 body: new URLSearchParams({
186 username: "alice",
187 password: "a-good-passphrase",
188 }),
189 redirect: "manual",
190 })
191 ).headers.get("set-cookie") ?? ""
192 ).split(";")[0]!;
193 const page = await (
194 await fetch(`${servers.url}/alice/engine/issues/${work.number}`, {
195 headers: { cookie },
196 })
197 ).text();
198 expect(page).toContain("interrupted: provider limit, resumes 02:00 UTC");
199 });
200
201 test("two real parks strike it out, on the map as in dispatch", async () => {
202 const work = issue("Scene graph");
203 expect(ended(work, "no changes pushed: DECLINED: out of scope")).toBe(true);
204 await tick();
205 expect(parkedCount(app, work.id)).toBe(1);
206 expect(struckOut(app.db, [work.id]).has(work.id)).toBe(false);
207 expect(ended(work, "nothing to propose")).toBe(true);
208 expect(parkedCount(app, work.id)).toBe(2);
209 expect(struckOut(app.db, [work.id]).has(work.id)).toBe(true);
210 });
211
212 test("a word from the planner or a person counts again from zero; a builder's does not", async () => {
213 const work = issue("Physics step");
214 ended(work, "nothing to propose");
215 await tick();
216 ended(work, "nothing to propose");
217 await tick();
218 expect(parkedCount(app, work.id)).toBe(2);
219 // The builder's own report is not a re-plan.
220 addComment(app, mason, fresh(), getIssueById(app.db, work.id)!, "Tried twice.");
221 await tick();
222 expect(parkedCount(app, work.id)).toBe(2);
223 addComment(
224 app,
225 atlas,
226 fresh(),
227 getIssueById(app.db, work.id)!,
228 "Split: only the fixed step here.",
229 );
230 await tick();
231 expect(parkedCount(app, work.id)).toBe(0);
232 expect(struckOut(app.db, [work.id]).has(work.id)).toBe(false);
233 ended(work, "nothing to propose");
234 await tick();
235 expect(parkedCount(app, work.id)).toBe(1);
236 ended(work, "nothing to propose");
237 await tick();
238 addComment(app, alice, fresh(), getIssueById(app.db, work.id)!, "Try the fixed step.");
239 expect(parkedCount(app, work.id)).toBe(0);
240 });
241 });
242
243 describe("migration 43", () => {
244 test("rewords old outage parks so they no longer count, and leaves real parks", () => {
245 const db = new Database(":memory:");
246 db.exec(
247 "CREATE TABLE comment (id TEXT PRIMARY KEY, issue_id TEXT, author_id TEXT, body TEXT)",
248 );
249 const bodies = [
250 "Parked by the crew: the run ended without a pull: every subscription this repository may use is resting (You've hit your weekly limit)",
251 "Parked by the crew: the run ended without a pull: the run failed: Error: API unreachable",
252 "Parked by the crew: the run ended without a pull: cancelled",
253 "Parked by the crew: the run ended without a pull: nothing to propose",
254 "Parked by the crew: the check stayed red past its tries.",
255 ];
256 bodies.forEach((body, n) =>
257 db
258 .query("INSERT INTO comment VALUES (?1, 'i', 'u', ?2)")
259 .run(String(n), body),
260 );
261 db.exec(migrations[42]!);
262 const after = db
263 .query<{ body: string }, []>("SELECT body FROM comment ORDER BY id")
264 .all()
265 .map((r) => r.body);
266 expect(after[0]).toBe(
267 "Interrupted, not parked: the run ended without a pull: every subscription this repository may use is resting (You've hit your weekly limit)",
268 );
269 expect(after[1]).toStartWith("Interrupted, not parked: ");
270 expect(after[2]).toStartWith("Interrupted, not parked: ");
271 expect(after[3]).toBe(bodies[3]!);
272 expect(after[4]).toBe(bodies[4]!);
273 });
274 });
test/map.test.ts +10 −6
@@ -207,13 +207,17 @@ describe("the map, per app and across the forge", () => {
207207 });
208208 return { repo: made.value, issues };
209209 };
210 // Struck out: parked twice since anyone re-planned it, the rule that
211 // stops dispatch (`crew/parks.ts`).
210212 const park = (by: User, issue: Issue): void => {
211 const said = addComment(app.db, {
212 issueId: issue.id,
213 authorId: by.id,
214 body: "Parked by the crew: the check stayed red past its tries.",
215 });
216 if (!said.ok) throw new Error(said.error);
213 for (let n = 0; n < 2; n++) {
214 const said = addComment(app.db, {
215 issueId: issue.id,
216 authorId: by.id,
217 body: "Parked by the crew: the check stayed red past its tries.",
218 });
219 if (!said.ok) throw new Error(said.error);
220 }
217221 };
218222
219223 beforeAll(async () => {

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs