trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRuns19g rEventsFlows

Usage and "limited until" reach the meter from the runner, not one CLI's event format #12255

Merged scoutcrew opened this

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

scout/forge:scout/12123-usage-and-limited-until-reach-the-meter → main 12 commits fork

  • Merged. scout/12123-usage-and-limited-until-reach-the-meter was merged into main as 5e7033f.

Review

Sign in to review this change.
10 files changed +322 −213
scripts/unused-exports-allow.json +1
@@ -56,6 +56,7 @@
5656 "src/core/usage.ts USAGE_BY",
5757 "src/core/usage.ts USAGE_OUTCOMES",
5858 "src/core/webhooks.ts deliveryBusy",
59 "src/crew/agent_runner.ts useRunner",
5960 "src/crew/ceiling_stagger.ts forgetStagger",
6061 "src/crew/conversation_confirm.ts CONFIRM_TTL_MS",
6162 "src/crew/first_look.ts useLookModel",
src/crew/agent_runner.ts +31 −2
@@ -79,7 +79,21 @@ export interface AgentRun {
7979 export interface Spent {
8080 readonly tokensIn: number;
8181 readonly tokensOut: number;
82 readonly costUsd: number;
82 /** Absent until the runner knows it (Claude reports it only at the end). */
83 readonly costUsd?: number | undefined;
84 }
85
86 /**
87 * The provider turned the run away. `limitedUntil` is the provider's own
88 * instant (ISO, no slack added) when it named one; a limit with no time is
89 * waited out by default. `window` says which allowance is full when known.
90 * What a CLI's rate-limit events look like never crosses the seam.
91 */
92 export interface RunnerLimit {
93 readonly limitedUntil?: string | undefined;
94 readonly window?: LimitWindow | undefined;
95 /** One line a person can read about it. */
96 readonly said?: string | undefined;
8397 }
8498
8599 export interface AgentOutcome {
@@ -102,6 +116,14 @@ export interface AgentOutcome {
102116 readonly contextWindow: number;
103117 /** `completed`, `max_budget_reached`, `timeout`, `cancelled`, `error`, `exit <n>`. */
104118 readonly reason: string;
119 /**
120 * Set when the end was the provider's refusal, not the agent's failure.
121 * A runner that can tell must set it: the crew pauses a run only on this,
122 * and reads no provider's wording itself (Claude Code's is `claude.ts`'s
123 * `limitedBy`). A runner that never sets it has its refusals judged as
124 * failures.
125 */
126 readonly limit?: RunnerLimit | undefined;
105127 }
106128
107129
@@ -131,9 +153,16 @@ const claude: AgentRunner = {
131153 detect: (binary) => detectClaude(binary),
132154 };
133155
156 let chosen: AgentRunner | undefined;
157
134158 /** The runner every run uses today. */
135159 export function currentRunner(): AgentRunner {
136 return claude;
160 return chosen ?? claude;
161 }
162
163 /** Swap the runner (tests drive a fake one through the crew); undefined restores Claude Code. */
164 export function useRunner(runner: AgentRunner | undefined): void {
165 chosen = runner;
137166 }
138167
139168 /** The forge tool names a brief should say for `runner` (#12128); the current runner by default. */
src/crew/claude.ts +154 −1
@@ -18,6 +18,8 @@ import { oomFirst } from "../apps/oom.ts";
1818 import { cgroupHost } from "../apps/cgroup.ts";
1919 import { openRunCell } from "./run-cell.ts";
2020 import { watchLabelled } from "../apps/contain.ts";
21 import { isNetworkError } from "./network.ts";
22 import type { LimitWindow } from "../core/standing.ts";
2123 import { childEnv, pump } from "../apps/process.ts";
2224 import { endedLine, liveTree } from "../runs/ended.ts";
2325 import { CANCEL_GRACE_MS } from "../runs/grace.ts";
@@ -28,6 +30,7 @@ import type {
2830 AgentOutcome,
2931 AgentRun,
3032 BUDGET_REACHED as SeamBudgetReached,
33 RunnerLimit,
3134 Spent,
3235 } from "./agent_runner.ts";
3336
@@ -138,6 +141,16 @@ export function claudeArgs(input: ClaudeInput): string[] {
138141
139142 /** Run Claude Code to completion. Never throws; every ending is an outcome. */
140143 export async function runClaude(input: ClaudeInput): Promise<ClaudeOutcome> {
144 return withLimit(await runClaudeRaw(input), new Date());
145 }
146
147 /** The outcome, marked with the provider's limit when that is how it ended. */
148 export function withLimit(outcome: ClaudeOutcome, now: Date): ClaudeOutcome {
149 const limit = limitedBy(outcome, now);
150 return limit === undefined ? outcome : { ...outcome, limit };
151 }
152
153 async function runClaudeRaw(input: ClaudeInput): Promise<ClaudeOutcome> {
141154 // Background work, with everything its shell starts: the forge's own
142155 // pages outrank it (#3319, `apps/background.ts`). A binary that is not
143156 // there is spawned bare, so it fails as itself (ENOENT, caught by the
@@ -339,7 +352,7 @@ export async function runClaude(input: ClaudeInput): Promise<ClaudeOutcome> {
339352 tokensIn += t.tokensIn;
340353 tokensOut += t.tokensOut;
341354 }
342 input.onUsage({ tokensIn, tokensOut, costUsd: 0 });
355 input.onUsage({ tokensIn, tokensOut });
343356 }
344357 }
345358 }
@@ -624,3 +637,143 @@ export async function detectClaude(
624637 };
625638 }
626639 }
640
641 /**
642 * Whether Claude Code's ending is the provider's refusal rather than the
643 * agent's failure, and what it said about when it lifts. It ends such a run
644 * with `api_error`; the message names the limit, and is checked too in case
645 * the ending is named differently. The seam carries only the result:
646 * `{ limitedUntil, window }`.
647 */
648 export function limitedBy(outcome: AgentOutcome, now: Date): RunnerLimit | undefined {
649 if (outcome.ok || isNetworkError(outcome)) return undefined;
650 const message = outcome.result;
651 const refused =
652 outcome.reason === "api_error" ||
653 /rate[ _-]?limit|session limit|usage limit|weekly limit|hit your \w+ limit|overloaded|too many requests|\b(?:429|529)\b/i.test(
654 message,
655 );
656 if (!refused) return undefined;
657 return limitFromWords(message, now, outcome.reason);
658 }
659
660 /** What a refusal's words say about when it lifts, as the seam's limit. */
661 export function limitFromWords(message: string, now: Date, fallback = ""): RunnerLimit {
662 const said = (message.split("\n").find((l) => l.trim() !== "") ?? fallback).slice(0, 160);
663 const match = RESET.exec(message);
664 const at = resetOf(message, now);
665 if (match === null || at === undefined) return { said };
666 // The week is named (`weekly limit`) or dated (`resets Sep 26 at 9pm`); a
667 // bare clock time is the 5-hour session window.
668 const weekly = match[1] !== undefined || /\bweek(?:ly)?\b/i.test(message);
669 const window: LimitWindow = weekly ? "week" : "5h";
670 return { limitedUntil: at.toISOString(), window, said };
671 }
672
673 const MONTHS = [
674 "jan",
675 "feb",
676 "mar",
677 "apr",
678 "may",
679 "jun",
680 "jul",
681 "aug",
682 "sep",
683 "oct",
684 "nov",
685 "dec",
686 ];
687
688 /**
689 * Claude Code says `resets 7:20am (UTC)` or `resets 2:20am (America/Chicago)`
690 * for a session's window, and `resets Sep 26 at 9pm (America/Chicago)` for
691 * the week's: that clock time in that zone, on that date or the next
692 * occurrence.
693 */
694 const RESET =
695 /resets?\s+(?:at\s+)?(?:([a-z]{3,9})\.?\s+(\d{1,2}),?\s+(?:at\s+)?)?(\d{1,2})(?::(\d{2}))?\s*(am|pm)\s*\(([^)]+)\)/i;
696
697 /** The reset instant a message names, exactly; undefined when it names none. */
698 function resetOf(message: string, now: Date): Date | undefined {
699 const match = RESET.exec(message);
700 if (match === null) return undefined;
701 const [, monthName, day, h, m, meridiem, zone] = match;
702 const hour = (Number(h) % 12) + (meridiem!.toLowerCase() === "pm" ? 12 : 0);
703 const minute = Number(m ?? "0");
704 const month =
705 monthName === undefined
706 ? -1
707 : MONTHS.indexOf(monthName.slice(0, 3).toLowerCase());
708 const date =
709 monthName !== undefined && month >= 0
710 ? { month: month + 1, day: Number(day) }
711 : undefined;
712 return nextWallTime(now, hour, minute, zone!, date);
713 }
714
715 /**
716 * The next instant at which a clock in `zone` reads `hour:minute`, strictly
717 * after `now`. The zone's offset is taken from the runtime's own tables,
718 * so daylight time is right without a list of rules here.
719 */
720 function nextWallTime(
721 now: Date,
722 hour: number,
723 minute: number,
724 zone: string,
725 date?: { month: number; day: number },
726 ): Date | undefined {
727 let parts: Intl.DateTimeFormat;
728 try {
729 parts = new Intl.DateTimeFormat("en-US", {
730 timeZone: zone,
731 hourCycle: "h23",
732 year: "numeric",
733 month: "numeric",
734 day: "numeric",
735 hour: "numeric",
736 minute: "numeric",
737 second: "numeric",
738 });
739 } catch {
740 return undefined; // not a zone the runtime knows
741 }
742 const wall = (date: Date): Record<string, number> => {
743 const out: Record<string, number> = {};
744 for (const p of parts.formatToParts(date))
745 if (p.type !== "literal") out[p.type] = Number(p.value);
746 return out;
747 };
748 // The instant whose wall clock in `zone` is `y-m-d hour:minute`: start
749 // from the UTC reading and correct by the zone's offset at that instant.
750 const instantOf = (y: number, mo: number, d: number): Date => {
751 let guess = new Date(Date.UTC(y, mo - 1, d, hour, minute));
752 for (let i = 0; i < 2; i++) {
753 const w = wall(guess);
754 const asUtc = Date.UTC(
755 w["year"]!,
756 w["month"]! - 1,
757 w["day"]!,
758 w["hour"]!,
759 w["minute"]!,
760 w["second"]!,
761 );
762 const want = Date.UTC(y, mo - 1, d, hour, minute);
763 guess = new Date(guess.getTime() + (want - asUtc));
764 }
765 return guess;
766 };
767 const today = wall(now);
768 if (date !== undefined) {
769 // A named date this year, or next year's if it has already passed.
770 const thisYear = instantOf(today["year"]!, date.month, date.day);
771 if (thisYear.getTime() > now.getTime()) return thisYear;
772 return instantOf(today["year"]! + 1, date.month, date.day);
773 }
774 const candidate = instantOf(today["year"]!, today["month"]!, today["day"]!);
775 if (candidate.getTime() > now.getTime()) return candidate;
776 const tomorrow = new Date(candidate.getTime() + 36 * 3_600_000);
777 const t = wall(tomorrow);
778 return instantOf(t["year"]!, t["month"]!, t["day"]!);
779 }
src/crew/conversation.ts +4 −4
@@ -38,7 +38,7 @@ import { redactValues, releaseValues } from "../kernel/log.ts";
3838 import { running } from "../kernel/version.ts";
3939 import { err, ok, type Result } from "../kernel/result.ts";
4040 import { appendRunLog } from "../runs/log.ts";
41 import { runAgent, type AgentOutcome } from "./agent_runner.ts";
41 import { runAgent, type AgentOutcome, type RunnerLimit } from "./agent_runner.ts";
4242 import { CONVERSATION_BRIEF } from "./conversation_brief.ts";
4343 import { isRateLimited, takeLimit, whenText } from "./limits.ts";
4444 import { credentialEnv, holdModelCredential, listenForSteer, steerRun } from "./runner.ts";
@@ -443,9 +443,9 @@ export function resumeInterrupted(app: App, options: ConversationOptions): Promi
443443 * never the provider's words. The forge's own token cannot rest, so the next
444444 * message is accepted; the line says the window is full, not that anything rests.
445445 */
446 function limitLine(app: App, picked: Pick, message: string): string {
446 function limitLine(app: App, picked: Pick, limit: RunnerLimit): string {
447447 const account = picked.kind === "ready" ? picked.subscription : undefined;
448 const { until } = takeLimit(app.db, account?.id, message, "", new Date());
448 const { until } = takeLimit(app.db, account?.id, limit, "", new Date());
449449 return account === undefined
450450 ? `The forge's own model window is full until ${whenText(until)}; try again then.`
451451 : `${account.name} is resting until ${whenText(until)}.`;
@@ -617,7 +617,7 @@ function begin(
617617 if (outcome === undefined) couldNot(failureLine("error"), () => false);
618618 else if (isRateLimited(outcome)) {
619619 const first = outcome.result.split("\n").find((l) => l.trim() !== "")?.trim() ?? "";
620 couldNot(limitLine(app, picked, outcome.result), (said) => said === outcome!.result || (first !== "" && said.includes(first)));
620 couldNot(limitLine(app, picked, outcome.limit), (said) => said === outcome!.result || (first !== "" && said.includes(first)));
621621 } else if (!outcome.ok) couldNot(failureLine(outcome.reason), (said) => said === outcome!.result);
622622 else if (answer.trim() !== "" && answer !== lastText) put("agent", answer);
623623 else if (answer.trim() === "" && lastText === "") couldNot(failureLine("error"), () => false);
src/crew/limits.ts +30 −146
@@ -8,8 +8,7 @@
88 import type { Database } from "bun:sqlite";
99 import { noteLimit, type LimitWindow } from "../core/standing.ts";
1010 import { PLATFORM, restSubscription } from "../core/subscriptions.ts";
11 import { isNetworkError } from "./network.ts";
12 import type { AgentOutcome } from "./agent_runner.ts";
11 import type { AgentOutcome, RunnerLimit } from "./agent_runner.ts";
1312
1413 /** How many times one task may be paused before the forge gives up on it. */
1514 export const MAX_PAUSES = 6;
@@ -21,83 +20,34 @@ const SLACK_MS = 60_000;
2120
2221 /**
2322 * Whether an outcome is the provider's refusal rather than the agent's
24 * failure. Claude Code ends such a run with `api_error`; the message names
25 * the limit, and is checked too in case the ending is named differently.
23 * failure: the runner says so with `limit` on the outcome. How a CLI words
24 * its refusals is that runner's business (`claude.ts`), not read here.
2625 */
27 export function isRateLimited(outcome: AgentOutcome): boolean {
28 if (outcome.ok || isNetworkError(outcome)) return false;
29 return (
30 outcome.reason === "api_error" ||
31 /rate[ _-]?limit|session limit|usage limit|weekly limit|hit your \w+ limit|overloaded|too many requests|\b(?:429|529)\b/i.test(
32 outcome.result,
33 )
34 );
35 }
36
37 const MONTHS = [
38 "jan",
39 "feb",
40 "mar",
41 "apr",
42 "may",
43 "jun",
44 "jul",
45 "aug",
46 "sep",
47 "oct",
48 "nov",
49 "dec",
50 ];
51
52 /**
53 * When to try again. Claude Code says `resets 7:20am (UTC)` or
54 * `resets 2:20am (America/Chicago)` for a session's window, and
55 * `resets Sep 26 at 9pm (America/Chicago)` for the week's; that clock time
56 * in that zone, on that date or the next occurrence, plus a minute.
57 * Anything else waits the default.
58 */
59 export function resumeAt(message: string, now: Date): Date {
60 const at = resetOf(message, now);
61 if (at === undefined) return new Date(now.getTime() + DEFAULT_WAIT_MS);
62 return new Date(at.getTime() + SLACK_MS);
26 export function isRateLimited(outcome: AgentOutcome): outcome is AgentOutcome & { readonly limit: RunnerLimit } {
27 return !outcome.ok && outcome.limit !== undefined;
6328 }
6429
6530 /**
66 * Which window the provider said is full, and the instant it resets — read
67 * from its own words, or undefined when it named no time. The week is named
68 * (`weekly limit`) or dated (`resets Sep 26 at 9pm`); a bare clock time is
69 * the 5-hour session window. No slack is added: this is the provider's time.
31 * When to try again: the runner's `limitedUntil` plus a minute. A limit
32 * that names no time waits the default.
33 */
34 export function resumeAt(limit: RunnerLimit | undefined, now: Date): Date {
35 const at = limit?.limitedUntil === undefined ? NaN : Date.parse(limit.limitedUntil);
36 if (Number.isNaN(at)) return new Date(now.getTime() + DEFAULT_WAIT_MS);
37 return new Date(at + SLACK_MS);
38 }
39
40 /**
41 * Which window the runner said is full, and the instant it resets; undefined
42 * when it named no time. No slack is added: this is the provider's time.
7043 */
7144 export function limitOf(
72 message: string,
73 now: Date,
45 limit: RunnerLimit | undefined,
7446 ): { window: LimitWindow; resetsAt: Date } | undefined {
75 const match = RESET.exec(message);
76 const resetsAt = resetOf(message, now);
77 if (match === null || resetsAt === undefined) return undefined;
78 const weekly = match[1] !== undefined || /\bweek(?:ly)?\b/i.test(message);
79 return { window: weekly ? "week" : "5h", resetsAt };
80 }
81
82 const RESET =
83 /resets?\s+(?:at\s+)?(?:([a-z]{3,9})\.?\s+(\d{1,2}),?\s+(?:at\s+)?)?(\d{1,2})(?::(\d{2}))?\s*(am|pm)\s*\(([^)]+)\)/i;
84
85 /** The reset instant a message names, exactly; undefined when it names none. */
86 function resetOf(message: string, now: Date): Date | undefined {
87 const match = RESET.exec(message);
88 if (match === null) return undefined;
89 const [, monthName, day, h, m, meridiem, zone] = match;
90 const hour = (Number(h) % 12) + (meridiem!.toLowerCase() === "pm" ? 12 : 0);
91 const minute = Number(m ?? "0");
92 const month =
93 monthName === undefined
94 ? -1
95 : MONTHS.indexOf(monthName.slice(0, 3).toLowerCase());
96 const date =
97 monthName !== undefined && month >= 0
98 ? { month: month + 1, day: Number(day) }
99 : undefined;
100 return nextWallTime(now, hour, minute, zone!, date);
47 if (limit?.limitedUntil === undefined) return undefined;
48 const resetsAt = new Date(limit.limitedUntil);
49 if (Number.isNaN(resetsAt.getTime())) return undefined;
50 return { window: limit.window ?? "5h", resetsAt };
10151 }
10252
10353 /**
@@ -110,91 +60,25 @@ function resetOf(message: string, now: Date): Date | undefined {
11060 export function takeLimit(
11161 db: Database,
11262 subscriptionId: string | undefined,
113 message: string,
63 limit: RunnerLimit,
11464 fallback: string,
11565 now: Date,
11666 ): { until: Date; limit: ReturnType<typeof limitOf>; said: string } {
117 const said = (message.split("\n").find((l) => l.trim() !== "") ?? fallback).slice(0, 160);
118 const until = resumeAt(message, now);
119 const limit = limitOf(message, now);
120 if (limit !== undefined)
67 const said = (limit.said ?? fallback).slice(0, 160);
68 const until = resumeAt(limit, now);
69 const named = limitOf(limit);
70 if (named !== undefined)
12171 noteLimit(db, subscriptionId ?? PLATFORM, {
122 window: limit.window,
123 resetsAt: limit.resetsAt.toISOString(),
72 window: named.window,
73 resetsAt: named.resetsAt.toISOString(),
12474 observedAt: now.toISOString(),
12575 said,
12676 });
12777 if (subscriptionId !== undefined) restSubscription(db, subscriptionId, until.toISOString(), said);
128 return { until, limit, said };
78 return { until, limit: named, said };
12979 }
13080
13181 /** `12:04 UTC` — how a resume time reads on a card. */
13282 export function whenText(at: Date): string {
13383 return `${at.toISOString().slice(11, 16)} UTC`;
13484 }
135
136 /**
137 * The next instant at which a clock in `zone` reads `hour:minute`, strictly
138 * after `now`. The zone's offset is taken from the runtime's own tables,
139 * so daylight time is right without a list of rules here.
140 */
141 function nextWallTime(
142 now: Date,
143 hour: number,
144 minute: number,
145 zone: string,
146 date?: { month: number; day: number },
147 ): Date | undefined {
148 let parts: Intl.DateTimeFormat;
149 try {
150 parts = new Intl.DateTimeFormat("en-US", {
151 timeZone: zone,
152 hourCycle: "h23",
153 year: "numeric",
154 month: "numeric",
155 day: "numeric",
156 hour: "numeric",
157 minute: "numeric",
158 second: "numeric",
159 });
160 } catch {
161 return undefined; // not a zone the runtime knows
162 }
163 const wall = (date: Date): Record<string, number> => {
164 const out: Record<string, number> = {};
165 for (const p of parts.formatToParts(date))
166 if (p.type !== "literal") out[p.type] = Number(p.value);
167 return out;
168 };
169 // The instant whose wall clock in `zone` is `y-m-d hour:minute`: start
170 // from the UTC reading and correct by the zone's offset at that instant.
171 const instantOf = (y: number, mo: number, d: number): Date => {
172 let guess = new Date(Date.UTC(y, mo - 1, d, hour, minute));
173 for (let i = 0; i < 2; i++) {
174 const w = wall(guess);
175 const asUtc = Date.UTC(
176 w["year"]!,
177 w["month"]! - 1,
178 w["day"]!,
179 w["hour"]!,
180 w["minute"]!,
181 w["second"]!,
182 );
183 const want = Date.UTC(y, mo - 1, d, hour, minute);
184 guess = new Date(guess.getTime() + (want - asUtc));
185 }
186 return guess;
187 };
188 const today = wall(now);
189 if (date !== undefined) {
190 // A named date this year, or next year's if it has already passed.
191 const thisYear = instantOf(today["year"]!, date.month, date.day);
192 if (thisYear.getTime() > now.getTime()) return thisYear;
193 return instantOf(today["year"]! + 1, date.month, date.day);
194 }
195 const candidate = instantOf(today["year"]!, today["month"]!, today["day"]!);
196 if (candidate.getTime() > now.getTime()) return candidate;
197 const tomorrow = new Date(candidate.getTime() + 36 * 3_600_000);
198 const t = wall(tomorrow);
199 return instantOf(t["year"]!, t["month"]!, t["day"]!);
200 }
src/crew/runner.ts +7 −7
@@ -1588,16 +1588,16 @@ async function work(
15881588 {
15891589 tokensIn: meter.tokensIn + so.tokensIn,
15901590 tokensOut: meter.tokensOut + so.tokensOut,
1591 costUsd: meter.costUsd + so.costUsd,
1591 costUsd: meter.costUsd + (so.costUsd ?? 0),
15921592 },
15931593 started,
1594 so.costUsd > 0,
1594 (so.costUsd ?? 0) > 0,
15951595 ),
15961596 }),
15971597 );
15981598 meter.tokensIn += spent.tokensIn;
15991599 meter.tokensOut += spent.tokensOut;
1600 meter.costUsd += spent.costUsd;
1600 meter.costUsd += spent.costUsd ?? 0;
16011601 return spent;
16021602 };
16031603 write(
@@ -1741,7 +1741,7 @@ async function work(
17411741 const { until, limit, said } = takeLimit(
17421742 app.db,
17431743 picked.kind === "ready" ? picked.subscription.id : undefined,
1744 result.result,
1744 result.limit,
17451745 result.reason,
17461746 now,
17471747 );
@@ -1772,18 +1772,18 @@ async function work(
17721772 }
17731773 }
17741774 if (run.attempt < MAX_PAUSES) {
1775 const at = resumeAt(result.result, new Date());
1775 const at = resumeAt(result.limit, new Date());
17761776 pausing = true;
17771777 return {
17781778 state: "paused",
1779 summary: `paused: ${pauseReason(picked.kind === "ready" ? picked.subscription.name : undefined, limitOf(result.result, new Date()), said)}; resumes ${whenText(at)}`,
1779 summary: `paused: ${pauseReason(picked.kind === "ready" ? picked.subscription.name : undefined, limitOf(result.limit), said)}; resumes ${whenText(at)}`,
17801780 resumeAt: at.toISOString(),
17811781 sessionId: result.sessionId,
17821782 costUsd: result.costUsd,
17831783 };
17841784 }
17851785 write(`paused ${MAX_PAUSES} times already; not waiting again`);
1786 limitLifts = resumeAt(result.result, new Date());
1786 limitLifts = resumeAt(result.limit, new Date());
17871787 }
17881788
17891789 watch.lap("harness");
test/crew-limits.test.ts +69 −33
@@ -5,7 +5,15 @@
55
66 import { describe, expect, test } from "bun:test";
77 import type { ClaudeOutcome } from "../src/crew/claude.ts";
8 import { isRateLimited, resumeAt, whenText } from "../src/crew/limits.ts";
8 import { limitFromWords, withLimit } from "../src/crew/claude.ts";
9 import { useRunner, type AgentRunner, type RunnerLimit } from "../src/crew/agent_runner.ts";
10 import { assignIssue, openIssue } from "../src/actions/issues.ts";
11 import { listRuns } from "../src/core/runs.ts";
12 import { agentRunSettled, startStandInCrew, stopStandInCrew } from "./helpers/crew-standin.ts";
13 import { isRateLimited, limitOf, resumeAt, whenText } from "../src/crew/limits.ts";
14
15 /** What the Claude runner reports for a refusal's words. */
16 const limitOfWords = (words: string, now: Date): RunnerLimit => limitFromWords(words, now);
917
1018 const outcome = (over: Partial<ClaudeOutcome>): ClaudeOutcome => ({
1119 ok: false,
@@ -23,68 +31,55 @@ const outcome = (over: Partial<ClaudeOutcome>): ClaudeOutcome => ({
2331
2432 describe("the provider's limit", () => {
2533 test("is told apart from the agent's own failure", () => {
26 expect(isRateLimited(outcome({ reason: "api_error" }))).toBe(true);
34 const rated = (o: ClaudeOutcome) => isRateLimited(withLimit(o, new Date()));
35 expect(rated(outcome({ reason: "api_error" }))).toBe(true);
2736 expect(
28 isRateLimited(
37 rated(
2938 outcome({
3039 result: "You've hit your session limit · resets 7:20am (UTC)",
3140 }),
3241 ),
3342 ).toBe(true);
34 expect(isRateLimited(outcome({ result: "Overloaded" }))).toBe(true);
35 expect(isRateLimited(outcome({ result: "429 Too Many Requests" }))).toBe(
43 expect(rated(outcome({ result: "Overloaded" }))).toBe(true);
44 expect(rated(outcome({ result: "429 Too Many Requests" }))).toBe(
3645 true,
3746 );
3847 expect(
39 isRateLimited(outcome({ result: "TypeError: x is not a function" })),
48 rated(outcome({ result: "TypeError: x is not a function" })),
4049 ).toBe(false);
41 expect(isRateLimited(outcome({ ok: true, reason: "completed" }))).toBe(
50 expect(rated(outcome({ ok: true, reason: "completed" }))).toBe(
4251 false,
4352 );
4453 });
4554
4655 test("resumes a minute after the reset the message names, in UTC", () => {
4756 const said = "You've hit your session limit · resets 7:20am (UTC)";
48 expect(resumeAt(said, new Date("2026-09-08T06:00:00Z")).toISOString()).toBe(
49 "2026-09-08T07:21:00.000Z",
50 );
57 const now = new Date("2026-09-08T06:00:00Z");
58 const later = new Date("2026-09-08T08:00:00Z");
59 expect(resumeAt(limitOfWords(said, now), now).toISOString()).toBe("2026-09-08T07:21:00.000Z");
5160 // Already past today's: tomorrow's.
52 expect(resumeAt(said, new Date("2026-09-08T08:00:00Z")).toISOString()).toBe(
53 "2026-09-09T07:21:00.000Z",
54 );
55 expect(
56 resumeAt(
57 "resets 2:20pm (UTC)",
58 new Date("2026-09-08T06:00:00Z"),
59 ).toISOString(),
60 ).toBe("2026-09-08T14:21:00.000Z");
61 expect(
62 resumeAt(
63 "resets 12:05am (UTC)",
64 new Date("2026-09-08T06:00:00Z"),
65 ).toISOString(),
66 ).toBe("2026-09-09T00:06:00.000Z");
61 expect(resumeAt(limitOfWords(said, later), later).toISOString()).toBe("2026-09-09T07:21:00.000Z");
62 expect(resumeAt(limitOfWords("resets 2:20pm (UTC)", now), now).toISOString()).toBe("2026-09-08T14:21:00.000Z");
63 expect(resumeAt(limitOfWords("resets 12:05am (UTC)", now), now).toISOString()).toBe("2026-09-09T00:06:00.000Z");
6764 });
6865
6966 test("honours the zone, daylight time included", () => {
7067 const said =
7168 "You've hit your session limit · resets 2:20am (America/Chicago)";
7269 // September: CDT, UTC−5.
73 expect(resumeAt(said, new Date("2026-09-08T06:00:00Z")).toISOString()).toBe(
74 "2026-09-08T07:21:00.000Z",
75 );
70 const september = new Date("2026-09-08T06:00:00Z");
71 expect(resumeAt(limitOfWords(said, september), september).toISOString()).toBe("2026-09-08T07:21:00.000Z");
7672 // January: CST, UTC−6.
77 expect(resumeAt(said, new Date("2026-01-10T06:00:00Z")).toISOString()).toBe(
78 "2026-01-10T08:21:00.000Z",
79 );
73 const january = new Date("2026-01-10T06:00:00Z");
74 expect(resumeAt(limitOfWords(said, january), january).toISOString()).toBe("2026-01-10T08:21:00.000Z");
8075 });
8176
8277 test("waits fifteen minutes when no reset is named or the zone is unknown", () => {
8378 const now = new Date("2026-09-08T06:00:00Z");
84 expect(resumeAt("Overloaded", now).toISOString()).toBe(
79 expect(resumeAt(limitOfWords("Overloaded", now), now).toISOString()).toBe(
8580 "2026-09-08T06:15:00.000Z",
8681 );
87 expect(resumeAt("resets 7:20am (Mars/Olympus)", now).toISOString()).toBe(
82 expect(resumeAt(limitOfWords("resets 7:20am (Mars/Olympus)", now), now).toISOString()).toBe(
8883 "2026-09-08T06:15:00.000Z",
8984 );
9085 });
@@ -92,4 +87,45 @@ describe("the provider's limit", () => {
9287 test("reads as a clock time on a card", () => {
9388 expect(whenText(new Date("2026-09-08T07:21:00Z"))).toBe("07:21 UTC");
9489 });
90
91 test("the seam's shape alone is read: a limit's time, whatever the runner", () => {
92 // A fake runner: no Claude wording, only { limitedUntil }.
93 const fake = outcome({ result: "slow down", reason: "quota", limit: { limitedUntil: "2026-09-08T09:00:00.000Z" } });
94 expect(isRateLimited(fake)).toBe(true);
95 expect(isRateLimited({ ...fake, limit: undefined })).toBe(false);
96 expect(resumeAt(fake.limit, new Date("2026-09-08T06:00:00Z")).toISOString()).toBe("2026-09-08T09:01:00.000Z");
97 expect(limitOf(fake.limit)).toEqual({ window: "5h", resetsAt: new Date("2026-09-08T09:00:00Z") });
98 expect(limitOf({})).toBeUndefined();
99 });
100
101 test("the crew pauses until the time a fake runner's limit names", async () => {
102 const limitedUntil = new Date(Date.now() + 3 * 3_600_000).toISOString();
103 const calls: number[] = [];
104 const fake: AgentRunner = {
105 detect: async () => ({ ok: true, version: "fake" }),
106 run: async () => {
107 calls.push(Date.now());
108 return { ...outcome({ result: "slow down", reason: "quota" }), limit: { limitedUntil, window: "week" } };
109 },
110 };
111 const standIn = await startStandInCrew({ tickMs: 50 });
112 useRunner(fake);
113 try {
114 const { app, alice, bot, repo } = standIn;
115 const issue = openIssue(app, alice, repo, { title: "Wait it out", body: "anything" });
116 if (!issue.ok) throw new Error(String(issue.error));
117 expect(assignIssue(app, alice, repo, issue.value, bot).ok).toBe(true);
118 const paused = await agentRunSettled(app, repo, (r) => r.issueId === issue.value.id && r.attempt === 0);
119 expect(paused.state).toBe("cancelled");
120 const waiting = listRuns(app.db, repo.id, { kind: "agent" }).find(
121 (r) => r.issueId === issue.value.id && r.state === "queued",
122 )!;
123 expect(waiting).toBeDefined();
124 expect(new Date(waiting.notBefore!).toISOString()).toBe(new Date(Date.parse(limitedUntil) + 60_000).toISOString());
125 expect(calls.length).toBe(1);
126 } finally {
127 useRunner(undefined);
128 await stopStandInCrew(standIn);
129 }
130 }, 60_000);
95131 });
test/crew-network.test.ts +4 −2
@@ -14,9 +14,9 @@ import { listPulls } from "../src/core/pulls.ts";
1414 import { listRuns, type Run } from "../src/core/runs.ts";
1515 import { MAX_PAUSES } from "../src/crew/limits.ts";
1616 import { isNetworkError, NETWORK_PAUSE_MS } from "../src/crew/network.ts";
17 import { isRateLimited } from "../src/crew/limits.ts";
17 import { isRateLimited as limited } from "../src/crew/limits.ts";
18 import { withLimit, type ClaudeOutcome } from "../src/crew/claude.ts";
1819 import { interruptionOf } from "../src/crew/interrupted.ts";
19 import type { ClaudeOutcome } from "../src/crew/claude.ts";
2020 import {
2121 agentRunSettled,
2222 startStandInCrew,
@@ -24,6 +24,8 @@ import {
2424 type StandInCrew,
2525 } from "./helpers/crew-standin.ts";
2626
27 const isRateLimited = (o: ClaudeOutcome): boolean => limited(withLimit(o, new Date()));
28
2729 const outcome = (over: Partial<ClaudeOutcome>): ClaudeOutcome => ({
2830 ok: false,
2931 sessionId: "",
test/standing.test.ts +11 −9
@@ -23,33 +23,35 @@ import { addSubscription, deleteSubscription } from "../src/core/subscriptions.t
2323 import { recordUsage } from "../src/core/usage.ts";
2424 import { createUser } from "../src/core/users.ts";
2525 import { limitOf, resumeAt } from "../src/crew/limits.ts";
26 import { limitFromWords } from "../src/crew/claude.ts";
27 import type { RunnerLimit } from "../src/crew/agent_runner.ts";
28
29 /** What the Claude runner makes of a refusal's words: the seam's limit. */
30 const refusal = (text: string, now: Date = new Date()): RunnerLimit => limitFromWords(text, now);
2631
2732 describe("which window the provider says is full", () => {
2833 test("a bare clock time is the 5-hour window, at that time exactly", () => {
2934 const now = new Date("2026-10-02T05:00:00Z");
3035 const said = "You've hit your session limit · resets 7:20am (UTC)";
31 expect(limitOf(said, now)).toEqual({
36 expect(limitOf(refusal(said, now))).toEqual({
3237 window: "5h",
3338 resetsAt: new Date("2026-10-02T07:20:00Z"),
3439 });
3540 // The pause still waits a minute past it.
36 expect(resumeAt(said, now).toISOString()).toBe("2026-10-02T07:21:00.000Z");
41 expect(resumeAt(refusal(said, now), now).toISOString()).toBe("2026-10-02T07:21:00.000Z");
3742 });
3843 test("a dated reset, or one named weekly, is the week's", () => {
3944 // 9pm in Chicago in September is daylight time, UTC−5: 02:00Z next day.
4045 expect(
41 limitOf(
42 "You've hit your weekly limit · resets Sep 26 at 9pm (America/Chicago)",
43 new Date("2026-09-24T00:00:00Z"),
44 ),
46 limitOf(refusal("You've hit your weekly limit · resets Sep 26 at 9pm (America/Chicago)", new Date("2026-09-24T00:00:00Z"))),
4547 ).toEqual({ window: "week", resetsAt: new Date("2026-09-27T02:00:00Z") });
4648 expect(
47 limitOf("Weekly limit reached · resets 3am (UTC)", new Date("2026-10-02T05:00:00Z")),
49 limitOf(refusal("Weekly limit reached · resets 3am (UTC)", new Date("2026-10-02T05:00:00Z"))),
4850 ).toEqual({ window: "week", resetsAt: new Date("2026-10-03T03:00:00Z") });
4951 });
5052 test("no time named is nothing read", () => {
51 expect(limitOf("rate limit exceeded", new Date())).toBeUndefined();
52 expect(limitOf("resets 7:20am (Not/AZone)", new Date())).toBeUndefined();
53 expect(limitOf(refusal("rate limit exceeded", new Date()))).toBeUndefined();
54 expect(limitOf(refusal("resets 7:20am (Not/AZone)", new Date()))).toBeUndefined();
5355 });
5456 });
5557
test/subscriptions.test.ts +11 −9
@@ -31,6 +31,11 @@ import {
3131 import { createUser, type User } from "../src/core/users.ts";
3232 import { startCrew, type CrewHandle } from "../src/crew/dispatch.ts";
3333 import { isRateLimited, resumeAt } from "../src/crew/limits.ts";
34 import { limitFromWords, withLimit } from "../src/crew/claude.ts";
35 import type { RunnerLimit } from "../src/crew/agent_runner.ts";
36
37 /** What the Claude runner makes of a refusal's words: the seam's limit. */
38 const refusal = (text: string, now: Date = new Date()): RunnerLimit => limitFromWords(text, now);
3439 import { serve, type Servers } from "../src/http/server.ts";
3540 import { stylesheetAsset } from "../src/web/asset.ts";
3641 import { adminCookie } from "./helpers/admin.ts";
@@ -240,28 +245,25 @@ describe("the pool", () => {
240245 describe("the provider's reset time", () => {
241246 const now = new Date("2026-09-23T02:10:00Z");
242247 test("names a date for the week's limit", () => {
243 const at = resumeAt(
244 "You've hit your weekly limit · resets Sep 26 at 9pm (America/Chicago)",
245 now,
246 );
248 const at = resumeAt(refusal("You've hit your weekly limit · resets Sep 26 at 9pm (America/Chicago)", now), now);
247249 // 9pm Chicago on the 26th is 02:00 UTC on the 27th, plus the minute.
248250 expect(at.toISOString()).toBe("2026-09-27T02:01:00.000Z");
249251 expect(
250 resumeAt("resets Sep 26, 2am (America/Chicago)", now).toISOString(),
252 resumeAt(refusal("resets Sep 26, 2am (America/Chicago)", now), now).toISOString(),
251253 ).toBe("2026-09-26T07:01:00.000Z");
252254 });
253255 test("a date already past means next year's", () => {
254 const at = resumeAt("resets Sep 20 at 9pm (UTC)", now);
256 const at = resumeAt(refusal("resets Sep 20 at 9pm (UTC)", now), now);
255257 expect(at.toISOString()).toBe("2027-09-20T21:01:00.000Z");
256258 });
257259 test("still reads a bare clock time", () => {
258 expect(resumeAt("resets 7am (UTC)", now).toISOString()).toBe(
260 expect(resumeAt(refusal("resets 7am (UTC)", now), now).toISOString()).toBe(
259261 "2026-09-23T07:01:00.000Z",
260262 );
261263 });
262264 test("the weekly wording counts as the provider's refusal", () => {
263265 expect(
264 isRateLimited({
266 isRateLimited(withLimit({
265267 ok: false,
266268 sessionId: "",
267269 result: "You've hit your weekly limit · resets Sep 26 at 9pm (UTC)",
@@ -272,7 +274,7 @@ describe("the provider's reset time", () => {
272274 contextTokens: 0,
273275 contextWindow: 0,
274276 reason: "error_during_execution",
275 }),
277 }, now)),
276278 ).toBe(true);
277279 });
278280 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs