Public: anyone can read this; the crew can change it.

Tests share process-wide state and eight are flaky: a pull's check can go red for what another test left behind #12599

Merged opened this in answer to

Check · Review · Land · Live done — Landed and live in production.

Merged

torque/forge:torque/12569-tests-share-process-wide-state-and-eight → main 5 commits fork

  • Merged. torque/12569-tests-share-process-wide-state-and-eight was merged into main as 2cdde14.

Review

Sign in to review this change.
59 files changed +679 −226
src/apps/proxy.ts +2 −2
@@ -70,7 +70,7 @@ import { holdPortRequest, holdPortSocket } from "./handoff.ts";
7070 import { secureFor } from "../http/cookie_secure.ts";
7171 import { currentLoopLag, forgeWaited, LAG_WINDOW_MS } from "../perf/loop_worst.ts";
7272 import { HOOK_MAX_BYTES, HOOK_PREFIX, hookPath, readCapped } from "./hook_relay.ts";
73 import { loopWatch } from "../perf/live.ts";
73 import { liveOf } from "../perf/live.ts";
7474 import { heldBy, noteRelayWait, stallTiming } from "./relay_wait.ts";
7575 import { cspNonce, htmlAnswer, readPage, rewritable, TOOLBAR_ASSET_PATH, TOOLBAR_ASSET_PREFIX, toolbarResponse, withToolbar, type PillWhere } from "./toolbar.ts";
7676
@@ -533,7 +533,7 @@ function relayUpstream(
533533 // What held a wait that was the forge's (#11000): the stalls of the lag's
534534 // window, read only then — at most 40 of them, so the common relay pays nothing.
535535 const now = performance.now();
536 const held = forgeWaited(lag, headersMs) ? heldBy(loopWatch.within(now - LAG_WINDOW_MS, now)) : [];
536 const held = forgeWaited(lag, headersMs) ? heldBy(liveOf(runtime.app).loopWatch.within(now - LAG_WINDOW_MS, now)) : [];
537537 // Production's slowest wait, for the app's speed card (`apps/relay_wait.ts`).
538538 if (target.env === "production") noteRelayWait(target.app.id, headersMs, lag, Date.now(), held);
539539 if (held.length > 0 && forgeAdmin(runtime, who)) stallFor.set(request, stallTiming(held));
src/apps/runtime.ts +1 −1
@@ -763,7 +763,7 @@ export class AppsRuntime {
763763 /** Rebuild every environment from git and the open pulls, then listen. */
764764 async start(): Promise<void> {
765765 this.cells = await cellsAvailable();
766 noteProtection("apps", this.cells);
766 noteProtection(this.app, "apps", this.cells);
767767 if (!this.cells.ok)
768768 log.warn("apps run uncontained here", { reason: this.cells.reason });
769769 // Deploys are failed; a check the last forge left is queued again while
src/checks/tiers.ts +2
@@ -48,6 +48,8 @@ export const RULE_TESTS: readonly string[] = [
4848 "test/spawn-pipes.test.ts",
4949 "test/hollow.test.ts",
5050 "test/release-boot.test.ts",
51 // No process-wide state a request writes to: it belongs to an app (#12569).
52 "test/module-state.test.ts",
5153 ];
5254
5355 /** What a test the times table does not name is assumed to take. */
src/crew/dispatch.ts +1 −1
@@ -230,7 +230,7 @@ export async function startCrew(
230230 // The default the Capacity page shows and goes back to (`core/capacity.ts`).
231231 noteBooted(app.db, "crew.concurrency", String(options.concurrency));
232232 const sandbox = await sandboxAvailable();
233 noteProtection("crew shells", sandbox);
233 noteProtection(app, "crew shells", sandbox);
234234 // Temp directories of runs a stopped forge never finished: old enough
235235 // that no forge beside this one (a rollout) can still be using them.
236236 try {
src/crew/sentinel.ts +6 −6
@@ -31,7 +31,7 @@ import { leaseLapsed, listRunning } from "../core/runs.ts";
3131 import { migrations } from "../core/schema.ts";
3232 import { getUserByName, type User } from "../core/users.ts";
3333 import { migrate } from "../kernel/migrate.ts";
34 import { createLog, recentErrors, redactSecrets } from "../kernel/log.ts";
34 import { createLog, recentErrors, redactSecrets, type ErrorLedger } from "../kernel/log.ts";
3535 import { spans } from "../kernel/degraded_history.ts";
3636 import { QUEUE_WAIT_MAX_MS } from "../kernel/vitals.ts";
3737 import { STALE_AFTER_MS } from "../runs/lease.ts";
@@ -374,9 +374,9 @@ export function runStreaks(db: Database): Finding[] {
374374 }
375375
376376 /** The same error line over and over in the pod's own log. */
377 export function repeatedErrors(now: number): Finding[] {
377 export function repeatedErrors(now: number, ledger?: ErrorLedger): Finding[] {
378378 // The sentinel's own failures are not a fault to file about itself.
379 const seen = recentErrors(100, now).latest.filter((l) => l.scope !== "sentinel");
379 const seen = recentErrors(100, now, ledger).latest.filter((l) => l.scope !== "sentinel");
380380 const groups = new Map<string, typeof seen>();
381381 for (const line of seen) {
382382 const k = `${line.scope}: ${oneLine(line.message).replace(/\d+/g, "N")}`;
@@ -576,7 +576,7 @@ function sharedLines(
576576 }
577577
578578 /** Everything the sentinel reads, in one pass. */
579 export function findFaults(db: Database, now = Date.now(), forgeRepoId?: string, dataDir?: string, ceilingUsd: number | null = null): Finding[] {
579 export function findFaults(db: Database, now = Date.now(), forgeRepoId?: string, dataDir?: string, ceilingUsd: number | null = null, ledger?: ErrorLedger): Finding[] {
580580 const found = [
581581 ...mainAhead(db, now),
582582 ...laneStopped(db, now, ceilingUsd),
@@ -585,7 +585,7 @@ export function findFaults(db: Database, now = Date.now(), forgeRepoId?: string,
585585 ...strayTriggers(db),
586586 ...ledgerStreaks(db, now),
587587 ...runStreaks(db),
588 ...repeatedErrors(now),
588 ...repeatedErrors(now, ledger),
589589 ...degradedCauses(db, now, forgeRepoId, dataDir),
590590 ];
591591 const keys = new Set<string>();
@@ -1029,7 +1029,7 @@ export function hunt(
10291029 let commented = 0;
10301030 const [owner = "", name = ""] = repoPath.split("/");
10311031 const repo = getRepo(app.db, owner, name);
1032 const findings = find(app.db, now, repo?.id, app.config.dataDir, forgeCeiling(app.config));
1032 const findings = find(app.db, now, repo?.id, app.config.dataDir, forgeCeiling(app.config), app.signals.errors);
10331033 const actor = sentinelActor(app.db);
10341034 if (repo !== undefined && actor !== undefined) {
10351035 settleShared(app, repo, actor, now);
src/http/api.ts +2 −1
@@ -8,6 +8,7 @@
88 * back to continue); errors are `{ error, message }`.
99 */
1010
11 import { liveOf } from "../perf/live.ts";
1112 import { reviewWords } from "../actions/review_words.ts";
1213 import { growth, MAX_DAYS as MAX_GROWTH_DAYS } from "../core/growth.ts";
1314 import { PAUSE_REFUSAL, pausedUntil } from "../core/planner_pause.ts";
@@ -3523,7 +3524,7 @@ export function apiRoutes(router: Router<Handler>): void {
35233524 router.add("GET", "/metrics", (ctx) => {
35243525 const denied = accountAdmin(ctx, "ro");
35253526 if (denied) return denied;
3526 return new Response(egressMetrics(egressRowsFor(ctx.app), egressTotalsFor(ctx.app)) + forgeMetrics(ctx.app.db, appSources(ctx.app)), {
3527 return new Response(egressMetrics(egressRowsFor(ctx.app), egressTotalsFor(ctx.app)) + forgeMetrics(ctx.app.db, appSources(ctx.app), Date.now(), liveOf(ctx.app).loopWatch), {
35273528 headers: {
35283529 "content-type": "text/plain; version=0.0.4; charset=utf-8",
35293530 "cache-control": "no-store",
src/http/app_turn.ts +12 −2
@@ -90,8 +90,18 @@ export function createTurn(
9090 };
9191 }
9292
93 /** The process's one gate: a hosted child answers on the same thread. */
94 export const turn: Turn = createTurn();
93 /**
94 * Each forge's own gate: a hosted child's requests come in through its
95 * parent's server, so they are the parent's. One gate for the process kept
96 * every open request of every forge in it, and in a test run a request one
97 * file left open made a later file's app wait.
98 */
99 const gates = new WeakMap<object, Turn>();
100 export function turnOf(app: object): Turn {
101 let gate = gates.get(app);
102 if (gate === undefined) gates.set(app, (gate = createTurn()));
103 return gate;
104 }
95105
96106 /**
97107 * How often each forge's own requests waited at the gate, for its /system.
src/http/mcp/tools/system.ts +4 −4
@@ -4,7 +4,7 @@ import { scriptErrorCounts } from "../../script_errors.ts";
44 import { violationsByDirective, windowStart } from "../../csp_report.ts";
55 import { countRead } from "../../../system/telemetry_reads.ts";
66 import { stallHour, TOP_ROUTES } from "../../../metrics/stalls.ts";
7 import { lagMinutes, loopWatch, perfRecord } from "../../../perf/live.ts";
7 import { liveOf } from "../../../perf/live.ts";
88 import { LAG_BAR_MS, overBar } from "../../../perf/lag_minutes.ts";
99 import { slowestRoutes } from "../../../perf/watch.ts";
1010 import { footing, riskLines } from "../../../system/footing.ts";
@@ -28,7 +28,7 @@ export const tool: Tool = {
2828 const reader = telemetryReader(ctx.app, user);
2929 if (!reader) return bad("forbidden", "only an admin of this forge, or an agent granted telemetry:read, reads its route timings");
3030 noteRead(ctx.app, user, reader, "system");
31 return good({ routes: slowestRoutes(perfRecord, Date.now()) });
31 return good({ routes: slowestRoutes(liveOf(ctx.app).perfRecord, Date.now()) });
3232 }
3333 if (args["kind"] === "size") {
3434 const reader = telemetryReader(ctx.app, user);
@@ -63,11 +63,11 @@ export const tool: Tool = {
6363 noteRead(ctx.app, user, reader, "system");
6464 const out = systemWithReaders(ctx.app, reader);
6565 countRead("system", user.username);
66 const stalls = stallHour(loopWatch.tally(performance.now()))
66 const stalls = stallHour(liveOf(ctx.app).loopWatch.tally(performance.now()))
6767 .slice(0, TOP_ROUTES)
6868 .map((r) => ({ cause: r.route, ms: Math.round(r.ms), count: r.count }));
6969 // The same 20 minutes /system draws (#11220); `what` is null when no stall past 200 ms was named.
70 const rows = lagMinutes.readings();
70 const rows = liveOf(ctx.app).lagMinutes.readings();
7171 const readings = rows.map((r) => ({ at: new Date(r.at).toISOString(), ms: Math.round(r.ms * 10) / 10, what: r.what }));
7272 // At most 0.8 s on the walk (#11800); the last measurement, or null, when it is slower.
7373 const largest = await largestDirs.read(ctx.app.config.dataDir);
src/http/server.ts +10 −10
@@ -553,7 +553,7 @@ import {
553553 takeDbTime,
554554 type RouteTiming,
555555 } from "../core/route_timing.ts";
556 import { captureStore, inFlight, loopWatch, perfRecord, readLedger, stallBlame } from "../perf/live.ts";
556 import { captureStore, inFlight, liveOf, readLedger } from "../perf/live.ts";
557557 import { childLabel, markChildRequest, noteChildRequest } from "../perf/child_time.ts";
558558 import { parseHostedChildName } from "../spawn/placement.ts";
559559 import { childHost } from "../spawn/door.ts";
@@ -572,7 +572,7 @@ import { parseServerTiming } from "../perf/record.ts";
572572 import { currentLoopLag, LAG_WINDOW_MS, startLoopWorst } from "../perf/loop_worst.ts";
573573 import { methodOf, noteUsage, roleOf, surfaceOf, takeUsage } from "./usage_tag.ts";
574574 import { askingOf, noteAsker, noteWaited, takeAsker } from "../perf/asking.ts";
575 import { turn, turnStatsOf } from "./app_turn.ts";
575 import { turnOf, turnStatsOf } from "./app_turn.ts";
576576 import { keepRead, repeatKey, repeatRead } from "./repeat_reads.ts";
577577 import { writeStamp } from "../core/write_clock.ts";
578578 import { bearerChallenge, oauthRoutes } from "./oauth.ts";
@@ -2675,7 +2675,7 @@ export async function serve(app: App): Promise<Servers> {
26752675 const counters = new Map<Database, UsageCounter>();
26762676 const timings = new Map<Database, RouteTiming>();
26772677 // The minute ledger's history: one write a minute, never on a request.
2678 const trafficWriter = createTrafficWriter(app.db, perfRecord.ledger, app.log);
2678 const trafficWriter = createTrafficWriter(app.db, liveOf(app).perfRecord.ledger, app.log);
26792679 // Page views are buffered by the beacon doors and written here; GitHub's traffic is pulled daily when a token is set.
26802680 const viewWriter = createViewWriter(app.db, app.log);
26812681 const githubPull = startGithubPull(app.db, app.log);
@@ -2698,7 +2698,7 @@ export async function serve(app: App): Promise<Servers> {
26982698 ): void => {
26992699 const tag = takeUsage(request);
27002700 const ms = performance.now() - started;
2701 turn.end(request);
2701 turnOf(app).end(request);
27022702 const asker = takeAsker(request);
27032703 askingOf(app).note(asker?.token, tag.pattern, Math.max(0, ms - (asker?.waitedMs ?? 0)));
27042704 finishedStrangerDeep(owner, request);
@@ -2710,7 +2710,7 @@ export async function serve(app: App): Promise<Servers> {
27102710 response.headers.set("server-timing", tag.role === "admin" ? withLag(header, currentLoopLag()) : header);
27112711 const end = performance.now();
27122712 const family = familyOf(request.headers.get("user-agent") ?? "");
2713 perfRecord.add(
2713 liveOf(app).perfRecord.add(
27142714 tag.pattern,
27152715 { at: Date.now(), ms, signedIn: tag.role !== "anon" && tag.role !== "crawler", crawler: wasCrawler(request), deepStranger: wasDeepStranger(request), internal: wasInternal(request), refused: response.status === 429 || wasShielded(request), phases: parseServerTiming(header) },
27162716 {
@@ -2721,7 +2721,7 @@ export async function serve(app: App): Promise<Servers> {
27212721 repo: response.status < 400 ? repoOf(tag.pattern, new URL(request.url).pathname) : null,
27222722 },
27232723 );
2724 loopWatch.finished(hostedChild === undefined ? tag.pattern : childLabel(hostedChild), started, end);
2724 liveOf(app).loopWatch.finished(hostedChild === undefined ? tag.pattern : childLabel(hostedChild), started, end);
27252725 if (hostedChild !== undefined) noteChildRequest(hostedChild, ms);
27262726 // Evidence of what was read, and the running capture, if any (never a signed-in person's).
27272727 const signedIn = tag.role !== "anon" && tag.role !== "crawler";
@@ -2808,7 +2808,7 @@ export async function serve(app: App): Promise<Servers> {
28082808 ): Promise<Response | undefined> => {
28092809 const started = performance.now();
28102810 noteStart(request, started);
2811 turn.begin(request, started);
2811 turnOf(app).begin(request, started);
28122812 // A read-only standby refuses every mutation here, before any route.
28132813 if (app.config.readOnly === true) {
28142814 const refused = readOnlyRefusal(request);
@@ -3102,7 +3102,7 @@ async function answerGet(
31023102 () => app.db.query("select 1").get(),
31033103 app.log,
31043104 wantsJson ? currentLoopLag() : undefined,
3105 wantsJson ? (stallBlame.worst(performance.now(), LAG_WINDOW_MS, 1)[0]?.what ?? null) : null,
3105 wantsJson ? (liveOf(app).stallBlame.worst(performance.now(), LAG_WINDOW_MS, 1)[0]?.what ?? null) : null,
31063106 );
31073107 noteDbTime(request, performance.now() - dbStart);
31083108 // The release proof learns which release answered, old pod or new; a
@@ -3313,9 +3313,9 @@ async function answerGet(
33133313 if (repeat !== undefined) {
33143314 const same = repeatRead(app.db, repeat, request, stamp);
33153315 if (same !== undefined) return withRateHeaders(same, verdict);
3316 turn.machine(request);
3316 turnOf(app).machine(request);
33173317 const waitFrom = performance.now();
3318 await turn.wait(turnStatsOf(app));
3318 await turnOf(app).wait(turnStatsOf(app));
33193319 noteWaited(request, performance.now() - waitFrom);
33203320 }
33213321 // A person's own write, from their own browser, is theirs to see on the page they open next: when
src/kernel/protection.ts +17 −12
@@ -7,29 +7,34 @@
77 * and this line in the foot of every page a site admin reads. Each part
88 * reports itself once, when it learns what the host can do; a part that
99 * works reports nothing, so a fully protected forge shows no line at all.
10 *
11 * Each forge keeps its own notes (`app` is the forge's app object): in a test
12 * run one file's note was another file's footer.
1013 */
1114
12 const lacking = new Map<string, string>();
15 const lacking = new WeakMap<object, Map<string, string>>();
1316
14 /** A part says what it lacks (`ok`: nothing) and why, e.g. `("apps", "bwrap is not installed")`. */
17 /**
18 * A part says what it lacks (`ok`: nothing) and why,
19 * e.g. `(app, "apps", "bwrap is not installed")`.
20 */
1521 export function noteProtection(
22 app: object,
1623 part: string,
1724 result: { ok: true } | { ok: false; reason: string },
1825 ): void {
19 if (result.ok) lacking.delete(part);
20 else lacking.set(part, result.reason);
26 let notes = lacking.get(app);
27 if (notes === undefined) lacking.set(app, (notes = new Map()));
28 if (result.ok) notes.delete(part);
29 else notes.set(part, result.reason);
2130 }
2231
2332 /** The one-line notice, or `undefined` when nothing is reduced. Plain text: escape it. */
24 export function protectionNotice(): string | undefined {
25 if (lacking.size === 0) return undefined;
26 const parts = [...lacking]
33 export function protectionNotice(app: object): string | undefined {
34 const notes = lacking.get(app);
35 if (notes === undefined || notes.size === 0) return undefined;
36 const parts = [...notes]
2737 .sort(([a], [b]) => a.localeCompare(b))
2838 .map(([part, reason]) => `${part} (${reason})`);
2939 return `Reduced protection: ${parts.join("; ")} run${parts.length === 1 ? "s" : ""} uncontained.`;
3040 }
31
32 /** For tests. */
33 export function resetProtection(): void {
34 lacking.clear();
35 }
src/kernel/stall_relay.ts added +37
@@ -0,0 +1,37 @@
1 /**
2 * A stall that arrives before the forge exists (#12569).
3 *
4 * The loop watch starts with the first line of the boot, and a forge's
5 * stall record is made with its app a few lines later. A stall in between
6 * waits here, at most `EARLY_KEPT` of them (the worst-ever boot is a few
7 * seconds of lag, not a flood: the oldest are dropped), and is read in as
8 * soon as the forge is made.
9 */
10
11 export const EARLY_KEPT = 64;
12
13 export interface StallRelay<T> {
14 /** A stall: read at once when the forge exists, else kept for it. */
15 note(late: number, at: number): void;
16 /** The forge exists: read the kept stalls in order, then every later one as it comes. */
17 attach(target: T): void;
18 /** How many wait for the forge. */
19 waiting(): number;
20 }
21
22 export function createStallRelay<T>(read: (target: T, late: number, at: number) => void): StallRelay<T> {
23 let target: { readonly t: T } | undefined;
24 const early: [number, number][] = [];
25 return {
26 note(late, at) {
27 if (target !== undefined) return read(target.t, late, at);
28 early.push([late, at]);
29 if (early.length > EARLY_KEPT) early.shift();
30 },
31 attach(t) {
32 target = { t };
33 for (const [late, at] of early.splice(0)) read(t, late, at);
34 },
35 waiting: () => early.length,
36 };
37 }
src/main.ts +14 −7
@@ -129,10 +129,11 @@ import { reapLeftoverBridges } from "./crew/leftovers.ts";
129129 import { crewRoot } from "./crew/workdir.ts";
130130 import { createLog } from "./kernel/log.ts";
131131 import { watchStalls } from "./kernel/loop_lag.ts";
132 import { createStallRelay } from "./kernel/stall_relay.ts";
132133 import { startPerfWatcher, TICK_MS } from "./perf/duty.ts";
133134 import { during } from "./perf/jobs.ts";
134135 import { stallLine } from "./perf/loop.ts";
135 import { bootStall, gitRecord, lagMinutes, loopWatch, perfRecord, stallBlame } from "./perf/live.ts";
136 import { bootStall, gitRecord, liveOf } from "./perf/live.ts";
136137 import { keepFlowReadings, KEPT_MS } from "./flow/kept.ts";
137138 import { startKeeping } from "./flow/keep.ts";
138139 import { keepForgePages } from "./flow/pages.ts";
@@ -269,13 +270,18 @@ if (parentProblem !== null) {
269270 // second is a warning that names its minute (kernel/loop_lag.ts, #6957),
270271 // draining included.
271272 const loopLog = createLog("loop");
272 watchStalls(loopLog, undefined, undefined, undefined, (late, at) => {
273 bootStall.note(late, at);
274 lagMinutes.note(late, at);
275 const stall = loopWatch.stalled(late, at);
273 // The loop watch is the forge's own (`liveOf(app)`), and the app is made a few lines below: a stall
274 // before then waits in the relay (bounded) and is read in as soon as the forge exists.
275 const stallRelay = createStallRelay((live: ReturnType<typeof liveOf>, late, at) => {
276 live.lagMinutes.note(late, at);
277 const stall = live.loopWatch.stalled(late, at);
276278 if (stall === null) return;
277279 loopLog.warn(stallLine(stall));
278 stallBlame.note(stall);
280 live.stallBlame.note(stall);
281 });
282 watchStalls(loopLog, undefined, undefined, undefined, (late, at) => {
283 bootStall.note(late, at);
284 stallRelay.note(late, at);
279285 });
280286
281287 const previewKey = previewIdentityKey(process.env);
@@ -363,6 +369,7 @@ const app = createApp({
363369 }),
364370 }),
365371 });
372 stallRelay.attach(liveOf(app));
366373
367374 // A statement that holds the thread past 200 ms names its caller (#6957).
368375 timeStatements(app.db, createLog("sql"));
@@ -397,7 +404,7 @@ if (readOnly) {
397404 const bootSignup = signupFromEnv(process.env.FORGE_SIGNUP);
398405 if (bootSignup.ok && bootSignup.value !== null)
399406 setSignup(app.db, bootSignup.value, BOOT_ACTOR);
400 startPerfWatcher(app, perfRecord, Date.now, TICK_MS, gitRecord);
407 startPerfWatcher(app, liveOf(app).perfRecord, Date.now, TICK_MS, gitRecord);
401408 // A release shipped by hand is recorded as live at boot, so live estimates and the base rate
402409 // resolve whichever way a release ships; estimates of "done" come due as time passes (#11955).
403410 const byHand = recordByHand(app.db);
src/metrics/forge.ts +4 −2
@@ -18,7 +18,7 @@ import type { EnvState } from "../apps/runtime.ts";
1818 import { EDGES_MS, readTotals } from "../core/route_timing.ts";
1919 import { closesFromBody } from "../core/pulls.ts";
2020 import { telemetryReadsFamily } from "../system/telemetry_reads.ts";
21 import { loopWatch } from "../perf/live.ts";
21 import { createLoopWatch, type LoopWatch } from "../perf/loop.ts";
2222 import { stallFamilies } from "./stalls.ts";
2323 import { openMetrics, type Family, type HistogramSample, type Sample } from "./openmetrics.ts";
2424
@@ -334,6 +334,8 @@ export function forgeMetrics(
334334 db: Database,
335335 sources: AppSources = { apps: [], cells: [] },
336336 now: number = Date.now(),
337 /** The forge's own loop watch (`liveOf(app).loopWatch`); none read means none stalled. */
338 loop: LoopWatch = createLoopWatch(),
337339 ): string {
338340 return openMetrics([
339341 durationFamily(db),
@@ -347,6 +349,6 @@ export function forgeMetrics(
347349 askToLiveFamily(db),
348350 mainRedFamily(db, now),
349351 telemetryReadsFamily(),
350 ...stallFamilies(loopWatch.tally(performance.now())),
352 ...stallFamilies(loop.tally(performance.now())),
351353 ]).replace(/# EOF\n$/, "");
352354 }
src/perf/crawlers.ts +2 −2
@@ -6,11 +6,11 @@
66
77 import type { App } from "../app.ts";
88 import { crawlRefusalScopes, crawlRefusals } from "../http/crawl.ts";
9 import { perfRecord } from "./live.ts";
9 import { liveOf } from "./live.ts";
1010 import { WINDOW_MS, crawlerShare } from "./record.ts";
1111
1212 export function crawlReport(app: App, now = Date.now()) {
13 const share = crawlerShare(perfRecord, now - WINDOW_MS);
13 const share = crawlerShare(liveOf(app).perfRecord, now - WINDOW_MS);
1414 return {
1515 window_minutes: WINDOW_MS / 60_000,
1616 requests: share.requests,
src/perf/live.ts +44 −14
@@ -1,4 +1,7 @@
1 /** The process's one record, git record (#8158) and loop watch (#7559): what the server feeds and the duty reads. */
1 /**
2 * Each forge's traffic record and loop watch (`liveOf`), and the process's git record (#8158):
3 * what the server feeds and the duty reads.
4 */
25
36 import { createBootStall } from "./boot_stall.ts";
47 import { createGitRecord } from "./git.ts";
@@ -13,26 +16,53 @@ import { createStallBlame } from "./stall_blame.ts";
1316 import { createPerfRecord } from "./record.ts";
1417 import { createLagMinutes } from "./lag_minutes.ts";
1518
16 export const perfRecord = createPerfRecord();
1719 /** The requests the server is answering now (#11478): begun and ended around each one in `http/server.ts`. */
1820 export const inFlight = createInFlight<Request>((request) => childRequestLabel(request) ?? patternOf(request));
19 /**
20 * A stall names the requests that ran across it, finished or still in flight
21 * (#11478); when none did, the background jobs that did (#10834). A job that
22 * held the loop for most of a stall is named ahead of the requests that waited
23 * behind it, and a name that only brushed the stall is left out (`loop.ts`).
24 */
25 export const loopWatch = createLoopWatch(undefined, jobWatch.across, inFlight.since, jobWatch.held, jobWatch.awaited);
2621 export const gitRecord = createGitRecord();
2722 /** The worst stall in this process's first five minutes, read by the release keeper (#8848). */
2823 export const bootStall = createBootStall();
2924 /** What was read from repositories (`reads.ts`), and the one bounded capture an admin may run (`capture.ts`). */
3025 export const readLedger = createReadLedger();
3126 export const captureStore = createCaptureStore();
32 /** Each stall over 200 ms with what ran (#11069): the last hour's worst on /system, the last 10 s's on `/healthz`. */
33 export const stallBlame = createStallBlame();
27
3428 /**
35 * The last 20 minutes' worst loop lag (#11220), each blamed on the worst stall
36 * `stallBlame` named in it: /system and the MCP `system` tool read it.
29 * What each forge's own server feeds and its pages read (#12569): the traffic
30 * record, the loop watch, the stall blame and the lag minutes. They were one
31 * set for the process, so in a test run a file's traffic or stall was in every
32 * later file's /system and /traffic, by the order the files ran in.
33 *
34 * The loop watch is told of jobs from `jobWatch` and of requests from
35 * `inFlight`, which stay the process's: a timer and a request in flight are
36 * the thread's, and the watch only reads them.
37 *
38 * A stall names the requests that ran across it, finished or still in flight
39 * (#11478); when none did, the background jobs that did (#10834). A job that
40 * held the loop for most of a stall is named ahead of the requests that waited
41 * behind it, and a name that only brushed the stall is left out (`loop.ts`).
42 * `stallBlame`: each stall over 200 ms with what ran (#11069): the last hour's
43 * worst on /system, the last 10 s's on `/healthz`. `lagMinutes`: the last 20
44 * minutes' worst loop lag (#11220), each blamed on the worst stall
45 * `stallBlame` named in it.
3746 */
38 export const lagMinutes = createLagMinutes((to, span) => stallBlame.worst(to, span, 1)[0]?.what ?? null);
47 export interface Live {
48 readonly perfRecord: ReturnType<typeof createPerfRecord>;
49 readonly loopWatch: ReturnType<typeof createLoopWatch>;
50 readonly stallBlame: ReturnType<typeof createStallBlame>;
51 readonly lagMinutes: ReturnType<typeof createLagMinutes>;
52 }
53
54 const lives = new WeakMap<object, Live>();
55 export function liveOf(app: object): Live {
56 let live = lives.get(app);
57 if (live === undefined) {
58 const stallBlame = createStallBlame();
59 live = {
60 perfRecord: createPerfRecord(),
61 loopWatch: createLoopWatch(undefined, jobWatch.across, inFlight.since, jobWatch.held, jobWatch.awaited),
62 stallBlame,
63 lagMinutes: createLagMinutes((to, span) => stallBlame.worst(to, span, 1)[0]?.what ?? null),
64 };
65 lives.set(app, live);
66 }
67 return live;
68 }
src/release/landings.ts +20 −9
@@ -98,17 +98,22 @@ interface LiveState {
9898 /** Commits in `head` and not in `commit` (`git rev-list --count commit..head`), asked once per pair. */
9999 readonly ahead?: number;
100100 }
101 let kept: LiveState | undefined;
101 /**
102 * What each forge's reads kept, by its database (one per forge): one for the
103 * process let a test's own repository, or a forge hosted beside another, show
104 * as the other's "This forge" on /changelog (#12569).
105 */
106 const keeps = new WeakMap<Database, LiveState>();
102107
103108 /** Keep what the running commit contains; a test says it directly. */
104 export function keepLive(own: string, oids: Iterable<string>, at: number, gap?: { commit: string; head: string; ahead: number }): void {
105 kept = { own, oids: new Set(oids), misses: new Set(), at, commit: gap?.commit ?? "", ...(gap === undefined ? {} : { head: gap.head, ahead: gap.ahead }) };
109 export function keepLive(db: Database, own: string, oids: Iterable<string>, at: number, gap?: { commit: string; head: string; ahead: number }): void {
110 keeps.set(db, { own, oids: new Set(oids), misses: new Set(), at, commit: gap?.commit ?? "", ...(gap === undefined ? {} : { head: gap.head, ahead: gap.ahead }) });
106111 }
107112
108113 /** Forget it, so the alarm is silent until the next read. */
109 export function forgetLive(): void {
110 kept = undefined;
111 found.clear();
114 export function forgetLive(db: Database): void {
115 keeps.delete(db);
116 founds.delete(db);
112117 }
113118
114119 function isLive(row: MergedRow, state: LiveState | undefined): boolean {
@@ -139,6 +144,7 @@ export function split(rows: readonly MergedRow[], state: LiveState | undefined):
139144
140145 /** The merged pulls still on their way and when the oldest merged, from what the last read kept; undefined when none or nothing was read. */
141146 export function waiting(db: Database): { readonly since: string; readonly count: number } | undefined {
147 const kept = keeps.get(db);
142148 if (kept === undefined) return undefined;
143149 const { onItsWay } = split(mergedRows(db), kept);
144150 const oldest = onItsWay.at(-1);
@@ -147,6 +153,7 @@ export function waiting(db: Database): { readonly since: string; readonly count:
147153
148154 /** The merged pulls not yet in the running commit, oldest first, with their merge commits; empty when nothing was read (#11507). */
149155 export function waitingRows(db: Database): MergedRow[] {
156 const kept = keeps.get(db);
150157 if (kept === undefined) return [];
151158 const rows = mergedRows(db);
152159 const way = new Set(split(rows, kept).onItsWay.map((l) => `${l.repo}#${l.number}`));
@@ -157,7 +164,7 @@ export function waitingRows(db: Database): MergedRow[] {
157164 const MISS_MS = 60_000;
158165
159166 /** The repository the running commit was built from, per running commit; "" when none held it at `at`. */
160 const found = new Map<string, { repo: string; at: number }>();
167 const founds = new WeakMap<Database, Map<string, { repo: string; at: number }>>();
161168
162169 /**
163170 * The forge's own repository: the one `FORGE_REPO` names or, where a deploy
@@ -171,6 +178,8 @@ async function ownRepo(app: App, rows: readonly MergedRow[], now = Date.now()):
171178 if (owner !== "" && name !== "") return getRepo(app.db, owner, name);
172179 const commit = running().commit;
173180 if (commit === null) return undefined;
181 let found = founds.get(app.db);
182 if (found === undefined) founds.set(app.db, (found = new Map()));
174183 const known = found.get(commit);
175184 if (known?.repo === "" && now - known.at < MISS_MS) return undefined;
176185 for (const named of known?.repo ? [known.repo] : candidates(rows)) {
@@ -206,6 +215,7 @@ async function askLive(app: App, rows: readonly MergedRow[]): Promise<void> {
206215 const named = `${repo.ownerName}/${repo.name}`;
207216 // For one running commit an ancestor stays one and a non-ancestor never becomes one:
208217 // only a new running commit voids what was kept.
218 const kept = keeps.get(app.db);
209219 const before = kept?.own === named && kept.commit === commit ? kept : undefined;
210220 let at = before?.at;
211221 if (at === undefined) {
@@ -240,7 +250,7 @@ async function askLive(app: App, rows: readonly MergedRow[]): Promise<void> {
240250 const v = n.ok ? Number(n.value.trim()) : NaN;
241251 if (Number.isSafeInteger(v) && v >= 0) ahead = v;
242252 }
243 kept = { own: named, oids, misses, at, commit, ...(tip === undefined ? {} : { head: tip }), ...(ahead === undefined ? {} : { ahead }) };
253 keeps.set(app.db, { own: named, oids, misses, at, commit, ...(tip === undefined ? {} : { head: tip }), ...(ahead === undefined ? {} : { ahead }) });
244254 }
245255
246256 export interface MainGap {
@@ -255,7 +265,7 @@ export interface MainGap {
255265 * git, one query. Undefined until a read found the forge's own main.
256266 */
257267 export function mainGap(db: Database): MainGap | undefined {
258 const k = kept;
268 const k = keeps.get(db);
259269 if (k === undefined || k.head === undefined) return undefined;
260270 if (k.head === k.commit) return { main: { sha: k.head }, behind: { commits: 0, pulls: 0 } };
261271 const own = k.own.toLowerCase();
@@ -283,6 +293,7 @@ async function buildersOfRepo(app: App, own: string, rows: readonly MergedRow[])
283293 export async function readLandings(app: App): Promise<Landings> {
284294 const rows = mergedRows(app.db);
285295 await askLive(app, rows).catch(() => undefined);
296 const kept = keeps.get(app.db);
286297 const { live, onItsWay } = split(rows, kept);
287298 const own = kept?.own;
288299 const credits = own === undefined ? new Map<string, Builders>() : await buildersOfRepo(app, own, rows).catch(() => new Map<string, Builders>());
src/tui/screens/dashboard.ts +1 −1
@@ -1283,7 +1283,7 @@ export function dashboardScreen(f: Session, ctx: Ctx): Screen {
12831283 read === undefined
12841284 ? undefined
12851285 : `Read as: ${exploreLine(read.query) || "everything"}${explored === undefined ? "" : ` · ${exploreCount(read.query, explored.total, f.user?.isAdmin === true)}${exploreFiltered(read.query) ? " · Esc clears" : ""}`}${mend === undefined ? "" : ` · Did you mean ${mend.line}?`}${note === "" ? "" : ` · ${note}`}`;
1286 const reduced = readAs ?? (f.user?.isAdmin === true ? protectionNotice() : undefined);
1286 const reduced = readAs ?? (f.user?.isAdmin === true ? protectionNotice(f.app) : undefined);
12871287 const [box, warn, list] = split(
12881288 main,
12891289 "v",
src/tui/screens/loopstalls.ts +6 −6
@@ -12,7 +12,7 @@
1212
1313 import { stallHour } from "../../metrics/stalls.ts";
1414 import { HOUR_MS, type RouteStall } from "../../perf/loop.ts";
15 import { lagMinutes, loopWatch, stallBlame } from "../../perf/live.ts";
15 import { liveOf } from "../../perf/live.ts";
1616 import { LAG_BAR_MS, LAG_READINGS, overBar, type LagReading } from "../../perf/lag_minutes.ts";
1717 import { WORST_STALLS, type BlamedStall } from "../../perf/stall_blame.ts";
1818 import type { Ctx, Screen } from "../app/app.ts";
@@ -34,9 +34,9 @@ export interface WorstNow {
3434 readonly wall: number;
3535 }
3636
37 function worstNow(): WorstNow {
37 function worstNow(session: Session): WorstNow {
3838 const mono = performance.now();
39 return { rows: stallBlame.worst(mono, HOUR_MS, WORST_STALLS), mono, wall: Date.now() };
39 return { rows: liveOf(session.app).stallBlame.worst(mono, HOUR_MS, WORST_STALLS), mono, wall: Date.now() };
4040 }
4141
4242 /**
@@ -46,8 +46,8 @@ function worstNow(): WorstNow {
4646 */
4747 export function stallLines(
4848 session: Session,
49 rows: readonly RouteStall[] = stallHour(loopWatch.tally(performance.now())),
50 worst: WorstNow = worstNow(),
49 rows: readonly RouteStall[] = stallHour(liveOf(session.app).loopWatch.tally(performance.now())),
50 worst: WorstNow = worstNow(session),
5151 ): string[] {
5252 if (!mayRead(session)) return [];
5353 const col = (s: string) => pad(s, 10, "right");
@@ -71,7 +71,7 @@ export function stallLines(
7171 * start in UTC, its worst ms, `over` when strictly past the bar, and what ran
7272 * (or "not named"). Empty for anyone the gate refuses.
7373 */
74 export function lagLines(session: Session, rows: readonly LagReading[] = lagMinutes.readings()): string[] {
74 export function lagLines(session: Session, rows: readonly LagReading[] = liveOf(session.app).lagMinutes.readings()): string[] {
7575 if (!mayRead(session)) return [];
7676 const head = `Loop lag, last ${String(LAG_READINGS)} minutes`;
7777 if (rows.length === 0) return [head, "No full minute measured yet."];
src/tui/screens/traffic.ts +2 −2
@@ -8,7 +8,7 @@
88 import type { Ctx, Screen } from "../app/app.ts";
99 import { binding } from "../app/keys.ts";
1010 import { trafficShare, TRAFFIC_WORDS } from "../../flow/kpi_words.ts";
11 import { perfRecord } from "../../perf/live.ts";
11 import { liveOf } from "../../perf/live.ts";
1212 import { TRAFFIC_CLASSES } from "../../perf/traffic.ts";
1313 import { trafficHistory } from "../../perf/traffic_store.ts";
1414 import type { Session } from "../session.ts";
@@ -24,7 +24,7 @@ export function trafficScreen(f: Session, ctx: Ctx): Screen {
2424 const state = new ListState();
2525 const lines = (): ListRow[] => {
2626 if (!may(f)) return [];
27 const t = trafficHistory(f.app.db, perfRecord.ledger, Date.now(), "15m");
27 const t = trafficHistory(f.app.db, liveOf(f.app).perfRecord.ledger, Date.now(), "15m");
2828 if (t.requests === 0) return [{ title: "No requests in the last 15 minutes", meta: "each finished minute is written down" }];
2929 const rows: ListRow[] = [
3030 { title: `${t.requests} requests, ${(t.ms / 1000).toFixed(1)} s of answering time`, meta: `${t.refused} refused, about ${t.addresses} client addresses` },
src/web/layout.ts +1 −1
@@ -707,7 +707,7 @@ function topbar(
707707 function footer(ctx: Ctx, own?: Html): Html {
708708 if (own !== undefined) return html`<footer class="foot">${own}</footer>`;
709709 const reduced =
710 ctx.auth.user?.isAdmin === true ? protectionNotice() : undefined;
710 ctx.auth.user?.isAdmin === true ? protectionNotice(ctx.app) : undefined;
711711 return html`<footer class="foot">
712712 <span class="mark"></span>
713713 <span
src/web/pages/scrapes.ts +2 −2
@@ -9,7 +9,7 @@
99 import { notePhase, takeSessionTime } from "../../core/route_timing.ts";
1010 import type { Ctx } from "../../http/context.ts";
1111 import { repoDir } from "../../git/repos.ts";
12 import { perfRecord, captureStore, readLedger } from "../../perf/live.ts";
12 import { liveOf, captureStore, readLedger } from "../../perf/live.ts";
1313 import { groupRequests, type Group } from "../../perf/actors.ts";
1414 import { CAPTURE_MAX, CAPTURE_MS } from "../../perf/capture.ts";
1515 import { networkTable, parseNetworks, saveNetworks, type Network, type NetworkTable } from "../../perf/network.ts";
@@ -154,7 +154,7 @@ export async function scrapesPage(ctx: Ctx): Promise<Response> {
154154 );
155155 }),
156156 );
157 const prefixes = trafficHistory(ctx.app.db, perfRecord.ledger, now, "24h").tops.prefixes;
157 const prefixes = trafficHistory(ctx.app.db, liveOf(ctx.app).perfRecord.ledger, now, "24h").tops.prefixes;
158158 const cap = captureStore.current(now);
159159 const groups = cap === null ? [] : groupRequests(cap.entries).filter((g) => g.size >= 3);
160160 const picked = groups.find((g) => g.id === q.get("group"));
src/web/pages/security.ts +2 −2
@@ -17,7 +17,7 @@ import { readBackup } from "../../flow/kpis.ts";
1717 import type { Ctx } from "../../http/context.ts";
1818 import { violationsByDirective } from "../../http/csp_report.ts";
1919 import { scriptErrorCounts } from "../../http/script_errors.ts";
20 import { perfRecord } from "../../perf/live.ts";
20 import { liveOf } from "../../perf/live.ts";
2121 import { getSignup } from "../../settings/signup.ts";
2222 import { trafficHistory } from "../../perf/traffic_store.ts";
2323 import { lastDrill } from "../../snapshot/drills.ts";
@@ -277,7 +277,7 @@ export function securityPage(ctx: Ctx): Response {
277277 const db = ctx.app.db;
278278 const dataDir = ctx.app.config.dataDir;
279279
280 const traffic = trafficHistory(db, perfRecord.ledger, nowMs, "15m");
280 const traffic = trafficHistory(db, liveOf(ctx.app).perfRecord.ledger, nowMs, "15m");
281281 const wrong = trafficAttention(traffic);
282282 const { admins, adminTokens, added } = peopleRows(db, since, nowIso);
283283 const { backup, backupAt } = readBackup(db, dataDir, nowMs);
src/web/pages/system.ts +4 −4
@@ -37,7 +37,7 @@ import { readerPaces, type ReaderPace } from "../../system/telemetry_reads.ts";
3737 import { slowPageLine, slowPages, type SlowPage } from "../../perf/watch.ts";
3838 import { stallHour } from "../../metrics/stalls.ts";
3939 import { HOUR_MS, type RouteStall } from "../../perf/loop.ts";
40 import { lagMinutes, loopWatch, stallBlame } from "../../perf/live.ts";
40 import { liveOf } from "../../perf/live.ts";
4141 import { childHours, childHourWords, type ChildHour } from "../../perf/child_time.ts";
4242 import { openChildren } from "../../spawn/host.ts";
4343 import { LAG_BAR_MS, LAG_READINGS, overBar, type LagReading } from "../../perf/lag_minutes.ts";
@@ -597,7 +597,7 @@ export async function systemPage(ctx: Ctx): Promise<Response> {
597597 const self = owner ? timePhase(ctx.request, "self", () => selfUse(realSelf(ctx.app.config.dataDir))) : null;
598598 const listed = owner ? await listChildren(ctx.app) : [];
599599 const children = childrenUse(listed, (bodyId) => childDirOf(ctx.app, bodyId));
600 const tally = loopWatch.tally(performance.now());
600 const tally = liveOf(ctx.app).loopWatch.tally(performance.now());
601601 const stalls = timePhase(ctx.request, "stalls", () => stallHour(tally));
602602 // The forges open in this process now, and any that ran here in the hour: each with what it took
603603 // of this thread.
@@ -606,7 +606,7 @@ export async function systemPage(ctx: Ctx): Promise<Response> {
606606 const mine = new Set(listed.map((c) => c.name));
607607 const inProcess = childHours(Date.now(), tally.hour, openChildren(ctx.app).map((c) => c.name)).filter((r) => mine.has(r.name));
608608 const nowMono = performance.now();
609 const worst = stallBlame.worst(nowMono, HOUR_MS, WORST_STALLS);
609 const worst = liveOf(ctx.app).stallBlame.worst(nowMono, HOUR_MS, WORST_STALLS);
610610 // Never waits on the cluster: the last reading, or the declared rows while the first read runs behind.
611611 const standsOn = footing.peek() ?? declaredFooting("the cluster is being read; reload in a moment");
612612 // Never waits on the walk: the last measurement, a new one started behind it when stale.
@@ -655,7 +655,7 @@ export async function systemPage(ctx: Ctx): Promise<Response> {
655655 ${footingSection(standsOn)}
656656 ${offsiteSection(timePhase(ctx.request, "offsite", () => readOffsiteLane(ctx.app.config.dataDir, ctx.app.db)))}
657657 ${featuresSection(timePhase(ctx.request, "features", () => featureHealth(ctx.app, reader)))}
658 ${lagSection(lagMinutes.readings())}
658 ${lagSection(liveOf(ctx.app).lagMinutes.readings())}
659659 <div id="sys-stalls">${stallSection(stalls, worstStallsTable(worst, nowMono, Date.now()))}</div>
660660 ${owner ? askingSection(askingOf(ctx.app).heavy(), turnStatsOf(ctx.app)) : html``}
661661 <div id="sys-slow-pages">${slowPagesSection(timePhase(ctx.request, "slowpages", () => slowPagesCached(ctx.app.db)))}</div>
src/web/pages/traffic.ts +2 −2
@@ -14,7 +14,7 @@ import {
1414 trafficShare,
1515 TRAFFIC_WORDS,
1616 } from "../../flow/kpi_words.ts";
17 import { perfRecord } from "../../perf/live.ts";
17 import { liveOf } from "../../perf/live.ts";
1818 import {
1919 TRAFFIC_CLASSES,
2020 type TopRow,
@@ -118,7 +118,7 @@ export function trafficPage(ctx: Ctx): Response {
118118 const rendering = performance.now();
119119 const now = Date.now();
120120 const chosen = windowOf(new URL(ctx.request.url).searchParams.get("window"));
121 const t = trafficHistory(ctx.app.db, perfRecord.ledger, now, chosen);
121 const t = trafficHistory(ctx.app.db, liveOf(ctx.app).perfRecord.ledger, now, chosen);
122122 const tops = t.tops;
123123 const wrong = trafficAttention(t);
124124 const span = TRAFFIC_WINDOWS[chosen].words;
src/web/studio/kpi_probe.ts +2 −2
@@ -39,7 +39,7 @@ import {
3939 visitorsValue,
4040 } from "../../flow/kpi_words.ts";
4141 import { HOURS, median } from "../../flow/throughput.ts";
42 import { perfRecord } from "../../perf/live.ts";
42 import { liveOf } from "../../perf/live.ts";
4343 import { scanTraffic, type PerfRecord } from "../../perf/record.ts";
4444 import { TRAFFIC_CLASSES } from "../../perf/traffic.ts";
4545 import { HOME_TTL_MS, cachedKpis } from "./home_cache.ts";
@@ -315,6 +315,6 @@ export function kpiProbe(app: App, now: Date = new Date()): KpiProbe {
315315 now,
316316 { kpis: cachedKpis(app.db, now, forge), live: liveNow() },
317317 forge,
318 perfRecord,
318 liveOf(app).perfRecord,
319319 );
320320 }
test/changelog-api.test.ts +2 −2
@@ -87,7 +87,7 @@ describe("GET /api/v1/changelog", () => {
8787 });
8888
8989 test("with a repository of its own the default is that repository, on every door", async () => {
90 keepLive("alice/forge", [], 0);
90 keepLive(app.db, "alice/forge", [], 0);
9191 try {
9292 expect(titles(await (await get("")).json())).toEqual(["Public landing"]);
9393 expect(titles(await mcp({ shipped: true }, admin))).toEqual(["Public landing"]);
@@ -95,7 +95,7 @@ describe("GET /api/v1/changelog", () => {
9595 expect(titles(await (await get("?repo=all")).json())).toEqual(["Other repo landing", "Public landing"]);
9696 expect((await get("")).headers.get("cache-control")).toBe("public, max-age=5");
9797 } finally {
98 forgetLive();
98 forgetLive(app.db);
9999 }
100100 });
101101
test/changelog-atom.test.ts +2 −2
@@ -41,11 +41,11 @@ describe("GET /changelog.atom", () => {
4141 insertPull.run("i2", made.value.defaultBranch, "2026-10-02T10:00:00.000Z", "oid-new");
4242 insertIssue.run("i3", made.value.id, 3, "small: j/k, enter, esc and ? on pulls, with test/keys-pulls.test.ts", u.value.id);
4343 insertPull.run("i3", made.value.defaultBranch, "2026-09-30T10:00:00.000Z", "oid-live3");
44 keepLive("alice/forge", ["oid-live", "oid-live3"], 0);
44 keepLive(app.db, "alice/forge", ["oid-live", "oid-live3"], 0);
4545 }, 30_000);
4646
4747 afterAll(async () => {
48 forgetLive();
48 forgetLive(app.db);
4949 if (saved === undefined) delete process.env["FORGE_COMMIT"];
5050 else process.env["FORGE_COMMIT"] = saved;
5151 servers.stop();
test/changelog-cache.test.ts +3 −3
@@ -46,11 +46,11 @@ describe("GET /changelog caching", () => {
4646 insertPull.run("i2", made.value.defaultBranch, "2026-10-02T10:00:00.000Z", "oid-2");
4747 insertIssue.run("i3", other.value.id, 1, "Elsewhere", u.value.id);
4848 insertPull.run("i3", other.value.defaultBranch, "2026-09-30T10:00:00.000Z", "oid-3");
49 keepLive("alice/forge", ["oid-1"], 0);
49 keepLive(app.db, "alice/forge", ["oid-1"], 0);
5050 }, 30_000);
5151
5252 afterAll(async () => {
53 forgetLive();
53 forgetLive(app.db);
5454 if (saved === undefined) delete process.env["FORGE_COMMIT"];
5555 else process.env["FORGE_COMMIT"] = saved;
5656 servers.stop();
@@ -131,7 +131,7 @@ describe("GET /changelog caching", () => {
131131
132132 test("the tag changes when a release lands", async () => {
133133 const before = (await get()).headers.get("etag") ?? "";
134 keepLive("alice/forge", ["oid-1", "oid-2"], 0);
134 keepLive(app.db, "alice/forge", ["oid-1", "oid-2"], 0);
135135 const after = await get("", { "if-none-match": before });
136136 expect(after.status).toBe(200);
137137 expect(after.headers.get("etag")).not.toBe(before);
test/changelog-etag.test.ts +3 −3
@@ -41,11 +41,11 @@ describe("GET /api/v1/changelog ETag", () => {
4141 insertPull.run("i2", made.value.defaultBranch, "2026-10-02T10:00:00.000Z", "oid-2");
4242 insertIssue.run("i3", other.value.id, 1, "Elsewhere", u.value.id);
4343 insertPull.run("i3", other.value.defaultBranch, "2026-09-30T10:00:00.000Z", "oid-3");
44 keepLive("alice/forge", ["oid-1"], 0);
44 keepLive(app.db, "alice/forge", ["oid-1"], 0);
4545 }, 30_000);
4646
4747 afterAll(async () => {
48 forgetLive();
48 forgetLive(app.db);
4949 if (saved === undefined) delete process.env["FORGE_COMMIT"];
5050 else process.env["FORGE_COMMIT"] = saved;
5151 servers.stop();
@@ -77,7 +77,7 @@ describe("GET /api/v1/changelog ETag", () => {
7777
7878 test("a newly shipped pull makes the next answer a 200 with a new tag", async () => {
7979 const before = (await get()).headers.get("etag") ?? "";
80 keepLive("alice/forge", ["oid-1", "oid-2"], 0);
80 keepLive(app.db, "alice/forge", ["oid-1", "oid-2"], 0);
8181 const after = await get("", { "if-none-match": before });
8282 expect(before).toMatch(/^W\//);
8383 expect(after.status).toBe(200);
test/changelog-timing.test.ts +2 −2
@@ -45,11 +45,11 @@ beforeAll(async () => {
4545 oids.push(`oid-${n}`);
4646 }
4747 })();
48 keepLive("alice/forge", oids, 0);
48 keepLive(app.db, "alice/forge", oids, 0);
4949 }, 60_000);
5050
5151 afterAll(async () => {
52 forgetLive();
52 forgetLive(app.db);
5353 if (saved === undefined) delete process.env["FORGE_COMMIT"];
5454 else process.env["FORGE_COMMIT"] = saved;
5555 servers.stop();
test/changelog.test.ts +11 −11
@@ -77,7 +77,7 @@ describe("changelog page", () => {
7777 else process.env[k] = v;
7878 }
7979 delete process.env["FORGE_VERSION"];
80 forgetLive();
80 forgetLive(app.db);
8181 servers.stop();
8282 app.close();
8383 await rm(dataDir, { recursive: true, force: true });
@@ -140,7 +140,7 @@ describe("changelog page", () => {
140140
141141 test("with no FORGE_REPO the repository holding the running commit is the forge's own", async () => {
142142 delete app.config.forgeRepo;
143 forgetLive();
143 forgetLive(app.db);
144144 try {
145145 const body = await (await fetch(`${servers.url}/changelog`)).text();
146146 expect(body.indexOf("On its way")).toBeLessThan(body.indexOf("Third thing"));
@@ -156,13 +156,13 @@ describe("changelog page", () => {
156156 } finally {
157157 process.env["FORGE_COMMIT"] = oids[1]!;
158158 app.config.forgeRepo = "alice/forge";
159 forgetLive();
159 forgetLive(app.db);
160160 }
161161 });
162162
163163 test("a private repository holding the running commit is never the forge's own", async () => {
164164 delete app.config.forgeRepo;
165 forgetLive();
165 forgetLive(app.db);
166166 app.db.run("UPDATE repo SET visibility = 'private' WHERE name = 'forge'");
167167 process.env["FORGE_COMMIT"] = oids[0]!;
168168 try {
@@ -172,7 +172,7 @@ describe("changelog page", () => {
172172 app.db.run("UPDATE repo SET visibility = 'public' WHERE name = 'forge'");
173173 process.env["FORGE_COMMIT"] = oids[1]!;
174174 app.config.forgeRepo = "alice/forge";
175 forgetLive();
175 forgetLive(app.db);
176176 }
177177 });
178178
@@ -182,7 +182,7 @@ describe("changelog page", () => {
182182 const day = new Date().toISOString().slice(0, 10);
183183 merge("fork", "Fork change", `${day}T23:00:00.000Z`, c.commitOid);
184184 delete app.config.forgeRepo;
185 forgetLive();
185 forgetLive(app.db);
186186 process.env["FORGE_COMMIT"] = c.commitOid;
187187 try {
188188 const body = await (await fetch(`${servers.url}/changelog`)).text();
@@ -191,7 +191,7 @@ describe("changelog page", () => {
191191 } finally {
192192 process.env["FORGE_COMMIT"] = oids[1]!;
193193 app.config.forgeRepo = "alice/forge";
194 forgetLive();
194 forgetLive(app.db);
195195 }
196196 });
197197
@@ -202,7 +202,7 @@ describe("changelog page", () => {
202202 });
203203 expect(fetched.ok).toBe(true);
204204 delete app.config.forgeRepo;
205 forgetLive();
205 forgetLive(app.db);
206206 try {
207207 const body = await (await fetch(`${servers.url}/changelog`)).text();
208208 expect(body.indexOf("On its way")).toBeLessThan(body.indexOf("Third thing"));
@@ -210,13 +210,13 @@ describe("changelog page", () => {
210210 } finally {
211211 app.config.forgeRepo = "alice/forge";
212212 app.db.run("DELETE FROM issue WHERE title = 'Fork change'");
213 forgetLive();
213 forgetLive(app.db);
214214 }
215215 });
216216
217217 test("a miss with no merged pulls yet is not remembered", async () => {
218218 delete app.config.forgeRepo;
219 forgetLive();
219 forgetLive(app.db);
220220 app.db.run("UPDATE repo SET visibility = 'private'");
221221 try {
222222 expect(await (await fetch(`${servers.url}/changelog`)).text()).toContain("Nothing has shipped yet");
@@ -226,7 +226,7 @@ describe("changelog page", () => {
226226 } finally {
227227 app.db.run("UPDATE repo SET visibility = 'public'");
228228 app.config.forgeRepo = "alice/forge";
229 forgetLive();
229 forgetLive(app.db);
230230 }
231231 });
232232
test/coauthors.test.ts +2 −2
@@ -192,7 +192,7 @@ describe("a squash landing", () => {
192192 const dir0 = repoDir(app.reposRoot, repo.ownerName, repo.name);
193193 const was = process.env["FORGE_COMMIT"];
194194 process.env["FORGE_COMMIT"] = (await runGitCapture(["rev-parse", "main"], { cwd: dir0 })).stdout.trim();
195 forgetLive();
195 forgetLive(app.db);
196196 const read = await readLandings(app);
197197 const byTitle = (t: string) => [...read.live, ...read.onItsWay].find((l) => l.title === t)!;
198198 expect(byTitle("Do credited").with).toEqual(["Jay Moran"]);
@@ -214,7 +214,7 @@ describe("a squash landing", () => {
214214 server.stop();
215215 if (was === undefined) delete process.env["FORGE_COMMIT"];
216216 else process.env["FORGE_COMMIT"] = was;
217 forgetLive();
217 forgetLive(app.db);
218218 }
219219 });
220220 });
test/flow-alarm-lines.test.ts +2 −2
@@ -29,8 +29,8 @@ async function seed(mergedMinAgo?: number) {
2929 const repo = await createRepo(app.db, app.reposRoot, { ownerId: u.value.id, name: "r", visibility: "public", description: "r" });
3030 if (!repo.ok) throw new Error(repo.error);
3131 // The running commit predates every merge here, so a merge is on its way.
32 if (mergedMinAgo === undefined) forgetLive();
33 else keepLive("alice/r", [], 0);
32 if (mergedMinAgo === undefined) forgetLive(app.db);
33 else keepLive(app.db, "alice/r", [], 0);
3434 if (mergedMinAgo !== undefined) {
3535 const at = new Date(Date.now() - mergedMinAgo * 60_000).toISOString();
3636 const issueId = id();
test/flow-alarm-main-ahead.test.ts +5 −5
@@ -66,7 +66,7 @@ afterAll(() => {
6666 for (const [k, v] of [["FORGE_SENTINEL_REPO", saved.sentinel], ["FORGE_COMMIT", saved.commit], ["FORGE_VERSION", saved.version]] as const)
6767 if (v === undefined) delete process.env[k];
6868 else process.env[k] = v;
69 forgetLive();
69 forgetLive(app.db);
7070 });
7171
7272 function check(state: "ok" | "failed", oid: string): void {
@@ -83,7 +83,7 @@ function merge(minAgo: number, oid: string): void {
8383 const reset = () => {
8484 app.db.query(`DELETE FROM run`).run();
8585 app.db.query(`DELETE FROM pull`).run();
86 keepLive("trevato/forge", [], NOW.getTime() - 600 * 60_000);
86 keepLive(app.db, "trevato/forge", [], NOW.getTime() - 600 * 60_000);
8787 };
8888
8989 describe("main ahead alarm", () => {
@@ -140,7 +140,7 @@ describe("main ahead alarm", () => {
140140 const open = () => app.db.query<{ title: string }, []>(`SELECT title FROM issue WHERE state = 'open' AND title LIKE 'Sentinel: Main is green and ahead%'`).all();
141141 expect(open()).toHaveLength(1);
142142 // Production catches up: it runs the head and the pull is in the running commit.
143 keepLive("trevato/forge", ["2".repeat(40)], NOW.getTime());
143 keepLive(app.db, "trevato/forge", ["2".repeat(40)], NOW.getTime());
144144 hunt(app, "trevato/forge", NOW.getTime() + 120_000);
145145 expect(open()).toHaveLength(0);
146146 });
@@ -196,7 +196,7 @@ describe("the main-ahead issue", () => {
196196 const repo = getRepo(app.db, "trevato", "forge")!;
197197 const said = addComment(app, owner, repo, getIssueById(app.db, filed[0]!.id)!, "Looking at the tick now.");
198198 expect(said.ok).toBe(true);
199 keepLive("trevato/forge", ["4".repeat(40)], NOW.getTime());
199 keepLive(app.db, "trevato/forge", ["4".repeat(40)], NOW.getTime());
200200 hunt(app, "trevato/forge", NOW.getTime() + 720_000);
201201 expect(open()).toHaveLength(1);
202202 });
@@ -362,7 +362,7 @@ describe("a tick never heard from (#11924)", () => {
362362 try {
363363 hunt(app, "trevato/forge", NOW.getTime() + 1_200_000);
364364 expect(open()).toHaveLength(1);
365 keepLive("trevato/forge", ["8".repeat(40)], NOW.getTime());
365 keepLive(app.db, "trevato/forge", ["8".repeat(40)], NOW.getTime());
366366 hunt(app, "trevato/forge", NOW.getTime() + 1_300_000);
367367 expect(open()).toHaveLength(0);
368368 } finally {
test/flow-alarm-no-release-1111.test.ts +3 −4
@@ -16,7 +16,7 @@ import { createUser } from "../src/core/users.ts";
1616 import { setNotifyUrl, startNotifier } from "../src/crew/notify.ts";
1717 import { firedAlarms } from "../src/flow/alarms.ts";
1818 import { id } from "../src/kernel/id.ts";
19 import { forgetLive, keepLive } from "../src/release/landings.ts";
19 import { keepLive } from "../src/release/landings.ts";
2020
2121 const T0 = Date.parse("2026-11-11T12:00:00.000Z");
2222 const at = (min: number) => new Date(T0 + min * 60_000).toISOString();
@@ -33,7 +33,6 @@ afterAll(() => {
3333 for (const [k, v] of Object.entries(env)) if (v === undefined) delete process.env[k];
3434 else process.env[k] = v;
3535 for (const d of dirs) rmSync(d, { recursive: true, force: true });
36 forgetLive();
3736 });
3837
3938 describe("no release on 2026-11-11", () => {
@@ -55,7 +54,7 @@ describe("no release on 2026-11-11", () => {
5554 };
5655 // The running commit holds only what merged before it.
5756 merge(own.value, -30, "a".repeat(40));
58 keepLive("alice/forge", ["a".repeat(40)], T0);
57 keepLive(app.db, "alice/forge", ["a".repeat(40)], T0);
5958 merge(own.value, 10, "b".repeat(40));
6059 merge(own.value, 40, "c".repeat(40));
6160 merge(own.value, 70, "d".repeat(40));
@@ -99,7 +98,7 @@ describe("no release on 2026-11-11", () => {
9998 expect(posts[0]).toContain("production runs 0.7.535 · 5e1d0c0");
10099
101100 // A release whose commit contains the merges clears it, and nothing more is said.
102 keepLive("alice/forge", ["a", "b", "c", "d"].map((c) => c.repeat(40)), T0 + 360 * 60_000);
101 keepLive(app.db, "alice/forge", ["a", "b", "c", "d"].map((c) => c.repeat(40)), T0 + 360 * 60_000);
103102 expect(releaseAlarms(370)).toHaveLength(0);
104103 await wakeAt(370);
105104 expect(posts).toHaveLength(1);
test/flow-alarm-no-release.test.ts +2 −2
@@ -23,7 +23,7 @@ async function seed() {
2323 return {
2424 app,
2525 /** The running commit was made at `at`: it contains every merge before it. */
26 release: (at: string) => keepLive("alice/r", [], Date.parse(at)),
26 release: (at: string) => keepLive(app.db, "alice/r", [], Date.parse(at)),
2727 merge: (at: string) => {
2828 const issueId = id();
2929 n++;
@@ -38,7 +38,7 @@ process.env["FORGE_COMMIT"] = "abcdef1234567890";
3838 describe("no release alarm", () => {
3939 test("says nothing until the running commit has been read", async () => {
4040 const s = await seed();
41 forgetLive();
41 forgetLive(s.app.db);
4242 s.merge(minAgo(500));
4343 expect(noReleaseAlarm(s.app.db, NOW).fired).toBe(false);
4444 });
test/forge-owns-its-state.test.ts added +56
@@ -0,0 +1,56 @@
1 import { afterEach, expect, test } from "bun:test";
2 import { mkdtempSync, rmSync } from "node:fs";
3 import { tmpdir } from "node:os";
4 import { join } from "node:path";
5 import { createApp } from "../src/app.ts";
6 import { turnOf, turnStatsOf } from "../src/http/app_turn.ts";
7 import { noteProtection, protectionNotice } from "../src/kernel/protection.ts";
8 import { liveOf } from "../src/perf/live.ts";
9 import { keepLive, mainGap } from "../src/release/landings.ts";
10
11 // Two forges in one process (#12569): what one is fed is not on the other. Each assertion
12 // fails if the store it reads goes back to one per process.
13 const dirs: string[] = [];
14 afterEach(() => {
15 for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true });
16 });
17 const forge = () => {
18 const dir = mkdtempSync(join(tmpdir(), "forge-own-state-"));
19 dirs.push(dir);
20 return createApp({ dataDir: dir, port: 0 });
21 };
22
23 test("traffic, a stall, a protection note and the kept live set of one forge are not visible from another", () => {
24 const a = forge();
25 const b = forge();
26
27 // traffic
28 liveOf(a).perfRecord.add("/x", { at: 1000, ms: 5, signedIn: false, phases: [] });
29 expect(liveOf(a).perfRecord.traffic("/x", 0).length).toBe(1);
30 expect(liveOf(b).perfRecord.traffic("/x", 0)).toEqual([]);
31 expect(liveOf(a)).toBe(liveOf(a));
32 expect(liveOf(b)).not.toBe(liveOf(a));
33
34 // a stall, and the lag minute it is read in
35 const stall = liveOf(a).loopWatch.stalled(1500, 5000);
36 expect(stall).not.toBeNull();
37 liveOf(a).stallBlame.note(stall!);
38 expect(liveOf(a).stallBlame.size()).toBe(1);
39 expect(liveOf(b).stallBlame.size()).toBe(0);
40 expect(liveOf(b).stallBlame.worst(5000, 60_000, 5)).toEqual([]);
41
42 // a protection note
43 noteProtection(a, "apps", { ok: false, reason: "bwrap is not installed" });
44 expect(protectionNotice(a)).toContain("bwrap");
45 expect(protectionNotice(b)).toBeUndefined();
46
47 // the turn gate and its count
48 turnStatsOf(a).waited = 3;
49 expect(turnStatsOf(b).waited).toBe(0);
50 expect(turnOf(b)).not.toBe(turnOf(a));
51
52 // the kept live set, by database
53 keepLive(a.db, "o/r", [], 1, { commit: "c1", head: "h1", ahead: 2 });
54 expect(mainGap(a.db)?.main.sha).toBe("h1");
55 expect(mainGap(b.db)).toBeUndefined();
56 });
test/helpers/stall-state.ts +6 −10
@@ -1,24 +1,20 @@
11 /**
2 * The process's stall watch, starting clean and left clean (#11069). The
3 * suite runs every file in one process, and `loopWatch`, `stallBlame` and the
4 * telemetry reader counts (`perf/live.ts`, `system/telemetry_reads.ts`) are
5 * that process's; a stall one file noted would otherwise be a row in the next
6 * file's /system. Call at a file's top level, as `quietLogs` is
2 * The telemetry reader counts, starting clean and left clean (#11069). The
3 * suite runs every file in one process, and the counts
4 * (`system/telemetry_reads.ts`) are that process's. The loop watch and the stall
5 * blame are each forge's (`liveOf`) and need no clearing. Call at a file's top level, as `quietLogs` is
76 * (`globals.ts`): it clears before the file's first test and after its last.
87 */
98
109 import { afterAll, beforeAll } from "bun:test";
11 import { loopWatch, stallBlame } from "../../src/perf/live.ts";
1210 import { resetTelemetryReads } from "../../src/system/telemetry_reads.ts";
1311
14 /** Forget every stall and telemetry read the process has noted. */
12 /** Forget every telemetry read the process has noted. */
1513 export function clearStallState(): void {
16 loopWatch.clear();
17 stallBlame.clear();
1814 resetTelemetryReads();
1915 }
2016
21 /** Clear the process's stall watch before this scope runs and after it. */
17 /** Clear the telemetry reads before this scope runs and after it. */
2218 export function freshStallState(): void {
2319 beforeAll(clearStallState);
2420 afterAll(clearStallState);
test/home-truth.test.ts +2 −2
@@ -117,7 +117,7 @@ test("homeTruth names the running commit apart from main's head, and how far beh
117117 process.env.FORGE_VERSION = "1.2.3";
118118
119119 process.env.FORGE_COMMIT = a;
120 forgetLive();
120 forgetLive(app.db);
121121 await readLandings(app);
122122 const behind = homeTruth(app, owner, now);
123123 expect(behind.live).toEqual({ sha: a, version: "1.2.3" });
@@ -137,6 +137,6 @@ test("homeTruth names the running commit apart from main's head, and how far beh
137137 }
138138 if (saved.repo === undefined) delete (app.config as { forgeRepo?: string }).forgeRepo;
139139 else (app.config as { forgeRepo?: string }).forgeRepo = saved.repo;
140 forgetLive();
140 forgetLive(app.db);
141141 }
142142 });
test/landings-asked-once.test.ts +1 −2
@@ -7,7 +7,7 @@ import { createApp } from "../src/app.ts";
77 import { createUser } from "../src/core/users.ts";
88 import { repoDir } from "../src/git/repos.ts";
99 import { id } from "../src/kernel/id.ts";
10 import { forgetLive, readLandings } from "../src/release/landings.ts";
10 import { readLandings } from "../src/release/landings.ts";
1111
1212 const dirs: string[] = [];
1313 const saved = { path: process.env.PATH, commit: process.env.FORGE_COMMIT };
@@ -15,7 +15,6 @@ afterEach(() => {
1515 process.env.PATH = saved.path;
1616 if (saved.commit === undefined) delete process.env.FORGE_COMMIT;
1717 else process.env.FORGE_COMMIT = saved.commit;
18 forgetLive();
1918 for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true });
2019 });
2120
test/loop-stall-by-route.test.ts +10 −10
@@ -11,7 +11,7 @@ import { serve, type Servers } from "../src/http/server.ts";
1111 import { watchStalls } from "../src/kernel/loop_lag.ts";
1212 import { openMetrics } from "../src/metrics/openmetrics.ts";
1313 import { stallFamilies, TOP_ROUTES } from "../src/metrics/stalls.ts";
14 import { lagMinutes, loopWatch, stallBlame } from "../src/perf/live.ts";
14 import { liveOf } from "../src/perf/live.ts";
1515 import { WORST_STALLS } from "../src/perf/stall_blame.ts";
1616 import { createLoopWatch, HOUR_MS, NO_REQUEST } from "../src/perf/loop.ts";
1717 import { createAgent } from "../src/actions/crew.ts";
@@ -144,8 +144,8 @@ test("GET /metrics names the route a stall ran across", async () => {
144144 // millisecond was only across a 400 ms stall, not its cause (`WITNESS_SHARE`), so the one
145145 // that held the loop is noted as the server would have noted it: 300 ms of the window.
146146 const at = performance.now();
147 loopWatch.finished("/api/v1/health", at - 300, at);
148 expect(loopWatch.stalled(400, at)).not.toBeNull();
147 liveOf(app).loopWatch.finished("/api/v1/health", at - 300, at);
148 expect(liveOf(app).loopWatch.stalled(400, at)).not.toBeNull();
149149 const text = await (await fetch(`${servers.url}/metrics`, { headers: { authorization: `Bearer ${ro}` } })).text();
150150 expect(text).toMatch(/^forge_loop_stalls_total\{route="\/api\/v1\/health"\} [1-9]/m);
151151 expect(text).toMatch(/^forge_loop_stall_hour_milliseconds\{route="\/api\/v1\/health"\} [1-9]/m);
@@ -229,12 +229,12 @@ test("the Stalls screen lists the hour's worst single stalls as /system does: wh
229229 test("the Stalls screen reads the process's stall blame by default, the five /system reads", () => {
230230 const now = performance.now();
231231 for (const [ms, r] of [[300, "/a"], [800, "/b"], [250, "/c"], [500, "/d"], [700, "/e"], [210, "/f"]] as const)
232 stallBlame.note({ at: now - 1_000, lateMs: ms, routes: [r], jobs: [] });
232 liveOf(app).stallBlame.note({ at: now - 1_000, lateMs: ms, routes: [r], jobs: [] });
233233 const lines = stallLines(new Session(app, identity(alice)), []);
234234 const named = lines.slice(lines.indexOf("") + 2).map((l) => l.split(/\s+/)[0] ?? "");
235235 expect(named).toEqual(["/b", "/e", "/d", "/a", "/c"]);
236 expect(stallBlame.worst(performance.now(), HOUR_MS, WORST_STALLS).map((s) => s.what)).toEqual(named);
237 stallBlame.clear();
236 expect(liveOf(app).stallBlame.worst(performance.now(), HOUR_MS, WORST_STALLS).map((s) => s.what)).toEqual(named);
237 liveOf(app).stallBlame.clear();
238238 });
239239
240240 test("the Stalls screen shows the last 20 minutes' loop lag as /system does: over count, each minute, what ran (#11220)", async () => {
@@ -258,9 +258,9 @@ test("the Stalls screen shows the last 20 minutes' loop lag as /system does: ove
258258 for (const who of [bob, bot, undefined]) expect(lagLines(new Session(app, identity(who)), rows)).toEqual([]);
259259
260260 // The screen itself draws the process's readings above the stalls.
261 lagMinutes.clear();
262 lagMinutes.note(250, 0);
263 lagMinutes.note(0, 60_000); // closes the first minute
261 liveOf(app).lagMinutes.clear();
262 liveOf(app).lagMinutes.note(250, 0);
263 liveOf(app).lagMinutes.note(0, 60_000); // closes the first minute
264264 const { app: tui, session } = createForgeApp(app, identity(alice), { terminal: headlessTerminal(120, 40), exitHooks: false });
265265 tui.start();
266266 await settleApp(tui);
@@ -268,7 +268,7 @@ test("the Stalls screen shows the last 20 minutes' loop lag as /system does: ove
268268 await settleApp(tui);
269269 const shown = tui.screen.toText();
270270 tui.stop();
271 lagMinutes.clear();
271 liveOf(app).lagMinutes.clear();
272272 expect(shown).toContain("Loop lag, last 20 minutes: 1 of 1 over 100 ms");
273273 expect(shown).toContain("Route (last hour)");
274274 expect(shown.indexOf("Loop lag")).toBeLessThan(shown.indexOf("Route (last hour)"));
test/mcp-system-routes.test.ts +4 −4
@@ -9,7 +9,7 @@ import { setTelemetryGrant } from "../src/actions/telemetry.ts";
99 import { createToken } from "../src/core/tokens.ts";
1010 import { createUser, getUserByName } from "../src/core/users.ts";
1111 import { serve, type Servers } from "../src/http/server.ts";
12 import { perfRecord } from "../src/perf/live.ts";
12 import { liveOf } from "../src/perf/live.ts";
1313 import { quietLogs } from "./helpers/globals.ts";
1414
1515 quietLogs();
@@ -48,11 +48,11 @@ const call = async (token: string) => {
4848
4949 test("routes lists the over-budget route first, with its budget and slowest phase; a non-admin is refused", async () => {
5050 // perfRecord is the process's own, and a shard-mate's requests leave samples (and patterns, up to the cap) in it.
51 perfRecord.clear();
51 liveOf(app).perfRecord.clear();
5252 const at = Date.now();
5353 for (let i = 0; i < 25; i++) {
54 perfRecord.add("/boards", { at, ms: 100, signedIn: true, phases: [{ name: "app", ms: 100, hit: false }] });
55 perfRecord.add("/explore", { at, ms: 2000, signedIn: true, phases: [{ name: "app", ms: 2000, hit: false }, { name: "read", ms: 1500, hit: false }] });
54 liveOf(app).perfRecord.add("/boards", { at, ms: 100, signedIn: true, phases: [{ name: "app", ms: 100, hit: false }] });
55 liveOf(app).perfRecord.add("/explore", { at, ms: 2000, signedIn: true, phases: [{ name: "app", ms: 2000, hit: false }, { name: "read", ms: 1500, hit: false }] });
5656 }
5757 const out = JSON.parse((await call(admin)).result.content[0].text);
5858 const mine = out.routes.filter((r: any) => r.route === "/boards" || r.route === "/explore");
test/mcp-system-stalls.test.ts +3 −3
@@ -8,7 +8,7 @@ import { createToken } from "../src/core/tokens.ts";
88 import { createUser } from "../src/core/users.ts";
99 import { serve, type Servers } from "../src/http/server.ts";
1010 import { during } from "../src/perf/jobs.ts";
11 import { loopWatch } from "../src/perf/live.ts";
11 import { liveOf } from "../src/perf/live.ts";
1212 import { LIMIT_MS } from "../src/perf/loop.ts";
1313 import { quietLogs } from "./helpers/globals.ts";
1414
@@ -37,14 +37,14 @@ afterAll(async () => {
3737 test("a stall under a named job is read through system, at most 10, most ms first", async () => {
3838 // The watch is the process's one: stalls and requests an earlier file left
3939 // would crowd the top 10 or be blamed ahead of the job. Start from nothing.
40 loopWatch.clear();
40 liveOf(app).loopWatch.clear();
4141 // The job holds the loop for a real share of the stall: one that only brushed it is not
4242 // named (`perf/loop.ts`).
4343 during("test-job-10907", () => {
4444 const until = performance.now() + 40;
4545 while (performance.now() < until);
4646 });
47 loopWatch.stalled(LIMIT_MS + 10, performance.now());
47 liveOf(app).loopWatch.stalled(LIMIT_MS + 10, performance.now());
4848 const res = await fetch(`${servers.url}/mcp`, {
4949 method: "POST",
5050 headers: { "content-type": "application/json", accept: "application/json, text/event-stream", authorization: `Bearer ${token}` },
test/module-state.baseline.txt added +219
@@ -0,0 +1,219 @@
1 # State that predates the rule (#12569), to be moved under an app, never added to.
2 src/actions/detach.ts:jobs
3 src/actions/features.ts:changed
4 src/actions/features.ts:declared
5 src/actions/forks.ts:standing
6 src/actions/pin_demos.ts:factsKept
7 src/actions/spawn.ts:inFlight
8 src/actions/upstream_follow.ts:checking
9 src/apps/api_spec.ts:asking
10 src/apps/api_spec.ts:kept
11 src/apps/check-memory.ts:declaredAt
12 src/apps/check-memory.ts:held
13 src/apps/contain.ts:diskOpen
14 src/apps/contain.ts:treeOpen
15 src/apps/contain.ts:watched
16 src/apps/cpu_duty.ts:stopped
17 src/apps/cron.ts:formatters
18 src/apps/egress.ts:meters
19 src/apps/egress.ts:moved
20 src/apps/egress.ts:npmByRepo
21 src/apps/egress.ts:tallies
22 src/apps/egress_refused.ts:books
23 src/apps/egress_refused.ts:cache
24 src/apps/feature_clock.ts:labels
25 src/apps/feature_schedule.ts:inFlight
26 src/apps/feedback.ts:proofs
27 src/apps/handoff.ts:ports
28 src/apps/pull_schedule.ts:declaredByTip
29 src/apps/pull_schedule.ts:inFlight
30 src/apps/relay_wait.ts:slowest
31 src/apps/timer_fire.ts:inFlight
32 src/billing/sync.ts:chains
33 src/checks/impact.ts:paths
34 src/checks/install.ts:local
35 src/checks/install_evict.ts:lastSweep
36 src/checks/install_evict.ts:sweeping
37 src/checks/queue.ts:lines
38 src/checks/queue.ts:watchers
39 src/checks/revert_red.ts:working
40 src/cloud/declared.ts:cache
41 src/contract/audit.ts:remembered
42 src/contract/served.ts:built
43 src/core/chapters.ts:frozen
44 src/core/contribute_token.ts:turns
45 src/core/credit.ts:authorCache
46 src/core/following.ts:relationCache
47 src/core/idempotency.ts:inFlight
48 src/core/parent_releases.ts:refreshing
49 src/core/pulls.ts:inflight
50 src/core/pulls.ts:lastByPull
51 src/core/pulls.ts:readings
52 src/core/readme_description.ts:lines
53 src/core/recombine.ts:running
54 src/core/recombine.ts:started
55 src/core/search.ts:grepCache
56 src/crew/approved_week.ts:judged
57 src/crew/autonomy.ts:asked
58 src/crew/autonomy.ts:checked
59 src/crew/autonomy.ts:cleanAt
60 src/crew/autonomy.ts:conflicted
61 src/crew/autonomy.ts:conflictedAt
62 src/crew/autonomy.ts:sweeping
63 src/crew/ceiling_stagger.ts:memo
64 src/crew/conversation.ts:live
65 src/crew/conversation.ts:queues
66 src/crew/conversation.ts:stoppers
67 src/crew/dispatch.ts:deferredFailures
68 src/crew/dispatch.ts:retriedChecks
69 src/crew/dispatch.ts:waitingOnMain
70 src/crew/finishing.ts:entered
71 src/crew/finishing.ts:read
72 src/crew/finishing.ts:said
73 src/crew/governed.ts:seen
74 src/crew/handover.ts:handing
75 src/crew/hot_files.ts:standing
76 src/crew/hot_files.ts:standingWideBy
77 src/crew/hot_files.ts:touched
78 src/crew/hot_history.ts:samples
79 src/crew/mirror.ts:chains
80 src/crew/notify.ts:lastTest
81 src/crew/plan.ts:found
82 src/crew/repeat_checks.ts:swept
83 src/crew/requested.ts:said
84 src/crew/review.ts:said
85 src/crew/runner.ts:steering
86 src/crew/running.ts:running
87 src/crew/waits.ts:last
88 src/crew/week_read.ts:remembered
89 src/crew/workdir.ts:claimed
90 src/design/mark.ts:cache
91 src/devices/print.ts:inFlight
92 src/display/config.ts:known
93 src/efficiency/installs.ts:seen
94 src/flow/alarm_drift.ts:cache
95 src/flow/keep.ts:keepers
96 src/flow/lanes.ts:open
97 src/flow/lanes.ts:viewers
98 src/flow/lanes_worker.ts:handles
99 src/flow/lanes_worker.ts:kept
100 src/flow/pages.ts:logged
101 src/flow/pages.ts:open
102 src/flow/pages.ts:workers
103 src/flow/pages_read.ts:said
104 src/flow/pages_worker.ts:envs
105 src/flow/pipeline.ts:waitingOpen
106 src/flow/pipeline_worker.ts:handles
107 src/flows/probe.ts:running
108 src/flows/start.ts:running
109 src/git/archive.ts:held
110 src/git/coauthors.ts:credited
111 src/git/diff-memory.ts:kept
112 src/git/diff-memory.ts:mergeBases
113 src/git/diff.ts:running
114 src/git/env.ts:bundles
115 src/git/own_refs.ts:reached
116 src/git/path_listing.ts:kept
117 src/git/pins_kept.ts:epochs
118 src/git/pins_kept.ts:kept
119 src/git/read.ts:pool
120 src/git/sync_link.ts:queues
121 src/git/tip.ts:packedOf
122 src/http/access_log.ts:pending
123 src/http/api/forge_view.ts:recent
124 src/http/archive_limit.ts:seen
125 src/http/compress.ts:kept
126 src/http/flow_stream.ts:pollers
127 src/http/rp.ts:registering
128 src/http/sse.ts:openStreams
129 src/kernel/full_disk.ts:owned
130 src/kernel/full_disk.ts:states
131 src/kernel/log.ts:forgeLedgers
132 src/kernel/log.ts:held
133 src/kernel/passport.ts:subkeys
134 src/kernel/probe.ts:probes
135 src/kernel/secret.ts:keyCache
136 src/kernel/secret.ts:previousCache
137 src/looks/capture.ts:browsers
138 src/looks/thumb.ts:open
139 src/machines/rc.ts:inflight
140 src/origins/credential.ts:refreshing
141 src/origins/jobs.ts:jobs
142 src/origins/jobs.ts:listeners
143 src/origins/mirror-pulls.ts:running
144 src/origins/providers/index.ts:cache
145 src/origins/sync.ts:inFlight
146 src/origins/token-reach.ts:kept
147 src/origins/work-branch.ts:tipsSeen
148 src/perf/child_time.ts:minutes
149 src/release/changelog.ts:cache
150 src/release/prove.ts:superseded
151 src/runs/activity.ts:lastAt
152 src/runs/lease.ts:held
153 src/runs/log.ts:listeners
154 src/runs/log.ts:pending
155 src/runs/log.ts:taps
156 src/runs/log_append.ts:made
157 src/runs/stoppable.ts:running
158 src/snapshot/bucket.ts:reads
159 src/snapshot/bucket.ts:runCounts
160 src/snapshot/bucket.ts:watching
161 src/spawn/app_links.ts:notOffered
162 src/spawn/app_links.ts:offerFailed
163 src/spawn/app_links.ts:said
164 src/spawn/archive.ts:busy
165 src/spawn/child_db.ts:busy
166 src/spawn/child_db.ts:hands
167 src/spawn/child_db.ts:line
168 src/spawn/funding.ts:askedAt
169 src/spawn/heartbeat.ts:turns
170 src/spawn/host.ts:waiters
171 src/spawn/own_code.ts:inUse
172 src/spawn/own_code.ts:queues
173 src/spawn/sealed.ts:read
174 src/spawn/usage.ts:disks
175 src/spawn/usage.ts:walking
176 src/system/telemetry_reads.ts:counts
177 src/system/telemetry_reads.ts:slots
178 src/tui/screens/mirror.ts:shown
179 src/tui/screens/mirror.ts:watched
180 src/tui/term/screen.ts:GLYPHS
181 src/tui/term/screen.ts:GLYPH_IDS
182 src/tui/term/style.ts:quantCache
183 src/tui/term/width.ts:widthCache
184 src/tui/ui/theme.ts:cache
185 src/web/asset.ts:kept
186 src/web/asset.ts:pageScripts
187 src/web/asset.ts:pageScriptsByName
188 src/web/asset.ts:pruned
189 src/web/code.ts:cache
190 src/web/code_rows_pool.ts:pool
191 src/web/code_rows_pool.ts:waiting
192 src/web/diff-cache.ts:kept
193 src/web/diff-cache.ts:rendered
194 src/web/feeds.ts:waveCache
195 src/web/head-reads.ts:kept
196 src/web/llms_tools.ts:sections
197 src/web/map_snapshot.ts:open
198 src/web/map_snapshot.ts:said
199 src/web/map_worker.ts:apps
200 src/web/markdown.ts:memo
201 src/web/og/raster.ts:open
202 src/web/pages/branches.ts:kept
203 src/web/pages/changelog.ts:rendered
204 src/web/pages/docs_link.ts:rendered
205 src/web/pages/pull.ts:fastByHead
206 src/web/pages/repo.ts:readmeCache
207 src/web/pages/talk.ts:open
208 src/web/render/gcode.ts:kept
209 src/web/render/model3d-client.ts:live
210 src/web/share_target.ts:SENT
211 src/web/standards.ts:discoveryBodies
212 src/web/standards.ts:llmsBodies
213 src/web/standards.ts:releasesBodies
214 src/web/standards.ts:sitemapBodies
215 src/web/studio/home_last.ts:drawing
216 src/web/studio/toolbar.ts:stats
217 src/apps/egress_refused.ts:said
218 src/core/readme_description.ts:tips
219 src/release/leftover_sweep.ts:removing
test/module-state.test.ts added +54
@@ -0,0 +1,54 @@
1 /**
2 * State belongs to a forge (an app object), not to the process (#12569): two
3 * forges in one `bun test` process, or one test after another, must not see
4 * each other's requests. A module under src/ may not hold a module-level
5 * Map, Set or array that code inside a function writes to; key the state by
6 * the app in a WeakMap (`askingOf(app)`, `liveOf(app)`) instead.
7 *
8 * `test/module-state.baseline.txt` lists what predates the rule, one
9 * `file:name` per line. It only shrinks: a new entry is refused, and an entry
10 * whose state has moved or gone must be deleted from the list.
11 *
12 * @tree-scan: reads every file under src/.
13 */
14
15 import { describe, expect, test } from "bun:test";
16 import { readFileSync } from "node:fs";
17 import { Glob } from "bun";
18
19 const DECL = /^(?:export )?(?:const|let) ([A-Za-z_$][\w$]*)(?:: [^=\n]+)? = (?:new (?:Map|Set)\b|\[\])/gm;
20
21 /** `file:name` of each module-level collection that indented code writes to. */
22 export function sharedState(): string[] {
23 const found: string[] = [];
24 for (const file of new Glob("src/**/*.ts").scanSync(".")) {
25 const text = readFileSync(file, "utf8");
26 for (const m of text.matchAll(DECL)) {
27 const name = m[1]!;
28 const writes = new RegExp(
29 `^\\s+.*\\b${name}\\.(set|add|push|unshift|splice|delete|pop|shift)\\(|^\\s+.*\\b${name}\\[[^\\]]+\\] =`,
30 "m",
31 );
32 if (writes.test(text)) found.push(`${file}:${name}`);
33 }
34 }
35 return found.sort();
36 }
37
38 const baseline = readFileSync(new URL("./module-state.baseline.txt", import.meta.url), "utf8")
39 .split("\n")
40 .filter((l) => l !== "" && !l.startsWith("#"));
41
42 describe("no process-wide state that requests write to", () => {
43 const now = sharedState();
44 test("a module adds no module-level Map, Set or array that a function writes to", () => {
45 expect(now.filter((s) => !baseline.includes(s))).toEqual([]);
46 });
47 test("the baseline names only state that still exists", () => {
48 expect(baseline.filter((s) => !now.includes(s))).toEqual([]);
49 });
50 test("the forge-scoped state of #12569 is not on the baseline", () => {
51 for (const gone of ["src/http/app_turn.ts", "src/perf/live.ts", "src/kernel/protection.ts"])
52 expect(baseline.filter((s) => s.startsWith(gone))).toEqual([]);
53 });
54 });
test/pages-budget.test.ts +2 −2
@@ -36,7 +36,7 @@ import { keepTick } from "../src/flow/keep.ts";
3636 import { readLanesInline, useLanesReader } from "../src/flow/lanes.ts";
3737 import { usePagesReader } from "../src/flow/pages.ts";
3838 import { pagesConfig, readPagePicture, type Picture } from "../src/flow/pages_read.ts";
39 import { perfRecord } from "../src/perf/live.ts";
39 import { liveOf } from "../src/perf/live.ts";
4040 import { buildApiRouter, buildRouter, serve, type Servers } from "../src/http/server.ts";
4141 import { quietLogs } from "./helpers/globals.ts";
4242 import { promote } from "./helpers/members.ts";
@@ -172,7 +172,7 @@ beforeAll(async () => {
172172 }, 60_000);
173173 afterAll(() => {
174174 // The walk fills the shared recorder to its pattern cap; leave it empty for the next file in this process.
175 perfRecord.clear();
175 liveOf(app).perfRecord.clear();
176176 servers.stop();
177177 app.close();
178178 rmSync(root, { recursive: true, force: true });
test/perf-lag-minutes.test.ts +6 −6
@@ -13,7 +13,7 @@ import { createToken } from "../src/core/tokens.ts";
1313 import { createUser } from "../src/core/users.ts";
1414 import { serve, type Servers } from "../src/http/server.ts";
1515 import { createLagMinutes, LAG_BAR_MS, LAG_MINUTE_MS, LAG_READINGS, overBar } from "../src/perf/lag_minutes.ts";
16 import { lagMinutes } from "../src/perf/live.ts";
16 import { liveOf } from "../src/perf/live.ts";
1717 import { lagSection } from "../src/web/pages/system.ts";
1818 import { quietLogs } from "./helpers/globals.ts";
1919
@@ -122,18 +122,18 @@ describe("the MCP system tool", () => {
122122 }, 30_000);
123123
124124 afterAll(async () => {
125 lagMinutes.clear();
125 liveOf(app).lagMinutes.clear();
126126 servers.stop();
127127 app.close();
128128 await rm(dataDir, { recursive: true, force: true });
129129 });
130130
131131 test("answers the same readings as {at, ms, what}, with the bar and the count over it", async () => {
132 lagMinutes.clear();
132 liveOf(app).lagMinutes.clear();
133133 const base = 1_000 * MIN;
134 lagMinutes.note(20, base);
135 lagMinutes.note(180.04, base + MIN);
136 lagMinutes.note(1, base + 2 * MIN);
134 liveOf(app).lagMinutes.note(20, base);
135 liveOf(app).lagMinutes.note(180.04, base + MIN);
136 liveOf(app).lagMinutes.note(1, base + 2 * MIN);
137137 const res = await fetch(`${servers.url}/mcp`, {
138138 method: "POST",
139139 headers: { "content-type": "application/json", accept: "application/json, text/event-stream", authorization: `Bearer ${token}` },
test/perf-loop-jobs.test.ts +3 −6
@@ -2,12 +2,8 @@
22
33 import { expect, test } from "bun:test";
44 import { createJobWatch, during } from "../src/perf/jobs.ts";
5 import { loopWatch } from "../src/perf/live.ts";
5 import { liveOf } from "../src/perf/live.ts";
66 import { createLoopWatch, NO_REQUEST, stallLine } from "../src/perf/loop.ts";
7 import { freshStallState } from "./helpers/stall-state.ts";
8
9 // The wired test notes a stall on the process's watch, which every file in this process shares.
10 freshStallState();
117
128 const block = (ms: number): void => {
139 const until = performance.now() + ms;
@@ -66,7 +62,8 @@ test("a loop stall while a job awaits and another blocks is tallied to the block
6662 await lane;
6763 });
6864
69 test("the process's loop watch reads the process's job watch", () => {
65 test("a forge's loop watch reads the process's job watch", () => {
66 const { loopWatch } = liveOf({});
7067 const before = loopWatch.tally(performance.now()).total.find((r) => r.route === "job:wired-test")?.count ?? 0;
7168 during("wired-test", () => block(300));
7269 const at = performance.now();
test/perf-server-hook.test.ts +3 −3
@@ -5,7 +5,7 @@ import { join } from "node:path";
55 import { createApp, type App } from "../src/app.ts";
66 import { serve, type Servers } from "../src/http/server.ts";
77 import { watchStalls } from "../src/kernel/loop_lag.ts";
8 import { perfRecord } from "../src/perf/live.ts";
8 import { liveOf } from "../src/perf/live.ts";
99 import { quietLogs } from "./helpers/globals.ts";
1010
1111 quietLogs();
@@ -25,10 +25,10 @@ afterAll(() => {
2525 });
2626
2727 test("a served request lands in the record under its route pattern; a client label hides nothing", async () => {
28 const before = perfRecord.traffic("/api/v1/health", 0).length;
28 const before = liveOf(app).perfRecord.traffic("/api/v1/health", 0).length;
2929 await fetch(`${servers.url}/api/v1/health`);
3030 await fetch(`${servers.url}/api/v1/health`, { headers: { "x-forge-synthetic": "1" } });
31 const after = perfRecord.traffic("/api/v1/health", 0);
31 const after = liveOf(app).perfRecord.traffic("/api/v1/health", 0);
3232 expect(after.length).toBe(before + 2);
3333 expect(after.at(-1)!.signedIn).toBe(false);
3434 expect(after.at(-1)!.phases.some((p) => p.name === "app")).toBe(true);
test/perf-stall-blame.test.ts +6 −6
@@ -13,7 +13,7 @@ import { createApp, type App } from "../src/app.ts";
1313 import { serve, type Servers } from "../src/http/server.ts";
1414 import { startLoopLag } from "../src/kernel/loop_lag.ts";
1515 import { createJobWatch } from "../src/perf/jobs.ts";
16 import { stallBlame } from "../src/perf/live.ts";
16 import { liveOf } from "../src/perf/live.ts";
1717 import { createLoopWatch, HOUR_MS, LIMIT_MS, type Stall } from "../src/perf/loop.ts";
1818 import { createStallBlame, STALL_KEPT, UNKNOWN, whatOf } from "../src/perf/stall_blame.ts";
1919 import { stallSection, worstStallsTable } from "../src/web/pages/system.ts";
@@ -139,14 +139,14 @@ describe("on /healthz", () => {
139139 });
140140
141141 afterAll(async () => {
142 stallBlame.clear();
142 liveOf(app).stallBlame.clear();
143143 servers.stop();
144144 app.close();
145145 await rm(dataDir, { recursive: true, force: true });
146146 });
147147
148148 test("loopWorstWhat is null with no stall, then names the last 10 s's worst", async () => {
149 stallBlame.clear();
149 liveOf(app).stallBlame.clear();
150150 const ask = async () =>
151151 (await (await fetch(`${servers.url}/healthz`, { headers: { accept: "application/json" } })).json()) as {
152152 ok: boolean;
@@ -159,9 +159,9 @@ describe("on /healthz", () => {
159159 expect(before.loopWorstWhat).toBeNull();
160160
161161 const now = performance.now();
162 stallBlame.note({ at: now - 60_000, lateMs: 900, routes: ["/too-old"], jobs: [] });
163 stallBlame.note({ at: now, lateMs: 300, routes: [], jobs: ["crew-tick"] });
164 stallBlame.note({ at: now, lateMs: 220, routes: ["/explore"], jobs: [] });
162 liveOf(app).stallBlame.note({ at: now - 60_000, lateMs: 900, routes: ["/too-old"], jobs: [] });
163 liveOf(app).stallBlame.note({ at: now, lateMs: 300, routes: [], jobs: ["crew-tick"] });
164 liveOf(app).stallBlame.note({ at: now, lateMs: 220, routes: ["/explore"], jobs: [] });
165165 expect((await ask()).loopWorstWhat).toBe("job:crew-tick");
166166 // A probe asking for nothing still hears the plain `ok` (#2917).
167167 expect(await (await fetch(`${servers.url}/healthz`)).text()).toBe("ok");
test/protection.test.ts +14 −14
@@ -1,33 +1,33 @@
11 import { mkdtemp, rm } from "node:fs/promises";
22 import { tmpdir } from "node:os";
33 import { join } from "node:path";
4 import { afterEach, expect, test } from "bun:test";
4 import { expect, test } from "bun:test";
55 import {
66 noteProtection,
77 protectionNotice,
8 resetProtection,
98 } from "../src/kernel/protection.ts";
109
11 afterEach(resetProtection);
12
1310 test("a fully protected forge has no notice", () => {
14 noteProtection("apps", { ok: true });
15 expect(protectionNotice()).toBeUndefined();
11 const app = {};
12 noteProtection(app, "apps", { ok: true });
13 expect(protectionNotice(app)).toBeUndefined();
1614 });
1715
1816 test("a part that lacks protection is named with its reason, in one line", () => {
19 noteProtection("apps", { ok: false, reason: "bwrap is not installed" });
20 noteProtection("crew shells", { ok: false, reason: "no sandbox on win32" });
21 const line = protectionNotice()!;
17 const app = {};
18 noteProtection(app, "apps", { ok: false, reason: "bwrap is not installed" });
19 noteProtection(app, "crew shells", { ok: false, reason: "no sandbox on win32" });
20 const line = protectionNotice(app)!;
2221 expect(line).toContain("apps (bwrap is not installed)");
2322 expect(line).toContain("crew shells (no sandbox on win32)");
2423 expect(line).not.toContain("\n");
2524 });
2625
2726 test("a part that recovers clears its own line", () => {
28 noteProtection("apps", { ok: false, reason: "x" });
29 noteProtection("apps", { ok: true });
30 expect(protectionNotice()).toBeUndefined();
27 const app = {};
28 noteProtection(app, "apps", { ok: false, reason: "x" });
29 noteProtection(app, "apps", { ok: true });
30 expect(protectionNotice(app)).toBeUndefined();
3131 });
3232
3333 // Run in a process of its own: the probes are cached per process, and the
@@ -48,13 +48,13 @@ test("AppsRuntime.start and startCrew each note what they lack", async () => {
4848 forgeUrl: "http://127.0.0.1:1",
4949 });
5050 await runtime.start();
51 const afterApps = protectionNotice();
51 const afterApps = protectionNotice(app);
5252 const crew = await startCrew(app, {
5353 forgeUrl: "http://127.0.0.1:1", localUrl: "http://127.0.0.1:1",
5454 binary: "/no/such/claude", concurrency: 1, budgetUsd: 1,
5555 runTimeoutMs: 60000, tickMs: 50,
5656 });
57 console.log(JSON.stringify({ afterApps, afterCrew: protectionNotice() }));
57 console.log(JSON.stringify({ afterApps, afterCrew: protectionNotice(app) }));
5858 await crew.stop();
5959 await runtime.stop();
6060 process.exit(0);
test/red-main.test.ts +2 −1
@@ -315,7 +315,8 @@ describe("a red main in another repository (#3992)", () => {
315315 span("red:r2");
316316 span("red:r1");
317317 expect(budgetBreaches(db, 2000, "r1").map((f) => f.key)).toEqual(["degraded:red-r1"]);
318 expect(findFaults(db, 2000, "r1").map((f) => f.key)).toEqual(["degraded:red-r1"]);
318 // A ledger of its own: the process's holds what earlier files logged (#12569).
319 expect(findFaults(db, 2000, "r1", undefined, null, new ErrorLedger()).map((f) => f.key)).toEqual(["degraded:red-r1"]);
319320 expect(budgetBreaches(db, 2000, "r2").map((f) => f.key)).toEqual(["degraded:red-r2"]);
320321 });
321322
test/relay-stall-name.test.ts +3 −3
@@ -30,7 +30,7 @@ import type { Repo } from "../src/core/repos.ts";
3030 import { createUser, type User } from "../src/core/users.ts";
3131 import { serve, type Servers } from "../src/http/server.ts";
3232 import { createJobWatch, jobWatch } from "../src/perf/jobs.ts";
33 import { loopWatch } from "../src/perf/live.ts";
33 import { liveOf } from "../src/perf/live.ts";
3434 import { createLoopWatch, type Stall } from "../src/perf/loop.ts";
3535 import { loopWorst } from "../src/perf/loop_worst.ts";
3636 import { deploySettled } from "./helpers/apps-forge.ts";
@@ -176,8 +176,8 @@ describe("forge-stall on the relayed answer", () => {
176176 loopWorst.note(ms, now);
177177 // A job that ran for a moment was only across the stall (`WITNESS_SHARE`, `perf/loop.ts`):
178178 // what is named is what covered it, so a request that did is noted as the server notes one.
179 loopWatch.finished("/api/v1/health", now - ms, now);
180 const s = loopWatch.stalled(ms, now)!;
179 liveOf(app).loopWatch.finished("/api/v1/health", now - ms, now);
180 const s = liveOf(app).loopWatch.stalled(ms, now)!;
181181 // A route that ran across it outranks the job (setup's requests often do);
182182 // the job's own naming is pinned above, with a stepped clock.
183183 return heldBy([s])[0]!.name;
test/spawn-in-process-visible.test.ts +7 −7
@@ -20,7 +20,7 @@ import {
2020 noteChildRequest,
2121 } from "../src/perf/child_time.ts";
2222 import { during, jobWatch } from "../src/perf/jobs.ts";
23 import { loopWatch } from "../src/perf/live.ts";
23 import { liveOf } from "../src/perf/live.ts";
2424 import { stallLine } from "../src/perf/loop.ts";
2525 import { clientRows, servedWindow } from "../src/spawn/clients.ts";
2626 import { closeHostedChildren, openChildren, provideCellsForTest } from "../src/spawn/host.ts";
@@ -226,12 +226,12 @@ describe("a parent holding a forge in its own process", () => {
226226 });
227227
228228 test("a request it answers is the stall watcher's child:<name>, and is counted against it", async () => {
229 loopWatch.clear();
229 liveOf(parent).loopWatch.clear();
230230 // Open it, so the next request is answered by a forge already in this process.
231231 await (await fetch(`${servers.url}/healthz`, { headers: { host: "homie--forge.test" } })).text();
232232 const child = openChildren(parent).find((c) => c.name === "homie")!.app;
233233 const had = childHours().find((r) => r.name === "homie")?.requests ?? 0;
234 loopWatch.clear();
234 liveOf(parent).loopWatch.clear();
235235 // The child's own request holds the parent's thread: its first statement takes 80 ms of
236236 // synchronous work, as a slow query or a git call of the child's would. A stall is named
237237 // for what held the loop, not for what was only across it (`WITNESS_SHARE`, `perf/loop.ts`).
@@ -244,22 +244,22 @@ describe("a parent holding a forge in its own process", () => {
244244 expect(release()).toBe(true);
245245 // The stall that request made names the child: before, it named the parent's route of the
246246 // same shape.
247 const stall = loopWatch.stalled(250, performance.now());
247 const stall = liveOf(parent).loopWatch.stalled(250, performance.now());
248248 expect(stall?.routes).toContain("child:homie");
249249 expect(stallLine(stall!)).toContain("running then: child:homie");
250250 expect(childHours().find((r) => r.name === "homie")?.requests).toBe(had + 1);
251251 // The parent's own request is still its route's.
252 loopWatch.clear();
252 liveOf(parent).loopWatch.clear();
253253 const mine = holdOnce(parent, 80);
254254 await (
255255 await fetch(`${servers.url}/healthz`, { headers: { host: "forge.test" } })
256256 ).text();
257257 expect(mine()).toBe(true);
258 const own = loopWatch.stalled(250, performance.now());
258 const own = liveOf(parent).loopWatch.stalled(250, performance.now());
259259 expect(own?.routes).toEqual(["/healthz"]);
260260 expect(own?.routes.some((r) => r.startsWith("child:"))).toBe(false);
261261 // The watch is the process's: the two stalls this test made up are not left in its hour.
262 loopWatch.clear();
262 liveOf(parent).loopWatch.clear();
263263 });
264264
265265 test("/forges says so on its row, in plain words", async () => {
test/stall-in-flight.test.ts +4 −4
@@ -10,7 +10,7 @@ import { createUser } from "../src/core/users.ts";
1010 import { serve, type Servers } from "../src/http/server.ts";
1111 import { stallFamilies } from "../src/metrics/stalls.ts";
1212 import { createInFlight } from "../src/perf/in_flight.ts";
13 import { inFlight, loopWatch } from "../src/perf/live.ts";
13 import { inFlight, liveOf } from "../src/perf/live.ts";
1414 import { blamed, createLoopWatch, LIMIT_MS, NO_REQUEST, stallLine } from "../src/perf/loop.ts";
1515 import { whatOf } from "../src/perf/stall_blame.ts";
1616 import { quietLogs } from "./helpers/globals.ts";
@@ -142,7 +142,7 @@ function describe_server() {
142142 };
143143
144144 test("a real request held open across a stall is in MCP `system` `stalls`; one begun after is not", async () => {
145 loopWatch.clear();
145 liveOf(app).loopWatch.clear();
146146 // The suite shares one process: a request another file left open (or its own server's
147147 // background traffic) is in the open set too, and comes and goes on its own. Watch only
148148 // this test's `/mcp`, never the size of the whole set.
@@ -171,11 +171,11 @@ function describe_server() {
171171 await Bun.sleep(LIMIT_MS + 60); // real-timer: the request must be older than the stall limit on the real clock
172172 // The wake: the loop held for longer than the stall limit, beginning after the request did.
173173 const now = performance.now();
174 const stall = loopWatch.stalled(LIMIT_MS + 10, now)!;
174 const stall = liveOf(app).loopWatch.stalled(LIMIT_MS + 10, now)!;
175175 const named = stall.inFlight ?? [];
176176 expect(named).toContain("/mcp");
177177 // A stall shorter than the request's age is charged; one reaching back before it began is not.
178 expect(loopWatch.stalled(60_000, now)!.inFlight ?? []).toEqual(inFlight.since(now - 60_000));
178 expect(liveOf(app).loopWatch.stalled(60_000, now)!.inFlight ?? []).toEqual(inFlight.since(now - 60_000));
179179 push!(new TextEncoder().encode('"id":9,"method":"tools/list"}'));
180180 close!();
181181 await (await held).text();
test/stall-relay.test.ts added +27
@@ -0,0 +1,27 @@
1 import { expect, test } from "bun:test";
2 import { createStallRelay, EARLY_KEPT } from "../src/kernel/stall_relay.ts";
3
4 test("a stall that arrives before the forge exists is read in order once it does", () => {
5 const seen: [string, number, number][] = [];
6 const relay = createStallRelay<string>((t, late, at) => seen.push([t, late, at]));
7 relay.note(1500, 10);
8 relay.note(2500, 20);
9 expect(seen).toEqual([]);
10 expect(relay.waiting()).toBe(2);
11 relay.attach("forge");
12 expect(seen).toEqual([["forge", 1500, 10], ["forge", 2500, 20]]);
13 expect(relay.waiting()).toBe(0);
14 relay.note(3000, 30);
15 expect(seen.at(-1)).toEqual(["forge", 3000, 30]);
16 });
17
18 test("the stalls kept for a forge that is not yet made are bounded, oldest dropped", () => {
19 const seen: number[] = [];
20 const relay = createStallRelay<null>((_t, late) => seen.push(late));
21 for (let i = 0; i < EARLY_KEPT + 10; i++) relay.note(i, i);
22 expect(relay.waiting()).toBe(EARLY_KEPT);
23 relay.attach(null);
24 expect(seen.length).toBe(EARLY_KEPT);
25 expect(seen[0]).toBe(10);
26 expect(seen.at(-1)).toBe(EARLY_KEPT + 9);
27 });
test/traffic-history.test.ts +2 −2
@@ -25,7 +25,7 @@ import {
2525 signatureOf,
2626 type TrafficClass,
2727 } from "../src/perf/traffic.ts";
28 import { perfRecord } from "../src/perf/live.ts";
28 import { liveOf } from "../src/perf/live.ts";
2929 import { createTrafficWriter, HOUR_TOPS, trafficHistory, windowOf } from "../src/perf/traffic_store.ts";
3030
3131 const quiet = { warn: () => {} };
@@ -171,7 +171,7 @@ test("what a request is sorted by: internal, country and signature", () => {
171171 test("the page offers the three windows, for an admin alone, and a loopback request is the forge's own", async () => {
172172 const cookie = `forge_session=${createSession(app.db, admin.id).token}`;
173173 await fetch(`${servers.url}/`, { redirect: "manual" });
174 const own = perfRecord.ledger.totals(Date.now()).classes.internal.requests;
174 const own = liveOf(app).perfRecord.ledger.totals(Date.now()).classes.internal.requests;
175175 expect(own).toBeGreaterThan(0);
176176 const page = await fetch(`${servers.url}/settings/traffic?window=24h`, { headers: { cookie } });
177177 expect(page.status).toBe(200);
test/tui-screens.test.ts +3 −3
@@ -33,7 +33,7 @@ import { getPull } from "../src/core/pulls.ts";
3333 import { getRepo, type Repo } from "../src/core/repos.ts";
3434 import { setSignup } from "../src/settings/signup.ts";
3535 import { getUserSetting } from "../src/core/settings.ts";
36 import { noteProtection, resetProtection } from "../src/kernel/protection.ts";
36 import { noteProtection } from "../src/kernel/protection.ts";
3737 import { getUserByName, type User } from "../src/core/users.ts";
3838 import { quietLogs } from "./helpers/globals.ts";
3939 import { parseGitCommand, splitCommand } from "../src/git/smart-ssh.ts";
@@ -232,7 +232,7 @@ describe("the terminal UI, against a real forge", () => {
232232 });
233233
234234 test("the dashboard tells a site admin, and only them, that protection is reduced", async () => {
235 noteProtection("apps", { ok: false, reason: "bwrap is not installed" });
235 noteProtection(forge, "apps", { ok: false, reason: "bwrap is not installed" });
236236 try {
237237 const admin = boot(forge, asAlice());
238238 await admin.settle();
@@ -245,7 +245,7 @@ describe("the terminal UI, against a real forge", () => {
245245 h.stop();
246246 }
247247 } finally {
248 resetProtection();
248 noteProtection(forge, "apps", { ok: true });
249249 }
250250 const h = boot(forge, asAlice());
251251 await h.settle();
test/version.test.ts +5 −5
@@ -18,7 +18,7 @@ import { createApp, type App } from "../src/app.ts";
1818 import { createToken } from "../src/core/tokens.ts";
1919 import { createUser, type User } from "../src/core/users.ts";
2020 import { serve, type Servers } from "../src/http/server.ts";
21 import { noteProtection, resetProtection } from "../src/kernel/protection.ts";
21 import { noteProtection } from "../src/kernel/protection.ts";
2222 import { running, runningLine, UNSTAMPED } from "../src/kernel/version.ts";
2323
2424 const ROOT = resolve(import.meta.dir, "..");
@@ -296,7 +296,7 @@ describe("where the stamp is read", () => {
296296 });
297297
298298 test("a forge with less than default protection says so to its admin alone, escaped", async () => {
299 noteProtection("apps", { ok: false, reason: "<b>no bwrap</b>" });
299 noteProtection(app, "apps", { ok: false, reason: "<b>no bwrap</b>" });
300300 try {
301301 const get = async (cookie?: string) =>
302302 await (
@@ -311,13 +311,13 @@ describe("where the stamp is read", () => {
311311 for (const other of [await get(await signIn("bob")), await get()])
312312 expect(other).not.toContain("reduced-protection");
313313 } finally {
314 resetProtection();
314 noteProtection(app, "apps", { ok: true });
315315 }
316316 });
317317
318318 test("once the host is protected again the admin's page shows no notice", async () => {
319 noteProtection("apps", { ok: false, reason: "x" });
320 noteProtection("apps", { ok: true });
319 noteProtection(app, "apps", { ok: false, reason: "x" });
320 noteProtection(app, "apps", { ok: true });
321321 const page = await (
322322 await fetch(`${base}/explore`, {
323323 headers: { cookie: await signIn("alice") },

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs