Public: anyone can read this; the crew can change it.

The job watch and child-time minutes belong to a forge, not the process, so one test file's jobs never show in another's #12619

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/12614-the-job-watch-and-child-time-minutes-bel → main 9 commits fork 1 thread · 1 unresolved

  • Merged. scout/12614-the-job-watch-and-child-time-minutes-bel was merged into main as b9164c9.

Review

Sign in to review this change.
44 files changed +300 −145
src/actions/ask_host.ts +1 −1
@@ -379,7 +379,7 @@ export function watch(app: App): void {
379379 refreshAsks(app).finally(() => {
380380 watching.delete(app);
381381 watch(app);
382 }),
382 }), undefined, app,
383383 ),
384384 LOOK_MS,
385385 );
src/apps/feature_schedule.ts +1 −1
@@ -198,7 +198,7 @@ export function startFeatureSchedule(app: App, tickMs: number = TICK_MS): { stop
198198 const tick = setInterval(() => {
199199 if (running) return;
200200 running = true;
201 void during("feature-schedule", () => fireDue(app)).finally(() => (running = false));
201 void during("feature-schedule", () => fireDue(app), undefined, app).finally(() => (running = false));
202202 }, tickMs);
203203 tick.unref();
204204 return { stop: () => clearInterval(tick) };
src/apps/pull_schedule.ts +1 −1
@@ -247,7 +247,7 @@ export function startPullSchedule(app: App, tickMs: number = TICK_MS): { stop():
247247 const tick = setInterval(() => {
248248 if (running) return;
249249 running = true;
250 void during("pull-schedule", () => pullDue(app)).finally(() => (running = false));
250 void during("pull-schedule", () => pullDue(app), undefined, app).finally(() => (running = false));
251251 }, tickMs);
252252 tick.unref();
253253 return { stop: () => clearInterval(tick) };
src/apps/runtime.ts +2 −2
@@ -804,7 +804,7 @@ export class AppsRuntime {
804804 this.pruneDirs(known);
805805 await this.sampleStorage();
806806 this.sweeper = setInterval(
807 () => void during("apps-sweep", () => this.sweep(), this.options.owner),
807 () => void during("apps-sweep", () => this.sweep(), this.options.owner, this.app),
808808 this.options.sweepEveryMs ?? SWEEP_EVERY_MS,
809809 );
810810 }
@@ -1840,7 +1840,7 @@ export class AppsRuntime {
18401840 live.commitOid,
18411841 oom ? "oom" : "restart",
18421842 );
1843 }, this.options.owner),
1843 }, this.options.owner, this.app),
18441844 delay,
18451845 );
18461846 })
src/apps/timer_service.ts +1 −1
@@ -78,7 +78,7 @@ export function startTimers(app: App, tickMs = 1000): { stop(): void } | undefin
7878 firing = false;
7979 }
8080 };
81 const tick = setInterval(() => void during("timers", cycle).catch((e) => app.log.warn("timers: could not fire", { error: String(e) })), tickMs);
81 const tick = setInterval(() => void during("timers", cycle, undefined, app).catch((e) => app.log.warn("timers: could not fire", { error: String(e) })), tickMs);
8282 tick.unref();
8383 return { stop: () => clearInterval(tick) };
8484 }
src/apps/tty_registry.ts +1 −1
@@ -80,7 +80,7 @@ export function openAppTty(app: App, open: OpenAppTty): AppTtyHandle {
8080 () =>
8181 during("app-tty-idle", () => {
8282 if (idleFor(entry, open.idleMs())) stop(app, entry);
83 }),
83 }, undefined, app),
8484 Math.max(10, Math.min(open.idleMs(), open.checkMs ?? CHECK_MS)),
8585 ),
8686 };
src/boards/memory.ts +2 −2
@@ -73,8 +73,8 @@ export function startMemorySampler(app: App, everyMs = MEMORY_EVERY_MS): MemoryS
7373 app.log.warn("memory sample failed", { error: String(error) });
7474 }
7575 };
76 during("memory-sampler", () => tick());
77 const timer = setInterval(() => during("memory-sampler", () => tick()), everyMs);
76 during("memory-sampler", () => tick(), undefined, app);
77 const timer = setInterval(() => during("memory-sampler", () => tick(), undefined, app), everyMs);
7878 timer.unref?.();
7979 return { tick, stop: () => clearInterval(timer) };
8080 }
src/boards/sampler.ts +2 −2
@@ -170,8 +170,8 @@ export function startSampler(
170170 app.log.warn("machine sample failed", { error: String(error) });
171171 }
172172 };
173 during("sampler", tick);
174 const timer = setInterval(() => during("sampler", tick), options.everyMs ?? FINE_MS);
173 during("sampler", tick, undefined, app);
174 const timer = setInterval(() => during("sampler", tick, undefined, app), options.everyMs ?? FINE_MS);
175175 timer.unref?.();
176176 return {
177177 tick,
src/checks/service.ts +4 −4
@@ -1989,7 +1989,7 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
19891989 // (`perf/child_time.ts`).
19901990 const owner = options.label === undefined ? undefined : childLabel(options.label);
19911991 const timer = setInterval(() => {
1992 if (!stopped) during("lane", schedule, owner);
1992 if (!stopped) during("lane", schedule, owner, app);
19931993 }, options.tickMs ?? TICK_MS);
19941994 // Main is verified after landing on a schedule, when the forge says how often.
19951995 const sweeper =
@@ -1997,14 +1997,14 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
19971997 ? undefined
19981998 : setInterval(() => {
19991999 if (!stopped)
2000 during("verify-sweep", askVerifications, owner).catch((error) =>
2000 during("verify-sweep", askVerifications, owner, app).catch((error) =>
20012001 log.error("verification not asked", { error: String(error) }),
20022002 );
20032003 }, Math.min(VERIFY_SWEEP_MS, options.verifyEveryMs));
20042004 // A gate's slot frees: the pull check held for it starts now, not at the tick.
20052005 const unwatch = watchGates(schedule);
2006 during("lane", schedule, owner);
2007 void during("lane-catch-up", catchUp, owner);
2006 during("lane", schedule, owner, app);
2007 void during("lane-catch-up", catchUp, owner, app);
20082008
20092009 /** One gate's run on the lane: the full tier on `commitOid`, traced, cut off by `controller`. */
20102010 async function gateOnce(
src/contract/probes.ts +2 −2
@@ -238,7 +238,7 @@ export function startProbes(
238238 if (!record) return;
239239 const timer = setTimeout(() => {
240240 timers.delete(timer);
241 if (!stopped) void during("probes", () => one(record));
241 if (!stopped) void during("probes", () => one(record), undefined, app);
242242 }, AFTER_DEPLOY_MS);
243243 timers.add(timer);
244244 });
@@ -247,7 +247,7 @@ export function startProbes(
247247 const scheduleNightly = (): void => {
248248 if (options.nightly === false) return;
249249 nightly = setTimeout(() => {
250 void track(during("probes", round)).finally(scheduleNightly);
250 void track(during("probes", round, undefined, app)).finally(scheduleNightly);
251251 }, untilNightly());
252252 };
253253 scheduleNightly();
src/crew/dispatch.ts +4 −4
@@ -304,7 +304,7 @@ export async function startCrew(
304304 ));
305305 }
306306 pump(app, ready, inFlight);
307 });
307 }, undefined, app);
308308 } catch (error) {
309309 log.error("crew tick failed", { error: String(error) });
310310 }
@@ -319,13 +319,13 @@ export async function startCrew(
319319 const pruned = pruneRunFiles(app.config.dataDir);
320320 if (pruned > 0) log.info("removed old run logs", { count: pruned });
321321 };
322 during("clone-sweep", sweepClones);
323 const cloneTimer = setInterval(() => !stopped && during("clone-sweep", sweepClones), 60 * 60_000);
322 during("clone-sweep", sweepClones, undefined, app);
323 const cloneTimer = setInterval(() => !stopped && during("clone-sweep", sweepClones, undefined, app), 60 * 60_000);
324324 during("crew", () => {
325325 requeueInterrupted(app);
326326 sweepAssigned(app);
327327 pump(app, ready, inFlight);
328 });
328 }, undefined, app);
329329
330330 const handle: CrewHandle = {
331331 ready: detected.ok,
src/crew/notify.ts +2 −2
@@ -596,7 +596,7 @@ export function startNotifier(app: App, options: NotifyOptions): Notifier {
596596 } catch (error) {
597597 app.log.error("notify failed", { error: String(error) });
598598 }
599 }));
599 }, undefined, app));
600600 }, options.clockMs ?? CLOCK_MS);
601601 clock.unref();
602602
@@ -613,7 +613,7 @@ export function startNotifier(app: App, options: NotifyOptions): Notifier {
613613 return queue;
614614 };
615615 const pager = setInterval(
616 () => void during("notify-page", pageAll),
616 () => void during("notify-page", pageAll, undefined, app),
617617 Math.min(options.clockMs ?? CLOCK_MS, PAGE_MS),
618618 );
619619 pager.unref();
src/crew/prompts.ts +1 −1
@@ -102,7 +102,7 @@ export function startPromptRoutine(
102102
103103 const timer = setInterval(() => {
104104 if (stopped) return;
105 during("prompts", runDue).catch((error) =>
105 during("prompts", runDue, undefined, app).catch((error) =>
106106 log.error("prompt sweep failed", { error: String(error) }),
107107 );
108108 }, options.everyMs ?? TICK_MS);
src/crew/routines.ts +1 −1
@@ -79,7 +79,7 @@ export function startRoutines(app: App, signal: AbortSignal): () => void {
7979 const loop = (wait: number): void => {
8080 timer = setTimeout(() => {
8181 if (signal.aborted) return;
82 void during("routines", () => runDue(app, new Date())).finally(() => loop(60_000));
82 void during("routines", () => runDue(app, new Date()), undefined, app).finally(() => loop(60_000));
8383 }, wait);
8484 timer.unref();
8585 };
src/engine/duties.ts +11 −11
@@ -52,19 +52,19 @@ export function startDuties(app: App, options: { onWorker: boolean }): () => voi
5252 pruneSessions(app.db);
5353 pruneTokens(app.db);
5454 };
55 timers.push(setInterval(() => during("delivery", delivery), 5_000));
55 timers.push(setInterval(() => during("delivery", delivery, undefined, app), 5_000));
5656 // Sweep anything queued while we were down.
57 during("delivery", delivery);
57 during("delivery", delivery, undefined, app);
5858 // Estimates of "done" come due as time passes (#11955).
59 timers.push(setInterval(() => during("estimates-due", () => void settleDue(app.db)), 300_000));
59 timers.push(setInterval(() => during("estimates-due", () => void settleDue(app.db), undefined, app), 300_000));
6060 // Accounts an agent started and nobody claimed sleep after a week and are
6161 // archived after a month (`actions/start_sweep.ts`, #10912): hourly, and on boot.
62 timers.push(setInterval(() => during("starts", () => void sweepStarts(app)), HOUR_MS));
63 during("starts", () => void sweepStarts(app));
62 timers.push(setInterval(() => during("starts", () => void sweepStarts(app), undefined, app), HOUR_MS));
63 during("starts", () => void sweepStarts(app), undefined, app);
6464 // A token a brokered sign-in sealed is swept by a timer a minute later
6565 // (`spawn/origin_broker.ts`), and a timer dies with its process: what a restart left past its
6666 // minute goes here, on boot.
67 during("origin-broker-sweep", () => pruneBroker(app));
67 during("origin-broker-sweep", () => pruneBroker(app), undefined, app);
6868 // An agent's hold that has stood 24 h is released each minute (#6918).
6969 timers.push(
7070 setInterval(
@@ -75,7 +75,7 @@ export function startDuties(app: App, options: { onWorker: boolean }): () => voi
7575 } catch (error) {
7676 app.log.warn("hold expiry failed", { error: String(error) });
7777 }
78 }),
78 }, undefined, app),
7979 60_000,
8080 ),
8181 );
@@ -89,7 +89,7 @@ export function startDuties(app: App, options: { onWorker: boolean }): () => voi
8989 } catch (error) {
9090 app.log.warn("model review failed", { error: String(error) });
9191 }
92 }),
92 }, undefined, app),
9393 HOUR_MS,
9494 ),
9595 );
@@ -105,7 +105,7 @@ export function startDuties(app: App, options: { onWorker: boolean }): () => voi
105105 } catch (error) {
106106 app.log.warn("week retrospective failed", { error: String(error) });
107107 }
108 }),
108 }, undefined, app),
109109 HOUR_MS,
110110 ),
111111 );
@@ -116,7 +116,7 @@ export function startDuties(app: App, options: { onWorker: boolean }): () => voi
116116 stops.push(startReaders(app, () => void adoptObserved(app.db)));
117117 // The one checkpointer: the serving thread's commits never pay for it (`main.ts`). A passive one
118118 // takes what no reader still needs and waits for none; a log nobody reads is reset by the next write.
119 timers.push(setInterval(() => during("checkpoint", () => app.db.exec("PRAGMA wal_checkpoint(PASSIVE)")), 2_000));
119 timers.push(setInterval(() => during("checkpoint", () => app.db.exec("PRAGMA wal_checkpoint(PASSIVE)"), undefined, app), 2_000));
120120 }
121121 return () => {
122122 for (const timer of timers) clearInterval(timer);
@@ -151,7 +151,7 @@ function startReaders(app: App, adopt: () => void): () => void {
151151 } catch (error) {
152152 app.log.warn("database size reading failed", { error: String(error) });
153153 }
154 }),
154 }, undefined, app),
155155 120_000,
156156 );
157157 return () => {
src/engine/host.ts +4 −2
@@ -69,6 +69,8 @@ export function startEngine(
6969 stallMs?: number;
7070 giveUpDeaths?: number;
7171 onGiveUp?: (what: string) => void;
72 /** The forge this engine serves: its restart and watch count on that forge's job watch. */
73 forge?: object;
7274 } = {},
7375 ): Engine {
7476 let worker: Worker | undefined;
@@ -155,7 +157,7 @@ export function startEngine(
155157 during("engine-restart", () => {
156158 restarts++;
157159 launch();
158 }),
160 }, undefined, options.forge),
159161 wait,
160162 );
161163 };
@@ -167,7 +169,7 @@ export function startEngine(
167169 clearInterval(watch);
168170 w.terminate();
169171 died("silent");
170 });
172 }, undefined, options.forge);
171173 }, BEAT_MS);
172174 watch.unref();
173175 w.onerror = (event: ErrorEvent) => died(String(event.message ?? event));
src/engine/run.ts +2 −2
@@ -71,7 +71,7 @@ export function runSplit(
7171 };
7272 const engine = optional<Engine | undefined>(
7373 "the engine's Worker",
74 () => startEngine(app.config, tail, { ...options, onGiveUp: givenUp }),
74 () => startEngine(app.config, tail, { ...options, onGiveUp: givenUp, forge: app }),
7575 (error) => {
7676 givenUp(`could not be started (${String(error).slice(0, 200)})`);
7777 return undefined;
@@ -82,7 +82,7 @@ export function runSplit(
8282 // watch and asks ledger to read.
8383 if (fell === undefined)
8484 observed = setInterval(
85 () => during("observed-publish", () => void publishObserved(app.db)),
85 () => during("observed-publish", () => void publishObserved(app.db), undefined, app),
8686 2_000,
8787 );
8888 return {
src/engine/worker.ts +1 −1
@@ -64,7 +64,7 @@ async function boot(init: EngineInit): Promise<void> {
6464 app.close();
6565 };
6666 // Alive: a thread stuck in one statement stops saying so, and the serving thread ends it (`host.ts`).
67 const beat = setInterval(() => during("engine-beat", () => postMessage("h")), BEAT_MS);
67 const beat = setInterval(() => during("engine-beat", () => postMessage("h"), undefined, app), BEAT_MS);
6868 const was = stop;
6969 stop = () => {
7070 clearInterval(beat);
src/flow/keeper.ts +1 −1
@@ -188,7 +188,7 @@ export function startKeeper(app: App, onLift: () => void, everyMs = 60_000, now:
188188 during("keeper", () => {
189189 prepare();
190190 tickKeeper(app, new Date(now()), onLift);
191 });
191 }, undefined, app);
192192 } catch (error) {
193193 log.warn("keeper tick failed", { error: String(error) });
194194 }
src/http/server.ts +2 −2
@@ -2643,7 +2643,7 @@ export async function serve(app: App): Promise<Servers> {
26432643 during("standing-sweep", () =>
26442644 sweepStanding(app.db, (error) =>
26452645 app.log.warn("standing sweep did not finish", { error: String(error) }),
2646 ),
2646 ), undefined, app,
26472647 ),
26482648 app.config.standingSweepMs ?? 5_000,
26492649 );
@@ -2727,7 +2727,7 @@ export async function serve(app: App): Promise<Servers> {
27272727 },
27282728 );
27292729 liveOf(app).loopWatch.finished(hostedChild === undefined ? tag.pattern : childLabel(hostedChild), started, end);
2730 if (hostedChild !== undefined) noteChildRequest(hostedChild, ms);
2730 if (hostedChild !== undefined) noteChildRequest(app, hostedChild, ms);
27312731 // Evidence of what was read, and the running capture, if any (never a signed-in person's).
27322732 const signedIn = tag.role !== "anon" && tag.role !== "crawler";
27332733 const cls = classOf({ signedIn, crawler: wasCrawler(request), deepStranger: wasDeepStranger(request), internal: wasInternal(request) });
src/looks/service.ts +2 −2
@@ -329,8 +329,8 @@ export function startLooks(app: App, options: LooksOptions): LooksHandle {
329329 // Production missing a picture at its head — one never taken, or a
330330 // thumb from before thumbs were the page — gets one while it is up.
331331 // A woken app says nothing on the bus, so this is how it is pictured.
332 const firstSweep = setTimeout(() => during("looks", sweep), FIRST_SWEEP_MS);
333 const sweeping = setInterval(() => during("looks", sweep), SWEEP_MS);
332 const firstSweep = setTimeout(() => during("looks", sweep, undefined, app), FIRST_SWEEP_MS);
333 const sweeping = setInterval(() => during("looks", sweep, undefined, app), SWEEP_MS);
334334 firstSweep.unref?.();
335335 sweeping.unref?.();
336336
src/main.ts +26 −26
@@ -415,7 +415,7 @@ if (byHand !== undefined) {
415415 settleLiveNow(app.db);
416416 }
417417 const flagTimer = setInterval(
418 () => during("flag-uses", () => flushFlagUses(app.db)),
418 () => during("flag-uses", () => flushFlagUses(app.db), undefined, app),
419419 10_000,
420420 );
421421 await reconcile(app);
@@ -738,13 +738,13 @@ function positive(value: string | undefined, fallback: number): number {
738738 // the sweep. The operator guard's sweep of stored origins rides it too: one
739739 // origin a tick, the never-checked first (so on boot), every other daily.
740740 const syncTimer = setInterval(() => {
741 during("sync", () => runSync(app));
742 void during("ahead", () => runAheadReads(app));
743 void during("guard-sweep", () => runGuardSweep(app));
741 during("sync", () => runSync(app), undefined, app);
742 void during("ahead", () => runAheadReads(app), undefined, app);
743 void during("guard-sweep", () => runGuardSweep(app), undefined, app);
744744 }, 60_000);
745745 // One pass on boot: a forge that was down has fetching to do.
746 during("sync", () => runSync(app));
747 void during("guard-sweep", () => runGuardSweep(app));
746 during("sync", () => runSync(app), undefined, app);
747 void during("guard-sweep", () => runGuardSweep(app), undefined, app);
748748
749749 // Which green pulls no longer merge as they stand, so the asks can leave
750750 // them out (`crew/stale_asks.ts`): git answers, off the request path, and an
@@ -811,28 +811,28 @@ const followTimer = setInterval(
811811 await checkDue(app);
812812 // The apps a spawn sent down: offered, taken and kept current (`spawn/app_follow.ts`).
813813 await followTick(app);
814 }),
814 }, undefined, app),
815815 60_000,
816816 );
817817 // An app landing on this forge is told to the children awake here that follow it at once.
818818 const landingsWatch = watchLandings(app);
819819 // A child born before its apps were linked is offered the link, and a child offered one takes it: at boot, then every tick.
820820 void followTick(app, true).catch((error) => app.log.error("could not link the apps", { error: String(error) }));
821 const staleTimer = setInterval(() => void during("asks", tickAsks), 120_000);
821 const staleTimer = setInterval(() => void during("asks", tickAsks, undefined, app), 120_000);
822822 // A desk note whose time is up goes off the home line at once, over the stream.
823 const deskTimer = setInterval(() => void during("desk", () => sweepDesk(app)), 30_000);
823 const deskTimer = setInterval(() => void during("desk", () => sweepDesk(app), undefined, app), 30_000);
824824 // Every paid entitlement is re-read from Stripe once a day, so a missed webhook heals (#11247).
825 const entitlementTimer = setInterval(() => void during("entitlements", () => sweepEntitlements(app)), 86_400_000);
826 void during("entitlements", () => sweepEntitlements(app));
825 const entitlementTimer = setInterval(() => void during("entitlements", () => sweepEntitlements(app), undefined, app), 86_400_000);
826 void during("entitlements", () => sweepEntitlements(app), undefined, app);
827827 // The cost ledger's chain is walked at boot and once a day (#12273), on this thread:
828828 // health reads the verdict from this thread's memory.
829829 const walkLedger = (): void =>
830830 void checkLedger(join(dataDir, "ledger"), (line) => app.log.info(line));
831831 const ledgerTimer = readOnly
832832 ? undefined
833 : setInterval(() => during("ledger", walkLedger), 86_400_000);
834 if (!readOnly) during("ledger", walkLedger);
835 void during("asks", tickAsks);
833 : setInterval(() => during("ledger", walkLedger, undefined, app), 86_400_000);
834 if (!readOnly) during("ledger", walkLedger, undefined, app);
835 void during("asks", tickAsks, undefined, app);
836836
837837 // Every start is counted, and one that follows a process that never said
838838 // goodbye is a crash the log could not record (kernel/boots.ts, #3708).
@@ -880,18 +880,18 @@ const nightlyScanTimer = setInterval(() => {
880880 const repo = getRepoById(app.db, id);
881881 if (repo !== undefined) await scanNightly(app, repo);
882882 }
883 }).catch((error) => app.log.warn("nightly scan failed", { error: String(error) }));
883 }, undefined, app).catch((error) => app.log.warn("nightly scan failed", { error: String(error) }));
884884 }, 3_600_000);
885885 // The cost ledger's chain goes to the `ledger` branch in signed batches: once an hour, and only
886886 // when there is something new (core/ledger_git.ts).
887887 const ledgerGitTimer = readOnly
888888 ? undefined
889889 : setInterval(
890 () => void during("ledger-git", () => ledgerGitTick(app, join(dataDir, "ledger"))).catch((error) => app.log.warn("ledger batch failed", { error: String(error) })),
890 () => void during("ledger-git", () => ledgerGitTick(app, join(dataDir, "ledger")), undefined, app).catch((error) => app.log.warn("ledger batch failed", { error: String(error) })),
891891 BATCH_EVERY_MS,
892892 );
893 const processTimer = setInterval(() => during("processes", processTick), 60_000);
894 during("processes", processTick);
893 const processTimer = setInterval(() => during("processes", processTick, undefined, app), 60_000);
894 during("processes", processTick, undefined, app);
895895 // A new version proves itself against the public URL; a failed one again, 5 min after and backing off to hourly (release/prove.ts).
896896 proveRunning({
897897 dataDir: app.config.dataDir,
@@ -926,14 +926,14 @@ if (app.config.forgeRepo) {
926926 read: ownTagsReader(app, app.config.forgeRepo),
927927 });
928928 setBehind(watch);
929 void during("release-behind", () => watch.tick());
929 void during("release-behind", () => watch.tick(), undefined, app);
930930 setInterval(
931 () => void during("release-behind", () => watch.tick()),
931 () => void during("release-behind", () => watch.tick(), undefined, app),
932932 CHECK_EVERY_MS,
933933 );
934934 // The no-release alarm reads what this keeps of the running commit (#7638).
935 void during("landings", () => readLandings(app));
936 setInterval(() => void during("landings", () => readLandings(app)), 60_000);
935 void during("landings", () => readLandings(app), undefined, app);
936 setInterval(() => void during("landings", () => readLandings(app), undefined, app), 60_000);
937937 }
938938 // The release keeper (#7557): cut on cadence, promote, verify under page load, roll back to the named
939939 // known-good image and hold. Off unless FORGE_RELEASE_KEEPER=on in a cluster (keeper/wire.ts).
@@ -956,10 +956,10 @@ if (keeperOn.on) {
956956 said = line;
957957 })
958958 .catch((error) => app.log.warn("release keeper step failed", { error: String(error) }));
959 during("release-keeper", step);
960 setInterval(() => during("release-keeper", step), 60_000);
959 during("release-keeper", step, undefined, app);
960 setInterval(() => during("release-keeper", step, undefined, app), 60_000);
961961 setInterval(
962 () => void during("roll-probe", () => rollProbe(ports)).catch(() => {}),
962 () => void during("roll-probe", () => rollProbe(ports), undefined, app).catch(() => {}),
963963 1000,
964964 );
965965 }
@@ -1011,7 +1011,7 @@ const leaseTimer = setInterval(() => during("lease", () => {
10111011 requeueAbandonedChecks(app, STALE_AFTER_MS);
10121012 // A grant whose run is no longer running is a crash's leftover: close it.
10131013 releaseOrphans(app.db);
1014 }), 30_000);
1014 }, undefined, app), 30_000);
10151015
10161016 // A release that read healthy beside this process takes over (#1663),
10171017 // and this one lets go of the port before it drains (#2040).
src/maintain/runner.ts +2 −2
@@ -89,8 +89,8 @@ export function startMaintain(app: App, options: { duties?: readonly Duty[]; eve
8989 }
9090 }
9191
92 const first = setTimeout(() => void during("maintain", () => tick()), FIRST_MS);
93 const every = setInterval(() => void during("maintain", () => tick()), options.everyMs ?? TICK_MS);
92 const first = setTimeout(() => void during("maintain", () => tick(), undefined, app), FIRST_MS);
93 const every = setInterval(() => void during("maintain", () => tick(), undefined, app), options.everyMs ?? TICK_MS);
9494 first.unref?.();
9595 every.unref?.();
9696 return {
src/nightly/service.ts +2 −2
@@ -133,7 +133,7 @@ export function startNightly(
133133 const waitRadar = (): void => {
134134 const timer = setTimeout(() => {
135135 timers.delete(timer);
136 void during("nightly", runRadarNight).finally(() => !abort.signal.aborted && waitRadar());
136 void during("nightly", runRadarNight, undefined, app).finally(() => !abort.signal.aborted && waitRadar());
137137 }, untilNext(RADAR_SLOT, now()) + 1_000);
138138 timers.add(timer);
139139 };
@@ -143,7 +143,7 @@ export function startNightly(
143143 const wait = (job: Job): void => {
144144 const timer = setTimeout(() => {
145145 timers.delete(timer);
146 void during("nightly", () => run(job.name)).finally(() => !abort.signal.aborted && wait(job));
146 void during("nightly", () => run(job.name), undefined, app).finally(() => !abort.signal.aborted && wait(job));
147147 // A second past the slot: a timer that fires early would read yesterday's.
148148 }, untilNext(job.slot, now()) + 1_000);
149149 timers.add(timer);
src/origins/mirror-pulls.ts +2 −2
@@ -336,7 +336,7 @@ export function startMirrorPulls(app: App): () => void {
336336 busy = false;
337337 });
338338 };
339 const timer = setInterval(() => during("mirror-pulls", tick), MIRROR_PULLS_EVERY_MS);
340 during("mirror-pulls", tick);
339 const timer = setInterval(() => during("mirror-pulls", tick, undefined, app), MIRROR_PULLS_EVERY_MS);
340 during("mirror-pulls", tick, undefined, app);
341341 return () => clearInterval(timer);
342342 }
src/perf/child_time.ts +32 −9
@@ -30,10 +30,31 @@ interface Minute {
3030 jobMs: number;
3131 }
3232
33 /** By child name, then by minute: at most sixty small rows a child. */
34 const minutes = new Map<string, Map<number, Minute>>();
33 type Minutes = Map<string, Map<number, Minute>>;
3534
36 function minuteOf(name: string, at: number): Minute {
35 /**
36 * By hosting forge, then child name, then minute: at most sixty small rows a
37 * child. Held per forge, as `askingOf` holds its reading (#12614): one map for
38 * the process showed one test file's children on every later file's forge.
39 */
40 const byHost = new WeakMap<object, Minutes>();
41 function minutesOf(host: object): Minutes {
42 let m = byHost.get(host);
43 if (m === undefined) byHost.set(host, (m = new Map()));
44 return m;
45 }
46
47 /** Which forge hosts each hosted child's `App`: its work is counted on that host. */
48 const hosts = new WeakMap<object, object>();
49 /** `child` is served on `host`'s thread (`spawn/child-apps.ts`). */
50 export function hostChild(child: object, host: object): void {
51 hosts.set(child, host);
52 }
53 /** The forge hosting `app`, when it is a hosted child; undefined for a forge of its own. */
54 export const hostOf = (app: object): object | undefined => hosts.get(app);
55
56 function minuteOf(host: object, name: string, at: number): Minute {
57 const minutes = minutesOf(host);
3758 let mine = minutes.get(name);
3859 if (mine === undefined) minutes.set(name, (mine = new Map()));
3960 const key = Math.floor(at / MINUTE_MS);
@@ -46,13 +67,14 @@ function minuteOf(name: string, at: number): Minute {
4667 return m;
4768 }
4869
49 /** A request this process answered for the hosted child `name`, open for `ms`. */
70 /** A request `host` answered for its hosted child `name`, open for `ms`. */
5071 export function noteChildRequest(
72 host: object,
5173 name: string,
5274 ms: number,
5375 at = Date.now(),
5476 ): void {
55 const m = minuteOf(name, at);
77 const m = minuteOf(host, name, at);
5678 m.requests++;
5779 m.requestMs += ms;
5880 }
@@ -61,9 +83,9 @@ export function noteChildRequest(
6183 * A timer slice that held the thread for `ms`, owned by `owner` (`child:<name>`); any other owner
6284 * is not a child's.
6385 */
64 export function noteChildJob(owner: string, ms: number, at = Date.now()): void {
86 export function noteChildJob(host: object, owner: string, ms: number, at = Date.now()): void {
6587 if (!owner.startsWith(CHILD_PREFIX)) return;
66 const m = minuteOf(owner.slice(CHILD_PREFIX.length), at);
88 const m = minuteOf(host, owner.slice(CHILD_PREFIX.length), at);
6789 m.jobs++;
6890 m.jobMs += ms;
6991 }
@@ -94,11 +116,12 @@ export function childOfRoute(route: string): string | undefined {
94116 }
95117
96118 /**
97 * Every child counted in the hour ending at `now`, most thread first.
119 * Every child `host` counted in the hour ending at `now`, most thread first.
98120 * `stalled` is the stall tally's hour (`loopWatch.tally(...).hour`); `also`
99121 * names children to list even when they did nothing (those open now).
100122 */
101123 export function childHours(
124 host: object,
102125 now = Date.now(),
103126 stalled: readonly {
104127 readonly route: string;
@@ -137,7 +160,7 @@ export function childHours(
137160 return r;
138161 };
139162 for (const name of also) row(name);
140 for (const [name, mine] of minutes) {
163 for (const [name, mine] of minutesOf(host)) {
141164 for (const [key, m] of mine) {
142165 if (key <= from || key > to) continue;
143166 const r = row(name);
src/perf/duty.ts +1 −1
@@ -57,7 +57,7 @@ export function startPerfWatcher(
5757 app.log.warn("performance watcher failed", { error: String(error) });
5858 }
5959 };
60 const timer = setInterval(() => during("perf-watcher", tick), everyMs);
60 const timer = setInterval(() => during("perf-watcher", tick, undefined, app), everyMs);
6161 timer.unref();
6262 return Object.assign(() => clearInterval(timer), { tick });
6363 }
src/perf/jobs.ts +59 −11
@@ -12,7 +12,7 @@
1212 * is `performance.now`, the stall watcher's.
1313 */
1414
15 import { noteChildJob } from "./child_time.ts";
15 import { hostOf, noteChildJob } from "./child_time.ts";
1616
1717 /** Finished slices kept to match a stall against: a tick a second fills it in about two minutes. */
1818 const KEPT = 256;
@@ -113,20 +113,68 @@ export function createJobWatch(now: () => number = () => performance.now()): Job
113113 };
114114 }
115115
116 /** The process's one job watch, read by the loop watch in `live.ts`. */
117 export const jobWatch = createJobWatch();
116 /** What reads a job watch: the loop watch asks these three (`live.ts`). */
117 export type JobReader = Pick<JobWatch, "across" | "held" | "awaited">;
118
118119 /**
119 * Run `fn` as background job `name` on the process's job watch. With `owner`
120 * (a forge hosted in this process, `child:<name>`) the job is named
121 * `<owner>:<name>`, so a stall it ran across names the child, and the slice
122 * it held the thread is counted against that child (`child_time.ts`).
120 * Jobs no forge was named for: the thread's own, run by code that holds no
outdated

unowned is still a process-wide singleton that every app-less during call shares, which is the leak #12614 names. Move the callers that have an app in scope onto jobWatchOf(app); keep this only for jobs that truly have no forge.

121 * forge (a git reader sweep, a cell watch, a store's flush timer). Every call
122 * with a forge in scope passes it, so this holds only what is truly no forge's.
123123 */
124 export const during = <T>(name: string, fn: () => T, owner?: string): T => {
125 if (owner === undefined) return jobWatch.during(name, fn);
124 const unowned = createJobWatch();
125 /**
126 * Each forge's own watch, as `askingOf` keeps its reading (#12614): one for
127 * the process put one test file's jobs in every later file's forge.
128 */
129 const watches = new WeakMap<object, JobWatch>();
130 export function jobWatchOf(app: object): JobWatch {
131 let watch = watches.get(app);
132 if (watch === undefined) watches.set(app, (watch = createJobWatch()));
133 return watch;
134 }
135
136 /**
137 * What a forge's loop watch reads (`live.ts`): its own jobs (with its hosted
138 * children's, counted on it) and the thread's unowned ones, which hold the
139 * loop under every forge. Another forge's are not read, so a test file's jobs
140 * are never named in a later file's stalls (#12569, #12614).
141 */
142 export function jobsOf(app: object): JobReader {
143 const both = (): JobWatch[] => [unowned, jobWatchOf(app)];
144 return {
145 across(from, to) {
146 // A slice that held the thread wins over an awaiting one, across both as within one.
147 const each = both().map((w) => ({ w, blocked: w.held(from, to).length > 0 }));
148 const any = each.some((e) => e.blocked);
149 return [...new Set(each.filter((e) => !any || e.blocked).flatMap((e) => e.w.across(from, to)))];
150 },
151 held(from, to) {
152 return both()
153 .flatMap((w) => w.held(from, to))
154 .sort((x, y) => y.ms - x.ms);
155 },
156 awaited(from, to) {
157 return [...new Set(both().flatMap((w) => w.awaited(from, to)))];
158 },
159 };
160 }
161
162 /**
163 * Run `fn` as background job `name`. With `app` it is counted on that
164 * forge's watch, or on the forge hosting it when it is a hosted child
165 * (`child_time.ts`); without, on the thread's own. With `owner` (a forge
166 * hosted in this process, `child:<name>`) the job is named `<owner>:<name>`,
167 * so a stall it ran across names the child, and the slice it held the
168 * thread is counted against that child on its host.
169 */
170 export const during = <T>(name: string, fn: () => T, owner?: string, app?: object): T => {
171 const forge = app === undefined ? undefined : (hostOf(app) ?? app);
172 const watch = forge === undefined ? unowned : jobWatchOf(forge);
173 if (owner === undefined) return watch.during(name, fn);
126174 const start = performance.now();
127175 try {
128 return jobWatch.during(`${owner}:${name}`, fn);
176 return watch.during(`${owner}:${name}`, fn);
129177 } finally {
130 noteChildJob(owner, performance.now() - start);
178 if (forge !== undefined) noteChildJob(forge, owner, performance.now() - start);
131179 }
132180 };
src/perf/live.ts +4 −3
@@ -5,7 +5,7 @@
55
66 import { createBootStall } from "./boot_stall.ts";
77 import { createGitRecord } from "./git.ts";
8 import { jobWatch } from "./jobs.ts";
8 import { jobsOf } from "./jobs.ts";
99 import { createLoopWatch } from "./loop.ts";
1010 import { createInFlight } from "./in_flight.ts";
1111 import { childRequestLabel } from "./child_time.ts";
@@ -31,7 +31,7 @@ export const captureStore = createCaptureStore();
3131 * set for the process, so in a test run a file's traffic or stall was in every
3232 * later file's /system and /traffic, by the order the files ran in.
3333 *
34 * The loop watch is told of jobs from `jobWatch` and of requests from
34 * The loop watch is told of jobs from `jobsOf` (this forge's watch and the thread's unowned one, #12614) and of requests from
3535 * `inFlight`, which stay the process's: a timer and a request in flight are
3636 * the thread's, and the watch only reads them.
3737 *
@@ -56,9 +56,10 @@ export function liveOf(app: object): Live {
5656 let live = lives.get(app);
5757 if (live === undefined) {
5858 const stallBlame = createStallBlame();
59 const jobs = jobsOf(app);
5960 live = {
6061 perfRecord: createPerfRecord(),
61 loopWatch: createLoopWatch(undefined, jobWatch.across, inFlight.since, jobWatch.held, jobWatch.awaited),
62 loopWatch: createLoopWatch(undefined, jobs.across, inFlight.since, jobs.held, jobs.awaited),
6263 stallBlame,
6364 lagMinutes: createLagMinutes((to, span) => stallBlame.worst(to, span, 1)[0]?.what ?? null),
6465 };
src/perf/stall_blame.ts +1 −1
@@ -6,7 +6,7 @@
66 * Who ran is the loop watch's finding (`loop.ts`): the route patterns whose
77 * request ran across the blocked window, then those still running at the
88 * wake as `<pattern> (in flight)` (#11478), else the background jobs marked
9 * with `jobWatch.during` (`jobs.ts`), else nothing. `what` is the first of
9 * with `during` (`jobs.ts`), else nothing. `what` is the first of
1010 * those — the longest-running finished route, the oldest open one, or
1111 * `job:<name>` — and `unknown` when nothing marked ran across the stall. A job that
1212 * was only awaiting across a stall nothing measured is `awaiting:<name>`: a suspect,
src/snapshot/service.ts +6 −6
@@ -432,7 +432,7 @@ export function startSnapshots(
432432 void during("snapshot", async () => {
433433 if (active === 0 && crewHolds()) return arm(holdRetryMs);
434434 await take();
435 }).catch(() => {});
435 }, undefined, app).catch(() => {});
436436 }, ms);
437437 }
438438 arm(options.nightlyInMs ?? untilNightly());
@@ -452,7 +452,7 @@ export function startSnapshots(
452452 const scheduleDrill = (): void => {
453453 if (options.nightly === false || stopped) return;
454454 drillTimer = setTimeout(() => {
455 during("drill", () => runIfDue()).finally(scheduleDrill);
455 during("drill", () => runIfDue(), undefined, app).finally(scheduleDrill);
456456 }, untilDrill());
457457 };
458458 // While the newest drill is red, look again when it is owed again.
@@ -463,7 +463,7 @@ export function startSnapshots(
463463 const newest = lastDrill(app.db);
464464 if (newest?.verdict !== "red") return;
465465 const wait = Math.max(catchUpMs, Date.parse(newest.at) + redRetryMs - Date.now());
466 redRetry = setTimeout(() => void during("drill", () => runIfDue()).finally(armRedRetry), wait);
466 redRetry = setTimeout(() => void during("drill", () => runIfDue(), undefined, app).finally(armRedRetry), wait);
467467 };
468468 // Owed and not yet kept: run it once. A restart after a green drill finds it
469469 // kept; one after a red drill runs it again, once it is past the boot floor.
@@ -478,7 +478,7 @@ export function startSnapshots(
478478 if (options.nightly !== false) {
479479 catchUp = setTimeout(
480480 () =>
481 void during("drill", () => runIfDue(options.bootRedFloorMs ?? BOOT_RED_FLOOR_MS))
481 void during("drill", () => runIfDue(options.bootRedFloorMs ?? BOOT_RED_FLOOR_MS), undefined, app)
482482 .finally(armRedRetry),
483483 catchUpMs,
484484 );
@@ -490,7 +490,7 @@ export function startSnapshots(
490490 const scheduleMirror = (first: boolean): void => {
491491 if (options.nightly === false || stopped) return;
492492 mirrorTimer = setTimeout(() => {
493 during("snapshot-mirror", mirrorNow)
493 during("snapshot-mirror", mirrorNow, undefined, app)
494494 .catch(() => {})
495495 .finally(() => scheduleMirror(false));
496496 }, first ? Math.min(30_000, everyMs) : everyMs);
@@ -503,7 +503,7 @@ export function startSnapshots(
503503 const scheduleCatchUp = (ms: number): void => {
504504 if (options.nightly === false || stopped) return;
505505 catchUpSnapshot = setTimeout(() => {
506 during("snapshot", takeIfDue)
506 during("snapshot", takeIfDue, undefined, app)
507507 .catch(() => {})
508508 .finally(() => {
509509 bootLookAt = undefined;
src/spawn/child-apps.ts +3 −1
@@ -15,7 +15,7 @@ import { listApps } from "../core/apps.ts";
1515 import { appsRuntime, AppsRuntime } from "../apps/runtime.ts";
1616 import { childHost } from "./door.ts";
1717 import { childPublicUrl, parentOrigin } from "./placement.ts";
18 import { childLabel } from "../perf/child_time.ts";
18 import { childLabel, hostChild } from "../perf/child_time.ts";
1919
2020 interface Running {
2121 readonly child: App;
@@ -41,6 +41,8 @@ export async function startChildApps(parent: App, child: App, name: string): Pro
4141 let held = running.get(parent);
4242 if (!held) running.set(parent, (held = new Map()));
4343 if (held.has(name)) return;
44 // Its timers and requests hold this thread: counted on the parent (`perf/child_time.ts`).
45 hostChild(child, parent);
4446 const runtime = new AppsRuntime(child, {
4547 domain: `--${name}--${childHost(parent)}`,
4648 scheme: own.options.scheme,
src/spawn/heartbeat.ts +1 −1
@@ -352,7 +352,7 @@ export async function sendBeat(
352352 /** Report now and then every BEAT_MS; returns the timer to clear. */
353353 export function startHeartbeat(app: App): ReturnType<typeof setInterval> {
354354 void sendBeat(app);
355 return setInterval(() => void during("heartbeat", () => sendBeat(app)), BEAT_MS);
355 return setInterval(() => void during("heartbeat", () => sendBeat(app), undefined, app), BEAT_MS);
356356 }
357357
358358 export type Standing = "reporting" | "quiet" | "unheard";
src/spawn/host.ts +3 −3
@@ -282,7 +282,7 @@ function wakeWhenDue(parent: App, ledger: ChildLedger, at: number, now = Date.no
282282 );
283283 };
284284 const timer = setTimeout(
285 () => void during("spawn-due-wake", due).catch(() => {}),
285 () => void during("spawn-due-wake", due, undefined, parent).catch(() => {}),
286286 Math.min(TIMER_MAX_MS, Math.max(0, at - now)),
287287 );
288288 timer.unref?.();
@@ -359,7 +359,7 @@ const floorWatchers = new WeakMap<App, ReturnType<typeof setInterval>>();
359359
360360 function keepWatchingFloor(parent: App): void {
361361 if (floorWatchers.has(parent)) return;
362 const timer = setInterval(() => void during("spawn-floor", () => sleepBelowFloor(parent)).catch(() => {}), FLOOR_EVERY_MS);
362 const timer = setInterval(() => void during("spawn-floor", () => sleepBelowFloor(parent), undefined, parent).catch(() => {}), FLOOR_EVERY_MS);
363363 timer.unref?.();
364364 floorWatchers.set(parent, timer);
365365 }
@@ -457,7 +457,7 @@ function keepSweeping(parent: App): void {
457457 const timer = setInterval(() => void during("spawn-idle", async () => {
458458 await sleepLapsed(parent);
459459 await sleepIdleChildren(parent);
460 }).catch(() => {}), every);
460 }, undefined, parent).catch(() => {}), every);
461461 timer.unref?.();
462462 sweepers.set(parent, timer);
463463 }
src/spawn/origin_broker.ts +1 −1
@@ -488,7 +488,7 @@ export async function brokerCallback(
488488 if (ready.changes !== 1) return back({ failure: "failed" });
489489 // A token waits a minute, not until somebody next asks: its row is swept when its time is up.
490490 setTimeout(
491 () => void during("origin-broker-sweep", () => sweepBroker(app)),
491 () => void during("origin-broker-sweep", () => sweepBroker(app), undefined, app),
492492 CODE_TTL_MS + 1_000,
493493 ).unref();
494494 // The audit names who and where. Never a token, never the code.
src/spawn/reach.ts +2 −2
@@ -296,8 +296,8 @@ export function startReachWatch(
296296 everyMs = REACH_EVERY_MS,
297297 asleep: (name: string) => boolean = () => false,
298298 ): ReturnType<typeof setInterval> {
299 const first = setTimeout(() => void during("reach", () => sweepReach(app, asleep)), 30_000);
300 const timer = setInterval(() => void during("reach", () => sweepReach(app, asleep)), everyMs);
299 const first = setTimeout(() => void during("reach", () => sweepReach(app, asleep), undefined, app), 30_000);
300 const timer = setInterval(() => void during("reach", () => sweepReach(app, asleep), undefined, app), everyMs);
301301 app.closing.signal.addEventListener("abort", () => clearTimeout(first), { once: true });
302302 return timer;
303303 }
src/spawn/upstream.ts +1 −1
@@ -214,7 +214,7 @@ function tellOwner(app: App, repo: Repo, fork: string, behind: number, commits =
214214 /** Check at boot and every hour; returns the timer to clear. */
215215 export function startUpstreamWatch(app: App): ReturnType<typeof setInterval> {
216216 void checkUpstream(app);
217 return setInterval(() => void during("upstream", () => checkUpstream(app)), CHECK_MS);
217 return setInterval(() => void during("upstream", () => checkUpstream(app), undefined, app), CHECK_MS);
218218 }
219219
220220 /** The state in one plain sentence, escaped by whoever renders it. */
src/system/retention.ts +5 −5
@@ -208,35 +208,35 @@ export function startRetention(app: App, which: { local: boolean; engine: boolea
208208 pending[0] = setTimeout(() => {
209209 if (stopped) return;
210210 if (held(backfill)) return;
211 during("landed-backfill", () => backfill(work(jobs.backfill, "deploy landing backfill")));
211 during("landed-backfill", () => backfill(work(jobs.backfill, "deploy landing backfill")), undefined, app);
212212 }, delay);
213213 };
214214 const live = (delay: number): void => {
215215 pending[1] = setTimeout(() => {
216216 if (stopped) return;
217217 if (held(live)) return;
218 during("retention-live", () => live(work(jobs.live, "live event prune")));
218 during("retention-live", () => live(work(jobs.live, "live event prune")), undefined, app);
219219 }, delay);
220220 };
221221 const events = (delay: number): void => {
222222 pending[2] = setTimeout(() => {
223223 if (stopped) return;
224224 if (held(events)) return;
225 during("retention-events", () => events(work(jobs.events, "event prune")));
225 during("retention-events", () => events(work(jobs.events, "event prune")), undefined, app);
226226 }, delay);
227227 };
228228 const samples = (delay: number): void => {
229229 pending[3] = setTimeout(() => {
230230 if (stopped) return;
231231 if (held(samples)) return;
232 during("retention-samples", () => samples(work(jobs.samples, "machine sample prune")));
232 during("retention-samples", () => samples(work(jobs.samples, "machine sample prune")), undefined, app);
233233 }, delay);
234234 };
235235 const deploys = (delay: number): void => {
236236 pending[4] = setTimeout(() => {
237237 if (stopped) return;
238238 if (held(deploys)) return;
239 during("retention-deploys", () => deploys(work(jobs.deploys, "deploy ledger trim")));
239 during("retention-deploys", () => deploys(work(jobs.deploys, "deploy ledger trim")), undefined, app);
240240 }, delay);
241241 };
242242 if (which.engine) {
src/web/pages/system.ts +1 −1
@@ -604,7 +604,7 @@ export async function systemPage(ctx: Ctx): Promise<Response> {
604604 // Only this forge's own: the count is the process's, and a process holds one forge everywhere but
605605 // a test.
606606 const mine = new Set(listed.map((c) => c.name));
607 const inProcess = childHours(Date.now(), tally.hour, openChildren(ctx.app).map((c) => c.name)).filter((r) => mine.has(r.name));
607 const inProcess = childHours(ctx.app, Date.now(), tally.hour, openChildren(ctx.app).map((c) => c.name)).filter((r) => mine.has(r.name));
608608 const nowMono = performance.now();
609609 const worst = liveOf(ctx.app).stallBlame.worst(nowMono, HOUR_MS, WORST_STALLS);
610610 // Never waits on the cluster: the last reading, or the declared rows while the first read runs behind.
test/mcp-system-stalls.test.ts +6 −5
@@ -38,13 +38,14 @@ test("a stall under a named job is read through system, at most 10, most ms firs
3838 // The watch is the process's one: stalls and requests an earlier file left
3939 // would crowd the top 10 or be blamed ahead of the job. Start from nothing.
4040 liveOf(app).loopWatch.clear();
41 // The job holds the loop for a real share of the stall: one that only brushed it is not
42 // named (`perf/loop.ts`).
41 // The job holds the loop for the whole stall: one that only brushed it is not named
42 // (`perf/loop.ts`), and a window wider than the job reaches back into the file before (#12614).
43 const from = performance.now();
4344 during("test-job-10907", () => {
44 const until = performance.now() + 40;
45 const until = from + LIMIT_MS + 10;
4546 while (performance.now() < until);
46 });
47 liveOf(app).loopWatch.stalled(LIMIT_MS + 10, performance.now());
47 }, undefined, app);
48 liveOf(app).loopWatch.stalled(LIMIT_MS + 10, Math.max(performance.now(), from + LIMIT_MS + 10));
4849 const res = await fetch(`${servers.url}/mcp`, {
4950 method: "POST",
5051 headers: { "content-type": "application/json", accept: "application/json, text/event-stream", authorization: `Bearer ${token}` },
test/module-state.baseline.txt −2
@@ -60,7 +60,6 @@ src/crew/autonomy.ts:cleanAt
6060 src/crew/autonomy.ts:conflicted
6161 src/crew/autonomy.ts:conflictedAt
6262 src/crew/autonomy.ts:sweeping
63 src/crew/ceiling_stagger.ts:memo
6463 src/crew/conversation.ts:live
6564 src/crew/conversation.ts:queues
6665 src/crew/conversation.ts:stoppers
@@ -145,7 +144,6 @@ src/origins/providers/index.ts:cache
145144 src/origins/sync.ts:inFlight
146145 src/origins/token-reach.ts:kept
147146 src/origins/work-branch.ts:tipsSeen
148 src/perf/child_time.ts:minutes
149147 src/release/changelog.ts:cache
150148 src/release/prove.ts:superseded
151149 src/runs/activity.ts:lastAt
test/perf-jobs-per-forge.test.ts added +76
@@ -0,0 +1,76 @@
1 /**
2 * The job watch and a hosted child's minutes are each forge's own (#12614):
3 * two forges booted in one process, as two test files are, never read each
4 * other's jobs or children, and each forge's loop watch reads its own.
5 */
6 import { afterAll, beforeAll, expect, test } from "bun:test";
7 import { mkdtemp, rm } from "node:fs/promises";
8 import { tmpdir } from "node:os";
9 import { join } from "node:path";
10 import { createApp, type App } from "../src/app.ts";
11 import { startMemorySampler } from "../src/boards/memory.ts";
12 import { childHours, hostChild, noteChildRequest } from "../src/perf/child_time.ts";
13 import { during, jobsOf, jobWatchOf } from "../src/perf/jobs.ts";
14
15 let dirs: string[] = [];
16 let one: App;
17 let other: App;
18
19 beforeAll(async () => {
20 dirs = [await mkdtemp(join(tmpdir(), "jobs-one-")), await mkdtemp(join(tmpdir(), "jobs-other-"))];
21 one = createApp({ dataDir: dirs[0]!, url: "https://one.test", port: 0 });
22 other = createApp({ dataDir: dirs[1]!, url: "https://other.test", port: 0 });
23 });
24
25 afterAll(async () => {
26 one?.close();
27 other?.close();
28 for (const d of dirs) await rm(d, { recursive: true, force: true });
29 });
30
31 const block = (ms: number) => {
32 const until = performance.now() + ms;
33 while (performance.now() < until);
34 };
35
36 test("a job and a child minute noted on one forge are not the other's", () => {
37 const before = performance.now();
38 during("own-sweep", () => block(2), undefined, one);
39 during("lane", () => block(2), "child:kid", one);
40 noteChildRequest(one, "kid", 40);
41 const after = performance.now();
42
43 expect(jobWatchOf(one).across(before, after)).toEqual(expect.arrayContaining(["own-sweep", "child:kid:lane"]));
44 expect(childHours(one).find((r) => r.name === "kid")).toMatchObject({ requests: 1, jobs: 1 });
45
46 expect(jobWatchOf(other).across(before, after)).toEqual([]);
47 expect(jobWatchOf(other).held(before, after)).toEqual([]);
48 expect(childHours(other)).toEqual([]);
49
50 // Each forge's stall watcher reads its own jobs, and not the other's.
51 expect(jobsOf(one).across(before, after)).toEqual(expect.arrayContaining(["own-sweep", "child:kid:lane"]));
52 expect(jobsOf(other).across(before, after)).toEqual([]);
53 });
54
55 test("a hosted child's job is counted on the forge hosting it", () => {
56 const child = {};
57 hostChild(child, other);
58 const before = performance.now();
59 during("apps-sweep", () => block(2), "child:hosted", child);
60 const after = performance.now();
61 expect(jobWatchOf(other).across(before, after)).toEqual(["child:hosted:apps-sweep"]);
62 expect(childHours(other).map((r) => r.name)).toEqual(["hosted"]);
63 expect(jobWatchOf(child).across(before, after)).toEqual([]);
64 expect(childHours(one).some((r) => r.name === "hosted")).toBe(false);
65 });
66
67 test("a forge's own timer, started with no owner, is read on that forge's watch and not another's", () => {
68 const before = performance.now();
69 const sampler = startMemorySampler(one, 3_600_000);
70 sampler.stop();
71 const after = performance.now();
72 expect(jobWatchOf(one).across(before, after)).toContain("memory-sampler");
73 expect(jobWatchOf(other).across(before, after)).not.toContain("memory-sampler");
74 expect(jobWatchOf(other).held(before, after)).toEqual([]);
75 expect(jobsOf(one).across(before, after)).toContain("memory-sampler");
76 });
test/relay-stall-name.test.ts +2 −2
@@ -29,7 +29,7 @@ import { getAppByRepo } from "../src/core/apps.ts";
2929 import type { Repo } from "../src/core/repos.ts";
3030 import { createUser, type User } from "../src/core/users.ts";
3131 import { serve, type Servers } from "../src/http/server.ts";
32 import { createJobWatch, jobWatch } from "../src/perf/jobs.ts";
32 import { createJobWatch, during } from "../src/perf/jobs.ts";
3333 import { liveOf } from "../src/perf/live.ts";
3434 import { createLoopWatch, type Stall } from "../src/perf/loop.ts";
3535 import { loopWorst } from "../src/perf/loop_worst.ts";
@@ -171,7 +171,7 @@ describe("forge-stall on the relayed answer", () => {
171171 * blocks the loop, so the test takes no time and the stall is exactly `ms`.
172172 */
173173 const stallAsJob = (name: string, ms: number) => {
174 jobWatch.during(name, () => {});
174 during(name, () => {});
175175 const now = performance.now();
176176 loopWorst.note(ms, now);
177177 // A job that ran for a moment was only across the stall (`WITNESS_SHARE`, `perf/loop.ts`):
test/spawn-in-process-visible.test.ts +16 −12
@@ -19,7 +19,7 @@ import {
1919 childOfRoute,
2020 noteChildRequest,
2121 } from "../src/perf/child_time.ts";
22 import { during, jobWatch } from "../src/perf/jobs.ts";
22 import { during, jobWatchOf } from "../src/perf/jobs.ts";
2323 import { liveOf } from "../src/perf/live.ts";
2424 import { stallLine } from "../src/perf/loop.ts";
2525 import { clientRows, servedWindow } from "../src/spawn/clients.ts";
@@ -118,10 +118,12 @@ describe("what a forge in this process took of the serving thread", () => {
118118 test("its requests and timer slices are counted against it by name, for an hour", () => {
119119 // Its own clock, a year off the other tests': what they count is outside this hour.
120120 const now = Date.parse("2025-11-13T12:00:00Z");
121 noteChildRequest("counted", 40, now - 10 * 60_000);
122 noteChildRequest("counted", 60, now - 5_000);
123 noteChildRequest("counted", 999, now - 61 * 60_000); // past the hour
121 const host = {};
122 noteChildRequest(host, "counted", 40, now - 10 * 60_000);
123 noteChildRequest(host, "counted", 60, now - 5_000);
124 noteChildRequest(host, "counted", 999, now - 61 * 60_000); // past the hour
124125 const rows = childHours(
126 host,
125127 now,
126128 [
127129 { route: "child:counted", ms: 640, count: 2 },
@@ -141,6 +143,7 @@ describe("what a forge in this process took of the serving thread", () => {
141143 });
142144
143145 test("a timer it owns is named for it in the job ring, so a stall across it reads child:<name>", () => {
146 const host = {};
144147 const before = performance.now();
145148 during(
146149 "sweep-of-a-test",
@@ -149,20 +152,21 @@ describe("what a forge in this process took of the serving thread", () => {
149152 while (performance.now() < until);
150153 },
151154 "child:timed",
155 host,
152156 );
153157 const after = performance.now();
154 expect(jobWatch.across(before, after)).toContain("child:timed:sweep-of-a-test");
158 expect(jobWatchOf(host).across(before, after)).toContain("child:timed:sweep-of-a-test");
155159 expect(stallLine({ at: after, lateMs: after - before, routes: [], jobs: ["child:timed:sweep-of-a-test"] })).toContain(
156160 "no request; jobs child:timed:sweep-of-a-test",
157161 );
158 const row = childHours().find((r) => r.name === "timed");
162 const row = childHours(host).find((r) => r.name === "timed");
159163 expect(row?.jobs).toBe(1);
160164 expect(row?.jobMs).toBeGreaterThan(0);
161165 // A job with no owner is the forge's own, and is counted against no child.
162 const rows = childHours().length;
163 during("sweep-of-a-test", () => {});
164 expect(childHours().length).toBe(rows);
165 expect(childHours().find((r) => r.name === "timed")?.jobs).toBe(1);
166 const rows = childHours(host).length;
167 during("sweep-of-a-test", () => {}, undefined, host);
168 expect(childHours(host).length).toBe(rows);
169 expect(childHours(host).find((r) => r.name === "timed")?.jobs).toBe(1);
166170 });
167171 });
168172
@@ -230,7 +234,7 @@ describe("a parent holding a forge in its own process", () => {
230234 // Open it, so the next request is answered by a forge already in this process.
231235 await (await fetch(`${servers.url}/healthz`, { headers: { host: "homie--forge.test" } })).text();
232236 const child = openChildren(parent).find((c) => c.name === "homie")!.app;
233 const had = childHours().find((r) => r.name === "homie")?.requests ?? 0;
237 const had = childHours(parent).find((r) => r.name === "homie")?.requests ?? 0;
234238 liveOf(parent).loopWatch.clear();
235239 // The child's own request holds the parent's thread: its first statement takes 80 ms of
236240 // synchronous work, as a slow query or a git call of the child's would. A stall is named
@@ -247,7 +251,7 @@ describe("a parent holding a forge in its own process", () => {
247251 const stall = liveOf(parent).loopWatch.stalled(250, performance.now());
248252 expect(stall?.routes).toContain("child:homie");
249253 expect(stallLine(stall!)).toContain("running then: child:homie");
250 expect(childHours().find((r) => r.name === "homie")?.requests).toBe(had + 1);
254 expect(childHours(parent).find((r) => r.name === "homie")?.requests).toBe(had + 1);
251255 // The parent's own request is still its route's.
252256 liveOf(parent).loopWatch.clear();
253257 const mine = holdOnce(parent, 80);

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs