Public: anyone can read this; the crew can change it.
Tests share process-wide state and eight are flaky: a pull's check can go red for what another test left behind #12599
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/12569-tests-share-process-wide-state-and-eight → main 5 commits fork
59 files
changed +679
−226
src/apps/proxy.ts +2 −2
| @@ -70,7 +70,7 @@ import { holdPortRequest, holdPortSocket } from "./handoff.ts"; | ||
| 70 | 70 | import { secureFor } from "../http/cookie_secure.ts"; |
| 71 | 71 | import { currentLoopLag, forgeWaited, LAG_WINDOW_MS } from "../perf/loop_worst.ts"; |
| 72 | 72 | import { HOOK_MAX_BYTES, HOOK_PREFIX, hookPath, readCapped } from "./hook_relay.ts"; |
| 73 | import { loopWatch } from "../perf/live.ts"; | |
| 73 | import { liveOf } from "../perf/live.ts"; | |
| 74 | 74 | import { heldBy, noteRelayWait, stallTiming } from "./relay_wait.ts"; |
| 75 | 75 | import { cspNonce, htmlAnswer, readPage, rewritable, TOOLBAR_ASSET_PATH, TOOLBAR_ASSET_PREFIX, toolbarResponse, withToolbar, type PillWhere } from "./toolbar.ts"; |
| 76 | 76 | |
| @@ -533,7 +533,7 @@ function relayUpstream( | ||
| 533 | 533 | // What held a wait that was the forge's (#11000): the stalls of the lag's |
| 534 | 534 | // window, read only then — at most 40 of them, so the common relay pays nothing. |
| 535 | 535 | const now = performance.now(); |
| 536 | const held = forgeWaited(lag, headersMs) ? heldBy(loopWatch.within(now - LAG_WINDOW_MS, now)) : []; | |
| 536 | const held = forgeWaited(lag, headersMs) ? heldBy(liveOf(runtime.app).loopWatch.within(now - LAG_WINDOW_MS, now)) : []; | |
| 537 | 537 | // Production's slowest wait, for the app's speed card (`apps/relay_wait.ts`). |
| 538 | 538 | if (target.env === "production") noteRelayWait(target.app.id, headersMs, lag, Date.now(), held); |
| 539 | 539 | if (held.length > 0 && forgeAdmin(runtime, who)) stallFor.set(request, stallTiming(held)); |
src/apps/runtime.ts +1 −1
| @@ -763,7 +763,7 @@ export class AppsRuntime { | ||
| 763 | 763 | /** Rebuild every environment from git and the open pulls, then listen. */ |
| 764 | 764 | async start(): Promise<void> { |
| 765 | 765 | this.cells = await cellsAvailable(); |
| 766 | noteProtection("apps", this.cells); | |
| 766 | noteProtection(this.app, "apps", this.cells); | |
| 767 | 767 | if (!this.cells.ok) |
| 768 | 768 | log.warn("apps run uncontained here", { reason: this.cells.reason }); |
| 769 | 769 | // Deploys are failed; a check the last forge left is queued again while |
src/checks/tiers.ts +2
| @@ -48,6 +48,8 @@ export const RULE_TESTS: readonly string[] = [ | ||
| 48 | 48 | "test/spawn-pipes.test.ts", |
| 49 | 49 | "test/hollow.test.ts", |
| 50 | 50 | "test/release-boot.test.ts", |
| 51 | // No process-wide state a request writes to: it belongs to an app (#12569). | |
| 52 | "test/module-state.test.ts", | |
| 51 | 53 | ]; |
| 52 | 54 | |
| 53 | 55 | /** What a test the times table does not name is assumed to take. */ |
src/crew/dispatch.ts +1 −1
| @@ -230,7 +230,7 @@ export async function startCrew( | ||
| 230 | 230 | // The default the Capacity page shows and goes back to (`core/capacity.ts`). |
| 231 | 231 | noteBooted(app.db, "crew.concurrency", String(options.concurrency)); |
| 232 | 232 | const sandbox = await sandboxAvailable(); |
| 233 | noteProtection("crew shells", sandbox); | |
| 233 | noteProtection(app, "crew shells", sandbox); | |
| 234 | 234 | // Temp directories of runs a stopped forge never finished: old enough |
| 235 | 235 | // that no forge beside this one (a rollout) can still be using them. |
| 236 | 236 | try { |
src/crew/sentinel.ts +6 −6
| @@ -31,7 +31,7 @@ import { leaseLapsed, listRunning } from "../core/runs.ts"; | ||
| 31 | 31 | import { migrations } from "../core/schema.ts"; |
| 32 | 32 | import { getUserByName, type User } from "../core/users.ts"; |
| 33 | 33 | import { migrate } from "../kernel/migrate.ts"; |
| 34 | import { createLog, recentErrors, redactSecrets } from "../kernel/log.ts"; | |
| 34 | import { createLog, recentErrors, redactSecrets, type ErrorLedger } from "../kernel/log.ts"; | |
| 35 | 35 | import { spans } from "../kernel/degraded_history.ts"; |
| 36 | 36 | import { QUEUE_WAIT_MAX_MS } from "../kernel/vitals.ts"; |
| 37 | 37 | import { STALE_AFTER_MS } from "../runs/lease.ts"; |
| @@ -374,9 +374,9 @@ export function runStreaks(db: Database): Finding[] { | ||
| 374 | 374 | } |
| 375 | 375 | |
| 376 | 376 | /** The same error line over and over in the pod's own log. */ |
| 377 | export function repeatedErrors(now: number): Finding[] { | |
| 377 | export function repeatedErrors(now: number, ledger?: ErrorLedger): Finding[] { | |
| 378 | 378 | // The sentinel's own failures are not a fault to file about itself. |
| 379 | const seen = recentErrors(100, now).latest.filter((l) => l.scope !== "sentinel"); | |
| 379 | const seen = recentErrors(100, now, ledger).latest.filter((l) => l.scope !== "sentinel"); | |
| 380 | 380 | const groups = new Map<string, typeof seen>(); |
| 381 | 381 | for (const line of seen) { |
| 382 | 382 | const k = `${line.scope}: ${oneLine(line.message).replace(/\d+/g, "N")}`; |
| @@ -576,7 +576,7 @@ function sharedLines( | ||
| 576 | 576 | } |
| 577 | 577 | |
| 578 | 578 | /** Everything the sentinel reads, in one pass. */ |
| 579 | export function findFaults(db: Database, now = Date.now(), forgeRepoId?: string, dataDir?: string, ceilingUsd: number | null = null): Finding[] { | |
| 579 | export function findFaults(db: Database, now = Date.now(), forgeRepoId?: string, dataDir?: string, ceilingUsd: number | null = null, ledger?: ErrorLedger): Finding[] { | |
| 580 | 580 | const found = [ |
| 581 | 581 | ...mainAhead(db, now), |
| 582 | 582 | ...laneStopped(db, now, ceilingUsd), |
| @@ -585,7 +585,7 @@ export function findFaults(db: Database, now = Date.now(), forgeRepoId?: string, | ||
| 585 | 585 | ...strayTriggers(db), |
| 586 | 586 | ...ledgerStreaks(db, now), |
| 587 | 587 | ...runStreaks(db), |
| 588 | ...repeatedErrors(now), | |
| 588 | ...repeatedErrors(now, ledger), | |
| 589 | 589 | ...degradedCauses(db, now, forgeRepoId, dataDir), |
| 590 | 590 | ]; |
| 591 | 591 | const keys = new Set<string>(); |
| @@ -1029,7 +1029,7 @@ export function hunt( | ||
| 1029 | 1029 | let commented = 0; |
| 1030 | 1030 | const [owner = "", name = ""] = repoPath.split("/"); |
| 1031 | 1031 | const repo = getRepo(app.db, owner, name); |
| 1032 | const findings = find(app.db, now, repo?.id, app.config.dataDir, forgeCeiling(app.config)); | |
| 1032 | const findings = find(app.db, now, repo?.id, app.config.dataDir, forgeCeiling(app.config), app.signals.errors); | |
| 1033 | 1033 | const actor = sentinelActor(app.db); |
| 1034 | 1034 | if (repo !== undefined && actor !== undefined) { |
| 1035 | 1035 | settleShared(app, repo, actor, now); |
src/http/api.ts +2 −1
| @@ -8,6 +8,7 @@ | ||
| 8 | 8 | * back to continue); errors are `{ error, message }`. |
| 9 | 9 | */ |
| 10 | 10 | |
| 11 | import { liveOf } from "../perf/live.ts"; | |
| 11 | 12 | import { reviewWords } from "../actions/review_words.ts"; |
| 12 | 13 | import { growth, MAX_DAYS as MAX_GROWTH_DAYS } from "../core/growth.ts"; |
| 13 | 14 | import { PAUSE_REFUSAL, pausedUntil } from "../core/planner_pause.ts"; |
| @@ -3523,7 +3524,7 @@ export function apiRoutes(router: Router<Handler>): void { | ||
| 3523 | 3524 | router.add("GET", "/metrics", (ctx) => { |
| 3524 | 3525 | const denied = accountAdmin(ctx, "ro"); |
| 3525 | 3526 | if (denied) return denied; |
| 3526 | return new Response(egressMetrics(egressRowsFor(ctx.app), egressTotalsFor(ctx.app)) + forgeMetrics(ctx.app.db, appSources(ctx.app)), { | |
| 3527 | return new Response(egressMetrics(egressRowsFor(ctx.app), egressTotalsFor(ctx.app)) + forgeMetrics(ctx.app.db, appSources(ctx.app), Date.now(), liveOf(ctx.app).loopWatch), { | |
| 3527 | 3528 | headers: { |
| 3528 | 3529 | "content-type": "text/plain; version=0.0.4; charset=utf-8", |
| 3529 | 3530 | "cache-control": "no-store", |
src/http/app_turn.ts +12 −2
| @@ -90,8 +90,18 @@ export function createTurn( | ||
| 90 | 90 | }; |
| 91 | 91 | } |
| 92 | 92 | |
| 93 | /** The process's one gate: a hosted child answers on the same thread. */ | |
| 94 | export const turn: Turn = createTurn(); | |
| 93 | /** | |
| 94 | * Each forge's own gate: a hosted child's requests come in through its | |
| 95 | * parent's server, so they are the parent's. One gate for the process kept | |
| 96 | * every open request of every forge in it, and in a test run a request one | |
| 97 | * file left open made a later file's app wait. | |
| 98 | */ | |
| 99 | const gates = new WeakMap<object, Turn>(); | |
| 100 | export function turnOf(app: object): Turn { | |
| 101 | let gate = gates.get(app); | |
| 102 | if (gate === undefined) gates.set(app, (gate = createTurn())); | |
| 103 | return gate; | |
| 104 | } | |
| 95 | 105 | |
| 96 | 106 | /** |
| 97 | 107 | * How often each forge's own requests waited at the gate, for its /system. |
src/http/mcp/tools/system.ts +4 −4
| @@ -4,7 +4,7 @@ import { scriptErrorCounts } from "../../script_errors.ts"; | ||
| 4 | 4 | import { violationsByDirective, windowStart } from "../../csp_report.ts"; |
| 5 | 5 | import { countRead } from "../../../system/telemetry_reads.ts"; |
| 6 | 6 | import { stallHour, TOP_ROUTES } from "../../../metrics/stalls.ts"; |
| 7 | import { lagMinutes, loopWatch, perfRecord } from "../../../perf/live.ts"; | |
| 7 | import { liveOf } from "../../../perf/live.ts"; | |
| 8 | 8 | import { LAG_BAR_MS, overBar } from "../../../perf/lag_minutes.ts"; |
| 9 | 9 | import { slowestRoutes } from "../../../perf/watch.ts"; |
| 10 | 10 | import { footing, riskLines } from "../../../system/footing.ts"; |
| @@ -28,7 +28,7 @@ export const tool: Tool = { | ||
| 28 | 28 | const reader = telemetryReader(ctx.app, user); |
| 29 | 29 | if (!reader) return bad("forbidden", "only an admin of this forge, or an agent granted telemetry:read, reads its route timings"); |
| 30 | 30 | noteRead(ctx.app, user, reader, "system"); |
| 31 | return good({ routes: slowestRoutes(perfRecord, Date.now()) }); | |
| 31 | return good({ routes: slowestRoutes(liveOf(ctx.app).perfRecord, Date.now()) }); | |
| 32 | 32 | } |
| 33 | 33 | if (args["kind"] === "size") { |
| 34 | 34 | const reader = telemetryReader(ctx.app, user); |
| @@ -63,11 +63,11 @@ export const tool: Tool = { | ||
| 63 | 63 | noteRead(ctx.app, user, reader, "system"); |
| 64 | 64 | const out = systemWithReaders(ctx.app, reader); |
| 65 | 65 | countRead("system", user.username); |
| 66 | const stalls = stallHour(loopWatch.tally(performance.now())) | |
| 66 | const stalls = stallHour(liveOf(ctx.app).loopWatch.tally(performance.now())) | |
| 67 | 67 | .slice(0, TOP_ROUTES) |
| 68 | 68 | .map((r) => ({ cause: r.route, ms: Math.round(r.ms), count: r.count })); |
| 69 | 69 | // The same 20 minutes /system draws (#11220); `what` is null when no stall past 200 ms was named. |
| 70 | const rows = lagMinutes.readings(); | |
| 70 | const rows = liveOf(ctx.app).lagMinutes.readings(); | |
| 71 | 71 | const readings = rows.map((r) => ({ at: new Date(r.at).toISOString(), ms: Math.round(r.ms * 10) / 10, what: r.what })); |
| 72 | 72 | // At most 0.8 s on the walk (#11800); the last measurement, or null, when it is slower. |
| 73 | 73 | const largest = await largestDirs.read(ctx.app.config.dataDir); |
src/http/server.ts +10 −10
| @@ -553,7 +553,7 @@ import { | ||
| 553 | 553 | takeDbTime, |
| 554 | 554 | type RouteTiming, |
| 555 | 555 | } from "../core/route_timing.ts"; |
| 556 | import { captureStore, inFlight, loopWatch, perfRecord, readLedger, stallBlame } from "../perf/live.ts"; | |
| 556 | import { captureStore, inFlight, liveOf, readLedger } from "../perf/live.ts"; | |
| 557 | 557 | import { childLabel, markChildRequest, noteChildRequest } from "../perf/child_time.ts"; |
| 558 | 558 | import { parseHostedChildName } from "../spawn/placement.ts"; |
| 559 | 559 | import { childHost } from "../spawn/door.ts"; |
| @@ -572,7 +572,7 @@ import { parseServerTiming } from "../perf/record.ts"; | ||
| 572 | 572 | import { currentLoopLag, LAG_WINDOW_MS, startLoopWorst } from "../perf/loop_worst.ts"; |
| 573 | 573 | import { methodOf, noteUsage, roleOf, surfaceOf, takeUsage } from "./usage_tag.ts"; |
| 574 | 574 | import { askingOf, noteAsker, noteWaited, takeAsker } from "../perf/asking.ts"; |
| 575 | import { turn, turnStatsOf } from "./app_turn.ts"; | |
| 575 | import { turnOf, turnStatsOf } from "./app_turn.ts"; | |
| 576 | 576 | import { keepRead, repeatKey, repeatRead } from "./repeat_reads.ts"; |
| 577 | 577 | import { writeStamp } from "../core/write_clock.ts"; |
| 578 | 578 | import { bearerChallenge, oauthRoutes } from "./oauth.ts"; |
| @@ -2675,7 +2675,7 @@ export async function serve(app: App): Promise<Servers> { | ||
| 2675 | 2675 | const counters = new Map<Database, UsageCounter>(); |
| 2676 | 2676 | const timings = new Map<Database, RouteTiming>(); |
| 2677 | 2677 | // The minute ledger's history: one write a minute, never on a request. |
| 2678 | const trafficWriter = createTrafficWriter(app.db, perfRecord.ledger, app.log); | |
| 2678 | const trafficWriter = createTrafficWriter(app.db, liveOf(app).perfRecord.ledger, app.log); | |
| 2679 | 2679 | // Page views are buffered by the beacon doors and written here; GitHub's traffic is pulled daily when a token is set. |
| 2680 | 2680 | const viewWriter = createViewWriter(app.db, app.log); |
| 2681 | 2681 | const githubPull = startGithubPull(app.db, app.log); |
| @@ -2698,7 +2698,7 @@ export async function serve(app: App): Promise<Servers> { | ||
| 2698 | 2698 | ): void => { |
| 2699 | 2699 | const tag = takeUsage(request); |
| 2700 | 2700 | const ms = performance.now() - started; |
| 2701 | turn.end(request); | |
| 2701 | turnOf(app).end(request); | |
| 2702 | 2702 | const asker = takeAsker(request); |
| 2703 | 2703 | askingOf(app).note(asker?.token, tag.pattern, Math.max(0, ms - (asker?.waitedMs ?? 0))); |
| 2704 | 2704 | finishedStrangerDeep(owner, request); |
| @@ -2710,7 +2710,7 @@ export async function serve(app: App): Promise<Servers> { | ||
| 2710 | 2710 | response.headers.set("server-timing", tag.role === "admin" ? withLag(header, currentLoopLag()) : header); |
| 2711 | 2711 | const end = performance.now(); |
| 2712 | 2712 | const family = familyOf(request.headers.get("user-agent") ?? ""); |
| 2713 | perfRecord.add( | |
| 2713 | liveOf(app).perfRecord.add( | |
| 2714 | 2714 | tag.pattern, |
| 2715 | 2715 | { at: Date.now(), ms, signedIn: tag.role !== "anon" && tag.role !== "crawler", crawler: wasCrawler(request), deepStranger: wasDeepStranger(request), internal: wasInternal(request), refused: response.status === 429 || wasShielded(request), phases: parseServerTiming(header) }, |
| 2716 | 2716 | { |
| @@ -2721,7 +2721,7 @@ export async function serve(app: App): Promise<Servers> { | ||
| 2721 | 2721 | repo: response.status < 400 ? repoOf(tag.pattern, new URL(request.url).pathname) : null, |
| 2722 | 2722 | }, |
| 2723 | 2723 | ); |
| 2724 | loopWatch.finished(hostedChild === undefined ? tag.pattern : childLabel(hostedChild), started, end); | |
| 2724 | liveOf(app).loopWatch.finished(hostedChild === undefined ? tag.pattern : childLabel(hostedChild), started, end); | |
| 2725 | 2725 | if (hostedChild !== undefined) noteChildRequest(hostedChild, ms); |
| 2726 | 2726 | // Evidence of what was read, and the running capture, if any (never a signed-in person's). |
| 2727 | 2727 | const signedIn = tag.role !== "anon" && tag.role !== "crawler"; |
| @@ -2808,7 +2808,7 @@ export async function serve(app: App): Promise<Servers> { | ||
| 2808 | 2808 | ): Promise<Response | undefined> => { |
| 2809 | 2809 | const started = performance.now(); |
| 2810 | 2810 | noteStart(request, started); |
| 2811 | turn.begin(request, started); | |
| 2811 | turnOf(app).begin(request, started); | |
| 2812 | 2812 | // A read-only standby refuses every mutation here, before any route. |
| 2813 | 2813 | if (app.config.readOnly === true) { |
| 2814 | 2814 | const refused = readOnlyRefusal(request); |
| @@ -3102,7 +3102,7 @@ async function answerGet( | ||
| 3102 | 3102 | () => app.db.query("select 1").get(), |
| 3103 | 3103 | app.log, |
| 3104 | 3104 | wantsJson ? currentLoopLag() : undefined, |
| 3105 | wantsJson ? (stallBlame.worst(performance.now(), LAG_WINDOW_MS, 1)[0]?.what ?? null) : null, | |
| 3105 | wantsJson ? (liveOf(app).stallBlame.worst(performance.now(), LAG_WINDOW_MS, 1)[0]?.what ?? null) : null, | |
| 3106 | 3106 | ); |
| 3107 | 3107 | noteDbTime(request, performance.now() - dbStart); |
| 3108 | 3108 | // The release proof learns which release answered, old pod or new; a |
| @@ -3313,9 +3313,9 @@ async function answerGet( | ||
| 3313 | 3313 | if (repeat !== undefined) { |
| 3314 | 3314 | const same = repeatRead(app.db, repeat, request, stamp); |
| 3315 | 3315 | if (same !== undefined) return withRateHeaders(same, verdict); |
| 3316 | turn.machine(request); | |
| 3316 | turnOf(app).machine(request); | |
| 3317 | 3317 | const waitFrom = performance.now(); |
| 3318 | await turn.wait(turnStatsOf(app)); | |
| 3318 | await turnOf(app).wait(turnStatsOf(app)); | |
| 3319 | 3319 | noteWaited(request, performance.now() - waitFrom); |
| 3320 | 3320 | } |
| 3321 | 3321 | // A person's own write, from their own browser, is theirs to see on the page they open next: when |
src/kernel/protection.ts +17 −12
| @@ -7,29 +7,34 @@ | ||
| 7 | 7 | * and this line in the foot of every page a site admin reads. Each part |
| 8 | 8 | * reports itself once, when it learns what the host can do; a part that |
| 9 | 9 | * works reports nothing, so a fully protected forge shows no line at all. |
| 10 | * | |
| 11 | * Each forge keeps its own notes (`app` is the forge's app object): in a test | |
| 12 | * run one file's note was another file's footer. | |
| 10 | 13 | */ |
| 11 | 14 | |
| 12 | const lacking = new Map<string, string>(); | |
| 15 | const lacking = new WeakMap<object, Map<string, string>>(); | |
| 13 | 16 | |
| 14 | /** A part says what it lacks (`ok`: nothing) and why, e.g. `("apps", "bwrap is not installed")`. */ | |
| 17 | /** | |
| 18 | * A part says what it lacks (`ok`: nothing) and why, | |
| 19 | * e.g. `(app, "apps", "bwrap is not installed")`. | |
| 20 | */ | |
| 15 | 21 | export function noteProtection( |
| 22 | app: object, | |
| 16 | 23 | part: string, |
| 17 | 24 | result: { ok: true } | { ok: false; reason: string }, |
| 18 | 25 | ): void { |
| 19 | if (result.ok) lacking.delete(part); | |
| 20 | else lacking.set(part, result.reason); | |
| 26 | let notes = lacking.get(app); | |
| 27 | if (notes === undefined) lacking.set(app, (notes = new Map())); | |
| 28 | if (result.ok) notes.delete(part); | |
| 29 | else notes.set(part, result.reason); | |
| 21 | 30 | } |
| 22 | 31 | |
| 23 | 32 | /** The one-line notice, or `undefined` when nothing is reduced. Plain text: escape it. */ |
| 24 | export function protectionNotice(): string | undefined { | |
| 25 | if (lacking.size === 0) return undefined; | |
| 26 | const parts = [...lacking] | |
| 33 | export function protectionNotice(app: object): string | undefined { | |
| 34 | const notes = lacking.get(app); | |
| 35 | if (notes === undefined || notes.size === 0) return undefined; | |
| 36 | const parts = [...notes] | |
| 27 | 37 | .sort(([a], [b]) => a.localeCompare(b)) |
| 28 | 38 | .map(([part, reason]) => `${part} (${reason})`); |
| 29 | 39 | return `Reduced protection: ${parts.join("; ")} run${parts.length === 1 ? "s" : ""} uncontained.`; |
| 30 | 40 | } |
| 31 | ||
| 32 | /** For tests. */ | |
| 33 | export function resetProtection(): void { | |
| 34 | lacking.clear(); | |
| 35 | } |
src/kernel/stall_relay.ts added +37
| @@ -0,0 +1,37 @@ | ||
| 1 | /** | |
| 2 | * A stall that arrives before the forge exists (#12569). | |
| 3 | * | |
| 4 | * The loop watch starts with the first line of the boot, and a forge's | |
| 5 | * stall record is made with its app a few lines later. A stall in between | |
| 6 | * waits here, at most `EARLY_KEPT` of them (the worst-ever boot is a few | |
| 7 | * seconds of lag, not a flood: the oldest are dropped), and is read in as | |
| 8 | * soon as the forge is made. | |
| 9 | */ | |
| 10 | ||
| 11 | export const EARLY_KEPT = 64; | |
| 12 | ||
| 13 | export interface StallRelay<T> { | |
| 14 | /** A stall: read at once when the forge exists, else kept for it. */ | |
| 15 | note(late: number, at: number): void; | |
| 16 | /** The forge exists: read the kept stalls in order, then every later one as it comes. */ | |
| 17 | attach(target: T): void; | |
| 18 | /** How many wait for the forge. */ | |
| 19 | waiting(): number; | |
| 20 | } | |
| 21 | ||
| 22 | export function createStallRelay<T>(read: (target: T, late: number, at: number) => void): StallRelay<T> { | |
| 23 | let target: { readonly t: T } | undefined; | |
| 24 | const early: [number, number][] = []; | |
| 25 | return { | |
| 26 | note(late, at) { | |
| 27 | if (target !== undefined) return read(target.t, late, at); | |
| 28 | early.push([late, at]); | |
| 29 | if (early.length > EARLY_KEPT) early.shift(); | |
| 30 | }, | |
| 31 | attach(t) { | |
| 32 | target = { t }; | |
| 33 | for (const [late, at] of early.splice(0)) read(t, late, at); | |
| 34 | }, | |
| 35 | waiting: () => early.length, | |
| 36 | }; | |
| 37 | } |
src/main.ts +14 −7
| @@ -129,10 +129,11 @@ import { reapLeftoverBridges } from "./crew/leftovers.ts"; | ||
| 129 | 129 | import { crewRoot } from "./crew/workdir.ts"; |
| 130 | 130 | import { createLog } from "./kernel/log.ts"; |
| 131 | 131 | import { watchStalls } from "./kernel/loop_lag.ts"; |
| 132 | import { createStallRelay } from "./kernel/stall_relay.ts"; | |
| 132 | 133 | import { startPerfWatcher, TICK_MS } from "./perf/duty.ts"; |
| 133 | 134 | import { during } from "./perf/jobs.ts"; |
| 134 | 135 | import { stallLine } from "./perf/loop.ts"; |
| 135 | import { bootStall, gitRecord, lagMinutes, loopWatch, perfRecord, stallBlame } from "./perf/live.ts"; | |
| 136 | import { bootStall, gitRecord, liveOf } from "./perf/live.ts"; | |
| 136 | 137 | import { keepFlowReadings, KEPT_MS } from "./flow/kept.ts"; |
| 137 | 138 | import { startKeeping } from "./flow/keep.ts"; |
| 138 | 139 | import { keepForgePages } from "./flow/pages.ts"; |
| @@ -269,13 +270,18 @@ if (parentProblem !== null) { | ||
| 269 | 270 | // second is a warning that names its minute (kernel/loop_lag.ts, #6957), |
| 270 | 271 | // draining included. |
| 271 | 272 | const loopLog = createLog("loop"); |
| 272 | watchStalls(loopLog, undefined, undefined, undefined, (late, at) => { | |
| 273 | bootStall.note(late, at); | |
| 274 | lagMinutes.note(late, at); | |
| 275 | const stall = loopWatch.stalled(late, at); | |
| 273 | // The loop watch is the forge's own (`liveOf(app)`), and the app is made a few lines below: a stall | |
| 274 | // before then waits in the relay (bounded) and is read in as soon as the forge exists. | |
| 275 | const stallRelay = createStallRelay((live: ReturnType<typeof liveOf>, late, at) => { | |
| 276 | live.lagMinutes.note(late, at); | |
| 277 | const stall = live.loopWatch.stalled(late, at); | |
| 276 | 278 | if (stall === null) return; |
| 277 | 279 | loopLog.warn(stallLine(stall)); |
| 278 | stallBlame.note(stall); | |
| 280 | live.stallBlame.note(stall); | |
| 281 | }); | |
| 282 | watchStalls(loopLog, undefined, undefined, undefined, (late, at) => { | |
| 283 | bootStall.note(late, at); | |
| 284 | stallRelay.note(late, at); | |
| 279 | 285 | }); |
| 280 | 286 | |
| 281 | 287 | const previewKey = previewIdentityKey(process.env); |
| @@ -363,6 +369,7 @@ const app = createApp({ | ||
| 363 | 369 | }), |
| 364 | 370 | }), |
| 365 | 371 | }); |
| 372 | stallRelay.attach(liveOf(app)); | |
| 366 | 373 | |
| 367 | 374 | // A statement that holds the thread past 200 ms names its caller (#6957). |
| 368 | 375 | timeStatements(app.db, createLog("sql")); |
| @@ -397,7 +404,7 @@ if (readOnly) { | ||
| 397 | 404 | const bootSignup = signupFromEnv(process.env.FORGE_SIGNUP); |
| 398 | 405 | if (bootSignup.ok && bootSignup.value !== null) |
| 399 | 406 | setSignup(app.db, bootSignup.value, BOOT_ACTOR); |
| 400 | startPerfWatcher(app, perfRecord, Date.now, TICK_MS, gitRecord); | |
| 407 | startPerfWatcher(app, liveOf(app).perfRecord, Date.now, TICK_MS, gitRecord); | |
| 401 | 408 | // A release shipped by hand is recorded as live at boot, so live estimates and the base rate |
| 402 | 409 | // resolve whichever way a release ships; estimates of "done" come due as time passes (#11955). |
| 403 | 410 | const byHand = recordByHand(app.db); |
src/metrics/forge.ts +4 −2
| @@ -18,7 +18,7 @@ import type { EnvState } from "../apps/runtime.ts"; | ||
| 18 | 18 | import { EDGES_MS, readTotals } from "../core/route_timing.ts"; |
| 19 | 19 | import { closesFromBody } from "../core/pulls.ts"; |
| 20 | 20 | import { telemetryReadsFamily } from "../system/telemetry_reads.ts"; |
| 21 | import { loopWatch } from "../perf/live.ts"; | |
| 21 | import { createLoopWatch, type LoopWatch } from "../perf/loop.ts"; | |
| 22 | 22 | import { stallFamilies } from "./stalls.ts"; |
| 23 | 23 | import { openMetrics, type Family, type HistogramSample, type Sample } from "./openmetrics.ts"; |
| 24 | 24 | |
| @@ -334,6 +334,8 @@ export function forgeMetrics( | ||
| 334 | 334 | db: Database, |
| 335 | 335 | sources: AppSources = { apps: [], cells: [] }, |
| 336 | 336 | now: number = Date.now(), |
| 337 | /** The forge's own loop watch (`liveOf(app).loopWatch`); none read means none stalled. */ | |
| 338 | loop: LoopWatch = createLoopWatch(), | |
| 337 | 339 | ): string { |
| 338 | 340 | return openMetrics([ |
| 339 | 341 | durationFamily(db), |
| @@ -347,6 +349,6 @@ export function forgeMetrics( | ||
| 347 | 349 | askToLiveFamily(db), |
| 348 | 350 | mainRedFamily(db, now), |
| 349 | 351 | telemetryReadsFamily(), |
| 350 | ...stallFamilies(loopWatch.tally(performance.now())), | |
| 352 | ...stallFamilies(loop.tally(performance.now())), | |
| 351 | 353 | ]).replace(/# EOF\n$/, ""); |
| 352 | 354 | } |
src/perf/crawlers.ts +2 −2
| @@ -6,11 +6,11 @@ | ||
| 6 | 6 | |
| 7 | 7 | import type { App } from "../app.ts"; |
| 8 | 8 | import { crawlRefusalScopes, crawlRefusals } from "../http/crawl.ts"; |
| 9 | import { perfRecord } from "./live.ts"; | |
| 9 | import { liveOf } from "./live.ts"; | |
| 10 | 10 | import { WINDOW_MS, crawlerShare } from "./record.ts"; |
| 11 | 11 | |
| 12 | 12 | export function crawlReport(app: App, now = Date.now()) { |
| 13 | const share = crawlerShare(perfRecord, now - WINDOW_MS); | |
| 13 | const share = crawlerShare(liveOf(app).perfRecord, now - WINDOW_MS); | |
| 14 | 14 | return { |
| 15 | 15 | window_minutes: WINDOW_MS / 60_000, |
| 16 | 16 | requests: share.requests, |
src/perf/live.ts +44 −14
| @@ -1,4 +1,7 @@ | ||
| 1 | /** The process's one record, git record (#8158) and loop watch (#7559): what the server feeds and the duty reads. */ | |
| 1 | /** | |
| 2 | * Each forge's traffic record and loop watch (`liveOf`), and the process's git record (#8158): | |
| 3 | * what the server feeds and the duty reads. | |
| 4 | */ | |
| 2 | 5 | |
| 3 | 6 | import { createBootStall } from "./boot_stall.ts"; |
| 4 | 7 | import { createGitRecord } from "./git.ts"; |
| @@ -13,26 +16,53 @@ import { createStallBlame } from "./stall_blame.ts"; | ||
| 13 | 16 | import { createPerfRecord } from "./record.ts"; |
| 14 | 17 | import { createLagMinutes } from "./lag_minutes.ts"; |
| 15 | 18 | |
| 16 | export const perfRecord = createPerfRecord(); | |
| 17 | 19 | /** The requests the server is answering now (#11478): begun and ended around each one in `http/server.ts`. */ |
| 18 | 20 | export const inFlight = createInFlight<Request>((request) => childRequestLabel(request) ?? patternOf(request)); |
| 19 | /** | |
| 20 | * A stall names the requests that ran across it, finished or still in flight | |
| 21 | * (#11478); when none did, the background jobs that did (#10834). A job that | |
| 22 | * held the loop for most of a stall is named ahead of the requests that waited | |
| 23 | * behind it, and a name that only brushed the stall is left out (`loop.ts`). | |
| 24 | */ | |
| 25 | export const loopWatch = createLoopWatch(undefined, jobWatch.across, inFlight.since, jobWatch.held, jobWatch.awaited); | |
| 26 | 21 | export const gitRecord = createGitRecord(); |
| 27 | 22 | /** The worst stall in this process's first five minutes, read by the release keeper (#8848). */ |
| 28 | 23 | export const bootStall = createBootStall(); |
| 29 | 24 | /** What was read from repositories (`reads.ts`), and the one bounded capture an admin may run (`capture.ts`). */ |
| 30 | 25 | export const readLedger = createReadLedger(); |
| 31 | 26 | export const captureStore = createCaptureStore(); |
| 32 | /** Each stall over 200 ms with what ran (#11069): the last hour's worst on /system, the last 10 s's on `/healthz`. */ | |
| 33 | export const stallBlame = createStallBlame(); | |
| 27 | ||
| 34 | 28 | /** |
| 35 | * The last 20 minutes' worst loop lag (#11220), each blamed on the worst stall | |
| 36 | * `stallBlame` named in it: /system and the MCP `system` tool read it. | |
| 29 | * What each forge's own server feeds and its pages read (#12569): the traffic | |
| 30 | * record, the loop watch, the stall blame and the lag minutes. They were one | |
| 31 | * set for the process, so in a test run a file's traffic or stall was in every | |
| 32 | * later file's /system and /traffic, by the order the files ran in. | |
| 33 | * | |
| 34 | * The loop watch is told of jobs from `jobWatch` and of requests from | |
| 35 | * `inFlight`, which stay the process's: a timer and a request in flight are | |
| 36 | * the thread's, and the watch only reads them. | |
| 37 | * | |
| 38 | * A stall names the requests that ran across it, finished or still in flight | |
| 39 | * (#11478); when none did, the background jobs that did (#10834). A job that | |
| 40 | * held the loop for most of a stall is named ahead of the requests that waited | |
| 41 | * behind it, and a name that only brushed the stall is left out (`loop.ts`). | |
| 42 | * `stallBlame`: each stall over 200 ms with what ran (#11069): the last hour's | |
| 43 | * worst on /system, the last 10 s's on `/healthz`. `lagMinutes`: the last 20 | |
| 44 | * minutes' worst loop lag (#11220), each blamed on the worst stall | |
| 45 | * `stallBlame` named in it. | |
| 37 | 46 | */ |
| 38 | export const lagMinutes = createLagMinutes((to, span) => stallBlame.worst(to, span, 1)[0]?.what ?? null); | |
| 47 | export interface Live { | |
| 48 | readonly perfRecord: ReturnType<typeof createPerfRecord>; | |
| 49 | readonly loopWatch: ReturnType<typeof createLoopWatch>; | |
| 50 | readonly stallBlame: ReturnType<typeof createStallBlame>; | |
| 51 | readonly lagMinutes: ReturnType<typeof createLagMinutes>; | |
| 52 | } | |
| 53 | ||
| 54 | const lives = new WeakMap<object, Live>(); | |
| 55 | export function liveOf(app: object): Live { | |
| 56 | let live = lives.get(app); | |
| 57 | if (live === undefined) { | |
| 58 | const stallBlame = createStallBlame(); | |
| 59 | live = { | |
| 60 | perfRecord: createPerfRecord(), | |
| 61 | loopWatch: createLoopWatch(undefined, jobWatch.across, inFlight.since, jobWatch.held, jobWatch.awaited), | |
| 62 | stallBlame, | |
| 63 | lagMinutes: createLagMinutes((to, span) => stallBlame.worst(to, span, 1)[0]?.what ?? null), | |
| 64 | }; | |
| 65 | lives.set(app, live); | |
| 66 | } | |
| 67 | return live; | |
| 68 | } |
src/release/landings.ts +20 −9
| @@ -98,17 +98,22 @@ interface LiveState { | ||
| 98 | 98 | /** Commits in `head` and not in `commit` (`git rev-list --count commit..head`), asked once per pair. */ |
| 99 | 99 | readonly ahead?: number; |
| 100 | 100 | } |
| 101 | let kept: LiveState | undefined; | |
| 101 | /** | |
| 102 | * What each forge's reads kept, by its database (one per forge): one for the | |
| 103 | * process let a test's own repository, or a forge hosted beside another, show | |
| 104 | * as the other's "This forge" on /changelog (#12569). | |
| 105 | */ | |
| 106 | const keeps = new WeakMap<Database, LiveState>(); | |
| 102 | 107 | |
| 103 | 108 | /** Keep what the running commit contains; a test says it directly. */ |
| 104 | export function keepLive(own: string, oids: Iterable<string>, at: number, gap?: { commit: string; head: string; ahead: number }): void { | |
| 105 | kept = { own, oids: new Set(oids), misses: new Set(), at, commit: gap?.commit ?? "", ...(gap === undefined ? {} : { head: gap.head, ahead: gap.ahead }) }; | |
| 109 | export function keepLive(db: Database, own: string, oids: Iterable<string>, at: number, gap?: { commit: string; head: string; ahead: number }): void { | |
| 110 | keeps.set(db, { own, oids: new Set(oids), misses: new Set(), at, commit: gap?.commit ?? "", ...(gap === undefined ? {} : { head: gap.head, ahead: gap.ahead }) }); | |
| 106 | 111 | } |
| 107 | 112 | |
| 108 | 113 | /** Forget it, so the alarm is silent until the next read. */ |
| 109 | export function forgetLive(): void { | |
| 110 | kept = undefined; | |
| 111 | found.clear(); | |
| 114 | export function forgetLive(db: Database): void { | |
| 115 | keeps.delete(db); | |
| 116 | founds.delete(db); | |
| 112 | 117 | } |
| 113 | 118 | |
| 114 | 119 | function isLive(row: MergedRow, state: LiveState | undefined): boolean { |
| @@ -139,6 +144,7 @@ export function split(rows: readonly MergedRow[], state: LiveState | undefined): | ||
| 139 | 144 | |
| 140 | 145 | /** The merged pulls still on their way and when the oldest merged, from what the last read kept; undefined when none or nothing was read. */ |
| 141 | 146 | export function waiting(db: Database): { readonly since: string; readonly count: number } | undefined { |
| 147 | const kept = keeps.get(db); | |
| 142 | 148 | if (kept === undefined) return undefined; |
| 143 | 149 | const { onItsWay } = split(mergedRows(db), kept); |
| 144 | 150 | const oldest = onItsWay.at(-1); |
| @@ -147,6 +153,7 @@ export function waiting(db: Database): { readonly since: string; readonly count: | ||
| 147 | 153 | |
| 148 | 154 | /** The merged pulls not yet in the running commit, oldest first, with their merge commits; empty when nothing was read (#11507). */ |
| 149 | 155 | export function waitingRows(db: Database): MergedRow[] { |
| 156 | const kept = keeps.get(db); | |
| 150 | 157 | if (kept === undefined) return []; |
| 151 | 158 | const rows = mergedRows(db); |
| 152 | 159 | const way = new Set(split(rows, kept).onItsWay.map((l) => `${l.repo}#${l.number}`)); |
| @@ -157,7 +164,7 @@ export function waitingRows(db: Database): MergedRow[] { | ||
| 157 | 164 | const MISS_MS = 60_000; |
| 158 | 165 | |
| 159 | 166 | /** The repository the running commit was built from, per running commit; "" when none held it at `at`. */ |
| 160 | const found = new Map<string, { repo: string; at: number }>(); | |
| 167 | const founds = new WeakMap<Database, Map<string, { repo: string; at: number }>>(); | |
| 161 | 168 | |
| 162 | 169 | /** |
| 163 | 170 | * The forge's own repository: the one `FORGE_REPO` names or, where a deploy |
| @@ -171,6 +178,8 @@ async function ownRepo(app: App, rows: readonly MergedRow[], now = Date.now()): | ||
| 171 | 178 | if (owner !== "" && name !== "") return getRepo(app.db, owner, name); |
| 172 | 179 | const commit = running().commit; |
| 173 | 180 | if (commit === null) return undefined; |
| 181 | let found = founds.get(app.db); | |
| 182 | if (found === undefined) founds.set(app.db, (found = new Map())); | |
| 174 | 183 | const known = found.get(commit); |
| 175 | 184 | if (known?.repo === "" && now - known.at < MISS_MS) return undefined; |
| 176 | 185 | for (const named of known?.repo ? [known.repo] : candidates(rows)) { |
| @@ -206,6 +215,7 @@ async function askLive(app: App, rows: readonly MergedRow[]): Promise<void> { | ||
| 206 | 215 | const named = `${repo.ownerName}/${repo.name}`; |
| 207 | 216 | // For one running commit an ancestor stays one and a non-ancestor never becomes one: |
| 208 | 217 | // only a new running commit voids what was kept. |
| 218 | const kept = keeps.get(app.db); | |
| 209 | 219 | const before = kept?.own === named && kept.commit === commit ? kept : undefined; |
| 210 | 220 | let at = before?.at; |
| 211 | 221 | if (at === undefined) { |
| @@ -240,7 +250,7 @@ async function askLive(app: App, rows: readonly MergedRow[]): Promise<void> { | ||
| 240 | 250 | const v = n.ok ? Number(n.value.trim()) : NaN; |
| 241 | 251 | if (Number.isSafeInteger(v) && v >= 0) ahead = v; |
| 242 | 252 | } |
| 243 | kept = { own: named, oids, misses, at, commit, ...(tip === undefined ? {} : { head: tip }), ...(ahead === undefined ? {} : { ahead }) }; | |
| 253 | keeps.set(app.db, { own: named, oids, misses, at, commit, ...(tip === undefined ? {} : { head: tip }), ...(ahead === undefined ? {} : { ahead }) }); | |
| 244 | 254 | } |
| 245 | 255 | |
| 246 | 256 | export interface MainGap { |
| @@ -255,7 +265,7 @@ export interface MainGap { | ||
| 255 | 265 | * git, one query. Undefined until a read found the forge's own main. |
| 256 | 266 | */ |
| 257 | 267 | export function mainGap(db: Database): MainGap | undefined { |
| 258 | const k = kept; | |
| 268 | const k = keeps.get(db); | |
| 259 | 269 | if (k === undefined || k.head === undefined) return undefined; |
| 260 | 270 | if (k.head === k.commit) return { main: { sha: k.head }, behind: { commits: 0, pulls: 0 } }; |
| 261 | 271 | const own = k.own.toLowerCase(); |
| @@ -283,6 +293,7 @@ async function buildersOfRepo(app: App, own: string, rows: readonly MergedRow[]) | ||
| 283 | 293 | export async function readLandings(app: App): Promise<Landings> { |
| 284 | 294 | const rows = mergedRows(app.db); |
| 285 | 295 | await askLive(app, rows).catch(() => undefined); |
| 296 | const kept = keeps.get(app.db); | |
| 286 | 297 | const { live, onItsWay } = split(rows, kept); |
| 287 | 298 | const own = kept?.own; |
| 288 | 299 | const credits = own === undefined ? new Map<string, Builders>() : await buildersOfRepo(app, own, rows).catch(() => new Map<string, Builders>()); |
src/tui/screens/dashboard.ts +1 −1
| @@ -1283,7 +1283,7 @@ export function dashboardScreen(f: Session, ctx: Ctx): Screen { | ||
| 1283 | 1283 | read === undefined |
| 1284 | 1284 | ? undefined |
| 1285 | 1285 | : `Read as: ${exploreLine(read.query) || "everything"}${explored === undefined ? "" : ` · ${exploreCount(read.query, explored.total, f.user?.isAdmin === true)}${exploreFiltered(read.query) ? " · Esc clears" : ""}`}${mend === undefined ? "" : ` · Did you mean ${mend.line}?`}${note === "" ? "" : ` · ${note}`}`; |
| 1286 | const reduced = readAs ?? (f.user?.isAdmin === true ? protectionNotice() : undefined); | |
| 1286 | const reduced = readAs ?? (f.user?.isAdmin === true ? protectionNotice(f.app) : undefined); | |
| 1287 | 1287 | const [box, warn, list] = split( |
| 1288 | 1288 | main, |
| 1289 | 1289 | "v", |
src/tui/screens/loopstalls.ts +6 −6
| @@ -12,7 +12,7 @@ | ||
| 12 | 12 | |
| 13 | 13 | import { stallHour } from "../../metrics/stalls.ts"; |
| 14 | 14 | import { HOUR_MS, type RouteStall } from "../../perf/loop.ts"; |
| 15 | import { lagMinutes, loopWatch, stallBlame } from "../../perf/live.ts"; | |
| 15 | import { liveOf } from "../../perf/live.ts"; | |
| 16 | 16 | import { LAG_BAR_MS, LAG_READINGS, overBar, type LagReading } from "../../perf/lag_minutes.ts"; |
| 17 | 17 | import { WORST_STALLS, type BlamedStall } from "../../perf/stall_blame.ts"; |
| 18 | 18 | import type { Ctx, Screen } from "../app/app.ts"; |
| @@ -34,9 +34,9 @@ export interface WorstNow { | ||
| 34 | 34 | readonly wall: number; |
| 35 | 35 | } |
| 36 | 36 | |
| 37 | function worstNow(): WorstNow { | |
| 37 | function worstNow(session: Session): WorstNow { | |
| 38 | 38 | const mono = performance.now(); |
| 39 | return { rows: stallBlame.worst(mono, HOUR_MS, WORST_STALLS), mono, wall: Date.now() }; | |
| 39 | return { rows: liveOf(session.app).stallBlame.worst(mono, HOUR_MS, WORST_STALLS), mono, wall: Date.now() }; | |
| 40 | 40 | } |
| 41 | 41 | |
| 42 | 42 | /** |
| @@ -46,8 +46,8 @@ function worstNow(): WorstNow { | ||
| 46 | 46 | */ |
| 47 | 47 | export function stallLines( |
| 48 | 48 | session: Session, |
| 49 | rows: readonly RouteStall[] = stallHour(loopWatch.tally(performance.now())), | |
| 50 | worst: WorstNow = worstNow(), | |
| 49 | rows: readonly RouteStall[] = stallHour(liveOf(session.app).loopWatch.tally(performance.now())), | |
| 50 | worst: WorstNow = worstNow(session), | |
| 51 | 51 | ): string[] { |
| 52 | 52 | if (!mayRead(session)) return []; |
| 53 | 53 | const col = (s: string) => pad(s, 10, "right"); |
| @@ -71,7 +71,7 @@ export function stallLines( | ||
| 71 | 71 | * start in UTC, its worst ms, `over` when strictly past the bar, and what ran |
| 72 | 72 | * (or "not named"). Empty for anyone the gate refuses. |
| 73 | 73 | */ |
| 74 | export function lagLines(session: Session, rows: readonly LagReading[] = lagMinutes.readings()): string[] { | |
| 74 | export function lagLines(session: Session, rows: readonly LagReading[] = liveOf(session.app).lagMinutes.readings()): string[] { | |
| 75 | 75 | if (!mayRead(session)) return []; |
| 76 | 76 | const head = `Loop lag, last ${String(LAG_READINGS)} minutes`; |
| 77 | 77 | if (rows.length === 0) return [head, "No full minute measured yet."]; |
src/tui/screens/traffic.ts +2 −2
| @@ -8,7 +8,7 @@ | ||
| 8 | 8 | import type { Ctx, Screen } from "../app/app.ts"; |
| 9 | 9 | import { binding } from "../app/keys.ts"; |
| 10 | 10 | import { trafficShare, TRAFFIC_WORDS } from "../../flow/kpi_words.ts"; |
| 11 | import { perfRecord } from "../../perf/live.ts"; | |
| 11 | import { liveOf } from "../../perf/live.ts"; | |
| 12 | 12 | import { TRAFFIC_CLASSES } from "../../perf/traffic.ts"; |
| 13 | 13 | import { trafficHistory } from "../../perf/traffic_store.ts"; |
| 14 | 14 | import type { Session } from "../session.ts"; |
| @@ -24,7 +24,7 @@ export function trafficScreen(f: Session, ctx: Ctx): Screen { | ||
| 24 | 24 | const state = new ListState(); |
| 25 | 25 | const lines = (): ListRow[] => { |
| 26 | 26 | if (!may(f)) return []; |
| 27 | const t = trafficHistory(f.app.db, perfRecord.ledger, Date.now(), "15m"); | |
| 27 | const t = trafficHistory(f.app.db, liveOf(f.app).perfRecord.ledger, Date.now(), "15m"); | |
| 28 | 28 | if (t.requests === 0) return [{ title: "No requests in the last 15 minutes", meta: "each finished minute is written down" }]; |
| 29 | 29 | const rows: ListRow[] = [ |
| 30 | 30 | { title: `${t.requests} requests, ${(t.ms / 1000).toFixed(1)} s of answering time`, meta: `${t.refused} refused, about ${t.addresses} client addresses` }, |
src/web/layout.ts +1 −1
| @@ -707,7 +707,7 @@ function topbar( | ||
| 707 | 707 | function footer(ctx: Ctx, own?: Html): Html { |
| 708 | 708 | if (own !== undefined) return html`<footer class="foot">${own}</footer>`; |
| 709 | 709 | const reduced = |
| 710 | ctx.auth.user?.isAdmin === true ? protectionNotice() : undefined; | |
| 710 | ctx.auth.user?.isAdmin === true ? protectionNotice(ctx.app) : undefined; | |
| 711 | 711 | return html`<footer class="foot"> |
| 712 | 712 | <span class="mark"></span> |
| 713 | 713 | <span |
src/web/pages/scrapes.ts +2 −2
| @@ -9,7 +9,7 @@ | ||
| 9 | 9 | import { notePhase, takeSessionTime } from "../../core/route_timing.ts"; |
| 10 | 10 | import type { Ctx } from "../../http/context.ts"; |
| 11 | 11 | import { repoDir } from "../../git/repos.ts"; |
| 12 | import { perfRecord, captureStore, readLedger } from "../../perf/live.ts"; | |
| 12 | import { liveOf, captureStore, readLedger } from "../../perf/live.ts"; | |
| 13 | 13 | import { groupRequests, type Group } from "../../perf/actors.ts"; |
| 14 | 14 | import { CAPTURE_MAX, CAPTURE_MS } from "../../perf/capture.ts"; |
| 15 | 15 | import { networkTable, parseNetworks, saveNetworks, type Network, type NetworkTable } from "../../perf/network.ts"; |
| @@ -154,7 +154,7 @@ export async function scrapesPage(ctx: Ctx): Promise<Response> { | ||
| 154 | 154 | ); |
| 155 | 155 | }), |
| 156 | 156 | ); |
| 157 | const prefixes = trafficHistory(ctx.app.db, perfRecord.ledger, now, "24h").tops.prefixes; | |
| 157 | const prefixes = trafficHistory(ctx.app.db, liveOf(ctx.app).perfRecord.ledger, now, "24h").tops.prefixes; | |
| 158 | 158 | const cap = captureStore.current(now); |
| 159 | 159 | const groups = cap === null ? [] : groupRequests(cap.entries).filter((g) => g.size >= 3); |
| 160 | 160 | const picked = groups.find((g) => g.id === q.get("group")); |
src/web/pages/security.ts +2 −2
| @@ -17,7 +17,7 @@ import { readBackup } from "../../flow/kpis.ts"; | ||
| 17 | 17 | import type { Ctx } from "../../http/context.ts"; |
| 18 | 18 | import { violationsByDirective } from "../../http/csp_report.ts"; |
| 19 | 19 | import { scriptErrorCounts } from "../../http/script_errors.ts"; |
| 20 | import { perfRecord } from "../../perf/live.ts"; | |
| 20 | import { liveOf } from "../../perf/live.ts"; | |
| 21 | 21 | import { getSignup } from "../../settings/signup.ts"; |
| 22 | 22 | import { trafficHistory } from "../../perf/traffic_store.ts"; |
| 23 | 23 | import { lastDrill } from "../../snapshot/drills.ts"; |
| @@ -277,7 +277,7 @@ export function securityPage(ctx: Ctx): Response { | ||
| 277 | 277 | const db = ctx.app.db; |
| 278 | 278 | const dataDir = ctx.app.config.dataDir; |
| 279 | 279 | |
| 280 | const traffic = trafficHistory(db, perfRecord.ledger, nowMs, "15m"); | |
| 280 | const traffic = trafficHistory(db, liveOf(ctx.app).perfRecord.ledger, nowMs, "15m"); | |
| 281 | 281 | const wrong = trafficAttention(traffic); |
| 282 | 282 | const { admins, adminTokens, added } = peopleRows(db, since, nowIso); |
| 283 | 283 | const { backup, backupAt } = readBackup(db, dataDir, nowMs); |
src/web/pages/system.ts +4 −4
| @@ -37,7 +37,7 @@ import { readerPaces, type ReaderPace } from "../../system/telemetry_reads.ts"; | ||
| 37 | 37 | import { slowPageLine, slowPages, type SlowPage } from "../../perf/watch.ts"; |
| 38 | 38 | import { stallHour } from "../../metrics/stalls.ts"; |
| 39 | 39 | import { HOUR_MS, type RouteStall } from "../../perf/loop.ts"; |
| 40 | import { lagMinutes, loopWatch, stallBlame } from "../../perf/live.ts"; | |
| 40 | import { liveOf } from "../../perf/live.ts"; | |
| 41 | 41 | import { childHours, childHourWords, type ChildHour } from "../../perf/child_time.ts"; |
| 42 | 42 | import { openChildren } from "../../spawn/host.ts"; |
| 43 | 43 | import { LAG_BAR_MS, LAG_READINGS, overBar, type LagReading } from "../../perf/lag_minutes.ts"; |
| @@ -597,7 +597,7 @@ export async function systemPage(ctx: Ctx): Promise<Response> { | ||
| 597 | 597 | const self = owner ? timePhase(ctx.request, "self", () => selfUse(realSelf(ctx.app.config.dataDir))) : null; |
| 598 | 598 | const listed = owner ? await listChildren(ctx.app) : []; |
| 599 | 599 | const children = childrenUse(listed, (bodyId) => childDirOf(ctx.app, bodyId)); |
| 600 | const tally = loopWatch.tally(performance.now()); | |
| 600 | const tally = liveOf(ctx.app).loopWatch.tally(performance.now()); | |
| 601 | 601 | const stalls = timePhase(ctx.request, "stalls", () => stallHour(tally)); |
| 602 | 602 | // The forges open in this process now, and any that ran here in the hour: each with what it took |
| 603 | 603 | // of this thread. |
| @@ -606,7 +606,7 @@ export async function systemPage(ctx: Ctx): Promise<Response> { | ||
| 606 | 606 | const mine = new Set(listed.map((c) => c.name)); |
| 607 | 607 | const inProcess = childHours(Date.now(), tally.hour, openChildren(ctx.app).map((c) => c.name)).filter((r) => mine.has(r.name)); |
| 608 | 608 | const nowMono = performance.now(); |
| 609 | const worst = stallBlame.worst(nowMono, HOUR_MS, WORST_STALLS); | |
| 609 | const worst = liveOf(ctx.app).stallBlame.worst(nowMono, HOUR_MS, WORST_STALLS); | |
| 610 | 610 | // Never waits on the cluster: the last reading, or the declared rows while the first read runs behind. |
| 611 | 611 | const standsOn = footing.peek() ?? declaredFooting("the cluster is being read; reload in a moment"); |
| 612 | 612 | // Never waits on the walk: the last measurement, a new one started behind it when stale. |
| @@ -655,7 +655,7 @@ export async function systemPage(ctx: Ctx): Promise<Response> { | ||
| 655 | 655 | ${footingSection(standsOn)} |
| 656 | 656 | ${offsiteSection(timePhase(ctx.request, "offsite", () => readOffsiteLane(ctx.app.config.dataDir, ctx.app.db)))} |
| 657 | 657 | ${featuresSection(timePhase(ctx.request, "features", () => featureHealth(ctx.app, reader)))} |
| 658 | ${lagSection(lagMinutes.readings())} | |
| 658 | ${lagSection(liveOf(ctx.app).lagMinutes.readings())} | |
| 659 | 659 | <div id="sys-stalls">${stallSection(stalls, worstStallsTable(worst, nowMono, Date.now()))}</div> |
| 660 | 660 | ${owner ? askingSection(askingOf(ctx.app).heavy(), turnStatsOf(ctx.app)) : html``} |
| 661 | 661 | <div id="sys-slow-pages">${slowPagesSection(timePhase(ctx.request, "slowpages", () => slowPagesCached(ctx.app.db)))}</div> |
src/web/pages/traffic.ts +2 −2
| @@ -14,7 +14,7 @@ import { | ||
| 14 | 14 | trafficShare, |
| 15 | 15 | TRAFFIC_WORDS, |
| 16 | 16 | } from "../../flow/kpi_words.ts"; |
| 17 | import { perfRecord } from "../../perf/live.ts"; | |
| 17 | import { liveOf } from "../../perf/live.ts"; | |
| 18 | 18 | import { |
| 19 | 19 | TRAFFIC_CLASSES, |
| 20 | 20 | type TopRow, |
| @@ -118,7 +118,7 @@ export function trafficPage(ctx: Ctx): Response { | ||
| 118 | 118 | const rendering = performance.now(); |
| 119 | 119 | const now = Date.now(); |
| 120 | 120 | const chosen = windowOf(new URL(ctx.request.url).searchParams.get("window")); |
| 121 | const t = trafficHistory(ctx.app.db, perfRecord.ledger, now, chosen); | |
| 121 | const t = trafficHistory(ctx.app.db, liveOf(ctx.app).perfRecord.ledger, now, chosen); | |
| 122 | 122 | const tops = t.tops; |
| 123 | 123 | const wrong = trafficAttention(t); |
| 124 | 124 | const span = TRAFFIC_WINDOWS[chosen].words; |
src/web/studio/kpi_probe.ts +2 −2
| @@ -39,7 +39,7 @@ import { | ||
| 39 | 39 | visitorsValue, |
| 40 | 40 | } from "../../flow/kpi_words.ts"; |
| 41 | 41 | import { HOURS, median } from "../../flow/throughput.ts"; |
| 42 | import { perfRecord } from "../../perf/live.ts"; | |
| 42 | import { liveOf } from "../../perf/live.ts"; | |
| 43 | 43 | import { scanTraffic, type PerfRecord } from "../../perf/record.ts"; |
| 44 | 44 | import { TRAFFIC_CLASSES } from "../../perf/traffic.ts"; |
| 45 | 45 | import { HOME_TTL_MS, cachedKpis } from "./home_cache.ts"; |
| @@ -315,6 +315,6 @@ export function kpiProbe(app: App, now: Date = new Date()): KpiProbe { | ||
| 315 | 315 | now, |
| 316 | 316 | { kpis: cachedKpis(app.db, now, forge), live: liveNow() }, |
| 317 | 317 | forge, |
| 318 | perfRecord, | |
| 318 | liveOf(app).perfRecord, | |
| 319 | 319 | ); |
| 320 | 320 | } |
test/changelog-api.test.ts +2 −2
| @@ -87,7 +87,7 @@ describe("GET /api/v1/changelog", () => { | ||
| 87 | 87 | }); |
| 88 | 88 | |
| 89 | 89 | test("with a repository of its own the default is that repository, on every door", async () => { |
| 90 | keepLive("alice/forge", [], 0); | |
| 90 | keepLive(app.db, "alice/forge", [], 0); | |
| 91 | 91 | try { |
| 92 | 92 | expect(titles(await (await get("")).json())).toEqual(["Public landing"]); |
| 93 | 93 | expect(titles(await mcp({ shipped: true }, admin))).toEqual(["Public landing"]); |
| @@ -95,7 +95,7 @@ describe("GET /api/v1/changelog", () => { | ||
| 95 | 95 | expect(titles(await (await get("?repo=all")).json())).toEqual(["Other repo landing", "Public landing"]); |
| 96 | 96 | expect((await get("")).headers.get("cache-control")).toBe("public, max-age=5"); |
| 97 | 97 | } finally { |
| 98 | forgetLive(); | |
| 98 | forgetLive(app.db); | |
| 99 | 99 | } |
| 100 | 100 | }); |
| 101 | 101 |
test/changelog-atom.test.ts +2 −2
| @@ -41,11 +41,11 @@ describe("GET /changelog.atom", () => { | ||
| 41 | 41 | insertPull.run("i2", made.value.defaultBranch, "2026-10-02T10:00:00.000Z", "oid-new"); |
| 42 | 42 | insertIssue.run("i3", made.value.id, 3, "small: j/k, enter, esc and ? on pulls, with test/keys-pulls.test.ts", u.value.id); |
| 43 | 43 | insertPull.run("i3", made.value.defaultBranch, "2026-09-30T10:00:00.000Z", "oid-live3"); |
| 44 | keepLive("alice/forge", ["oid-live", "oid-live3"], 0); | |
| 44 | keepLive(app.db, "alice/forge", ["oid-live", "oid-live3"], 0); | |
| 45 | 45 | }, 30_000); |
| 46 | 46 | |
| 47 | 47 | afterAll(async () => { |
| 48 | forgetLive(); | |
| 48 | forgetLive(app.db); | |
| 49 | 49 | if (saved === undefined) delete process.env["FORGE_COMMIT"]; |
| 50 | 50 | else process.env["FORGE_COMMIT"] = saved; |
| 51 | 51 | servers.stop(); |
test/changelog-cache.test.ts +3 −3
| @@ -46,11 +46,11 @@ describe("GET /changelog caching", () => { | ||
| 46 | 46 | insertPull.run("i2", made.value.defaultBranch, "2026-10-02T10:00:00.000Z", "oid-2"); |
| 47 | 47 | insertIssue.run("i3", other.value.id, 1, "Elsewhere", u.value.id); |
| 48 | 48 | insertPull.run("i3", other.value.defaultBranch, "2026-09-30T10:00:00.000Z", "oid-3"); |
| 49 | keepLive("alice/forge", ["oid-1"], 0); | |
| 49 | keepLive(app.db, "alice/forge", ["oid-1"], 0); | |
| 50 | 50 | }, 30_000); |
| 51 | 51 | |
| 52 | 52 | afterAll(async () => { |
| 53 | forgetLive(); | |
| 53 | forgetLive(app.db); | |
| 54 | 54 | if (saved === undefined) delete process.env["FORGE_COMMIT"]; |
| 55 | 55 | else process.env["FORGE_COMMIT"] = saved; |
| 56 | 56 | servers.stop(); |
| @@ -131,7 +131,7 @@ describe("GET /changelog caching", () => { | ||
| 131 | 131 | |
| 132 | 132 | test("the tag changes when a release lands", async () => { |
| 133 | 133 | const before = (await get()).headers.get("etag") ?? ""; |
| 134 | keepLive("alice/forge", ["oid-1", "oid-2"], 0); | |
| 134 | keepLive(app.db, "alice/forge", ["oid-1", "oid-2"], 0); | |
| 135 | 135 | const after = await get("", { "if-none-match": before }); |
| 136 | 136 | expect(after.status).toBe(200); |
| 137 | 137 | expect(after.headers.get("etag")).not.toBe(before); |
test/changelog-etag.test.ts +3 −3
| @@ -41,11 +41,11 @@ describe("GET /api/v1/changelog ETag", () => { | ||
| 41 | 41 | insertPull.run("i2", made.value.defaultBranch, "2026-10-02T10:00:00.000Z", "oid-2"); |
| 42 | 42 | insertIssue.run("i3", other.value.id, 1, "Elsewhere", u.value.id); |
| 43 | 43 | insertPull.run("i3", other.value.defaultBranch, "2026-09-30T10:00:00.000Z", "oid-3"); |
| 44 | keepLive("alice/forge", ["oid-1"], 0); | |
| 44 | keepLive(app.db, "alice/forge", ["oid-1"], 0); | |
| 45 | 45 | }, 30_000); |
| 46 | 46 | |
| 47 | 47 | afterAll(async () => { |
| 48 | forgetLive(); | |
| 48 | forgetLive(app.db); | |
| 49 | 49 | if (saved === undefined) delete process.env["FORGE_COMMIT"]; |
| 50 | 50 | else process.env["FORGE_COMMIT"] = saved; |
| 51 | 51 | servers.stop(); |
| @@ -77,7 +77,7 @@ describe("GET /api/v1/changelog ETag", () => { | ||
| 77 | 77 | |
| 78 | 78 | test("a newly shipped pull makes the next answer a 200 with a new tag", async () => { |
| 79 | 79 | const before = (await get()).headers.get("etag") ?? ""; |
| 80 | keepLive("alice/forge", ["oid-1", "oid-2"], 0); | |
| 80 | keepLive(app.db, "alice/forge", ["oid-1", "oid-2"], 0); | |
| 81 | 81 | const after = await get("", { "if-none-match": before }); |
| 82 | 82 | expect(before).toMatch(/^W\//); |
| 83 | 83 | expect(after.status).toBe(200); |
test/changelog-timing.test.ts +2 −2
| @@ -45,11 +45,11 @@ beforeAll(async () => { | ||
| 45 | 45 | oids.push(`oid-${n}`); |
| 46 | 46 | } |
| 47 | 47 | })(); |
| 48 | keepLive("alice/forge", oids, 0); | |
| 48 | keepLive(app.db, "alice/forge", oids, 0); | |
| 49 | 49 | }, 60_000); |
| 50 | 50 | |
| 51 | 51 | afterAll(async () => { |
| 52 | forgetLive(); | |
| 52 | forgetLive(app.db); | |
| 53 | 53 | if (saved === undefined) delete process.env["FORGE_COMMIT"]; |
| 54 | 54 | else process.env["FORGE_COMMIT"] = saved; |
| 55 | 55 | servers.stop(); |
test/changelog.test.ts +11 −11
| @@ -77,7 +77,7 @@ describe("changelog page", () => { | ||
| 77 | 77 | else process.env[k] = v; |
| 78 | 78 | } |
| 79 | 79 | delete process.env["FORGE_VERSION"]; |
| 80 | forgetLive(); | |
| 80 | forgetLive(app.db); | |
| 81 | 81 | servers.stop(); |
| 82 | 82 | app.close(); |
| 83 | 83 | await rm(dataDir, { recursive: true, force: true }); |
| @@ -140,7 +140,7 @@ describe("changelog page", () => { | ||
| 140 | 140 | |
| 141 | 141 | test("with no FORGE_REPO the repository holding the running commit is the forge's own", async () => { |
| 142 | 142 | delete app.config.forgeRepo; |
| 143 | forgetLive(); | |
| 143 | forgetLive(app.db); | |
| 144 | 144 | try { |
| 145 | 145 | const body = await (await fetch(`${servers.url}/changelog`)).text(); |
| 146 | 146 | expect(body.indexOf("On its way")).toBeLessThan(body.indexOf("Third thing")); |
| @@ -156,13 +156,13 @@ describe("changelog page", () => { | ||
| 156 | 156 | } finally { |
| 157 | 157 | process.env["FORGE_COMMIT"] = oids[1]!; |
| 158 | 158 | app.config.forgeRepo = "alice/forge"; |
| 159 | forgetLive(); | |
| 159 | forgetLive(app.db); | |
| 160 | 160 | } |
| 161 | 161 | }); |
| 162 | 162 | |
| 163 | 163 | test("a private repository holding the running commit is never the forge's own", async () => { |
| 164 | 164 | delete app.config.forgeRepo; |
| 165 | forgetLive(); | |
| 165 | forgetLive(app.db); | |
| 166 | 166 | app.db.run("UPDATE repo SET visibility = 'private' WHERE name = 'forge'"); |
| 167 | 167 | process.env["FORGE_COMMIT"] = oids[0]!; |
| 168 | 168 | try { |
| @@ -172,7 +172,7 @@ describe("changelog page", () => { | ||
| 172 | 172 | app.db.run("UPDATE repo SET visibility = 'public' WHERE name = 'forge'"); |
| 173 | 173 | process.env["FORGE_COMMIT"] = oids[1]!; |
| 174 | 174 | app.config.forgeRepo = "alice/forge"; |
| 175 | forgetLive(); | |
| 175 | forgetLive(app.db); | |
| 176 | 176 | } |
| 177 | 177 | }); |
| 178 | 178 | |
| @@ -182,7 +182,7 @@ describe("changelog page", () => { | ||
| 182 | 182 | const day = new Date().toISOString().slice(0, 10); |
| 183 | 183 | merge("fork", "Fork change", `${day}T23:00:00.000Z`, c.commitOid); |
| 184 | 184 | delete app.config.forgeRepo; |
| 185 | forgetLive(); | |
| 185 | forgetLive(app.db); | |
| 186 | 186 | process.env["FORGE_COMMIT"] = c.commitOid; |
| 187 | 187 | try { |
| 188 | 188 | const body = await (await fetch(`${servers.url}/changelog`)).text(); |
| @@ -191,7 +191,7 @@ describe("changelog page", () => { | ||
| 191 | 191 | } finally { |
| 192 | 192 | process.env["FORGE_COMMIT"] = oids[1]!; |
| 193 | 193 | app.config.forgeRepo = "alice/forge"; |
| 194 | forgetLive(); | |
| 194 | forgetLive(app.db); | |
| 195 | 195 | } |
| 196 | 196 | }); |
| 197 | 197 | |
| @@ -202,7 +202,7 @@ describe("changelog page", () => { | ||
| 202 | 202 | }); |
| 203 | 203 | expect(fetched.ok).toBe(true); |
| 204 | 204 | delete app.config.forgeRepo; |
| 205 | forgetLive(); | |
| 205 | forgetLive(app.db); | |
| 206 | 206 | try { |
| 207 | 207 | const body = await (await fetch(`${servers.url}/changelog`)).text(); |
| 208 | 208 | expect(body.indexOf("On its way")).toBeLessThan(body.indexOf("Third thing")); |
| @@ -210,13 +210,13 @@ describe("changelog page", () => { | ||
| 210 | 210 | } finally { |
| 211 | 211 | app.config.forgeRepo = "alice/forge"; |
| 212 | 212 | app.db.run("DELETE FROM issue WHERE title = 'Fork change'"); |
| 213 | forgetLive(); | |
| 213 | forgetLive(app.db); | |
| 214 | 214 | } |
| 215 | 215 | }); |
| 216 | 216 | |
| 217 | 217 | test("a miss with no merged pulls yet is not remembered", async () => { |
| 218 | 218 | delete app.config.forgeRepo; |
| 219 | forgetLive(); | |
| 219 | forgetLive(app.db); | |
| 220 | 220 | app.db.run("UPDATE repo SET visibility = 'private'"); |
| 221 | 221 | try { |
| 222 | 222 | expect(await (await fetch(`${servers.url}/changelog`)).text()).toContain("Nothing has shipped yet"); |
| @@ -226,7 +226,7 @@ describe("changelog page", () => { | ||
| 226 | 226 | } finally { |
| 227 | 227 | app.db.run("UPDATE repo SET visibility = 'public'"); |
| 228 | 228 | app.config.forgeRepo = "alice/forge"; |
| 229 | forgetLive(); | |
| 229 | forgetLive(app.db); | |
| 230 | 230 | } |
| 231 | 231 | }); |
| 232 | 232 |
test/coauthors.test.ts +2 −2
| @@ -192,7 +192,7 @@ describe("a squash landing", () => { | ||
| 192 | 192 | const dir0 = repoDir(app.reposRoot, repo.ownerName, repo.name); |
| 193 | 193 | const was = process.env["FORGE_COMMIT"]; |
| 194 | 194 | process.env["FORGE_COMMIT"] = (await runGitCapture(["rev-parse", "main"], { cwd: dir0 })).stdout.trim(); |
| 195 | forgetLive(); | |
| 195 | forgetLive(app.db); | |
| 196 | 196 | const read = await readLandings(app); |
| 197 | 197 | const byTitle = (t: string) => [...read.live, ...read.onItsWay].find((l) => l.title === t)!; |
| 198 | 198 | expect(byTitle("Do credited").with).toEqual(["Jay Moran"]); |
| @@ -214,7 +214,7 @@ describe("a squash landing", () => { | ||
| 214 | 214 | server.stop(); |
| 215 | 215 | if (was === undefined) delete process.env["FORGE_COMMIT"]; |
| 216 | 216 | else process.env["FORGE_COMMIT"] = was; |
| 217 | forgetLive(); | |
| 217 | forgetLive(app.db); | |
| 218 | 218 | } |
| 219 | 219 | }); |
| 220 | 220 | }); |
test/flow-alarm-lines.test.ts +2 −2
| @@ -29,8 +29,8 @@ async function seed(mergedMinAgo?: number) { | ||
| 29 | 29 | const repo = await createRepo(app.db, app.reposRoot, { ownerId: u.value.id, name: "r", visibility: "public", description: "r" }); |
| 30 | 30 | if (!repo.ok) throw new Error(repo.error); |
| 31 | 31 | // The running commit predates every merge here, so a merge is on its way. |
| 32 | if (mergedMinAgo === undefined) forgetLive(); | |
| 33 | else keepLive("alice/r", [], 0); | |
| 32 | if (mergedMinAgo === undefined) forgetLive(app.db); | |
| 33 | else keepLive(app.db, "alice/r", [], 0); | |
| 34 | 34 | if (mergedMinAgo !== undefined) { |
| 35 | 35 | const at = new Date(Date.now() - mergedMinAgo * 60_000).toISOString(); |
| 36 | 36 | const issueId = id(); |
test/flow-alarm-main-ahead.test.ts +5 −5
| @@ -66,7 +66,7 @@ afterAll(() => { | ||
| 66 | 66 | for (const [k, v] of [["FORGE_SENTINEL_REPO", saved.sentinel], ["FORGE_COMMIT", saved.commit], ["FORGE_VERSION", saved.version]] as const) |
| 67 | 67 | if (v === undefined) delete process.env[k]; |
| 68 | 68 | else process.env[k] = v; |
| 69 | forgetLive(); | |
| 69 | forgetLive(app.db); | |
| 70 | 70 | }); |
| 71 | 71 | |
| 72 | 72 | function check(state: "ok" | "failed", oid: string): void { |
| @@ -83,7 +83,7 @@ function merge(minAgo: number, oid: string): void { | ||
| 83 | 83 | const reset = () => { |
| 84 | 84 | app.db.query(`DELETE FROM run`).run(); |
| 85 | 85 | app.db.query(`DELETE FROM pull`).run(); |
| 86 | keepLive("trevato/forge", [], NOW.getTime() - 600 * 60_000); | |
| 86 | keepLive(app.db, "trevato/forge", [], NOW.getTime() - 600 * 60_000); | |
| 87 | 87 | }; |
| 88 | 88 | |
| 89 | 89 | describe("main ahead alarm", () => { |
| @@ -140,7 +140,7 @@ describe("main ahead alarm", () => { | ||
| 140 | 140 | const open = () => app.db.query<{ title: string }, []>(`SELECT title FROM issue WHERE state = 'open' AND title LIKE 'Sentinel: Main is green and ahead%'`).all(); |
| 141 | 141 | expect(open()).toHaveLength(1); |
| 142 | 142 | // Production catches up: it runs the head and the pull is in the running commit. |
| 143 | keepLive("trevato/forge", ["2".repeat(40)], NOW.getTime()); | |
| 143 | keepLive(app.db, "trevato/forge", ["2".repeat(40)], NOW.getTime()); | |
| 144 | 144 | hunt(app, "trevato/forge", NOW.getTime() + 120_000); |
| 145 | 145 | expect(open()).toHaveLength(0); |
| 146 | 146 | }); |
| @@ -196,7 +196,7 @@ describe("the main-ahead issue", () => { | ||
| 196 | 196 | const repo = getRepo(app.db, "trevato", "forge")!; |
| 197 | 197 | const said = addComment(app, owner, repo, getIssueById(app.db, filed[0]!.id)!, "Looking at the tick now."); |
| 198 | 198 | expect(said.ok).toBe(true); |
| 199 | keepLive("trevato/forge", ["4".repeat(40)], NOW.getTime()); | |
| 199 | keepLive(app.db, "trevato/forge", ["4".repeat(40)], NOW.getTime()); | |
| 200 | 200 | hunt(app, "trevato/forge", NOW.getTime() + 720_000); |
| 201 | 201 | expect(open()).toHaveLength(1); |
| 202 | 202 | }); |
| @@ -362,7 +362,7 @@ describe("a tick never heard from (#11924)", () => { | ||
| 362 | 362 | try { |
| 363 | 363 | hunt(app, "trevato/forge", NOW.getTime() + 1_200_000); |
| 364 | 364 | expect(open()).toHaveLength(1); |
| 365 | keepLive("trevato/forge", ["8".repeat(40)], NOW.getTime()); | |
| 365 | keepLive(app.db, "trevato/forge", ["8".repeat(40)], NOW.getTime()); | |
| 366 | 366 | hunt(app, "trevato/forge", NOW.getTime() + 1_300_000); |
| 367 | 367 | expect(open()).toHaveLength(0); |
| 368 | 368 | } finally { |
test/flow-alarm-no-release-1111.test.ts +3 −4
| @@ -16,7 +16,7 @@ import { createUser } from "../src/core/users.ts"; | ||
| 16 | 16 | import { setNotifyUrl, startNotifier } from "../src/crew/notify.ts"; |
| 17 | 17 | import { firedAlarms } from "../src/flow/alarms.ts"; |
| 18 | 18 | import { id } from "../src/kernel/id.ts"; |
| 19 | import { forgetLive, keepLive } from "../src/release/landings.ts"; | |
| 19 | import { keepLive } from "../src/release/landings.ts"; | |
| 20 | 20 | |
| 21 | 21 | const T0 = Date.parse("2026-11-11T12:00:00.000Z"); |
| 22 | 22 | const at = (min: number) => new Date(T0 + min * 60_000).toISOString(); |
| @@ -33,7 +33,6 @@ afterAll(() => { | ||
| 33 | 33 | for (const [k, v] of Object.entries(env)) if (v === undefined) delete process.env[k]; |
| 34 | 34 | else process.env[k] = v; |
| 35 | 35 | for (const d of dirs) rmSync(d, { recursive: true, force: true }); |
| 36 | forgetLive(); | |
| 37 | 36 | }); |
| 38 | 37 | |
| 39 | 38 | describe("no release on 2026-11-11", () => { |
| @@ -55,7 +54,7 @@ describe("no release on 2026-11-11", () => { | ||
| 55 | 54 | }; |
| 56 | 55 | // The running commit holds only what merged before it. |
| 57 | 56 | merge(own.value, -30, "a".repeat(40)); |
| 58 | keepLive("alice/forge", ["a".repeat(40)], T0); | |
| 57 | keepLive(app.db, "alice/forge", ["a".repeat(40)], T0); | |
| 59 | 58 | merge(own.value, 10, "b".repeat(40)); |
| 60 | 59 | merge(own.value, 40, "c".repeat(40)); |
| 61 | 60 | merge(own.value, 70, "d".repeat(40)); |
| @@ -99,7 +98,7 @@ describe("no release on 2026-11-11", () => { | ||
| 99 | 98 | expect(posts[0]).toContain("production runs 0.7.535 · 5e1d0c0"); |
| 100 | 99 | |
| 101 | 100 | // A release whose commit contains the merges clears it, and nothing more is said. |
| 102 | keepLive("alice/forge", ["a", "b", "c", "d"].map((c) => c.repeat(40)), T0 + 360 * 60_000); | |
| 101 | keepLive(app.db, "alice/forge", ["a", "b", "c", "d"].map((c) => c.repeat(40)), T0 + 360 * 60_000); | |
| 103 | 102 | expect(releaseAlarms(370)).toHaveLength(0); |
| 104 | 103 | await wakeAt(370); |
| 105 | 104 | expect(posts).toHaveLength(1); |
test/flow-alarm-no-release.test.ts +2 −2
| @@ -23,7 +23,7 @@ async function seed() { | ||
| 23 | 23 | return { |
| 24 | 24 | app, |
| 25 | 25 | /** The running commit was made at `at`: it contains every merge before it. */ |
| 26 | release: (at: string) => keepLive("alice/r", [], Date.parse(at)), | |
| 26 | release: (at: string) => keepLive(app.db, "alice/r", [], Date.parse(at)), | |
| 27 | 27 | merge: (at: string) => { |
| 28 | 28 | const issueId = id(); |
| 29 | 29 | n++; |
| @@ -38,7 +38,7 @@ process.env["FORGE_COMMIT"] = "abcdef1234567890"; | ||
| 38 | 38 | describe("no release alarm", () => { |
| 39 | 39 | test("says nothing until the running commit has been read", async () => { |
| 40 | 40 | const s = await seed(); |
| 41 | forgetLive(); | |
| 41 | forgetLive(s.app.db); | |
| 42 | 42 | s.merge(minAgo(500)); |
| 43 | 43 | expect(noReleaseAlarm(s.app.db, NOW).fired).toBe(false); |
| 44 | 44 | }); |
test/forge-owns-its-state.test.ts added +56
| @@ -0,0 +1,56 @@ | ||
| 1 | import { afterEach, expect, test } from "bun:test"; | |
| 2 | import { mkdtempSync, rmSync } from "node:fs"; | |
| 3 | import { tmpdir } from "node:os"; | |
| 4 | import { join } from "node:path"; | |
| 5 | import { createApp } from "../src/app.ts"; | |
| 6 | import { turnOf, turnStatsOf } from "../src/http/app_turn.ts"; | |
| 7 | import { noteProtection, protectionNotice } from "../src/kernel/protection.ts"; | |
| 8 | import { liveOf } from "../src/perf/live.ts"; | |
| 9 | import { keepLive, mainGap } from "../src/release/landings.ts"; | |
| 10 | ||
| 11 | // Two forges in one process (#12569): what one is fed is not on the other. Each assertion | |
| 12 | // fails if the store it reads goes back to one per process. | |
| 13 | const dirs: string[] = []; | |
| 14 | afterEach(() => { | |
| 15 | for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }); | |
| 16 | }); | |
| 17 | const forge = () => { | |
| 18 | const dir = mkdtempSync(join(tmpdir(), "forge-own-state-")); | |
| 19 | dirs.push(dir); | |
| 20 | return createApp({ dataDir: dir, port: 0 }); | |
| 21 | }; | |
| 22 | ||
| 23 | test("traffic, a stall, a protection note and the kept live set of one forge are not visible from another", () => { | |
| 24 | const a = forge(); | |
| 25 | const b = forge(); | |
| 26 | ||
| 27 | // traffic | |
| 28 | liveOf(a).perfRecord.add("/x", { at: 1000, ms: 5, signedIn: false, phases: [] }); | |
| 29 | expect(liveOf(a).perfRecord.traffic("/x", 0).length).toBe(1); | |
| 30 | expect(liveOf(b).perfRecord.traffic("/x", 0)).toEqual([]); | |
| 31 | expect(liveOf(a)).toBe(liveOf(a)); | |
| 32 | expect(liveOf(b)).not.toBe(liveOf(a)); | |
| 33 | ||
| 34 | // a stall, and the lag minute it is read in | |
| 35 | const stall = liveOf(a).loopWatch.stalled(1500, 5000); | |
| 36 | expect(stall).not.toBeNull(); | |
| 37 | liveOf(a).stallBlame.note(stall!); | |
| 38 | expect(liveOf(a).stallBlame.size()).toBe(1); | |
| 39 | expect(liveOf(b).stallBlame.size()).toBe(0); | |
| 40 | expect(liveOf(b).stallBlame.worst(5000, 60_000, 5)).toEqual([]); | |
| 41 | ||
| 42 | // a protection note | |
| 43 | noteProtection(a, "apps", { ok: false, reason: "bwrap is not installed" }); | |
| 44 | expect(protectionNotice(a)).toContain("bwrap"); | |
| 45 | expect(protectionNotice(b)).toBeUndefined(); | |
| 46 | ||
| 47 | // the turn gate and its count | |
| 48 | turnStatsOf(a).waited = 3; | |
| 49 | expect(turnStatsOf(b).waited).toBe(0); | |
| 50 | expect(turnOf(b)).not.toBe(turnOf(a)); | |
| 51 | ||
| 52 | // the kept live set, by database | |
| 53 | keepLive(a.db, "o/r", [], 1, { commit: "c1", head: "h1", ahead: 2 }); | |
| 54 | expect(mainGap(a.db)?.main.sha).toBe("h1"); | |
| 55 | expect(mainGap(b.db)).toBeUndefined(); | |
| 56 | }); |
test/helpers/stall-state.ts +6 −10
| @@ -1,24 +1,20 @@ | ||
| 1 | 1 | /** |
| 2 | * The process's stall watch, starting clean and left clean (#11069). The | |
| 3 | * suite runs every file in one process, and `loopWatch`, `stallBlame` and the | |
| 4 | * telemetry reader counts (`perf/live.ts`, `system/telemetry_reads.ts`) are | |
| 5 | * that process's; a stall one file noted would otherwise be a row in the next | |
| 6 | * file's /system. Call at a file's top level, as `quietLogs` is | |
| 2 | * The telemetry reader counts, starting clean and left clean (#11069). The | |
| 3 | * suite runs every file in one process, and the counts | |
| 4 | * (`system/telemetry_reads.ts`) are that process's. The loop watch and the stall | |
| 5 | * blame are each forge's (`liveOf`) and need no clearing. Call at a file's top level, as `quietLogs` is | |
| 7 | 6 | * (`globals.ts`): it clears before the file's first test and after its last. |
| 8 | 7 | */ |
| 9 | 8 | |
| 10 | 9 | import { afterAll, beforeAll } from "bun:test"; |
| 11 | import { loopWatch, stallBlame } from "../../src/perf/live.ts"; | |
| 12 | 10 | import { resetTelemetryReads } from "../../src/system/telemetry_reads.ts"; |
| 13 | 11 | |
| 14 | /** Forget every stall and telemetry read the process has noted. */ | |
| 12 | /** Forget every telemetry read the process has noted. */ | |
| 15 | 13 | export function clearStallState(): void { |
| 16 | loopWatch.clear(); | |
| 17 | stallBlame.clear(); | |
| 18 | 14 | resetTelemetryReads(); |
| 19 | 15 | } |
| 20 | 16 | |
| 21 | /** Clear the process's stall watch before this scope runs and after it. */ | |
| 17 | /** Clear the telemetry reads before this scope runs and after it. */ | |
| 22 | 18 | export function freshStallState(): void { |
| 23 | 19 | beforeAll(clearStallState); |
| 24 | 20 | afterAll(clearStallState); |
test/home-truth.test.ts +2 −2
| @@ -117,7 +117,7 @@ test("homeTruth names the running commit apart from main's head, and how far beh | ||
| 117 | 117 | process.env.FORGE_VERSION = "1.2.3"; |
| 118 | 118 | |
| 119 | 119 | process.env.FORGE_COMMIT = a; |
| 120 | forgetLive(); | |
| 120 | forgetLive(app.db); | |
| 121 | 121 | await readLandings(app); |
| 122 | 122 | const behind = homeTruth(app, owner, now); |
| 123 | 123 | expect(behind.live).toEqual({ sha: a, version: "1.2.3" }); |
| @@ -137,6 +137,6 @@ test("homeTruth names the running commit apart from main's head, and how far beh | ||
| 137 | 137 | } |
| 138 | 138 | if (saved.repo === undefined) delete (app.config as { forgeRepo?: string }).forgeRepo; |
| 139 | 139 | else (app.config as { forgeRepo?: string }).forgeRepo = saved.repo; |
| 140 | forgetLive(); | |
| 140 | forgetLive(app.db); | |
| 141 | 141 | } |
| 142 | 142 | }); |
test/landings-asked-once.test.ts +1 −2
| @@ -7,7 +7,7 @@ import { createApp } from "../src/app.ts"; | ||
| 7 | 7 | import { createUser } from "../src/core/users.ts"; |
| 8 | 8 | import { repoDir } from "../src/git/repos.ts"; |
| 9 | 9 | import { id } from "../src/kernel/id.ts"; |
| 10 | import { forgetLive, readLandings } from "../src/release/landings.ts"; | |
| 10 | import { readLandings } from "../src/release/landings.ts"; | |
| 11 | 11 | |
| 12 | 12 | const dirs: string[] = []; |
| 13 | 13 | const saved = { path: process.env.PATH, commit: process.env.FORGE_COMMIT }; |
| @@ -15,7 +15,6 @@ afterEach(() => { | ||
| 15 | 15 | process.env.PATH = saved.path; |
| 16 | 16 | if (saved.commit === undefined) delete process.env.FORGE_COMMIT; |
| 17 | 17 | else process.env.FORGE_COMMIT = saved.commit; |
| 18 | forgetLive(); | |
| 19 | 18 | for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }); |
| 20 | 19 | }); |
| 21 | 20 |
test/loop-stall-by-route.test.ts +10 −10
| @@ -11,7 +11,7 @@ import { serve, type Servers } from "../src/http/server.ts"; | ||
| 11 | 11 | import { watchStalls } from "../src/kernel/loop_lag.ts"; |
| 12 | 12 | import { openMetrics } from "../src/metrics/openmetrics.ts"; |
| 13 | 13 | import { stallFamilies, TOP_ROUTES } from "../src/metrics/stalls.ts"; |
| 14 | import { lagMinutes, loopWatch, stallBlame } from "../src/perf/live.ts"; | |
| 14 | import { liveOf } from "../src/perf/live.ts"; | |
| 15 | 15 | import { WORST_STALLS } from "../src/perf/stall_blame.ts"; |
| 16 | 16 | import { createLoopWatch, HOUR_MS, NO_REQUEST } from "../src/perf/loop.ts"; |
| 17 | 17 | import { createAgent } from "../src/actions/crew.ts"; |
| @@ -144,8 +144,8 @@ test("GET /metrics names the route a stall ran across", async () => { | ||
| 144 | 144 | // millisecond was only across a 400 ms stall, not its cause (`WITNESS_SHARE`), so the one |
| 145 | 145 | // that held the loop is noted as the server would have noted it: 300 ms of the window. |
| 146 | 146 | const at = performance.now(); |
| 147 | loopWatch.finished("/api/v1/health", at - 300, at); | |
| 148 | expect(loopWatch.stalled(400, at)).not.toBeNull(); | |
| 147 | liveOf(app).loopWatch.finished("/api/v1/health", at - 300, at); | |
| 148 | expect(liveOf(app).loopWatch.stalled(400, at)).not.toBeNull(); | |
| 149 | 149 | const text = await (await fetch(`${servers.url}/metrics`, { headers: { authorization: `Bearer ${ro}` } })).text(); |
| 150 | 150 | expect(text).toMatch(/^forge_loop_stalls_total\{route="\/api\/v1\/health"\} [1-9]/m); |
| 151 | 151 | expect(text).toMatch(/^forge_loop_stall_hour_milliseconds\{route="\/api\/v1\/health"\} [1-9]/m); |
| @@ -229,12 +229,12 @@ test("the Stalls screen lists the hour's worst single stalls as /system does: wh | ||
| 229 | 229 | test("the Stalls screen reads the process's stall blame by default, the five /system reads", () => { |
| 230 | 230 | const now = performance.now(); |
| 231 | 231 | for (const [ms, r] of [[300, "/a"], [800, "/b"], [250, "/c"], [500, "/d"], [700, "/e"], [210, "/f"]] as const) |
| 232 | stallBlame.note({ at: now - 1_000, lateMs: ms, routes: [r], jobs: [] }); | |
| 232 | liveOf(app).stallBlame.note({ at: now - 1_000, lateMs: ms, routes: [r], jobs: [] }); | |
| 233 | 233 | const lines = stallLines(new Session(app, identity(alice)), []); |
| 234 | 234 | const named = lines.slice(lines.indexOf("") + 2).map((l) => l.split(/\s+/)[0] ?? ""); |
| 235 | 235 | expect(named).toEqual(["/b", "/e", "/d", "/a", "/c"]); |
| 236 | expect(stallBlame.worst(performance.now(), HOUR_MS, WORST_STALLS).map((s) => s.what)).toEqual(named); | |
| 237 | stallBlame.clear(); | |
| 236 | expect(liveOf(app).stallBlame.worst(performance.now(), HOUR_MS, WORST_STALLS).map((s) => s.what)).toEqual(named); | |
| 237 | liveOf(app).stallBlame.clear(); | |
| 238 | 238 | }); |
| 239 | 239 | |
| 240 | 240 | test("the Stalls screen shows the last 20 minutes' loop lag as /system does: over count, each minute, what ran (#11220)", async () => { |
| @@ -258,9 +258,9 @@ test("the Stalls screen shows the last 20 minutes' loop lag as /system does: ove | ||
| 258 | 258 | for (const who of [bob, bot, undefined]) expect(lagLines(new Session(app, identity(who)), rows)).toEqual([]); |
| 259 | 259 | |
| 260 | 260 | // The screen itself draws the process's readings above the stalls. |
| 261 | lagMinutes.clear(); | |
| 262 | lagMinutes.note(250, 0); | |
| 263 | lagMinutes.note(0, 60_000); // closes the first minute | |
| 261 | liveOf(app).lagMinutes.clear(); | |
| 262 | liveOf(app).lagMinutes.note(250, 0); | |
| 263 | liveOf(app).lagMinutes.note(0, 60_000); // closes the first minute | |
| 264 | 264 | const { app: tui, session } = createForgeApp(app, identity(alice), { terminal: headlessTerminal(120, 40), exitHooks: false }); |
| 265 | 265 | tui.start(); |
| 266 | 266 | await settleApp(tui); |
| @@ -268,7 +268,7 @@ test("the Stalls screen shows the last 20 minutes' loop lag as /system does: ove | ||
| 268 | 268 | await settleApp(tui); |
| 269 | 269 | const shown = tui.screen.toText(); |
| 270 | 270 | tui.stop(); |
| 271 | lagMinutes.clear(); | |
| 271 | liveOf(app).lagMinutes.clear(); | |
| 272 | 272 | expect(shown).toContain("Loop lag, last 20 minutes: 1 of 1 over 100 ms"); |
| 273 | 273 | expect(shown).toContain("Route (last hour)"); |
| 274 | 274 | expect(shown.indexOf("Loop lag")).toBeLessThan(shown.indexOf("Route (last hour)")); |
test/mcp-system-routes.test.ts +4 −4
| @@ -9,7 +9,7 @@ import { setTelemetryGrant } from "../src/actions/telemetry.ts"; | ||
| 9 | 9 | import { createToken } from "../src/core/tokens.ts"; |
| 10 | 10 | import { createUser, getUserByName } from "../src/core/users.ts"; |
| 11 | 11 | import { serve, type Servers } from "../src/http/server.ts"; |
| 12 | import { perfRecord } from "../src/perf/live.ts"; | |
| 12 | import { liveOf } from "../src/perf/live.ts"; | |
| 13 | 13 | import { quietLogs } from "./helpers/globals.ts"; |
| 14 | 14 | |
| 15 | 15 | quietLogs(); |
| @@ -48,11 +48,11 @@ const call = async (token: string) => { | ||
| 48 | 48 | |
| 49 | 49 | test("routes lists the over-budget route first, with its budget and slowest phase; a non-admin is refused", async () => { |
| 50 | 50 | // perfRecord is the process's own, and a shard-mate's requests leave samples (and patterns, up to the cap) in it. |
| 51 | perfRecord.clear(); | |
| 51 | liveOf(app).perfRecord.clear(); | |
| 52 | 52 | const at = Date.now(); |
| 53 | 53 | for (let i = 0; i < 25; i++) { |
| 54 | perfRecord.add("/boards", { at, ms: 100, signedIn: true, phases: [{ name: "app", ms: 100, hit: false }] }); | |
| 55 | perfRecord.add("/explore", { at, ms: 2000, signedIn: true, phases: [{ name: "app", ms: 2000, hit: false }, { name: "read", ms: 1500, hit: false }] }); | |
| 54 | liveOf(app).perfRecord.add("/boards", { at, ms: 100, signedIn: true, phases: [{ name: "app", ms: 100, hit: false }] }); | |
| 55 | liveOf(app).perfRecord.add("/explore", { at, ms: 2000, signedIn: true, phases: [{ name: "app", ms: 2000, hit: false }, { name: "read", ms: 1500, hit: false }] }); | |
| 56 | 56 | } |
| 57 | 57 | const out = JSON.parse((await call(admin)).result.content[0].text); |
| 58 | 58 | const mine = out.routes.filter((r: any) => r.route === "/boards" || r.route === "/explore"); |
test/mcp-system-stalls.test.ts +3 −3
| @@ -8,7 +8,7 @@ import { createToken } from "../src/core/tokens.ts"; | ||
| 8 | 8 | import { createUser } from "../src/core/users.ts"; |
| 9 | 9 | import { serve, type Servers } from "../src/http/server.ts"; |
| 10 | 10 | import { during } from "../src/perf/jobs.ts"; |
| 11 | import { loopWatch } from "../src/perf/live.ts"; | |
| 11 | import { liveOf } from "../src/perf/live.ts"; | |
| 12 | 12 | import { LIMIT_MS } from "../src/perf/loop.ts"; |
| 13 | 13 | import { quietLogs } from "./helpers/globals.ts"; |
| 14 | 14 | |
| @@ -37,14 +37,14 @@ afterAll(async () => { | ||
| 37 | 37 | test("a stall under a named job is read through system, at most 10, most ms first", async () => { |
| 38 | 38 | // The watch is the process's one: stalls and requests an earlier file left |
| 39 | 39 | // would crowd the top 10 or be blamed ahead of the job. Start from nothing. |
| 40 | loopWatch.clear(); | |
| 40 | liveOf(app).loopWatch.clear(); | |
| 41 | 41 | // The job holds the loop for a real share of the stall: one that only brushed it is not |
| 42 | 42 | // named (`perf/loop.ts`). |
| 43 | 43 | during("test-job-10907", () => { |
| 44 | 44 | const until = performance.now() + 40; |
| 45 | 45 | while (performance.now() < until); |
| 46 | 46 | }); |
| 47 | loopWatch.stalled(LIMIT_MS + 10, performance.now()); | |
| 47 | liveOf(app).loopWatch.stalled(LIMIT_MS + 10, performance.now()); | |
| 48 | 48 | const res = await fetch(`${servers.url}/mcp`, { |
| 49 | 49 | method: "POST", |
| 50 | 50 | headers: { "content-type": "application/json", accept: "application/json, text/event-stream", authorization: `Bearer ${token}` }, |
test/module-state.baseline.txt added +219
| @@ -0,0 +1,219 @@ | ||
| 1 | # State that predates the rule (#12569), to be moved under an app, never added to. | |
| 2 | src/actions/detach.ts:jobs | |
| 3 | src/actions/features.ts:changed | |
| 4 | src/actions/features.ts:declared | |
| 5 | src/actions/forks.ts:standing | |
| 6 | src/actions/pin_demos.ts:factsKept | |
| 7 | src/actions/spawn.ts:inFlight | |
| 8 | src/actions/upstream_follow.ts:checking | |
| 9 | src/apps/api_spec.ts:asking | |
| 10 | src/apps/api_spec.ts:kept | |
| 11 | src/apps/check-memory.ts:declaredAt | |
| 12 | src/apps/check-memory.ts:held | |
| 13 | src/apps/contain.ts:diskOpen | |
| 14 | src/apps/contain.ts:treeOpen | |
| 15 | src/apps/contain.ts:watched | |
| 16 | src/apps/cpu_duty.ts:stopped | |
| 17 | src/apps/cron.ts:formatters | |
| 18 | src/apps/egress.ts:meters | |
| 19 | src/apps/egress.ts:moved | |
| 20 | src/apps/egress.ts:npmByRepo | |
| 21 | src/apps/egress.ts:tallies | |
| 22 | src/apps/egress_refused.ts:books | |
| 23 | src/apps/egress_refused.ts:cache | |
| 24 | src/apps/feature_clock.ts:labels | |
| 25 | src/apps/feature_schedule.ts:inFlight | |
| 26 | src/apps/feedback.ts:proofs | |
| 27 | src/apps/handoff.ts:ports | |
| 28 | src/apps/pull_schedule.ts:declaredByTip | |
| 29 | src/apps/pull_schedule.ts:inFlight | |
| 30 | src/apps/relay_wait.ts:slowest | |
| 31 | src/apps/timer_fire.ts:inFlight | |
| 32 | src/billing/sync.ts:chains | |
| 33 | src/checks/impact.ts:paths | |
| 34 | src/checks/install.ts:local | |
| 35 | src/checks/install_evict.ts:lastSweep | |
| 36 | src/checks/install_evict.ts:sweeping | |
| 37 | src/checks/queue.ts:lines | |
| 38 | src/checks/queue.ts:watchers | |
| 39 | src/checks/revert_red.ts:working | |
| 40 | src/cloud/declared.ts:cache | |
| 41 | src/contract/audit.ts:remembered | |
| 42 | src/contract/served.ts:built | |
| 43 | src/core/chapters.ts:frozen | |
| 44 | src/core/contribute_token.ts:turns | |
| 45 | src/core/credit.ts:authorCache | |
| 46 | src/core/following.ts:relationCache | |
| 47 | src/core/idempotency.ts:inFlight | |
| 48 | src/core/parent_releases.ts:refreshing | |
| 49 | src/core/pulls.ts:inflight | |
| 50 | src/core/pulls.ts:lastByPull | |
| 51 | src/core/pulls.ts:readings | |
| 52 | src/core/readme_description.ts:lines | |
| 53 | src/core/recombine.ts:running | |
| 54 | src/core/recombine.ts:started | |
| 55 | src/core/search.ts:grepCache | |
| 56 | src/crew/approved_week.ts:judged | |
| 57 | src/crew/autonomy.ts:asked | |
| 58 | src/crew/autonomy.ts:checked | |
| 59 | src/crew/autonomy.ts:cleanAt | |
| 60 | src/crew/autonomy.ts:conflicted | |
| 61 | src/crew/autonomy.ts:conflictedAt | |
| 62 | src/crew/autonomy.ts:sweeping | |
| 63 | src/crew/ceiling_stagger.ts:memo | |
| 64 | src/crew/conversation.ts:live | |
| 65 | src/crew/conversation.ts:queues | |
| 66 | src/crew/conversation.ts:stoppers | |
| 67 | src/crew/dispatch.ts:deferredFailures | |
| 68 | src/crew/dispatch.ts:retriedChecks | |
| 69 | src/crew/dispatch.ts:waitingOnMain | |
| 70 | src/crew/finishing.ts:entered | |
| 71 | src/crew/finishing.ts:read | |
| 72 | src/crew/finishing.ts:said | |
| 73 | src/crew/governed.ts:seen | |
| 74 | src/crew/handover.ts:handing | |
| 75 | src/crew/hot_files.ts:standing | |
| 76 | src/crew/hot_files.ts:standingWideBy | |
| 77 | src/crew/hot_files.ts:touched | |
| 78 | src/crew/hot_history.ts:samples | |
| 79 | src/crew/mirror.ts:chains | |
| 80 | src/crew/notify.ts:lastTest | |
| 81 | src/crew/plan.ts:found | |
| 82 | src/crew/repeat_checks.ts:swept | |
| 83 | src/crew/requested.ts:said | |
| 84 | src/crew/review.ts:said | |
| 85 | src/crew/runner.ts:steering | |
| 86 | src/crew/running.ts:running | |
| 87 | src/crew/waits.ts:last | |
| 88 | src/crew/week_read.ts:remembered | |
| 89 | src/crew/workdir.ts:claimed | |
| 90 | src/design/mark.ts:cache | |
| 91 | src/devices/print.ts:inFlight | |
| 92 | src/display/config.ts:known | |
| 93 | src/efficiency/installs.ts:seen | |
| 94 | src/flow/alarm_drift.ts:cache | |
| 95 | src/flow/keep.ts:keepers | |
| 96 | src/flow/lanes.ts:open | |
| 97 | src/flow/lanes.ts:viewers | |
| 98 | src/flow/lanes_worker.ts:handles | |
| 99 | src/flow/lanes_worker.ts:kept | |
| 100 | src/flow/pages.ts:logged | |
| 101 | src/flow/pages.ts:open | |
| 102 | src/flow/pages.ts:workers | |
| 103 | src/flow/pages_read.ts:said | |
| 104 | src/flow/pages_worker.ts:envs | |
| 105 | src/flow/pipeline.ts:waitingOpen | |
| 106 | src/flow/pipeline_worker.ts:handles | |
| 107 | src/flows/probe.ts:running | |
| 108 | src/flows/start.ts:running | |
| 109 | src/git/archive.ts:held | |
| 110 | src/git/coauthors.ts:credited | |
| 111 | src/git/diff-memory.ts:kept | |
| 112 | src/git/diff-memory.ts:mergeBases | |
| 113 | src/git/diff.ts:running | |
| 114 | src/git/env.ts:bundles | |
| 115 | src/git/own_refs.ts:reached | |
| 116 | src/git/path_listing.ts:kept | |
| 117 | src/git/pins_kept.ts:epochs | |
| 118 | src/git/pins_kept.ts:kept | |
| 119 | src/git/read.ts:pool | |
| 120 | src/git/sync_link.ts:queues | |
| 121 | src/git/tip.ts:packedOf | |
| 122 | src/http/access_log.ts:pending | |
| 123 | src/http/api/forge_view.ts:recent | |
| 124 | src/http/archive_limit.ts:seen | |
| 125 | src/http/compress.ts:kept | |
| 126 | src/http/flow_stream.ts:pollers | |
| 127 | src/http/rp.ts:registering | |
| 128 | src/http/sse.ts:openStreams | |
| 129 | src/kernel/full_disk.ts:owned | |
| 130 | src/kernel/full_disk.ts:states | |
| 131 | src/kernel/log.ts:forgeLedgers | |
| 132 | src/kernel/log.ts:held | |
| 133 | src/kernel/passport.ts:subkeys | |
| 134 | src/kernel/probe.ts:probes | |
| 135 | src/kernel/secret.ts:keyCache | |
| 136 | src/kernel/secret.ts:previousCache | |
| 137 | src/looks/capture.ts:browsers | |
| 138 | src/looks/thumb.ts:open | |
| 139 | src/machines/rc.ts:inflight | |
| 140 | src/origins/credential.ts:refreshing | |
| 141 | src/origins/jobs.ts:jobs | |
| 142 | src/origins/jobs.ts:listeners | |
| 143 | src/origins/mirror-pulls.ts:running | |
| 144 | src/origins/providers/index.ts:cache | |
| 145 | src/origins/sync.ts:inFlight | |
| 146 | src/origins/token-reach.ts:kept | |
| 147 | src/origins/work-branch.ts:tipsSeen | |
| 148 | src/perf/child_time.ts:minutes | |
| 149 | src/release/changelog.ts:cache | |
| 150 | src/release/prove.ts:superseded | |
| 151 | src/runs/activity.ts:lastAt | |
| 152 | src/runs/lease.ts:held | |
| 153 | src/runs/log.ts:listeners | |
| 154 | src/runs/log.ts:pending | |
| 155 | src/runs/log.ts:taps | |
| 156 | src/runs/log_append.ts:made | |
| 157 | src/runs/stoppable.ts:running | |
| 158 | src/snapshot/bucket.ts:reads | |
| 159 | src/snapshot/bucket.ts:runCounts | |
| 160 | src/snapshot/bucket.ts:watching | |
| 161 | src/spawn/app_links.ts:notOffered | |
| 162 | src/spawn/app_links.ts:offerFailed | |
| 163 | src/spawn/app_links.ts:said | |
| 164 | src/spawn/archive.ts:busy | |
| 165 | src/spawn/child_db.ts:busy | |
| 166 | src/spawn/child_db.ts:hands | |
| 167 | src/spawn/child_db.ts:line | |
| 168 | src/spawn/funding.ts:askedAt | |
| 169 | src/spawn/heartbeat.ts:turns | |
| 170 | src/spawn/host.ts:waiters | |
| 171 | src/spawn/own_code.ts:inUse | |
| 172 | src/spawn/own_code.ts:queues | |
| 173 | src/spawn/sealed.ts:read | |
| 174 | src/spawn/usage.ts:disks | |
| 175 | src/spawn/usage.ts:walking | |
| 176 | src/system/telemetry_reads.ts:counts | |
| 177 | src/system/telemetry_reads.ts:slots | |
| 178 | src/tui/screens/mirror.ts:shown | |
| 179 | src/tui/screens/mirror.ts:watched | |
| 180 | src/tui/term/screen.ts:GLYPHS | |
| 181 | src/tui/term/screen.ts:GLYPH_IDS | |
| 182 | src/tui/term/style.ts:quantCache | |
| 183 | src/tui/term/width.ts:widthCache | |
| 184 | src/tui/ui/theme.ts:cache | |
| 185 | src/web/asset.ts:kept | |
| 186 | src/web/asset.ts:pageScripts | |
| 187 | src/web/asset.ts:pageScriptsByName | |
| 188 | src/web/asset.ts:pruned | |
| 189 | src/web/code.ts:cache | |
| 190 | src/web/code_rows_pool.ts:pool | |
| 191 | src/web/code_rows_pool.ts:waiting | |
| 192 | src/web/diff-cache.ts:kept | |
| 193 | src/web/diff-cache.ts:rendered | |
| 194 | src/web/feeds.ts:waveCache | |
| 195 | src/web/head-reads.ts:kept | |
| 196 | src/web/llms_tools.ts:sections | |
| 197 | src/web/map_snapshot.ts:open | |
| 198 | src/web/map_snapshot.ts:said | |
| 199 | src/web/map_worker.ts:apps | |
| 200 | src/web/markdown.ts:memo | |
| 201 | src/web/og/raster.ts:open | |
| 202 | src/web/pages/branches.ts:kept | |
| 203 | src/web/pages/changelog.ts:rendered | |
| 204 | src/web/pages/docs_link.ts:rendered | |
| 205 | src/web/pages/pull.ts:fastByHead | |
| 206 | src/web/pages/repo.ts:readmeCache | |
| 207 | src/web/pages/talk.ts:open | |
| 208 | src/web/render/gcode.ts:kept | |
| 209 | src/web/render/model3d-client.ts:live | |
| 210 | src/web/share_target.ts:SENT | |
| 211 | src/web/standards.ts:discoveryBodies | |
| 212 | src/web/standards.ts:llmsBodies | |
| 213 | src/web/standards.ts:releasesBodies | |
| 214 | src/web/standards.ts:sitemapBodies | |
| 215 | src/web/studio/home_last.ts:drawing | |
| 216 | src/web/studio/toolbar.ts:stats | |
| 217 | src/apps/egress_refused.ts:said | |
| 218 | src/core/readme_description.ts:tips | |
| 219 | src/release/leftover_sweep.ts:removing |
test/module-state.test.ts added +54
| @@ -0,0 +1,54 @@ | ||
| 1 | /** | |
| 2 | * State belongs to a forge (an app object), not to the process (#12569): two | |
| 3 | * forges in one `bun test` process, or one test after another, must not see | |
| 4 | * each other's requests. A module under src/ may not hold a module-level | |
| 5 | * Map, Set or array that code inside a function writes to; key the state by | |
| 6 | * the app in a WeakMap (`askingOf(app)`, `liveOf(app)`) instead. | |
| 7 | * | |
| 8 | * `test/module-state.baseline.txt` lists what predates the rule, one | |
| 9 | * `file:name` per line. It only shrinks: a new entry is refused, and an entry | |
| 10 | * whose state has moved or gone must be deleted from the list. | |
| 11 | * | |
| 12 | * @tree-scan: reads every file under src/. | |
| 13 | */ | |
| 14 | ||
| 15 | import { describe, expect, test } from "bun:test"; | |
| 16 | import { readFileSync } from "node:fs"; | |
| 17 | import { Glob } from "bun"; | |
| 18 | ||
| 19 | const DECL = /^(?:export )?(?:const|let) ([A-Za-z_$][\w$]*)(?:: [^=\n]+)? = (?:new (?:Map|Set)\b|\[\])/gm; | |
| 20 | ||
| 21 | /** `file:name` of each module-level collection that indented code writes to. */ | |
| 22 | export function sharedState(): string[] { | |
| 23 | const found: string[] = []; | |
| 24 | for (const file of new Glob("src/**/*.ts").scanSync(".")) { | |
| 25 | const text = readFileSync(file, "utf8"); | |
| 26 | for (const m of text.matchAll(DECL)) { | |
| 27 | const name = m[1]!; | |
| 28 | const writes = new RegExp( | |
| 29 | `^\\s+.*\\b${name}\\.(set|add|push|unshift|splice|delete|pop|shift)\\(|^\\s+.*\\b${name}\\[[^\\]]+\\] =`, | |
| 30 | "m", | |
| 31 | ); | |
| 32 | if (writes.test(text)) found.push(`${file}:${name}`); | |
| 33 | } | |
| 34 | } | |
| 35 | return found.sort(); | |
| 36 | } | |
| 37 | ||
| 38 | const baseline = readFileSync(new URL("./module-state.baseline.txt", import.meta.url), "utf8") | |
| 39 | .split("\n") | |
| 40 | .filter((l) => l !== "" && !l.startsWith("#")); | |
| 41 | ||
| 42 | describe("no process-wide state that requests write to", () => { | |
| 43 | const now = sharedState(); | |
| 44 | test("a module adds no module-level Map, Set or array that a function writes to", () => { | |
| 45 | expect(now.filter((s) => !baseline.includes(s))).toEqual([]); | |
| 46 | }); | |
| 47 | test("the baseline names only state that still exists", () => { | |
| 48 | expect(baseline.filter((s) => !now.includes(s))).toEqual([]); | |
| 49 | }); | |
| 50 | test("the forge-scoped state of #12569 is not on the baseline", () => { | |
| 51 | for (const gone of ["src/http/app_turn.ts", "src/perf/live.ts", "src/kernel/protection.ts"]) | |
| 52 | expect(baseline.filter((s) => s.startsWith(gone))).toEqual([]); | |
| 53 | }); | |
| 54 | }); |
test/pages-budget.test.ts +2 −2
| @@ -36,7 +36,7 @@ import { keepTick } from "../src/flow/keep.ts"; | ||
| 36 | 36 | import { readLanesInline, useLanesReader } from "../src/flow/lanes.ts"; |
| 37 | 37 | import { usePagesReader } from "../src/flow/pages.ts"; |
| 38 | 38 | import { pagesConfig, readPagePicture, type Picture } from "../src/flow/pages_read.ts"; |
| 39 | import { perfRecord } from "../src/perf/live.ts"; | |
| 39 | import { liveOf } from "../src/perf/live.ts"; | |
| 40 | 40 | import { buildApiRouter, buildRouter, serve, type Servers } from "../src/http/server.ts"; |
| 41 | 41 | import { quietLogs } from "./helpers/globals.ts"; |
| 42 | 42 | import { promote } from "./helpers/members.ts"; |
| @@ -172,7 +172,7 @@ beforeAll(async () => { | ||
| 172 | 172 | }, 60_000); |
| 173 | 173 | afterAll(() => { |
| 174 | 174 | // The walk fills the shared recorder to its pattern cap; leave it empty for the next file in this process. |
| 175 | perfRecord.clear(); | |
| 175 | liveOf(app).perfRecord.clear(); | |
| 176 | 176 | servers.stop(); |
| 177 | 177 | app.close(); |
| 178 | 178 | rmSync(root, { recursive: true, force: true }); |
test/perf-lag-minutes.test.ts +6 −6
| @@ -13,7 +13,7 @@ import { createToken } from "../src/core/tokens.ts"; | ||
| 13 | 13 | import { createUser } from "../src/core/users.ts"; |
| 14 | 14 | import { serve, type Servers } from "../src/http/server.ts"; |
| 15 | 15 | import { createLagMinutes, LAG_BAR_MS, LAG_MINUTE_MS, LAG_READINGS, overBar } from "../src/perf/lag_minutes.ts"; |
| 16 | import { lagMinutes } from "../src/perf/live.ts"; | |
| 16 | import { liveOf } from "../src/perf/live.ts"; | |
| 17 | 17 | import { lagSection } from "../src/web/pages/system.ts"; |
| 18 | 18 | import { quietLogs } from "./helpers/globals.ts"; |
| 19 | 19 | |
| @@ -122,18 +122,18 @@ describe("the MCP system tool", () => { | ||
| 122 | 122 | }, 30_000); |
| 123 | 123 | |
| 124 | 124 | afterAll(async () => { |
| 125 | lagMinutes.clear(); | |
| 125 | liveOf(app).lagMinutes.clear(); | |
| 126 | 126 | servers.stop(); |
| 127 | 127 | app.close(); |
| 128 | 128 | await rm(dataDir, { recursive: true, force: true }); |
| 129 | 129 | }); |
| 130 | 130 | |
| 131 | 131 | test("answers the same readings as {at, ms, what}, with the bar and the count over it", async () => { |
| 132 | lagMinutes.clear(); | |
| 132 | liveOf(app).lagMinutes.clear(); | |
| 133 | 133 | const base = 1_000 * MIN; |
| 134 | lagMinutes.note(20, base); | |
| 135 | lagMinutes.note(180.04, base + MIN); | |
| 136 | lagMinutes.note(1, base + 2 * MIN); | |
| 134 | liveOf(app).lagMinutes.note(20, base); | |
| 135 | liveOf(app).lagMinutes.note(180.04, base + MIN); | |
| 136 | liveOf(app).lagMinutes.note(1, base + 2 * MIN); | |
| 137 | 137 | const res = await fetch(`${servers.url}/mcp`, { |
| 138 | 138 | method: "POST", |
| 139 | 139 | headers: { "content-type": "application/json", accept: "application/json, text/event-stream", authorization: `Bearer ${token}` }, |
test/perf-loop-jobs.test.ts +3 −6
| @@ -2,12 +2,8 @@ | ||
| 2 | 2 | |
| 3 | 3 | import { expect, test } from "bun:test"; |
| 4 | 4 | import { createJobWatch, during } from "../src/perf/jobs.ts"; |
| 5 | import { loopWatch } from "../src/perf/live.ts"; | |
| 5 | import { liveOf } from "../src/perf/live.ts"; | |
| 6 | 6 | import { createLoopWatch, NO_REQUEST, stallLine } from "../src/perf/loop.ts"; |
| 7 | import { freshStallState } from "./helpers/stall-state.ts"; | |
| 8 | ||
| 9 | // The wired test notes a stall on the process's watch, which every file in this process shares. | |
| 10 | freshStallState(); | |
| 11 | 7 | |
| 12 | 8 | const block = (ms: number): void => { |
| 13 | 9 | const until = performance.now() + ms; |
| @@ -66,7 +62,8 @@ test("a loop stall while a job awaits and another blocks is tallied to the block | ||
| 66 | 62 | await lane; |
| 67 | 63 | }); |
| 68 | 64 | |
| 69 | test("the process's loop watch reads the process's job watch", () => { | |
| 65 | test("a forge's loop watch reads the process's job watch", () => { | |
| 66 | const { loopWatch } = liveOf({}); | |
| 70 | 67 | const before = loopWatch.tally(performance.now()).total.find((r) => r.route === "job:wired-test")?.count ?? 0; |
| 71 | 68 | during("wired-test", () => block(300)); |
| 72 | 69 | const at = performance.now(); |
test/perf-server-hook.test.ts +3 −3
| @@ -5,7 +5,7 @@ import { join } from "node:path"; | ||
| 5 | 5 | import { createApp, type App } from "../src/app.ts"; |
| 6 | 6 | import { serve, type Servers } from "../src/http/server.ts"; |
| 7 | 7 | import { watchStalls } from "../src/kernel/loop_lag.ts"; |
| 8 | import { perfRecord } from "../src/perf/live.ts"; | |
| 8 | import { liveOf } from "../src/perf/live.ts"; | |
| 9 | 9 | import { quietLogs } from "./helpers/globals.ts"; |
| 10 | 10 | |
| 11 | 11 | quietLogs(); |
| @@ -25,10 +25,10 @@ afterAll(() => { | ||
| 25 | 25 | }); |
| 26 | 26 | |
| 27 | 27 | test("a served request lands in the record under its route pattern; a client label hides nothing", async () => { |
| 28 | const before = perfRecord.traffic("/api/v1/health", 0).length; | |
| 28 | const before = liveOf(app).perfRecord.traffic("/api/v1/health", 0).length; | |
| 29 | 29 | await fetch(`${servers.url}/api/v1/health`); |
| 30 | 30 | await fetch(`${servers.url}/api/v1/health`, { headers: { "x-forge-synthetic": "1" } }); |
| 31 | const after = perfRecord.traffic("/api/v1/health", 0); | |
| 31 | const after = liveOf(app).perfRecord.traffic("/api/v1/health", 0); | |
| 32 | 32 | expect(after.length).toBe(before + 2); |
| 33 | 33 | expect(after.at(-1)!.signedIn).toBe(false); |
| 34 | 34 | expect(after.at(-1)!.phases.some((p) => p.name === "app")).toBe(true); |
test/perf-stall-blame.test.ts +6 −6
| @@ -13,7 +13,7 @@ import { createApp, type App } from "../src/app.ts"; | ||
| 13 | 13 | import { serve, type Servers } from "../src/http/server.ts"; |
| 14 | 14 | import { startLoopLag } from "../src/kernel/loop_lag.ts"; |
| 15 | 15 | import { createJobWatch } from "../src/perf/jobs.ts"; |
| 16 | import { stallBlame } from "../src/perf/live.ts"; | |
| 16 | import { liveOf } from "../src/perf/live.ts"; | |
| 17 | 17 | import { createLoopWatch, HOUR_MS, LIMIT_MS, type Stall } from "../src/perf/loop.ts"; |
| 18 | 18 | import { createStallBlame, STALL_KEPT, UNKNOWN, whatOf } from "../src/perf/stall_blame.ts"; |
| 19 | 19 | import { stallSection, worstStallsTable } from "../src/web/pages/system.ts"; |
| @@ -139,14 +139,14 @@ describe("on /healthz", () => { | ||
| 139 | 139 | }); |
| 140 | 140 | |
| 141 | 141 | afterAll(async () => { |
| 142 | stallBlame.clear(); | |
| 142 | liveOf(app).stallBlame.clear(); | |
| 143 | 143 | servers.stop(); |
| 144 | 144 | app.close(); |
| 145 | 145 | await rm(dataDir, { recursive: true, force: true }); |
| 146 | 146 | }); |
| 147 | 147 | |
| 148 | 148 | test("loopWorstWhat is null with no stall, then names the last 10 s's worst", async () => { |
| 149 | stallBlame.clear(); | |
| 149 | liveOf(app).stallBlame.clear(); | |
| 150 | 150 | const ask = async () => |
| 151 | 151 | (await (await fetch(`${servers.url}/healthz`, { headers: { accept: "application/json" } })).json()) as { |
| 152 | 152 | ok: boolean; |
| @@ -159,9 +159,9 @@ describe("on /healthz", () => { | ||
| 159 | 159 | expect(before.loopWorstWhat).toBeNull(); |
| 160 | 160 | |
| 161 | 161 | const now = performance.now(); |
| 162 | stallBlame.note({ at: now - 60_000, lateMs: 900, routes: ["/too-old"], jobs: [] }); | |
| 163 | stallBlame.note({ at: now, lateMs: 300, routes: [], jobs: ["crew-tick"] }); | |
| 164 | stallBlame.note({ at: now, lateMs: 220, routes: ["/explore"], jobs: [] }); | |
| 162 | liveOf(app).stallBlame.note({ at: now - 60_000, lateMs: 900, routes: ["/too-old"], jobs: [] }); | |
| 163 | liveOf(app).stallBlame.note({ at: now, lateMs: 300, routes: [], jobs: ["crew-tick"] }); | |
| 164 | liveOf(app).stallBlame.note({ at: now, lateMs: 220, routes: ["/explore"], jobs: [] }); | |
| 165 | 165 | expect((await ask()).loopWorstWhat).toBe("job:crew-tick"); |
| 166 | 166 | // A probe asking for nothing still hears the plain `ok` (#2917). |
| 167 | 167 | expect(await (await fetch(`${servers.url}/healthz`)).text()).toBe("ok"); |
test/protection.test.ts +14 −14
| @@ -1,33 +1,33 @@ | ||
| 1 | 1 | import { mkdtemp, rm } from "node:fs/promises"; |
| 2 | 2 | import { tmpdir } from "node:os"; |
| 3 | 3 | import { join } from "node:path"; |
| 4 | import { afterEach, expect, test } from "bun:test"; | |
| 4 | import { expect, test } from "bun:test"; | |
| 5 | 5 | import { |
| 6 | 6 | noteProtection, |
| 7 | 7 | protectionNotice, |
| 8 | resetProtection, | |
| 9 | 8 | } from "../src/kernel/protection.ts"; |
| 10 | 9 | |
| 11 | afterEach(resetProtection); | |
| 12 | ||
| 13 | 10 | test("a fully protected forge has no notice", () => { |
| 14 | noteProtection("apps", { ok: true }); | |
| 15 | expect(protectionNotice()).toBeUndefined(); | |
| 11 | const app = {}; | |
| 12 | noteProtection(app, "apps", { ok: true }); | |
| 13 | expect(protectionNotice(app)).toBeUndefined(); | |
| 16 | 14 | }); |
| 17 | 15 | |
| 18 | 16 | test("a part that lacks protection is named with its reason, in one line", () => { |
| 19 | noteProtection("apps", { ok: false, reason: "bwrap is not installed" }); | |
| 20 | noteProtection("crew shells", { ok: false, reason: "no sandbox on win32" }); | |
| 21 | const line = protectionNotice()!; | |
| 17 | const app = {}; | |
| 18 | noteProtection(app, "apps", { ok: false, reason: "bwrap is not installed" }); | |
| 19 | noteProtection(app, "crew shells", { ok: false, reason: "no sandbox on win32" }); | |
| 20 | const line = protectionNotice(app)!; | |
| 22 | 21 | expect(line).toContain("apps (bwrap is not installed)"); |
| 23 | 22 | expect(line).toContain("crew shells (no sandbox on win32)"); |
| 24 | 23 | expect(line).not.toContain("\n"); |
| 25 | 24 | }); |
| 26 | 25 | |
| 27 | 26 | test("a part that recovers clears its own line", () => { |
| 28 | noteProtection("apps", { ok: false, reason: "x" }); | |
| 29 | noteProtection("apps", { ok: true }); | |
| 30 | expect(protectionNotice()).toBeUndefined(); | |
| 27 | const app = {}; | |
| 28 | noteProtection(app, "apps", { ok: false, reason: "x" }); | |
| 29 | noteProtection(app, "apps", { ok: true }); | |
| 30 | expect(protectionNotice(app)).toBeUndefined(); | |
| 31 | 31 | }); |
| 32 | 32 | |
| 33 | 33 | // Run in a process of its own: the probes are cached per process, and the |
| @@ -48,13 +48,13 @@ test("AppsRuntime.start and startCrew each note what they lack", async () => { | ||
| 48 | 48 | forgeUrl: "http://127.0.0.1:1", |
| 49 | 49 | }); |
| 50 | 50 | await runtime.start(); |
| 51 | const afterApps = protectionNotice(); | |
| 51 | const afterApps = protectionNotice(app); | |
| 52 | 52 | const crew = await startCrew(app, { |
| 53 | 53 | forgeUrl: "http://127.0.0.1:1", localUrl: "http://127.0.0.1:1", |
| 54 | 54 | binary: "/no/such/claude", concurrency: 1, budgetUsd: 1, |
| 55 | 55 | runTimeoutMs: 60000, tickMs: 50, |
| 56 | 56 | }); |
| 57 | console.log(JSON.stringify({ afterApps, afterCrew: protectionNotice() })); | |
| 57 | console.log(JSON.stringify({ afterApps, afterCrew: protectionNotice(app) })); | |
| 58 | 58 | await crew.stop(); |
| 59 | 59 | await runtime.stop(); |
| 60 | 60 | process.exit(0); |
test/red-main.test.ts +2 −1
| @@ -315,7 +315,8 @@ describe("a red main in another repository (#3992)", () => { | ||
| 315 | 315 | span("red:r2"); |
| 316 | 316 | span("red:r1"); |
| 317 | 317 | expect(budgetBreaches(db, 2000, "r1").map((f) => f.key)).toEqual(["degraded:red-r1"]); |
| 318 | expect(findFaults(db, 2000, "r1").map((f) => f.key)).toEqual(["degraded:red-r1"]); | |
| 318 | // A ledger of its own: the process's holds what earlier files logged (#12569). | |
| 319 | expect(findFaults(db, 2000, "r1", undefined, null, new ErrorLedger()).map((f) => f.key)).toEqual(["degraded:red-r1"]); | |
| 319 | 320 | expect(budgetBreaches(db, 2000, "r2").map((f) => f.key)).toEqual(["degraded:red-r2"]); |
| 320 | 321 | }); |
| 321 | 322 |
test/relay-stall-name.test.ts +3 −3
| @@ -30,7 +30,7 @@ import type { Repo } from "../src/core/repos.ts"; | ||
| 30 | 30 | import { createUser, type User } from "../src/core/users.ts"; |
| 31 | 31 | import { serve, type Servers } from "../src/http/server.ts"; |
| 32 | 32 | import { createJobWatch, jobWatch } from "../src/perf/jobs.ts"; |
| 33 | import { loopWatch } from "../src/perf/live.ts"; | |
| 33 | import { liveOf } from "../src/perf/live.ts"; | |
| 34 | 34 | import { createLoopWatch, type Stall } from "../src/perf/loop.ts"; |
| 35 | 35 | import { loopWorst } from "../src/perf/loop_worst.ts"; |
| 36 | 36 | import { deploySettled } from "./helpers/apps-forge.ts"; |
| @@ -176,8 +176,8 @@ describe("forge-stall on the relayed answer", () => { | ||
| 176 | 176 | loopWorst.note(ms, now); |
| 177 | 177 | // A job that ran for a moment was only across the stall (`WITNESS_SHARE`, `perf/loop.ts`): |
| 178 | 178 | // what is named is what covered it, so a request that did is noted as the server notes one. |
| 179 | loopWatch.finished("/api/v1/health", now - ms, now); | |
| 180 | const s = loopWatch.stalled(ms, now)!; | |
| 179 | liveOf(app).loopWatch.finished("/api/v1/health", now - ms, now); | |
| 180 | const s = liveOf(app).loopWatch.stalled(ms, now)!; | |
| 181 | 181 | // A route that ran across it outranks the job (setup's requests often do); |
| 182 | 182 | // the job's own naming is pinned above, with a stepped clock. |
| 183 | 183 | return heldBy([s])[0]!.name; |
test/spawn-in-process-visible.test.ts +7 −7
| @@ -20,7 +20,7 @@ import { | ||
| 20 | 20 | noteChildRequest, |
| 21 | 21 | } from "../src/perf/child_time.ts"; |
| 22 | 22 | import { during, jobWatch } from "../src/perf/jobs.ts"; |
| 23 | import { loopWatch } from "../src/perf/live.ts"; | |
| 23 | import { liveOf } from "../src/perf/live.ts"; | |
| 24 | 24 | import { stallLine } from "../src/perf/loop.ts"; |
| 25 | 25 | import { clientRows, servedWindow } from "../src/spawn/clients.ts"; |
| 26 | 26 | import { closeHostedChildren, openChildren, provideCellsForTest } from "../src/spawn/host.ts"; |
| @@ -226,12 +226,12 @@ describe("a parent holding a forge in its own process", () => { | ||
| 226 | 226 | }); |
| 227 | 227 | |
| 228 | 228 | test("a request it answers is the stall watcher's child:<name>, and is counted against it", async () => { |
| 229 | loopWatch.clear(); | |
| 229 | liveOf(parent).loopWatch.clear(); | |
| 230 | 230 | // Open it, so the next request is answered by a forge already in this process. |
| 231 | 231 | await (await fetch(`${servers.url}/healthz`, { headers: { host: "homie--forge.test" } })).text(); |
| 232 | 232 | const child = openChildren(parent).find((c) => c.name === "homie")!.app; |
| 233 | 233 | const had = childHours().find((r) => r.name === "homie")?.requests ?? 0; |
| 234 | loopWatch.clear(); | |
| 234 | liveOf(parent).loopWatch.clear(); | |
| 235 | 235 | // The child's own request holds the parent's thread: its first statement takes 80 ms of |
| 236 | 236 | // synchronous work, as a slow query or a git call of the child's would. A stall is named |
| 237 | 237 | // for what held the loop, not for what was only across it (`WITNESS_SHARE`, `perf/loop.ts`). |
| @@ -244,22 +244,22 @@ describe("a parent holding a forge in its own process", () => { | ||
| 244 | 244 | expect(release()).toBe(true); |
| 245 | 245 | // The stall that request made names the child: before, it named the parent's route of the |
| 246 | 246 | // same shape. |
| 247 | const stall = loopWatch.stalled(250, performance.now()); | |
| 247 | const stall = liveOf(parent).loopWatch.stalled(250, performance.now()); | |
| 248 | 248 | expect(stall?.routes).toContain("child:homie"); |
| 249 | 249 | expect(stallLine(stall!)).toContain("running then: child:homie"); |
| 250 | 250 | expect(childHours().find((r) => r.name === "homie")?.requests).toBe(had + 1); |
| 251 | 251 | // The parent's own request is still its route's. |
| 252 | loopWatch.clear(); | |
| 252 | liveOf(parent).loopWatch.clear(); | |
| 253 | 253 | const mine = holdOnce(parent, 80); |
| 254 | 254 | await ( |
| 255 | 255 | await fetch(`${servers.url}/healthz`, { headers: { host: "forge.test" } }) |
| 256 | 256 | ).text(); |
| 257 | 257 | expect(mine()).toBe(true); |
| 258 | const own = loopWatch.stalled(250, performance.now()); | |
| 258 | const own = liveOf(parent).loopWatch.stalled(250, performance.now()); | |
| 259 | 259 | expect(own?.routes).toEqual(["/healthz"]); |
| 260 | 260 | expect(own?.routes.some((r) => r.startsWith("child:"))).toBe(false); |
| 261 | 261 | // The watch is the process's: the two stalls this test made up are not left in its hour. |
| 262 | loopWatch.clear(); | |
| 262 | liveOf(parent).loopWatch.clear(); | |
| 263 | 263 | }); |
| 264 | 264 | |
| 265 | 265 | test("/forges says so on its row, in plain words", async () => { |
test/stall-in-flight.test.ts +4 −4
| @@ -10,7 +10,7 @@ import { createUser } from "../src/core/users.ts"; | ||
| 10 | 10 | import { serve, type Servers } from "../src/http/server.ts"; |
| 11 | 11 | import { stallFamilies } from "../src/metrics/stalls.ts"; |
| 12 | 12 | import { createInFlight } from "../src/perf/in_flight.ts"; |
| 13 | import { inFlight, loopWatch } from "../src/perf/live.ts"; | |
| 13 | import { inFlight, liveOf } from "../src/perf/live.ts"; | |
| 14 | 14 | import { blamed, createLoopWatch, LIMIT_MS, NO_REQUEST, stallLine } from "../src/perf/loop.ts"; |
| 15 | 15 | import { whatOf } from "../src/perf/stall_blame.ts"; |
| 16 | 16 | import { quietLogs } from "./helpers/globals.ts"; |
| @@ -142,7 +142,7 @@ function describe_server() { | ||
| 142 | 142 | }; |
| 143 | 143 | |
| 144 | 144 | test("a real request held open across a stall is in MCP `system` `stalls`; one begun after is not", async () => { |
| 145 | loopWatch.clear(); | |
| 145 | liveOf(app).loopWatch.clear(); | |
| 146 | 146 | // The suite shares one process: a request another file left open (or its own server's |
| 147 | 147 | // background traffic) is in the open set too, and comes and goes on its own. Watch only |
| 148 | 148 | // this test's `/mcp`, never the size of the whole set. |
| @@ -171,11 +171,11 @@ function describe_server() { | ||
| 171 | 171 | await Bun.sleep(LIMIT_MS + 60); // real-timer: the request must be older than the stall limit on the real clock |
| 172 | 172 | // The wake: the loop held for longer than the stall limit, beginning after the request did. |
| 173 | 173 | const now = performance.now(); |
| 174 | const stall = loopWatch.stalled(LIMIT_MS + 10, now)!; | |
| 174 | const stall = liveOf(app).loopWatch.stalled(LIMIT_MS + 10, now)!; | |
| 175 | 175 | const named = stall.inFlight ?? []; |
| 176 | 176 | expect(named).toContain("/mcp"); |
| 177 | 177 | // A stall shorter than the request's age is charged; one reaching back before it began is not. |
| 178 | expect(loopWatch.stalled(60_000, now)!.inFlight ?? []).toEqual(inFlight.since(now - 60_000)); | |
| 178 | expect(liveOf(app).loopWatch.stalled(60_000, now)!.inFlight ?? []).toEqual(inFlight.since(now - 60_000)); | |
| 179 | 179 | push!(new TextEncoder().encode('"id":9,"method":"tools/list"}')); |
| 180 | 180 | close!(); |
| 181 | 181 | await (await held).text(); |
test/stall-relay.test.ts added +27
| @@ -0,0 +1,27 @@ | ||
| 1 | import { expect, test } from "bun:test"; | |
| 2 | import { createStallRelay, EARLY_KEPT } from "../src/kernel/stall_relay.ts"; | |
| 3 | ||
| 4 | test("a stall that arrives before the forge exists is read in order once it does", () => { | |
| 5 | const seen: [string, number, number][] = []; | |
| 6 | const relay = createStallRelay<string>((t, late, at) => seen.push([t, late, at])); | |
| 7 | relay.note(1500, 10); | |
| 8 | relay.note(2500, 20); | |
| 9 | expect(seen).toEqual([]); | |
| 10 | expect(relay.waiting()).toBe(2); | |
| 11 | relay.attach("forge"); | |
| 12 | expect(seen).toEqual([["forge", 1500, 10], ["forge", 2500, 20]]); | |
| 13 | expect(relay.waiting()).toBe(0); | |
| 14 | relay.note(3000, 30); | |
| 15 | expect(seen.at(-1)).toEqual(["forge", 3000, 30]); | |
| 16 | }); | |
| 17 | ||
| 18 | test("the stalls kept for a forge that is not yet made are bounded, oldest dropped", () => { | |
| 19 | const seen: number[] = []; | |
| 20 | const relay = createStallRelay<null>((_t, late) => seen.push(late)); | |
| 21 | for (let i = 0; i < EARLY_KEPT + 10; i++) relay.note(i, i); | |
| 22 | expect(relay.waiting()).toBe(EARLY_KEPT); | |
| 23 | relay.attach(null); | |
| 24 | expect(seen.length).toBe(EARLY_KEPT); | |
| 25 | expect(seen[0]).toBe(10); | |
| 26 | expect(seen.at(-1)).toBe(EARLY_KEPT + 9); | |
| 27 | }); |
test/traffic-history.test.ts +2 −2
| @@ -25,7 +25,7 @@ import { | ||
| 25 | 25 | signatureOf, |
| 26 | 26 | type TrafficClass, |
| 27 | 27 | } from "../src/perf/traffic.ts"; |
| 28 | import { perfRecord } from "../src/perf/live.ts"; | |
| 28 | import { liveOf } from "../src/perf/live.ts"; | |
| 29 | 29 | import { createTrafficWriter, HOUR_TOPS, trafficHistory, windowOf } from "../src/perf/traffic_store.ts"; |
| 30 | 30 | |
| 31 | 31 | const quiet = { warn: () => {} }; |
| @@ -171,7 +171,7 @@ test("what a request is sorted by: internal, country and signature", () => { | ||
| 171 | 171 | test("the page offers the three windows, for an admin alone, and a loopback request is the forge's own", async () => { |
| 172 | 172 | const cookie = `forge_session=${createSession(app.db, admin.id).token}`; |
| 173 | 173 | await fetch(`${servers.url}/`, { redirect: "manual" }); |
| 174 | const own = perfRecord.ledger.totals(Date.now()).classes.internal.requests; | |
| 174 | const own = liveOf(app).perfRecord.ledger.totals(Date.now()).classes.internal.requests; | |
| 175 | 175 | expect(own).toBeGreaterThan(0); |
| 176 | 176 | const page = await fetch(`${servers.url}/settings/traffic?window=24h`, { headers: { cookie } }); |
| 177 | 177 | expect(page.status).toBe(200); |
test/tui-screens.test.ts +3 −3
| @@ -33,7 +33,7 @@ import { getPull } from "../src/core/pulls.ts"; | ||
| 33 | 33 | import { getRepo, type Repo } from "../src/core/repos.ts"; |
| 34 | 34 | import { setSignup } from "../src/settings/signup.ts"; |
| 35 | 35 | import { getUserSetting } from "../src/core/settings.ts"; |
| 36 | import { noteProtection, resetProtection } from "../src/kernel/protection.ts"; | |
| 36 | import { noteProtection } from "../src/kernel/protection.ts"; | |
| 37 | 37 | import { getUserByName, type User } from "../src/core/users.ts"; |
| 38 | 38 | import { quietLogs } from "./helpers/globals.ts"; |
| 39 | 39 | import { parseGitCommand, splitCommand } from "../src/git/smart-ssh.ts"; |
| @@ -232,7 +232,7 @@ describe("the terminal UI, against a real forge", () => { | ||
| 232 | 232 | }); |
| 233 | 233 | |
| 234 | 234 | test("the dashboard tells a site admin, and only them, that protection is reduced", async () => { |
| 235 | noteProtection("apps", { ok: false, reason: "bwrap is not installed" }); | |
| 235 | noteProtection(forge, "apps", { ok: false, reason: "bwrap is not installed" }); | |
| 236 | 236 | try { |
| 237 | 237 | const admin = boot(forge, asAlice()); |
| 238 | 238 | await admin.settle(); |
| @@ -245,7 +245,7 @@ describe("the terminal UI, against a real forge", () => { | ||
| 245 | 245 | h.stop(); |
| 246 | 246 | } |
| 247 | 247 | } finally { |
| 248 | resetProtection(); | |
| 248 | noteProtection(forge, "apps", { ok: true }); | |
| 249 | 249 | } |
| 250 | 250 | const h = boot(forge, asAlice()); |
| 251 | 251 | await h.settle(); |
test/version.test.ts +5 −5
| @@ -18,7 +18,7 @@ import { createApp, type App } from "../src/app.ts"; | ||
| 18 | 18 | import { createToken } from "../src/core/tokens.ts"; |
| 19 | 19 | import { createUser, type User } from "../src/core/users.ts"; |
| 20 | 20 | import { serve, type Servers } from "../src/http/server.ts"; |
| 21 | import { noteProtection, resetProtection } from "../src/kernel/protection.ts"; | |
| 21 | import { noteProtection } from "../src/kernel/protection.ts"; | |
| 22 | 22 | import { running, runningLine, UNSTAMPED } from "../src/kernel/version.ts"; |
| 23 | 23 | |
| 24 | 24 | const ROOT = resolve(import.meta.dir, ".."); |
| @@ -296,7 +296,7 @@ describe("where the stamp is read", () => { | ||
| 296 | 296 | }); |
| 297 | 297 | |
| 298 | 298 | test("a forge with less than default protection says so to its admin alone, escaped", async () => { |
| 299 | noteProtection("apps", { ok: false, reason: "<b>no bwrap</b>" }); | |
| 299 | noteProtection(app, "apps", { ok: false, reason: "<b>no bwrap</b>" }); | |
| 300 | 300 | try { |
| 301 | 301 | const get = async (cookie?: string) => |
| 302 | 302 | await ( |
| @@ -311,13 +311,13 @@ describe("where the stamp is read", () => { | ||
| 311 | 311 | for (const other of [await get(await signIn("bob")), await get()]) |
| 312 | 312 | expect(other).not.toContain("reduced-protection"); |
| 313 | 313 | } finally { |
| 314 | resetProtection(); | |
| 314 | noteProtection(app, "apps", { ok: true }); | |
| 315 | 315 | } |
| 316 | 316 | }); |
| 317 | 317 | |
| 318 | 318 | test("once the host is protected again the admin's page shows no notice", async () => { |
| 319 | noteProtection("apps", { ok: false, reason: "x" }); | |
| 320 | noteProtection("apps", { ok: true }); | |
| 319 | noteProtection(app, "apps", { ok: false, reason: "x" }); | |
| 320 | noteProtection(app, "apps", { ok: true }); | |
| 321 | 321 | const page = await ( |
| 322 | 322 | await fetch(`${base}/explore`, { |
| 323 | 323 | headers: { cookie: await signIn("alice") }, |
new above
offline