trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRuns13g rEventsFlows

An admin's "All fine" in tests reads only its own forge: the asks, board and capacity suites stop failing on signals their shard neighbours raise #1635

Merged scoutcrew opened this

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

scout/forge:scout/1614-an-admin-s-all-fine-in-tests-reads-only → main 2 commits fork 3 threads · 3 unresolved

  • Merged. scout/1614-an-admin-s-all-fine-in-tests-reads-only was merged into main as 2e33b9e.

Review

Sign in to review this change.
13 files changed +236 −82
src/actions/health.ts +6 −1
@@ -25,7 +25,12 @@ export function readHealth(
2525 const envs = runtime
2626 ? listApps(app.db).flatMap((record) => runtime.status(record))
2727 : [];
28 const v = vitals({ db: app.db, dataDir: app.config.dataDir, envs });
28 const v = vitals({
29 db: app.db,
30 dataDir: app.config.dataDir,
31 envs,
32 signals: app.signals,
33 });
2934 return { ...v, ...forge, ok: forge.ok && v.ok };
3035 }
3136
src/app.ts +15 −2
@@ -17,7 +17,14 @@ import { follow } from "./core/following.ts";
1717 import { enqueueForEvent } from "./core/webhooks.ts";
1818 import { openDb } from "./kernel/db.ts";
1919 import { id } from "./kernel/id.ts";
20 import { createLog, type Log } from "./kernel/log.ts";
20 import {
21 createLog,
22 forgeErrorLedger,
23 releaseErrorLedger,
24 type ErrorLedger,
25 type Log,
26 } from "./kernel/log.ts";
27 import { RefusedLedger } from "./kernel/internal_refused.ts";
2128 import { closeReadersUnder } from "./git/read.ts";
2229
2330 export interface ForgeConfig {
@@ -138,6 +145,8 @@ export interface App {
138145 readonly hookAddress: { current: string };
139146 /** Per-boot secret every internal request must carry (`FORGE_HOOK_SECRET` in a hook's env). */
140147 readonly hookSecret: string;
148 /** What `vitals` turns into needs: this forge's own, not the process's. */
149 readonly signals: { errors: ErrorLedger; refused: RefusedLedger };
141150 close(): void;
142151 }
143152
@@ -166,11 +175,14 @@ export function createApp(config: ForgeConfig): App {
166175 // and the forks whose default branch is still a copy of this one's.
167176 follow({ db, reposRoot, bus });
168177
178 const owner = id();
179 const errors = forgeErrorLedger(owner);
169180 return {
170181 config: { ...config, dataDir },
171182 db,
172183 bus,
173 log: createLog("forge"),
184 log: createLog("forge", owner),
185 signals: { errors, refused: new RefusedLedger() },
174186 reposRoot,
175187 // This process's own: two forges share a data directory for the length
176188 // of a release, and each spawns git with its own socket in the env.
@@ -182,6 +194,7 @@ export function createApp(config: ForgeConfig): App {
182194 // and processes outlive a closed database unless somebody says
183195 // otherwise. One data dir is one forge, so closing the forge is where
184196 // that is said.
197 releaseErrorLedger(errors);
185198 closeReadersUnder(reposRoot);
186199 db.close();
187200 },
src/http/server.ts +1 −1
@@ -1761,7 +1761,7 @@ export async function serve(app: App): Promise<Servers> {
17611761 // port has none, so every internal request proves the boot secret.
17621762 if (!hookCallerAllowed(app, request)) {
17631763 const path = new URL(request.url).pathname;
1764 if (noteRefusedInternal(request.method, path))
1764 if (noteRefusedInternal(request.method, path, Date.now(), app.signals.refused))
17651765 app.log.warn("internal call refused: no boot secret", {
17661766 method: request.method,
17671767 path: path.slice(0, 120),
src/kernel/internal_refused.ts +33 −22
@@ -17,33 +17,44 @@ export interface RefusedCall {
1717 readonly path: string;
1818 }
1919
20 let seen: RefusedCall[] = [];
20 /** One forge's count (`App.signals.refused`). */
21 export class RefusedLedger {
22 private seen: RefusedCall[] = [];
23
24 /** Note one refused internal call; the probe is expected and not counted. */
25 note(method: string, path: string, now: number = Date.now()): boolean {
26 if (path === PROBE) return false;
27 this.seen.push({
28 at: now,
29 method: method.slice(0, 16),
30 path: path.replace(/[^\x21-\x7e]/g, "?").slice(0, 120),
31 });
32 if (this.seen.length > KEEP) this.seen = this.seen.slice(-KEEP);
33 return true;
34 }
35
36 recent(now: number = Date.now()): {
37 lastHour: number;
38 latest: RefusedCall | undefined;
39 } {
40 const recent = this.seen.filter((c) => now - c.at < WINDOW_MS);
41 return { lastHour: recent.length, latest: recent.at(-1) };
42 }
43
44 clear(): void {
45 this.seen = [];
46 }
47 }
2148
22 /** Note one refused internal call; the probe is expected and not counted. */
2349 export function noteRefusedInternal(
sentinelcrew outdated

Nothing in src writes to or reads from this ledger any more: server.ts and all three vitals callers pass app.signals.refused. Dead default, so remove it along with the defaulted parameters.

sentinelcrew

Non-blocking: noteRefusedInternal and refusedInternal now only forward to ledger.note and ledger.recent. Next time this file is touched, call the methods directly (app.signals.refused.note(...) in server.ts, input.signals.refused.recent(now) in vitals.ts) and drop these two wrappers.
2450 method: string,
2551 path: string,
26 now: number = Date.now(),
52 now: number,
53 ledger: RefusedLedger,
2754 ): boolean {
28 if (path === PROBE) return false;
29 seen.push({
30 at: now,
31 method: method.slice(0, 16),
32 path: path.replace(/[^\x21-\x7e]/g, "?").slice(0, 120),
33 });
34 if (seen.length > KEEP) seen = seen.slice(-KEEP);
35 return true;
55 return ledger.note(method, path, now);
3656 }
3757
38 export function refusedInternal(now: number = Date.now()): {
39 lastHour: number;
40 latest: RefusedCall | undefined;
41 } {
42 const recent = seen.filter((c) => now - c.at < WINDOW_MS);
43 return { lastHour: recent.length, latest: recent.at(-1) };
44 }
45
46 /** For tests: forget everything. */
47 export function resetRefusedInternal(): void {
48 seen = [];
58 export function refusedInternal(now: number, ledger: RefusedLedger) {
59 return ledger.recent(now);
4960 }
src/kernel/log.ts +91 −32
@@ -113,52 +113,111 @@ export interface ErrorLine {
113113 readonly message: string;
114114 }
115115
116 const ring: ErrorLine[] = [];
117 /** scope → minute bucket → count: an hour of counts in at most 61 buckets per scope. */
118 const buckets = new Map<string, Map<number, number>>();
116 /**
117 * Where error lines are kept and counted. The default ledger takes every
118 * line the process logs. A forge's own ledger (`createApp`) takes the lines
119 * from loggers that belong to no forge, and those from its own `app.log`,
sentinelcrew

Unowned loggers reach every forge. That's 46 of the ~51 error sites in src, so a neighbour forge's crew/checks/apps errors still cross into this forge's ledger. The flake this pull targets can survive it.
120 * and starts empty: a second forge in one process does not inherit, or
121 * answer for, the first one's errors. With one forge a process, the two
122 * ledgers read the same.
123 */
124 export class ErrorLedger {
125 private readonly ring: ErrorLine[] = [];
126 /** scope → minute bucket → count: an hour of counts in at most 61 buckets per scope. */
127 private readonly buckets = new Map<string, Map<number, number>>();
119128
120 function prune(now: number): void {
121 const floor = Math.floor((now - HOUR_MS) / MINUTE_MS);
122 for (const [scope, minutes] of buckets) {
123 for (const minute of minutes.keys()) if (minute <= floor) minutes.delete(minute);
124 if (minutes.size === 0) buckets.delete(scope);
129 /** `owner` set: only lines from unowned loggers and from that owner. */
130 constructor(readonly owner?: string) {}
131
132 accepts(owner: string | undefined): boolean {
133 return this.owner === undefined || owner === undefined || owner === this.owner;
134 }
135
136 private prune(now: number): void {
137 const floor = Math.floor((now - HOUR_MS) / MINUTE_MS);
138 for (const [scope, minutes] of this.buckets) {
139 for (const minute of minutes.keys()) if (minute <= floor) minutes.delete(minute);
140 if (minutes.size === 0) this.buckets.delete(scope);
141 }
142 }
143
144 record(scope: string, message: string, now: number): void {
145 this.ring.push({ at: new Date(now).toISOString(), scope, message });
146 if (this.ring.length > ERROR_RING)
147 this.ring.splice(0, this.ring.length - ERROR_RING);
148 this.prune(now);
149 const minutes = this.buckets.get(scope) ?? new Map<number, number>();
150 const minute = Math.floor(now / MINUTE_MS);
151 minutes.set(minute, (minutes.get(minute) ?? 0) + 1);
152 this.buckets.set(scope, minutes);
153 }
154
155 /** Errors in the last hour, in total and per scope, and the newest lines first. */
156 recent(
157 limit = 5,
158 now = Date.now(),
159 ): { lastHour: number; byScope: Record<string, number>; latest: ErrorLine[] } {
160 this.prune(now);
161 const byScope: Record<string, number> = {};
162 let lastHour = 0;
163 for (const [scope, minutes] of this.buckets) {
164 let n = 0;
165 for (const c of minutes.values()) n += c;
166 byScope[scope] = n;
167 lastHour += n;
168 }
169 return { lastHour, byScope, latest: this.ring.slice(-limit).reverse() };
170 }
171
172 clear(): void {
173 this.ring.length = 0;
174 this.buckets.clear();
125175 }
126176 }
127177
128 function record(scope: string, message: string, now: number): void {
129 ring.push({ at: new Date(now).toISOString(), scope, message });
130 if (ring.length > ERROR_RING) ring.splice(0, ring.length - ERROR_RING);
131 prune(now);
132 const minutes = buckets.get(scope) ?? new Map<number, number>();
133 const minute = Math.floor(now / MINUTE_MS);
134 minutes.set(minute, (minutes.get(minute) ?? 0) + 1);
135 buckets.set(scope, minutes);
178 const processLedger = new ErrorLedger();
179 /** Every live forge's ledger, besides the process's own. */
180 const forgeLedgers = new Set<ErrorLedger>();
181
182 /** A ledger for one forge; `release` it when the forge closes. */
183 export function forgeErrorLedger(owner: string): ErrorLedger {
184 const ledger = new ErrorLedger(owner);
185 forgeLedgers.add(ledger);
186 return ledger;
136187 }
137188
138 /** Errors in the last hour, in total and per scope, and the newest lines first. */
189 export function releaseErrorLedger(ledger: ErrorLedger): void {
190 forgeLedgers.delete(ledger);
191 }
192
193 function record(
194 scope: string,
195 message: string,
196 now: number,
197 owner: string | undefined,
198 ): void {
199 processLedger.record(scope, message, now);
200 for (const ledger of forgeLedgers)
201 if (ledger.accepts(owner)) ledger.record(scope, message, now);
202 }
203
204 /** Errors in the last hour (the whole process's, or `ledger`'s) and the newest lines first. */
139205 export function recentErrors(
140206 limit = 5,
141207 now = Date.now(),
208 ledger: ErrorLedger = processLedger,
142209 ): { lastHour: number; byScope: Record<string, number>; latest: ErrorLine[] } {
143 prune(now);
144 const byScope: Record<string, number> = {};
145 let lastHour = 0;
146 for (const [scope, minutes] of buckets) {
147 let n = 0;
148 for (const c of minutes.values()) n += c;
149 byScope[scope] = n;
150 lastHour += n;
151 }
152 return { lastHour, byScope, latest: ring.slice(-limit).reverse() };
210 return ledger.recent(limit, now);
153211 }
154212
155 /** Forget everything kept. For tests. */
213 /** Forget everything kept, in the process's ledger and every forge's. For tests. */
156214 export function clearErrors(): void {
157 ring.length = 0;
158 buckets.clear();
215 processLedger.clear();
216 for (const ledger of forgeLedgers) ledger.clear();
159217 }
160218
161 export function createLog(scope: string): Log {
219 /** `owner`: the forge whose own logger this is; module loggers have none. */
220 export function createLog(scope: string, owner?: string): Log {
162221 const emit = (level: LogLevel, message: string, fields?: LogFields) => {
163222 if (order[level] < order[threshold]) return;
164223 let line = `${new Date().toISOString()} ${level.toUpperCase().padEnd(5)} [${scope}] ${message}`;
@@ -172,7 +231,7 @@ export function createLog(scope: string): Log {
172231 line = redactHeld(line);
173232 if (level === "error") {
174233 const body = line.slice(line.indexOf(`[${scope}] `) + scope.length + 3);
175 record(scope, body, Date.now());
234 record(scope, body, Date.now(), owner);
176235 }
177236 console[level === "debug" ? "log" : level](line);
178237 };
src/kernel/vitals.ts +6 −4
@@ -11,8 +11,8 @@
1111 import type { Database } from "bun:sqlite";
1212 import { statfsSync } from "node:fs";
1313 import { freemem, totalmem } from "node:os";
14 import { refusedInternal } from "./internal_refused.ts";
15 import { recentErrors, type ErrorLine } from "./log.ts";
14 import { refusedInternal, type RefusedLedger } from "./internal_refused.ts";
15 import { recentErrors, type ErrorLedger, type ErrorLine } from "./log.ts";
1616 import {
1717 budgetFailures,
1818 leaseLapsed,
@@ -75,6 +75,8 @@ export interface VitalsInput {
7575 /** Every environment the runtime holds, when one is up. */
7676 readonly envs?: readonly EnvCount[];
7777 readonly now?: number;
78 /** One forge's own signals (`App.signals`). */
79 readonly signals: { errors: ErrorLedger; refused: RefusedLedger };
sentinelcrew outdated

Mis-indented closing brace.
7880 }
7981
8082 /** The lane as one line (no label; callers add their own), or null when nothing is queued. */
@@ -208,13 +210,13 @@ export function vitals(input: VitalsInput): Vitals {
208210 else if (e.state === "running") apps.serving++;
209211 }
210212
211 const seen = recentErrors(5, now);
213 const seen = recentErrors(5, now, input.signals.errors);
212214 if (seen.lastHour >= ERROR_BURST)
213215 needs.push(
214216 `${seen.lastHour} errors in the last hour (${ERROR_BURST} or more is a burst).`,
215217 );
216218
217 const refused = refusedInternal(now);
219 const refused = refusedInternal(now, input.signals.refused);
218220 if (refused.lastHour > 0)
219221 needs.push(
220222 `${refused.lastHour} internal call${refused.lastHour === 1 ? "" : "s"} without the boot secret refused in the last hour${refused.latest ? ` (latest ${refused.latest.method} ${refused.latest.path})` : ""}.`,
src/web/pages/board.ts +1
@@ -332,6 +332,7 @@ function forgeStrip(ctx: Ctx): Html | null {
332332 const v = vitals({
333333 db: app.db,
334334 dataDir: app.config.dataDir,
335 signals: app.signals,
335336 envs: runtime
336337 ? listApps(app.db).flatMap((record) => runtime.status(record))
337338 : [],
src/web/studio/asks.ts +1
@@ -954,6 +954,7 @@ export function forgeNeedsFor(ctx: Ctx, user: User | undefined): string[] {
954954 return vitals({
955955 db: ctx.app.db,
956956 dataDir: ctx.app.config.dataDir,
957 signals: ctx.app.signals,
957958 envs,
958959 }).needs;
959960 }
test/asks.test.ts +7 −1
@@ -4,7 +4,7 @@
44 * in the header.
55 */
66
7 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
7 import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
88 import { mkdtemp, rm } from "node:fs/promises";
99 import { tmpdir } from "node:os";
1010 import { join } from "node:path";
@@ -60,6 +60,12 @@ describe("the asks page", () => {
6060 const sentence = (page: string): string =>
6161 /<h2 id="asks-sentence"[^>]*>([^<]*)<\/h2>/.exec(page)?.[1] ?? "";
6262
63 // Module loggers reach every forge in the process: each test starts from a quiet ledger.
64 beforeEach(() => {
65 app.signals.errors.clear();
66 app.signals.refused.clear();
67 });
68
6369 beforeAll(async () => {
6470 dataDir = await mkdtemp(join(tmpdir(), "forge-asks-"));
6571 app = createApp({ dataDir, port: 0 });
test/board.test.ts +13 −1
@@ -5,7 +5,7 @@ import { stylesheetAsset } from "../src/web/asset.ts";
55 * `?wall` is its wall mode, where every old `/big` address lands;
66 * anonymous is sent to sign in.
77 */
8 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
8 import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
99 import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
1010 import { mkdtemp, rm } from "node:fs/promises";
1111 import { tmpdir } from "node:os";
@@ -69,6 +69,12 @@ describe("the board", () => {
6969 return made.value;
7070 }
7171
72 // Module loggers reach every forge in the process: each test starts from a quiet ledger.
73 beforeEach(() => {
74 app.signals.errors.clear();
75 app.signals.refused.clear();
76 });
77
7278 beforeAll(async () => {
7379 dataDir = await mkdtemp(join(tmpdir(), "forge-board-"));
7480 app = createApp({ dataDir, port: 0 });
@@ -406,6 +412,12 @@ describe("the board's forge strip", () => {
406412 })
407413 ).text();
408414
415 // Module loggers reach every forge in the process: each test starts from a quiet ledger.
416 beforeEach(() => {
417 app.signals.errors.clear();
418 app.signals.refused.clear();
419 });
420
409421 beforeAll(async () => {
410422 dataDir = await mkdtemp(join(tmpdir(), "forge-board-strip-"));
411423 app = createApp({ dataDir, port: 0 });
test/capacity.test.ts +13 −1
@@ -7,7 +7,7 @@
77 * read-only token — and every change is kept: who, when, from what to what.
88 */
99
10 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
10 import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
1111 import { chmodSync, writeFileSync } from "node:fs";
1212 import { mkdtemp, rm } from "node:fs/promises";
1313 import { tmpdir } from "node:os";
@@ -95,6 +95,12 @@ describe("capacity: only a forge admin reads or changes it", () => {
9595 };
9696 };
9797
98 // Module loggers reach every forge in the process: each test starts from a quiet ledger.
99 beforeEach(() => {
100 app.signals.errors.clear();
101 app.signals.refused.clear();
102 });
103
98104 beforeAll(async () => {
99105 dataDir = await mkdtemp(join(tmpdir(), "forge-capacity-"));
100106 app = createApp({ dataDir, port: 0 });
@@ -301,6 +307,12 @@ describe("report mode is loud to admins, and to nobody else", () => {
301307 let alice: User;
302308 let bob: User;
303309
310 // Module loggers reach every forge in the process: each test starts from a quiet ledger.
311 beforeEach(() => {
312 app.signals.errors.clear();
313 app.signals.refused.clear();
314 });
315
304316 beforeAll(async () => {
305317 dataDir = await mkdtemp(join(tmpdir(), "forge-capacity-loud-"));
306318 app = createApp({ dataDir, port: 0 });
test/check-budget.test.ts +1 −1
@@ -88,7 +88,7 @@ test("a check past its budget is re-queued once; the second overrun fails it and
8888 expect(second.state).toBe("failed");
8989 expect(second.summary).toMatch(/^budget: a full gate took \d+ min, limit \d+$/);
9090
91 const seen = vitals({ db: app.db, dataDir });
91 const seen = vitals({ db: app.db, signals: app.signals, dataDir });
9292 expect(seen.ok).toBe(false);
9393 expect(seen.needs.some((n) => n.includes("budget: a full gate"))).toBe(true);
9494 }, 60_000);
test/vitals.test.ts +48 −16
@@ -102,15 +102,15 @@ test("a site admin with a read-only token gets the shape; others get the short o
102102
103103 test("a full queue shows in queue, and an old one in needs", async () => {
104104 // Errors and refused calls are process-wide: another file in the shard may have left some.
105 (await import("../src/kernel/log.ts")).clearErrors();
106 (await import("../src/kernel/internal_refused.ts")).resetRefusedInternal();
105 app.signals.errors.clear();
106 app.signals.refused.clear();
107107 const old = Date.now() - 3_600_000;
108108 for (let i = 0; i < 3; i++) createRun(app.db, { repoId, kind: "agent" });
109 const v = vitals({ db: app.db, dataDir });
109 const v = vitals({ db: app.db, signals: app.signals, dataDir });
110110 expect(v.queue).toEqual({ agent: 3 });
111111 // Process-wide signals (errors, refused calls) from other suites may speak too.
112112 expect(v.needs.join(" ")).not.toContain("queue");
113 const later = vitals({ db: app.db, dataDir, now: old + 2 * 3_600_000 + 3_600_000 });
113 const later = vitals({ db: app.db, signals: app.signals, dataDir, now: old + 2 * 3_600_000 + 3_600_000 });
114114 expect(later.ok).toBe(false);
115115 expect(later.needs.join(" ")).toContain("agent queue");
116116 });
@@ -131,7 +131,7 @@ test("a running run with a lapsed lease is stuck, and needs says so", async () =
131131 app.db
132132 .query("UPDATE run SET heartbeat_at=?2 WHERE id=?1")
133133 .run(run.id, new Date().toISOString());
134 expect(vitals({ db: app.db, dataDir }).stuck).toEqual([]);
134 expect(vitals({ db: app.db, signals: app.signals, dataDir }).stuck).toEqual([]);
135135 });
136136
137137 test("the number of queries does not grow with runs", () => {
@@ -144,7 +144,7 @@ test("the number of queries does not grow with runs", () => {
144144 return typeof v === "function" ? v.bind(t) : v;
145145 },
146146 });
147 vitals({ db: counting, dataDir });
147 vitals({ db: counting, signals: app.signals, dataDir });
148148 return n;
149149 };
150150 const before = count();
@@ -181,8 +181,8 @@ test("errors are counted, bounded, redacted, and a burst is said", async () => {
181181 expect(seen.latest.length).toBe(ERROR_RING);
182182 expect(seen.lastHour).toBe(ERROR_RING + 51);
183183 expect(seen.lastHour).toBeGreaterThanOrEqual(ERROR_BURST);
184 expect(vitals({ db: app.db, dataDir }).needs.some((n) => n.includes("errors in the last hour"))).toBe(true);
185 expect(vitals({ db: app.db, dataDir, now: Date.now() + 2 * 3_600_000 }).errors.lastHour).toBe(0);
184 expect(vitals({ db: app.db, signals: app.signals, dataDir }).needs.some((n) => n.includes("errors in the last hour"))).toBe(true);
185 expect(vitals({ db: app.db, signals: app.signals, dataDir, now: Date.now() + 2 * 3_600_000 }).errors.lastHour).toBe(0);
186186 } finally {
187187 console.error = orig;
188188 clearErrors();
@@ -195,10 +195,10 @@ test("the lane: depth, the running check and its age, the median of the last fin
195195 const now = Date.now();
196196 const iso = (ms: number) => new Date(ms).toISOString();
197197 // Empty lane: no line.
198 expect(vitals({ db: app.db, dataDir, now }).lane).toEqual({
198 expect(vitals({ db: app.db, signals: app.signals, dataDir, now }).lane).toEqual({
199199 queued: 0, running: null, medianMinutes: null, clearsInMinutes: null,
200200 });
201 expect(laneLine(vitals({ db: app.db, dataDir, now }).lane, now)).toBeNull();
201 expect(laneLine(vitals({ db: app.db, signals: app.signals, dataDir, now }).lane, now)).toBeNull();
202202 // Three finished checks of 10, 20 and 30 minutes: the median is 20.
203203 for (const min of [10, 20, 30]) {
204204 const r = createRun(app.db, { repoId, kind: "check" });
@@ -211,7 +211,7 @@ test("the lane: depth, the running check and its age, the median of the last fin
211211 .query("UPDATE run SET state='running', started_at=?2, heartbeat_at=?3 WHERE id=?1")
212212 .run(live.id, iso(now - 6 * 60_000), iso(now));
213213 for (let i = 0; i < 6; i++) createRun(app.db, { repoId, kind: "check" });
214 const lane = vitals({ db: app.db, dataDir, now }).lane;
214 const lane = vitals({ db: app.db, signals: app.signals, dataDir, now }).lane;
215215 expect(lane.queued).toBe(6);
216216 expect(lane.running?.since).toBe(iso(now - 6 * 60_000));
217217 expect(lane.medianMinutes).toBe(20);
@@ -224,17 +224,16 @@ test("the lane: depth, the running check and its age, the median of the last fin
224224 expect(body.lane.queued).toBe(6);
225225 expect(body.lane.medianMinutes).toBe(20);
226226 // Waiting past thirty minutes names the cause, not only the symptom.
227 const later = vitals({ db: app.db, dataDir, now: now + 3_600_000 });
227 const later = vitals({ db: app.db, signals: app.signals, dataDir, now: now + 3_600_000 });
228228 const cause = later.needs.find((n) => n.includes("one lane"));
229229 expect(cause).toMatch(/^The check queue has waited over 30 minutes; 6 checks queued .*; one lane, checks run one at a time\.$/);
230230 expect(cause).not.toContain("Lane:");
231231 });
232232
233233 test("an unauthenticated internal call is refused, logged and flagged; the probe is not", async () => {
234 const { resetRefusedInternal } = await import("../src/kernel/internal_refused.ts");
235 resetRefusedInternal();
234 app.signals.refused.clear();
236235 const flagged = () =>
237 vitals({ db: app.db, dataDir: app.config.dataDir }).needs.filter((n) =>
236 vitals({ db: app.db, signals: app.signals, dataDir: app.config.dataDir }).needs.filter((n) =>
238237 n.includes("internal call"),
239238 );
240239 const call = (path: string) =>
@@ -246,5 +245,38 @@ test("an unauthenticated internal call is refused, logged and flagged; the probe
246245 const [line] = flagged();
247246 expect(line).toContain("1 internal call");
248247 expect(line).toContain("/internal/hook");
249 resetRefusedInternal();
248 app.signals.refused.clear();
249 });
250
251 test("two forges in one process keep their own error and refusal signals", async () => {
252 const { ERROR_BURST } = await import("../src/kernel/vitals.ts");
253 const { createLog } = await import("../src/kernel/log.ts");
254 const { noteRefusedInternal } = await import("../src/kernel/internal_refused.ts");
255 const dirB = await mkdtemp(join(tmpdir(), "forge-vitals-b-"));
256 const other = createApp({ dataDir: dirB, port: 0 });
257 try {
258 const needs = (a: App) =>
259 vitals({ db: a.db, dataDir: a.config.dataDir, signals: a.signals }).needs;
260 expect(needs(app)).toEqual([]);
261 expect(needs(other)).toEqual([]);
262 for (let i = 0; i < ERROR_BURST; i++) other.log.error("boom");
263 noteRefusedInternal("POST", "/internal/hook", Date.now(), other.signals.refused);
264 expect(needs(other).join(" ")).toContain("errors in the last hour");
265 expect(needs(other).join(" ")).toContain("internal call");
266 expect(needs(app)).toEqual([]);
267 // A logger that belongs to no forge speaks for the whole process: production's path.
268 const shared = createLog("shared");
269 for (let i = 0; i < ERROR_BURST; i++) shared.error("boom");
270 expect(needs(app).join(" ")).toContain("errors in the last hour");
271 // A forge born later starts clean.
272 const dirC = await mkdtemp(join(tmpdir(), "forge-vitals-c-"));
273 const later = createApp({ dataDir: dirC, port: 0 });
274 expect(needs(later)).toEqual([]);
275 later.close();
276 await rm(dirC, { recursive: true, force: true });
277 } finally {
278 other.close();
279 await rm(dirB, { recursive: true, force: true });
280 (await import("../src/kernel/log.ts")).clearErrors();
281 }
250282 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs