Public: anyone can read this; the crew can change it.

A defect the crew files from a failed probe carries the URL, status, bytes and time it saw #13061

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/13033-a-defect-the-crew-files-from-a-failed-pr → main 7 commits fork 3 threads · 3 unresolved

  • Merged. mason/13033-a-defect-the-crew-files-from-a-failed-pr was merged into main as fe54e72.

Review

Sign in to review this change.
24 files changed +445 −37
scripts/hunt.ts +2 −1
@@ -4,6 +4,7 @@
44 * Never run on a person's machine in a loop: it is one pass, then it ends.
55 */
66
7 import { evidenceOf } from "../src/hunt/finding.ts";
78 import { huntTarget } from "../src/hunt/hunt.ts";
89
910 const target = process.argv[2];
@@ -12,5 +13,5 @@ if (!target || !/^https?:\/\//.test(target)) {
1213 process.exit(2);
1314 }
1415 const report = await huntTarget(target.replace(/\/$/, ""));
15 console.log(JSON.stringify(report, null, 2));
16 console.log(JSON.stringify({ ...report, findings: report.findings.map((f) => ({ ...f, evidence: evidenceOf(f) })) }, null, 2));
1617 process.exit(report.findings.length === 0 ? 0 : 1);
src/contract/probe.ts +10 −1
@@ -24,6 +24,7 @@ import type { AppRecord } from "../core/apps.ts";
2424 import type { Repo } from "../core/repos.ts";
2525 import type { Cell, Contract, Verb } from "./can.ts";
2626 import type { Standing } from "./audit.ts";
27 import { readBounded } from "../hunt/probe.ts";
2728
2829 /** The probe's identities: what the forge can sign an app a visitor as. */
2930 export type Identity = "anon" | "user" | "admin";
@@ -44,6 +45,9 @@ export interface Knock {
4445 | { readonly kind: "no_door"; readonly status: number }
4546 | { readonly kind: "unprobed"; readonly why: string }
4647 | { readonly kind: "error"; readonly message: string };
48 /** Milliseconds the door took and the bytes of its body; absent when it was not asked. */
49 readonly ms?: number;
50 readonly bytes?: number;
4751 /** The verdict for this knock: does the door agree with the table? */
4852 readonly verdict: "ok" | "mismatch" | "unprobed";
4953 }
@@ -149,6 +153,8 @@ export async function probe(input: ProbeInput): Promise<Probed | undefined> {
149153 })),
150154 };
151155 let result: Knock["result"];
156 let measured: { ms: number; bytes: number } | undefined;
157 const asked = performance.now();
152158 try {
153159 const res = await doFetch(url, {
154160 method: door.method,
@@ -157,7 +163,9 @@ export async function probe(input: ProbeInput): Promise<Probed | undefined> {
157163 signal: AbortSignal.timeout(input.timeoutMs ?? 5_000),
158164 redirect: "manual",
159165 });
outdated

Unbounded arrayBuffer() on a response from the probed app. Cap the read, or cancel the body and use the declared length.

160 await res.body?.cancel();
166 // Bounded: a probed app cannot make the prober hold more than readBounded's cap.
167 const { size: bytes } = await readBounded(res);
168 measured = { ms: performance.now() - asked, bytes };
161169 // A door refuses with 401 or 403 before it looks for a row, so a 404
162170 // in JSON is the action saying "no such row": past the door. A 404
163171 // that is not JSON is the router: there is no such door.
@@ -184,6 +192,7 @@ export async function probe(input: ProbeInput): Promise<Probed | undefined> {
184192 door: `${door.method} ${path}`,
185193 cell: cell.answer,
186194 result,
195 ...(measured === undefined ? {} : { ms: Math.round(measured.ms * 10) / 10, bytes: measured.bytes }),
187196 verdict,
188197 });
189198 }
src/contract/probes.ts +11
@@ -23,6 +23,7 @@ import { createLog } from "../kernel/log.ts";
2323 import { audit } from "./audit.ts";
2424 import { probe, type Knock, type ProbeInput, type Probed } from "./probe.ts";
2525 import { during } from "../perf/jobs.ts";
26 import { evidenceBlock, type Evidence } from "../maintain/evidence.ts";
2627
2728 const log = createLog("probes");
2829
@@ -146,6 +147,14 @@ export async function probeApp(
146147 return row;
147148 }
148149
150 /** What the first disagreeing door answered: its path, status, bytes and milliseconds. */
151 function seenOf(probed: Probed): Evidence {
152 const k = probed.knocks.find((n) => n.verdict === "mismatch");
153 if (k === undefined) return { at: probed.at };
154 const status = k.result.kind === "admitted" || k.result.kind === "refused" ? k.result.status : undefined;
155 return { where: k.door?.split(" ")[1], status, bytes: k.bytes, ms: k.ms, at: probed.at };
156 }
157
149158 /** One issue while the app disobeys; closed by the first clean probe after. */
150159 function reconcileIssue(app: App, repo: Repo, probed: Probed): void {
151160 const open = probeIssue(app, repo);
@@ -161,6 +170,8 @@ function reconcileIssue(app: App, repo: Repo, probed: Probed): void {
161170 ...probed.said.map((s) => `- ${s}`),
162171 "",
163172 "The table is the policy. Fix the door, not the table; the app's own `src/contract.test.ts` acts as every identity at every door and should have caught this. This issue closes itself when the next probe comes back clean.",
173 "",
174 evidenceBlock({ ...seenOf(probed), release: probed.commitOid.slice(0, 12) || undefined }),
164175 ].join("\n");
165176 const opened = openIssue(app, as, repo, { title: PROBE_ISSUE_TITLE, body });
166177 if (!opened.ok)
src/core/v1_proof.ts +20 −5
@@ -16,6 +16,14 @@ export interface Result {
1616 readonly item: string;
1717 readonly status: Status;
1818 readonly detail: string;
19 /** On a fail, the last read the probe made: the URL, its status, the bytes read and the ms. */
20 readonly seen?: Seen;
21 }
22 export interface Seen {
23 readonly url: string;
24 readonly status: number;
25 readonly bytes: number;
26 readonly ms: number;
1927 }
2028 interface Got {
2129 readonly status: number;
@@ -233,18 +241,25 @@ export function ageWords(seconds: number): string {
233241 /** Runs every probe against `base` with GET only; a probe that throws is a fail. */
234242 export async function prove(base: string, fetcher: Fetch): Promise<Result[]> {
235243 const root = base.replace(/\/+$/, "");
236 const get: Get = async (path) => {
237 const res = await fetcher(root + path, { method: "GET", redirect: "manual", signal: AbortSignal.timeout(15_000) });
238 return { status: res.status, headers: res.headers, text: await res.text(), location: res.headers.get("location") };
239 };
240244 // Side by side: each probe is one GET at a time, so the proof takes the slowest, not the sum.
241245 return Promise.all(
242246 probes.map(async (p): Promise<Result> => {
247 let last: Seen | undefined;
248 const get: Get = async (path) => {
249 const url = root + path;
250 const asked = performance.now();
251 last = { url, status: 0, bytes: 0, ms: 0 };
252 const res = await fetcher(url, { method: "GET", redirect: "manual", signal: AbortSignal.timeout(15_000) });
253 const text = await res.text();
254 last = { url, status: res.status, bytes: Buffer.byteLength(text), ms: Math.round((performance.now() - asked) * 10) / 10 };
255 return { status: res.status, headers: res.headers, text, location: res.headers.get("location") };
256 };
243257 try {
244258 const r = await p.run(get);
245259 return r instanceof Unproven ? { issue: p.issue, item: p.item, status: "unproven", detail: r.why } : { issue: p.issue, item: p.item, status: "pass", detail: r };
246260 } catch (e) {
247 return { issue: p.issue, item: p.item, status: "fail", detail: e instanceof Fail ? e.message : `probe error: ${String(e)}` };
261 const detail = e instanceof Fail ? e.message : `probe error: ${String(e)}`;
262 return { issue: p.issue, item: p.item, status: "fail", detail, ...(last === undefined ? {} : { seen: last }) };
248263 }
249264 }),
250265 );
src/crew/sentinel.ts +8
@@ -53,6 +53,8 @@ import { closeCaughtUp, mainAhead } from "./sentinel_main_ahead.ts";
5353 import { closeLaneMoving, laneStopped } from "./sentinel_lane.ts";
5454 import type { FindingKind } from "../hunt/finding.ts";
5555 import { addLabels, kindLabels } from "../hunt/label.ts";
56 import { evidenceBlock, type Evidence } from "../maintain/evidence.ts";
57 import { seenCause } from "./sentinel_seen.ts";
5658
5759 const log = createLog("sentinel");
5860
@@ -90,6 +92,8 @@ export interface Finding {
9092 readonly endedAt?: string;
9193 /** The vitals cause a budget-breach or degraded finding reads. */
9294 readonly cause?: string;
95 /** What the detector read, for the issue's evidence block (#13033): path and measurement. */
96 readonly seen?: Evidence;
9397 }
9498
9599 const QUOTE_MAX = 200;
@@ -449,6 +453,7 @@ export function degradedCauses(db: Database, now: number, forgeRepoId?: string,
449453 lastOpenedAt: new Date(d.lastOpened).toISOString(),
450454 ...(Number.isFinite(d.ended) ? { endedAt: new Date(d.ended).toISOString() } : {}),
451455 cause,
456 seen: seenCause(cause, d.href, d.lastOpened, dataDir, now),
452457 });
453458 }
454459 return out;
@@ -521,6 +526,7 @@ export function budgetBreaches(db: Database, now: number, forgeRepoId?: string,
521526 : "Diagnose the cause from the evidence and hand the mechanical fix to a builder as its own issue.",
522527 urgent: true,
523528 cause: s.cause,
529 seen: seenCause(s.cause, s.href, s.openedAt, dataDir, now),
524530 });
525531 }
526532 return out;
@@ -657,6 +663,8 @@ const bodyOf = (f: Finding, now: string): string =>
657663 `**What to look for when fixed:** ${f.lookFor}`,
658664 ...(f.fix === undefined ? [] : ["", `**Mechanical fix:** ${f.fix}`]),
659665 "",
666 evidenceBlock({ ...f.seen, at: now }),
667 "",
660668 `Found by the sentinel at ${now}.`,
661669 ].join("\n");
662670
src/crew/sentinel_seen.ts added +31
@@ -0,0 +1,31 @@
1 /**
2 * What the sentinel read when it filed a degraded cause (#13033), for the evidence block of the
3 * issue: the path of the page that says it (`href`), and the number the cause is about where the
4 * forge holds one: the free bytes of a full disk, the newest snapshot's size for a backup. A
5 * cause that is a state, not a measurement, has no status or milliseconds: they print `n/a`.
6 */
7
8 import { readDisk } from "../kernel/disk_reading.ts";
9 import type { Evidence } from "../maintain/evidence.ts";
10 import { backupVerdict } from "../snapshot/bucket.ts";
11
12 const BACKUP_CAUSE = /^(backup|snapshot|mirror|drill)/;
13
14 /** The newest snapshot's size across the lanes, when a lane has read one. */
15 function newestSnapshotBytes(dataDir: string, now: number): number | undefined {
16 const lanes = backupVerdict(dataDir, undefined, now).lanes.filter((l) => l.newestBytes !== null && l.newestAt !== null);
17 lanes.sort((a, b) => (b.newestAt ?? "").localeCompare(a.newestAt ?? ""));
18 return lanes[0]?.newestBytes ?? undefined;
19 }
20
21 export function seenCause(cause: string, href: string, at: number, dataDir: string | undefined, now: number): Evidence {
22 const base: Evidence = { where: href, at: new Date(at).toISOString() };
23 if (dataDir === undefined) return base;
24 try {
25 if (cause.startsWith("disk")) return { ...base, bytes: readDisk(dataDir).freeBytes };
26 if (BACKUP_CAUSE.test(cause)) return { ...base, bytes: newestSnapshotBytes(dataDir, now) };
27 } catch {
28 // A volume that cannot be read is the cause itself: the block prints n/a for its size.
29 }
30 return base;
31 }
src/efficiency/finding.ts +4 −2
@@ -1,5 +1,6 @@
11 /** The words of a finding and its comments (#7558): the cause, the number, the baseline, the code path. */
22
3 import { evidenceBlock } from "../maintain/evidence.ts";
34 import { show, type Cause } from "./causes.ts";
45 import type { HourVerdict, Why } from "./judge.ts";
56
@@ -18,12 +19,13 @@ export function numbers(c: Cause, v: HourVerdict, why: Why): string {
1819 return `${show(c, v.value)} in the hour of ${at(v.hour)}; ${base}; ${rule}.`;
1920 }
2021
21 export function bodyOf(c: Cause, v: HourVerdict, why: Why): string {
22 export function bodyOf(c: Cause, v: HourVerdict, why: Why, now: Date = new Date()): string {
2223 return (
2324 `The efficiency watcher (duty \`efficiency-watcher\`) found **${c.label}**: ${numbers(c, v, why)}\n\n` +
2425 `Code path that starts the cost: ${c.path}.\n\n` +
2526 `This issue stands while the cost does: it gets a comment each hour it is still bad, and is closed with a note once it has been back under for 24 hours. ` +
26 `The hourly series is the \`efficiency\` board source.`
27 `The hourly series is the \`efficiency\` board source.\n\n` +
28 evidenceBlock({ where: `efficiency board: ${c.id}`, bytes: c.unit === "bytes" ? v.value : undefined, at: now.toISOString() })
2729 );
2830 }
2931
src/efficiency/watch.ts +1 −1
@@ -52,7 +52,7 @@ export function watch(app: App, now: number, readings: Readings = readNow(app.db
5252 }
5353 continue;
5454 }
55 const filed = openIssue(app, actor, repo, { title: titleOf(c), body: bodyOf(c, stands.now, stands.why) });
55 const filed = openIssue(app, actor, repo, { title: titleOf(c), body: bodyOf(c, stands.now, stands.why, new Date(now)) });
5656 if (filed.ok) {
5757 putStanding(db, { cause: c.id, issueId: filed.value.id, lastBad: hour });
5858 actions.push(`filed #${filed.value.number} (${c.id})`);
src/http/server.ts +6 −1
@@ -2730,6 +2730,11 @@ export async function serve(app: App): Promise<Servers> {
27302730 const readWriter = createReadWriter(app.db, readLedger, app.log);
27312731 // The worst event-loop lag of the last 10 s, on every response (#10288).
27322732 startLoopWorst();
2733 /** The body size the response declared, when it declared one: a stream has none. */
2734 const declaredBytes = (response: Response): { bytes?: number } => {
2735 const n = Number(response.headers.get("content-length") ?? NaN);
2736 return Number.isFinite(n) ? { bytes: n } : {};
2737 };
27332738 const counted = (
27342739 owner: App,
27352740 request: Request,
@@ -2757,7 +2762,7 @@ export async function serve(app: App): Promise<Servers> {
27572762 const family = familyOf(request.headers.get("user-agent") ?? "");
27582763 liveOf(app).perfRecord.add(
27592764 tag.pattern,
2760 { at: Date.now(), ms, signedIn: tag.role !== "anon" && tag.role !== "crawler", crawler: wasCrawler(request), deepStranger: wasDeepStranger(request), internal: wasInternal(request), refused: response.status === 429 || wasShielded(request), phases: parseServerTiming(header) },
2765 { at: Date.now(), ms, signedIn: tag.role !== "anon" && tag.role !== "crawler", crawler: wasCrawler(request), deepStranger: wasDeepStranger(request), internal: wasInternal(request), refused: response.status === 429 || wasShielded(request), status: response.status, ...declaredBytes(response), phases: parseServerTiming(header) },
27612766 {
27622767 address: clientAddress(request),
27632768 family,
src/hunt/contract.ts +3 −3
@@ -19,17 +19,17 @@ export async function contract(p: Probe): Promise<Finding[]> {
1919 try {
2020 parsed = JSON.parse(doc.body) as Doc;
2121 } catch {
22 return [found("contract.openapi", "/contract/openapi.json", "the OpenAPI document is not JSON", seenLine("/contract/openapi.json", doc.status))];
22 return [found("contract.openapi", "/contract/openapi.json", "the OpenAPI document is not JSON", seenLine("/contract/openapi.json", doc.status), doc)];
2323 }
2424 for (const [path, ops] of Object.entries(parsed.paths ?? {})) {
2525 const get = ops.get;
2626 if (get === undefined || /[{}]/.test(path)) continue;
2727 const r = await p.get(path);
28 if (r.status === 0 || r.status >= 500) out.push(found("contract.no-500", path, `${path} answers ${r.status}`, seenLine(path, r.status)));
28 if (r.status === 0 || r.status >= 500) out.push(found("contract.no-500", path, `${path} answers ${r.status}`, seenLine(path, r.status), r));
2929 else if (get.responses && Object.keys(get.responses).length > 0) {
3030 const code = String(r.status);
3131 if (!(code in get.responses) && !REFUSALS.has(code) && !("default" in get.responses))
32 out.push(found("contract.documented", path, `${path} answers ${r.status}, which its entry does not document`, `${seenLine(path, r.status)}; documented: ${Object.keys(get.responses).join(", ")}`));
32 out.push(found("contract.documented", path, `${path} answers ${r.status}, which its entry does not document`, `${seenLine(path, r.status)}; documented: ${Object.keys(get.responses).join(", ")}`, r));
3333 }
3434 }
3535 return out;
src/hunt/door.ts +4 −4
@@ -16,16 +16,16 @@ export async function frontDoor(p: Probe): Promise<Finding[]> {
1616 const r = await p.get(path);
1717 const sent = r.status >= 300 && r.status < 400 && /\/signin(\?|$)/.test(r.location ?? "");
1818 if (!sent)
19 out.push(found("door.signin-redirect", path, `${path} does not send an anonymous visitor to sign-in`, seenLine(path, r.status, r.location)));
19 out.push(found("door.signin-redirect", path, `${path} does not send an anonymous visitor to sign-in`, seenLine(path, r.status, r.location), r));
2020 else if (!(r.location ?? "").includes("next="))
21 out.push(found("door.signin-next", path, `${path} sends to sign-in without a way back`, seenLine(path, r.status, r.location)));
21 out.push(found("door.signin-next", path, `${path} sends to sign-in without a way back`, seenLine(path, r.status, r.location), r));
2222 }
2323 for (const path of PUBLIC_PAGES) {
2424 const r = await p.get(path);
25 if (r.status !== 200) out.push(found("door.public-page", path, `${path} does not answer 200 to an anonymous visitor`, seenLine(path, r.status, r.location)));
25 if (r.status !== 200) out.push(found("door.public-page", path, `${path} does not answer 200 to an anonymous visitor`, seenLine(path, r.status, r.location), r));
2626 }
2727 const missing = "/no-such-page-hunter";
2828 const r = await p.get(missing);
29 if (r.status !== 404) out.push(found("door.unknown-404", missing, "an unknown path does not answer 404", seenLine(missing, r.status, r.location)));
29 if (r.status !== 404) out.push(found("door.unknown-404", missing, "an unknown path does not answer 404", seenLine(missing, r.status, r.location), r));
3030 return out;
3131 }
src/hunt/finding.ts +12 −1
@@ -1,5 +1,8 @@
11 /** What a detector says: plain data, one issue per `key` (the sentinel's rule). */
22
3 import { evidenceBlock, type Evidence } from "../maintain/evidence.ts";
4 import type { Seen } from "./probe.ts";
5
36 /**
47 * What a finding is, so it is filed already sorted (#8856): a security hole or
58 * a route that answers broken is a `high` bug (the 4 h clock in
@@ -50,15 +53,23 @@ export interface Finding {
5053 title: string;
5154 /** The minimal repro: the request and what came back. */
5255 evidence: string;
56 /** What the detector read, for the evidence block of the issue (#13033). */
57 seen?: Evidence;
5358 }
5459
55 export const found = (detector: Detector, subject: string, title: string, evidence: string): Finding => ({
60 /** `r` is the answer the detector read: its path, status, bytes and ms go in the block. */
61 export const found = (detector: Detector, subject: string, title: string, evidence: string, r?: Seen): Finding => ({
5662 detector,
5763 kind: DETECTOR_KINDS[detector],
5864 key: `${detector}:${subject}`,
5965 title,
6066 evidence,
67 ...(r === undefined ? {} : { seen: { where: r.path, status: r.status === 0 ? undefined : r.status, bytes: r.bytes, ms: r.ms } }),
6168 });
6269
70 /** The text an issue is filed with: the repro, then the evidence block of what was read. */
71 export const evidenceOf = (f: Finding, at?: string): string =>
72 `${f.evidence}\n\n${evidenceBlock({ ...f.seen, ...(at === undefined ? {} : { at }) })}`;
73
6374 export const seenLine = (path: string, status: number, location?: string | null): string =>
6475 `GET ${path} → ${status}${location ? ` → ${location}` : ""}`;
src/hunt/probe.ts +15 −9
@@ -12,6 +12,9 @@ export interface Seen {
1212 location: string | null;
1313 type: string;
1414 body: string;
15 /** Bytes of the body as read (bounded), and milliseconds to the answer. */
16 bytes: number;
17 ms: number;
1518 }
1619
1720 export interface Probe {
@@ -32,20 +35,21 @@ export function inside(base: string, href: string): string | null {
3235
3336 const BODY_MAX = 400_000;
3437
35 /** At most BODY_MAX bytes of the body; the rest is never read. */
36 async function readBounded(res: Response): Promise<string> {
38 /** At most BODY_MAX bytes of the body, and how many were read; the rest is never read. */
39 export async function readBounded(res: Response, max = BODY_MAX): Promise<{ bytes: Uint8Array; size: number }> {
3740 const reader = res.body?.getReader();
38 if (!reader) return "";
41 if (!reader) return { bytes: new Uint8Array(0), size: 0 };
3942 const chunks: Uint8Array[] = [];
4043 let size = 0;
41 while (size < BODY_MAX) {
44 while (size < max) {
4245 const { done, value } = await reader.read();
4346 if (done) break;
4447 chunks.push(value);
4548 size += value.length;
4649 }
4750 await reader.cancel().catch(() => {});
48 return new TextDecoder().decode(Buffer.concat(chunks).subarray(0, BODY_MAX));
51 const bytes = Buffer.concat(chunks).subarray(0, max);
52 return { bytes, size: bytes.length };
4953 }
5054
5155 export function probe(base: string, fetcher: Fetcher = fetch as Fetcher): Probe {
@@ -53,6 +57,7 @@ export function probe(base: string, fetcher: Fetcher = fetch as Fetcher): Probe
5357 async get(path) {
5458 const target = inside(base, path);
5559 if (target === null) throw new Error(`refusing a path off the target: ${path}`);
60 const asked = performance.now();
5661 try {
5762 const res = await fetcher(new URL(target, base).href, {
5863 method: "GET",
@@ -61,12 +66,13 @@ export function probe(base: string, fetcher: Fetcher = fetch as Fetcher): Probe
6166 signal: AbortSignal.timeout(10_000),
6267 });
6368 const type = res.headers.get("content-type") ?? "";
64 const body = /text|json|xml/.test(type) ? await readBounded(res) : "";
65 await res.body?.cancel().catch(() => {});
66 return { path: target, status: res.status, location: res.headers.get("location"), type, body };
69 // Bytes are what was read (bounded), never the target's own Content-Length claim.
70 const read = await readBounded(res);
outdated

bytes comes from the target's Content-Length header, not what was read; use the bytes actually read.

71 const body = /text|json|xml/.test(type) ? new TextDecoder().decode(read.bytes) : "";
72 return { path: target, status: res.status, location: res.headers.get("location"), type, body, bytes: read.size, ms: performance.now() - asked };
6773 } catch {
6874 // A dead or hung route is a result (status 0), not a crash that loses the pass.
69 return { path: target, status: 0, location: null, type: "", body: "" };
75 return { path: target, status: 0, location: null, type: "", body: "", bytes: 0, ms: performance.now() - asked };
7076 }
7177 },
7278 };
src/maintain/evidence.ts added +54
@@ -0,0 +1,54 @@
1 /**
2 * The evidence block a defect carries (#13033): where the duty looked, what
3 * answered, how many bytes, how long it took, when, and which release it ran
4 * against. One fenced block, one line per field, so the next builder reads
5 * what the duty saw instead of reproducing it. A field the duty does not have
6 * is printed `n/a`, never omitted, so the shape is always the same.
7 */
8
9 import { running } from "../kernel/version.ts";
10
11 export interface Evidence {
12 /** The URL or path the duty read (a route, a data path); n/a when it read neither. */
13 readonly where?: string | undefined;
14 readonly status?: number | undefined;
15 readonly bytes?: number | undefined;
16 readonly ms?: number | undefined;
17 /** ISO time of the reading; defaults to now. */
18 readonly at?: string | undefined;
19 /** The release this process runs; defaults to the stamped commit or version. */
20 readonly release?: string | undefined;
21 }
22
23 export const NA = "n/a";
24
25 const FIELDS = ["url", "status", "bytes", "ms", "time", "release"] as const;
26 export type EvidenceField = (typeof FIELDS)[number];
27
28 const number = (n: number | undefined): string => (n === undefined || !Number.isFinite(n) ? NA : String(Math.round(n * 10) / 10));
29 /** One line of text: a newline or a fence in a value cannot break out of the block. */
30 const line = (s: string | undefined): string => {
31 const t = (s ?? "").replace(/[\r\n`]+/g, " ").trim();
32 return t === "" ? NA : t.slice(0, 300);
33 };
34
35 /** The release the process runs, as a short commit or the version. */
36 export function releaseNow(env: Readonly<Record<string, string | undefined>> = process.env): string {
37 const r = running(env);
38 return r.commit !== null ? r.commit.slice(0, 12) : r.version;
39 }
40
41 /** The fenced block, with all six lines (the issue's five facts; the URL or path is `url`). */
42 export function evidenceBlock(e: Evidence, now: () => Date = () => new Date()): string {
43 const at = e.at !== undefined && !Number.isNaN(Date.parse(e.at)) ? new Date(Date.parse(e.at)).toISOString() : now().toISOString();
44 return [
45 "```evidence",
46 `url: ${line(e.where)}`,
47 `status: ${number(e.status)}`,

e.at is printed raw after a lenient Date.parse check; normalise through toISOString().

48 `bytes: ${number(e.bytes)}`,
49 `ms: ${number(e.ms)}`,
50 `time: ${at}`,
51 `release: ${line(e.release ?? releaseNow())}`,
52 "```",
53 ].join("\n");
54 }
src/maintain/proven.ts +2 −1
@@ -10,6 +10,7 @@ import { EventBus } from "../core/events.ts";
1010 import { addComment, createIssue, getIssueById } from "../core/issues.ts";
1111 import { line, prove, type Proof, type Result } from "../core/v1_proof.ts";
1212 import { watcherId } from "../rules/bot.ts";
13 import { evidenceBlock } from "./evidence.ts";
1314
1415 export const defectTitle = (r: Pick<Result, "issue" | "item">): string => `Defect of #${r.issue}: ${r.item}`;
1516
@@ -40,7 +41,7 @@ export function reportFailures(db: Database, repoId: string, results: readonly R
4041 const bot = watcherId(db);
4142 for (const r of results) {
4243 if (r.status !== "fail") continue;
43 const quote = `> ${line(r).replace(/\r?\n/g, " ")}`;
44 const quote = `> ${line(r).replace(/\r?\n/g, " ")}\n\n${evidenceBlock({ ...(r.seen === undefined ? {} : { where: r.seen.url, status: r.seen.status, bytes: r.seen.bytes, ms: r.seen.ms }), at: at.toISOString() })}`;
4445 const open = openDefect(db, repoId, r.issue);
4546 if (open !== undefined) {
4647 const c = addComment(db, { issueId: open, authorId: bot, body: `Still failing on ${at.toISOString().slice(0, 10)}:\n\n${quote}` });
src/perf/duty.ts +2 −1
@@ -13,6 +13,7 @@ import { openIssue } from "../actions/issues.ts";
1313 import { getUserById } from "../core/users.ts";
1414 import { getRepo } from "../core/repos.ts";
1515 import { listIssues } from "../core/issues.ts";
16 import { evidenceOf } from "../hunt/finding.ts";
1617 import { addLabels, kindLabels } from "../hunt/label.ts";
1718 import { createGitRecord } from "./git.ts";
1819 import type { PerfRecord } from "./record.ts";
@@ -50,7 +51,7 @@ export function startPerfWatcher(
5051 const findings = [...evaluate(record, now()), ...evaluateGit(gitRecord, now())];
5152 for (const f of fresh(findings, filed)) {
5253 if (open.has(f.title)) continue;
53 const made = openIssue(app, actor, repo, { title: f.title, body: f.evidence });
54 const made = openIssue(app, actor, repo, { title: f.title, body: evidenceOf(f, new Date(now()).toISOString()) });
5455 if (made.ok) addLabels(app, repo.id, actor.id, made.value, kindLabels(app, repo.id, f.kind));
5556 }
5657 } catch (error) {
src/perf/record.ts +3
@@ -35,6 +35,9 @@ export interface Sample {
3535 readonly internal?: boolean;
3636 /** Answered 429. */
3737 readonly refused?: boolean;
38 /** The status it answered and the bytes its body declared, for a filed defect (#13033). */
39 readonly status?: number;
40 readonly bytes?: number;
3841 readonly phases: readonly PhaseSample[];
3942 }
4043
src/perf/watch.ts +16 −7
@@ -6,10 +6,11 @@
66 */
77
88 import { found, type Finding } from "../hunt/finding.ts";
9 import type { Evidence } from "../maintain/evidence.ts";
910 import type { Database } from "bun:sqlite";
1011 import { dayOf } from "./day.ts";
1112 import { budgetFor, gitBudgetFor, PAGE_BUDGET } from "./budgets.ts";
12 import { percentile, summarize, WINDOW_MS, type PerfRecord, type RouteStat } from "./record.ts";
13 import { percentile, summarize, WINDOW_MS, type PerfRecord, type RouteStat, type Sample } from "./record.ts";
1314
1415 export { WINDOW_MS };
1516 export const MIN_REQUESTS = 20;
@@ -38,6 +39,12 @@ export function slowestRoutes(record: PerfRecord, now: number, limit = 10): Slow
3839 return rows.sort((a, b) => b.over - a.over).slice(0, limit).map((r) => r.row);
3940 }
4041
42 /** The status and bytes of the request that stood at the route's p95. */
43 function atP95(samples: readonly Sample[], p95: number): Pick<Evidence, "status" | "bytes" | "ms"> {
44 const at = samples.reduce((best, s) => (Math.abs(s.ms - p95) < Math.abs(best.ms - p95) ? s : best));
45 return { status: at.status, bytes: at.bytes, ms: p95 };
46 }
47
4148 export function evaluate(record: PerfRecord, now: number): Finding[] {
4249 const out: Finding[] = [];
4350 for (const pattern of record.patterns()) {
@@ -63,15 +70,16 @@ export function evaluate(record: PerfRecord, now: number): Finding[] {
6370 : `Cache: ${slow.name} hit ${slow.hits} of ${slow.n} requests (${Math.round((slow.hits / slow.n) * 100)}%).`,
6471 );
6572 }
66 out.push(
67 found(
73 out.push({
74 ...found(
6875 "performance-watcher",
6976 `${pattern}:${phase}`,
7077 // Stable: the numbers move every tick, the title is how "already open" matches.
7178 `${pattern} is over its budget: ${phase}`,
7279 lines.join("\n"),
7380 ),
74 );
81 seen: { where: pattern, ...atP95(samples, stat.p95) },
82 });
7583 }
7684 return out;
7785 }
@@ -88,8 +96,8 @@ export function evaluateGit(record: PerfRecord, now: number): Finding[] {
8896 const stat = summarize(record.traffic(key, now - WINDOW_MS));
8997 if (stat === null || stat.n < MIN_REQUESTS || stat.p95 <= budget.p95Ms) continue;
9098 if (now - (record.first(key) ?? now) < WINDOW_MS * 0.9) continue;
91 out.push(
92 found(
99 out.push({
100 ...found(
93101 "performance-watcher",
94102 key,
95103 `${key} is over its budget`,
@@ -98,7 +106,8 @@ export function evaluateGit(record: PerfRecord, now: number): Finding[] {
98106 `Subcommand: ${key}, ${stat.n} processes in 15 min: p50 ${Math.round(stat.p50)} ms, p95 ${Math.round(stat.p95)} ms, p99 ${Math.round(stat.p99)} ms.`,
99107 ].join("\n"),
100108 ),
101 );
109 seen: { where: key, ms: stat.p95 },
110 });
102111 }
103112 return out;
104113 }
test/helpers/evidence.ts added +19
@@ -0,0 +1,19 @@
1 import type { EvidenceField } from "../../src/maintain/evidence.ts";
2
3 const FIELDS: readonly EvidenceField[] = ["url", "status", "bytes", "ms", "time", "release"];
4
5 /**
6 * The fields a body's evidence block names, for a test or a reader; undefined when the body has
7 * no block.
8 */
9 export function readEvidence(body: string): Record<EvidenceField, string> | undefined {
10 const m = /```evidence\n([\s\S]*?)\n```/.exec(body);
11 if (m === null) return undefined;
12 const out: Partial<Record<EvidenceField, string>> = {};
13 for (const l of (m[1] ?? "").split("\n")) {
14 const i = l.indexOf(": ");
15 const k = l.slice(0, i) as EvidenceField;
16 if (FIELDS.includes(k)) out[k] = l.slice(i + 2);
17 }
18 return FIELDS.every((f) => out[f] !== undefined) ? (out as Record<EvidenceField, string>) : undefined;
19 }
test/hunt.test.ts +9
@@ -145,4 +145,13 @@ describe("more detectors", () => {
145145 expect(seen.body.length).toBe(400_000);
146146 expect(pulled).toBeLessThan(1_000_000);
147147 });
148
149 test("bytes are what was read, not what the target's Content-Length claims (#13033)", async () => {
150 const liar: Fetcher = async () => new Response("hello", { headers: { "content-type": "text/plain", "content-length": "999999" } });
151 const seen = await probe(BASE, liar).get("/");
152 expect(seen.bytes).toBe(5);
153 const binary: Fetcher = async () => new Response(new Uint8Array(7), { headers: { "content-type": "image/png" } });
154 const img = await probe(BASE, binary).get("/");
155 expect([img.body, img.bytes]).toEqual(["", 7]);
156 });
148157 });
test/maintain-evidence.test.ts added +184
@@ -0,0 +1,184 @@
1 /**
2 * Every defect a duty files carries the evidence block (#13033): url, status,
3 * bytes, ms, time and release, each printed, `n/a` where the duty has none.
4 * One test per filing: the perf watcher (route and git), the efficiency
5 * watcher, the sentinel (production probe, backup and disk findings file
6 * through it) and the contract probe.
7 */
8 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
9 import { mkdtempSync, rmSync } from "node:fs";
10 import { tmpdir } from "node:os";
11 import { join } from "node:path";
12 import { createRepo } from "../src/actions/repos.ts";
13 import { newApp } from "../src/actions/apps.ts";
14 import { createApp, type App } from "../src/app.ts";
15 import type { AppsRuntime } from "../src/apps/runtime.ts";
16 import { probeApp, probeIssue } from "../src/contract/probes.ts";
17 import { listIssues } from "../src/core/issues.ts";
18 import { getRepo } from "../src/core/repos.ts";
19 import { createUser, type User } from "../src/core/users.ts";
20 import { CAUSES } from "../src/efficiency/causes.ts";
21 import { bodyOf as efficiencyBody } from "../src/efficiency/finding.ts";
22 import { hunt as sentinelHunt } from "../src/crew/sentinel.ts";
23 import { huntTarget } from "../src/hunt/hunt.ts";
24 import { evidenceOf } from "../src/hunt/finding.ts";
25 import { provideDiskForTest } from "../src/kernel/disk_reading.ts";
26 import { record as recordDegraded } from "../src/kernel/degraded_history.ts";
27 import { keepReads } from "../src/snapshot/bucket.ts";
28 import { evidenceBlock } from "../src/maintain/evidence.ts";
29 import { readEvidence } from "./helpers/evidence.ts";
30 import { startPerfWatcher } from "../src/perf/duty.ts";
31 import { createGitRecord } from "../src/perf/git.ts";
32 import { createPerfRecord, WINDOW_MS } from "../src/perf/record.ts";
33 import { quietLogs } from "./helpers/globals.ts";
34 import { proveAll } from "./helpers/verified.ts";
35
36 quietLogs();
37
38 const NOW = Date.parse("2026-11-15T10:00:00.000Z");
39 const ISO = new Date(NOW).toISOString();
40
41 describe("the shared formatter", () => {
42 test("prints all six lines, n/a for what the duty does not have", () => {
43 const block = evidenceBlock({ where: "/explore", status: 503, bytes: 1234, ms: 88.26, at: ISO, release: "abc123def456" });
44 expect(block.startsWith("```evidence\n")).toBe(true);
45 expect(block.endsWith("\n```")).toBe(true);
46 expect(readEvidence(block)).toEqual({ url: "/explore", status: "503", bytes: "1234", ms: "88.3", time: ISO, release: "abc123def456" });
47 const bare = readEvidence(evidenceBlock({ at: ISO, release: "dev" }));
48 expect(bare).toEqual({ url: "n/a", status: "n/a", bytes: "n/a", ms: "n/a", time: ISO, release: "dev" });
49 });
50 test("a value cannot close the fence or add a line", () => {
51 const block = evidenceBlock({ where: "/x\n```\n# injected", at: ISO, release: "dev" });
52 expect(block.split("\n")).toHaveLength(8);
53 expect(readEvidence(block)?.url).not.toContain("`");
54 });
55 test("the time is printed normalised, never as the caller wrote it", () => {
56 const block = evidenceBlock({ at: "2026-11-15 10:00:00Z ```", release: "dev" });
57 const t = readEvidence(block)?.time ?? "";
58 if (!Number.isNaN(Date.parse("2026-11-15 10:00:00Z ```"))) expect(t).toBe(new Date(Date.parse("2026-11-15 10:00:00Z ```")).toISOString());
59 expect(t).toMatch(/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3}Z$/);
60 expect(readEvidence(evidenceBlock({ at: "Sun, 15 Nov 2026 10:00:00 GMT", release: "dev" }))?.time).toBe(ISO);
61 });
62 test("the release is the stamped commit or version by default", () => {
63 expect(readEvidence(evidenceBlock({ at: ISO }))?.release).not.toBe("n/a");
64 });
65 });
66
67 describe("each duty's filing carries the block", () => {
68 let root = "";
69 let app: App;
70 let alice: User;
71 const forge = () => getRepo(app.db, "alice", "forge")!;
72
73 beforeAll(async () => {
74 root = mkdtempSync(join(tmpdir(), "forge-evidence-"));
75 app = createApp({ dataDir: root, port: 0, forgeRepo: "alice/forge" });
76 const u = await createUser(app.db, {
77 origin: "seed", trusted: true, username: "alice", email: "[email protected]",
78 password: "a-good-passphrase", adminIfFirst: true,
79 });
80 if (!u.ok) throw new Error(u.error);
81 alice = u.value;
82 proveAll(app);
83 const made = await createRepo(app, alice, { name: "forge" });
84 if (!made.ok) throw new Error(made.error);
85 });
86 afterAll(() => {
87 app.close();
88 rmSync(root, { recursive: true, force: true });
89 });
90
91 test("the performance watcher: a slow route and a slow git subcommand", async () => {
92 const record = createPerfRecord();
93 const git = createGitRecord();
94 for (let i = 0; i < 100; i++) {
95 const at = NOW - WINDOW_MS + (i * WINDOW_MS) / 100;
96 record.add("/", { at, ms: 4600, status: 200, bytes: 48213, signedIn: false, phases: [{ name: "kpis", ms: 4400, hit: false }] });
97 git.add("git rev-parse", { at, ms: 9000, signedIn: false, phases: [] });
98 }
99 const stop = startPerfWatcher(app, record, () => NOW, 5, git);
100 await Bun.sleep(80); // real-timer: the watcher ticks on its own 5 ms interval
101 stop();
102 const issues = listIssues(app.db, forge().id, { state: "open" });
103 const route = issues.find((i) => i.title === "/ is over its budget: kpis");
104 const sub = issues.find((i) => i.title.startsWith("git rev-parse"));
105 expect(route).toBeDefined();
106 expect(sub).toBeDefined();
107 const r = readEvidence(route!.body);
108 expect(r).toBeDefined();
109 expect(r!.url).toBe("/");
110 expect(Number(r!.ms)).toBeGreaterThan(4000);
111 expect(r!.status).toBe("200");
112 expect(r!.bytes).toBe("48213");
113 expect(r!.time).toBe(ISO);
114 expect(readEvidence(sub!.body)?.url).toBe("git rev-parse");
115 });
116
117 test("the efficiency watcher", () => {
118 const c = CAUSES.find((x) => x.unit === "bytes")!;
119 const body = efficiencyBody(c, { hour: NOW, value: 5 * 1024 ** 3, baseline: 1024 ** 3, doubled: true, fell: false, overCeiling: true }, "ceiling", new Date(NOW));
120 const e = readEvidence(body);
121 expect(e).toBeDefined();
122 expect(e!.bytes).toBe(String(5 * 1024 ** 3));
123 expect(e!.url).toContain(c.id);
124 expect(e!.status).toBe("n/a");
125 expect(e!.time).toBe(ISO);
126 });
127
128 test("the sentinel's disk finding carries the path and the free bytes it read", () => {
129 provideDiskForTest(() => ({ freeBytes: 123_456_789, totalBytes: 10 ** 12 }));
130 try {
131 recordDegraded(app.db, [{ cause: "disk-red", sentence: "The data disk is nearly full.", href: "/settings/capacity" }], NOW - 60 * 60_000);
132 sentinelHunt(app, "alice/forge", NOW);
133 } finally {
134 provideDiskForTest(undefined);
135 }
136 const issue = listIssues(app.db, forge().id, { state: "open" }).find((i) => i.title.includes("disk-red"));
137 expect(issue).toBeDefined();
138 const e = readEvidence(issue!.body);
139 expect(e).toMatchObject({ url: "/settings/capacity", status: "n/a", bytes: "123456789", ms: "n/a", time: ISO });
140 expect(e!.release).not.toBe("n/a");
141 });
142
143 test("the sentinel's backup finding carries the path and the newest snapshot's size", () => {
144 keepReads(
145 root,
146 [{ lane: "primary", where: "s3://bucket/forge", offBox: true, newestAt: "2026-11-10T00:00:00.000Z", newestName: "snap.tar", newestBytes: 7_340_032, count: 3, readAt: ISO }],
147 NOW,
148 );
149 recordDegraded(app.db, [{ cause: "backup", sentence: "The newest backup is five days old.", href: "/settings/backups" }], NOW - 60 * 60_000);
150 sentinelHunt(app, "alice/forge", NOW);
151 const issue = listIssues(app.db, forge().id, { state: "open" }).find((i) => i.title.includes("backup was degraded"));
152 expect(issue).toBeDefined();
153 expect(readEvidence(issue!.body)).toMatchObject({ url: "/settings/backups", status: "n/a", bytes: "7340032", ms: "n/a", time: ISO });
154 });
155
156 test("the production probe: the hunter's finding carries the path, status, bytes and ms it saw", async () => {
157 const fetcher = async (url: string): Promise<Response> =>
158 new URL(url).pathname === "/healthz"
159 ? new Response("not well", { status: 503, headers: { "content-type": "text/plain" } })
160 : new Response("fine", { status: 200, headers: { "content-type": "text/plain" } });
161 const report = await huntTarget("https://prod.example", fetcher);
162 const f = report.findings.find((x) => x.key === "door.public-page:/healthz");
163 expect(f).toBeDefined();
164 const e = readEvidence(evidenceOf(f!, ISO));
165 expect(e).toMatchObject({ url: "/healthz", status: "503", bytes: "8", time: ISO });
166 expect(Number(e!.ms)).toBeGreaterThanOrEqual(0);
167 expect(e!.ms).not.toBe("n/a");
168 });
169
170 test("the contract probe", async () => {
171 const out = await newApp(app, alice, { name: "doors", visibility: "public" });
172 if (!out.ok) throw new Error(String(out.error));
173 await probeApp(app, {} as AppsRuntime, out.value.app, async (input) => ({
174 repo: input.repo, commitOid: "abcdef0123456789", at: ISO, verdict: "mismatch",
175 knocks: [{ resource: "note", verb: "delete", as: "anon", door: "DELETE /api/notes/0", cell: "no", result: { kind: "admitted", status: 404 }, ms: 12.5, bytes: 31, verdict: "mismatch" }],
176 said: ["note.delete as anon: the table says no, the app said 404"],
177 }));
178 const issue = probeIssue(app, out.value.repo);
179 expect(issue).toBeDefined();
180 const e = readEvidence(issue!.body);
181 expect(e).toBeDefined();
182 expect(e).toEqual({ url: "/api/notes/0", status: "404", bytes: "31", ms: "12.5", time: ISO, release: "abcdef012345" });
183 });
184 });
test/maintain-v1-proven.test.ts +20
@@ -12,6 +12,7 @@ import { createUser } from "../src/core/users.ts";
1212 import { probes, type Result } from "../src/core/v1_proof.ts";
1313 import { v1Proven } from "../src/maintain/duties/v1_proven.ts";
1414 import { duties } from "../src/maintain/registry.ts";
15 import { readEvidence } from "./helpers/evidence.ts";
1516 import { quietLogs } from "./helpers/globals.ts";
1617
1718 quietLogs();
@@ -116,3 +117,22 @@ test("no public origin: nothing is called", async () => {
116117 test("the duty is registered, weekly", () => {
117118 expect(duties.find((d) => d.name === "v1-proven")?.intervalMs).toBe(7 * 86_400_000);
118119 });
120
121 test("a filed defect and its still-failing comment carry the evidence block of what the probe read (#13033)", async () => {
122 app.db.query("UPDATE issue SET state = 'closed' WHERE repo_id = ?1 AND title LIKE 'Defect of #%'").run(repoId);
123 const before = defects().length;
124 const bad = probes[0]!;
125 const seen = { url: "https://forge.example/", status: 500, bytes: 12, ms: 41.5 };
126 const proof = async (): Promise<Result[]> =>
127 probes.map((p) => (p.issue === bad.issue ? { issue: p.issue, item: p.item, status: "fail", detail: "/ answers 500, not 200", seen } : { issue: p.issue, item: p.item, status: "pass", detail: "ok" }));
128 await run(undefined, undefined, proof);
129 const made = defects().slice(before);
130 expect(made).toHaveLength(1);
131 const body = app.db.query<{ body: string }, [string]>("SELECT body FROM issue WHERE id = ?1").get(made[0]!.id)!.body;
132 const ev = readEvidence(body)!;
133 expect(ev).toMatchObject({ url: seen.url, status: "500", bytes: "12", ms: "41.5", time: NOW.toISOString() });
134 expect(ev.release).not.toBe("");
135 await run(undefined, undefined, proof);
136 const comment = listComments(app.db, made[0]!.id).at(-1)!;
137 expect(readEvidence(comment.body)).toMatchObject({ url: seen.url, status: "500", bytes: "12" });
138 });
test/probe.test.ts +2
@@ -126,6 +126,8 @@ describe("the probe asks the running app whether it obeys its table", () => {
126126 // Every yes/no cell across five doors and three identities was asked.
127127 expect(asked.length).toBeGreaterThanOrEqual(10);
128128 expect(asked.every((k) => k.verdict === "ok")).toBe(true);
129 // Each asked door says how long it took and how many bytes it sent (#13033).
130 expect(asked.every((k) => typeof k.ms === "number" && k.ms >= 0 && typeof k.bytes === "number")).toBe(true);
129131 // The own cells were not: they need a row that names the actor.
130132 for (const k of out.knocks.filter((k) => k.cell === "own"))
131133 expect(k.result.kind).toBe("unprobed");
test/v1-proven.test.ts +7
@@ -109,6 +109,13 @@ describe("v1-proven", () => {
109109 expect(line(results.find((r) => r.issue === issue)!)).toStartWith(`fail #${issue}`);
110110 });
111111
112 test("a fail carries the last read the probe made: its URL, status, bytes and ms (#13033)", async () => {
113 const r = (await prove("https://f.test", stub({ "/healthz": () => new Response("down", { status: 503 }) }))).find((x) => x.issue === 11539)!;
114 expect(r.status).toBe("fail");
115 expect(r.seen).toMatchObject({ url: "https://f.test/healthz", status: 503, bytes: 4 });
116 expect(r.seen!.ms).toBeGreaterThanOrEqual(0);
117 });
118
112119 test("the schedule probe fails naming the oldest age, is unproven without the field, and a network error is a fail", async () => {
113120 const late = (await prove("https://f.test", stub({ "/api/v1/health": () => health({ overdue: 2, oldestSeconds: 3 * 3600 + 720 }) }))).find((x) => x.issue === 11595)!;
114121 expect(line(late)).toBe("fail #11595 no schedule is past its period: 2 schedule(s) overdue, the oldest by 3 h 12 min");

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs