Public: anyone can read this; the crew can change it.
A defect the crew files from a failed probe carries the URL, status, bytes and time it saw #13061
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/13033-a-defect-the-crew-files-from-a-failed-pr → main 7 commits fork 3 threads · 3 unresolved
24 files
changed +445
−37
scripts/hunt.ts +2 −1
| @@ -4,6 +4,7 @@ | ||
| 4 | 4 | * Never run on a person's machine in a loop: it is one pass, then it ends. |
| 5 | 5 | */ |
| 6 | 6 | |
| 7 | import { evidenceOf } from "../src/hunt/finding.ts"; | |
| 7 | 8 | import { huntTarget } from "../src/hunt/hunt.ts"; |
| 8 | 9 | |
| 9 | 10 | const target = process.argv[2]; |
| @@ -12,5 +13,5 @@ if (!target || !/^https?:\/\//.test(target)) { | ||
| 12 | 13 | process.exit(2); |
| 13 | 14 | } |
| 14 | 15 | const report = await huntTarget(target.replace(/\/$/, "")); |
| 15 | console.log(JSON.stringify(report, null, 2)); | |
| 16 | console.log(JSON.stringify({ ...report, findings: report.findings.map((f) => ({ ...f, evidence: evidenceOf(f) })) }, null, 2)); | |
| 16 | 17 | process.exit(report.findings.length === 0 ? 0 : 1); |
src/contract/probe.ts +10 −1
| @@ -24,6 +24,7 @@ import type { AppRecord } from "../core/apps.ts"; | ||
| 24 | 24 | import type { Repo } from "../core/repos.ts"; |
| 25 | 25 | import type { Cell, Contract, Verb } from "./can.ts"; |
| 26 | 26 | import type { Standing } from "./audit.ts"; |
| 27 | import { readBounded } from "../hunt/probe.ts"; | |
| 27 | 28 | |
| 28 | 29 | /** The probe's identities: what the forge can sign an app a visitor as. */ |
| 29 | 30 | export type Identity = "anon" | "user" | "admin"; |
| @@ -44,6 +45,9 @@ export interface Knock { | ||
| 44 | 45 | | { readonly kind: "no_door"; readonly status: number } |
| 45 | 46 | | { readonly kind: "unprobed"; readonly why: string } |
| 46 | 47 | | { readonly kind: "error"; readonly message: string }; |
| 48 | /** Milliseconds the door took and the bytes of its body; absent when it was not asked. */ | |
| 49 | readonly ms?: number; | |
| 50 | readonly bytes?: number; | |
| 47 | 51 | /** The verdict for this knock: does the door agree with the table? */ |
| 48 | 52 | readonly verdict: "ok" | "mismatch" | "unprobed"; |
| 49 | 53 | } |
| @@ -149,6 +153,8 @@ export async function probe(input: ProbeInput): Promise<Probed | undefined> { | ||
| 149 | 153 | })), |
| 150 | 154 | }; |
| 151 | 155 | let result: Knock["result"]; |
| 156 | let measured: { ms: number; bytes: number } | undefined; | |
| 157 | const asked = performance.now(); | |
| 152 | 158 | try { |
| 153 | 159 | const res = await doFetch(url, { |
| 154 | 160 | method: door.method, |
| @@ -157,7 +163,9 @@ export async function probe(input: ProbeInput): Promise<Probed | undefined> { | ||
| 157 | 163 | signal: AbortSignal.timeout(input.timeoutMs ?? 5_000), |
| 158 | 164 | redirect: "manual", |
| 159 | 165 | }); |
| 160 | await res.body?.cancel(); | |
| 166 | // Bounded: a probed app cannot make the prober hold more than readBounded's cap. | |
| 167 | const { size: bytes } = await readBounded(res); | |
| 168 | measured = { ms: performance.now() - asked, bytes }; | |
| 161 | 169 | // A door refuses with 401 or 403 before it looks for a row, so a 404 |
| 162 | 170 | // in JSON is the action saying "no such row": past the door. A 404 |
| 163 | 171 | // that is not JSON is the router: there is no such door. |
| @@ -184,6 +192,7 @@ export async function probe(input: ProbeInput): Promise<Probed | undefined> { | ||
| 184 | 192 | door: `${door.method} ${path}`, |
| 185 | 193 | cell: cell.answer, |
| 186 | 194 | result, |
| 195 | ...(measured === undefined ? {} : { ms: Math.round(measured.ms * 10) / 10, bytes: measured.bytes }), | |
| 187 | 196 | verdict, |
| 188 | 197 | }); |
| 189 | 198 | } |
src/contract/probes.ts +11
| @@ -23,6 +23,7 @@ import { createLog } from "../kernel/log.ts"; | ||
| 23 | 23 | import { audit } from "./audit.ts"; |
| 24 | 24 | import { probe, type Knock, type ProbeInput, type Probed } from "./probe.ts"; |
| 25 | 25 | import { during } from "../perf/jobs.ts"; |
| 26 | import { evidenceBlock, type Evidence } from "../maintain/evidence.ts"; | |
| 26 | 27 | |
| 27 | 28 | const log = createLog("probes"); |
| 28 | 29 | |
| @@ -146,6 +147,14 @@ export async function probeApp( | ||
| 146 | 147 | return row; |
| 147 | 148 | } |
| 148 | 149 | |
| 150 | /** What the first disagreeing door answered: its path, status, bytes and milliseconds. */ | |
| 151 | function seenOf(probed: Probed): Evidence { | |
| 152 | const k = probed.knocks.find((n) => n.verdict === "mismatch"); | |
| 153 | if (k === undefined) return { at: probed.at }; | |
| 154 | const status = k.result.kind === "admitted" || k.result.kind === "refused" ? k.result.status : undefined; | |
| 155 | return { where: k.door?.split(" ")[1], status, bytes: k.bytes, ms: k.ms, at: probed.at }; | |
| 156 | } | |
| 157 | ||
| 149 | 158 | /** One issue while the app disobeys; closed by the first clean probe after. */ |
| 150 | 159 | function reconcileIssue(app: App, repo: Repo, probed: Probed): void { |
| 151 | 160 | const open = probeIssue(app, repo); |
| @@ -161,6 +170,8 @@ function reconcileIssue(app: App, repo: Repo, probed: Probed): void { | ||
| 161 | 170 | ...probed.said.map((s) => `- ${s}`), |
| 162 | 171 | "", |
| 163 | 172 | "The table is the policy. Fix the door, not the table; the app's own `src/contract.test.ts` acts as every identity at every door and should have caught this. This issue closes itself when the next probe comes back clean.", |
| 173 | "", | |
| 174 | evidenceBlock({ ...seenOf(probed), release: probed.commitOid.slice(0, 12) || undefined }), | |
| 164 | 175 | ].join("\n"); |
| 165 | 176 | const opened = openIssue(app, as, repo, { title: PROBE_ISSUE_TITLE, body }); |
| 166 | 177 | if (!opened.ok) |
src/core/v1_proof.ts +20 −5
| @@ -16,6 +16,14 @@ export interface Result { | ||
| 16 | 16 | readonly item: string; |
| 17 | 17 | readonly status: Status; |
| 18 | 18 | readonly detail: string; |
| 19 | /** On a fail, the last read the probe made: the URL, its status, the bytes read and the ms. */ | |
| 20 | readonly seen?: Seen; | |
| 21 | } | |
| 22 | export interface Seen { | |
| 23 | readonly url: string; | |
| 24 | readonly status: number; | |
| 25 | readonly bytes: number; | |
| 26 | readonly ms: number; | |
| 19 | 27 | } |
| 20 | 28 | interface Got { |
| 21 | 29 | readonly status: number; |
| @@ -233,18 +241,25 @@ export function ageWords(seconds: number): string { | ||
| 233 | 241 | /** Runs every probe against `base` with GET only; a probe that throws is a fail. */ |
| 234 | 242 | export async function prove(base: string, fetcher: Fetch): Promise<Result[]> { |
| 235 | 243 | const root = base.replace(/\/+$/, ""); |
| 236 | const get: Get = async (path) => { | |
| 237 | const res = await fetcher(root + path, { method: "GET", redirect: "manual", signal: AbortSignal.timeout(15_000) }); | |
| 238 | return { status: res.status, headers: res.headers, text: await res.text(), location: res.headers.get("location") }; | |
| 239 | }; | |
| 240 | 244 | // Side by side: each probe is one GET at a time, so the proof takes the slowest, not the sum. |
| 241 | 245 | return Promise.all( |
| 242 | 246 | probes.map(async (p): Promise<Result> => { |
| 247 | let last: Seen | undefined; | |
| 248 | const get: Get = async (path) => { | |
| 249 | const url = root + path; | |
| 250 | const asked = performance.now(); | |
| 251 | last = { url, status: 0, bytes: 0, ms: 0 }; | |
| 252 | const res = await fetcher(url, { method: "GET", redirect: "manual", signal: AbortSignal.timeout(15_000) }); | |
| 253 | const text = await res.text(); | |
| 254 | last = { url, status: res.status, bytes: Buffer.byteLength(text), ms: Math.round((performance.now() - asked) * 10) / 10 }; | |
| 255 | return { status: res.status, headers: res.headers, text, location: res.headers.get("location") }; | |
| 256 | }; | |
| 243 | 257 | try { |
| 244 | 258 | const r = await p.run(get); |
| 245 | 259 | return r instanceof Unproven ? { issue: p.issue, item: p.item, status: "unproven", detail: r.why } : { issue: p.issue, item: p.item, status: "pass", detail: r }; |
| 246 | 260 | } catch (e) { |
| 247 | return { issue: p.issue, item: p.item, status: "fail", detail: e instanceof Fail ? e.message : `probe error: ${String(e)}` }; | |
| 261 | const detail = e instanceof Fail ? e.message : `probe error: ${String(e)}`; | |
| 262 | return { issue: p.issue, item: p.item, status: "fail", detail, ...(last === undefined ? {} : { seen: last }) }; | |
| 248 | 263 | } |
| 249 | 264 | }), |
| 250 | 265 | ); |
src/crew/sentinel.ts +8
| @@ -53,6 +53,8 @@ import { closeCaughtUp, mainAhead } from "./sentinel_main_ahead.ts"; | ||
| 53 | 53 | import { closeLaneMoving, laneStopped } from "./sentinel_lane.ts"; |
| 54 | 54 | import type { FindingKind } from "../hunt/finding.ts"; |
| 55 | 55 | import { addLabels, kindLabels } from "../hunt/label.ts"; |
| 56 | import { evidenceBlock, type Evidence } from "../maintain/evidence.ts"; | |
| 57 | import { seenCause } from "./sentinel_seen.ts"; | |
| 56 | 58 | |
| 57 | 59 | const log = createLog("sentinel"); |
| 58 | 60 | |
| @@ -90,6 +92,8 @@ export interface Finding { | ||
| 90 | 92 | readonly endedAt?: string; |
| 91 | 93 | /** The vitals cause a budget-breach or degraded finding reads. */ |
| 92 | 94 | readonly cause?: string; |
| 95 | /** What the detector read, for the issue's evidence block (#13033): path and measurement. */ | |
| 96 | readonly seen?: Evidence; | |
| 93 | 97 | } |
| 94 | 98 | |
| 95 | 99 | const QUOTE_MAX = 200; |
| @@ -449,6 +453,7 @@ export function degradedCauses(db: Database, now: number, forgeRepoId?: string, | ||
| 449 | 453 | lastOpenedAt: new Date(d.lastOpened).toISOString(), |
| 450 | 454 | ...(Number.isFinite(d.ended) ? { endedAt: new Date(d.ended).toISOString() } : {}), |
| 451 | 455 | cause, |
| 456 | seen: seenCause(cause, d.href, d.lastOpened, dataDir, now), | |
| 452 | 457 | }); |
| 453 | 458 | } |
| 454 | 459 | return out; |
| @@ -521,6 +526,7 @@ export function budgetBreaches(db: Database, now: number, forgeRepoId?: string, | ||
| 521 | 526 | : "Diagnose the cause from the evidence and hand the mechanical fix to a builder as its own issue.", |
| 522 | 527 | urgent: true, |
| 523 | 528 | cause: s.cause, |
| 529 | seen: seenCause(s.cause, s.href, s.openedAt, dataDir, now), | |
| 524 | 530 | }); |
| 525 | 531 | } |
| 526 | 532 | return out; |
| @@ -657,6 +663,8 @@ const bodyOf = (f: Finding, now: string): string => | ||
| 657 | 663 | `**What to look for when fixed:** ${f.lookFor}`, |
| 658 | 664 | ...(f.fix === undefined ? [] : ["", `**Mechanical fix:** ${f.fix}`]), |
| 659 | 665 | "", |
| 666 | evidenceBlock({ ...f.seen, at: now }), | |
| 667 | "", | |
| 660 | 668 | `Found by the sentinel at ${now}.`, |
| 661 | 669 | ].join("\n"); |
| 662 | 670 |
src/crew/sentinel_seen.ts added +31
| @@ -0,0 +1,31 @@ | ||
| 1 | /** | |
| 2 | * What the sentinel read when it filed a degraded cause (#13033), for the evidence block of the | |
| 3 | * issue: the path of the page that says it (`href`), and the number the cause is about where the | |
| 4 | * forge holds one: the free bytes of a full disk, the newest snapshot's size for a backup. A | |
| 5 | * cause that is a state, not a measurement, has no status or milliseconds: they print `n/a`. | |
| 6 | */ | |
| 7 | ||
| 8 | import { readDisk } from "../kernel/disk_reading.ts"; | |
| 9 | import type { Evidence } from "../maintain/evidence.ts"; | |
| 10 | import { backupVerdict } from "../snapshot/bucket.ts"; | |
| 11 | ||
| 12 | const BACKUP_CAUSE = /^(backup|snapshot|mirror|drill)/; | |
| 13 | ||
| 14 | /** The newest snapshot's size across the lanes, when a lane has read one. */ | |
| 15 | function newestSnapshotBytes(dataDir: string, now: number): number | undefined { | |
| 16 | const lanes = backupVerdict(dataDir, undefined, now).lanes.filter((l) => l.newestBytes !== null && l.newestAt !== null); | |
| 17 | lanes.sort((a, b) => (b.newestAt ?? "").localeCompare(a.newestAt ?? "")); | |
| 18 | return lanes[0]?.newestBytes ?? undefined; | |
| 19 | } | |
| 20 | ||
| 21 | export function seenCause(cause: string, href: string, at: number, dataDir: string | undefined, now: number): Evidence { | |
| 22 | const base: Evidence = { where: href, at: new Date(at).toISOString() }; | |
| 23 | if (dataDir === undefined) return base; | |
| 24 | try { | |
| 25 | if (cause.startsWith("disk")) return { ...base, bytes: readDisk(dataDir).freeBytes }; | |
| 26 | if (BACKUP_CAUSE.test(cause)) return { ...base, bytes: newestSnapshotBytes(dataDir, now) }; | |
| 27 | } catch { | |
| 28 | // A volume that cannot be read is the cause itself: the block prints n/a for its size. | |
| 29 | } | |
| 30 | return base; | |
| 31 | } |
src/efficiency/finding.ts +4 −2
| @@ -1,5 +1,6 @@ | ||
| 1 | 1 | /** The words of a finding and its comments (#7558): the cause, the number, the baseline, the code path. */ |
| 2 | 2 | |
| 3 | import { evidenceBlock } from "../maintain/evidence.ts"; | |
| 3 | 4 | import { show, type Cause } from "./causes.ts"; |
| 4 | 5 | import type { HourVerdict, Why } from "./judge.ts"; |
| 5 | 6 | |
| @@ -18,12 +19,13 @@ export function numbers(c: Cause, v: HourVerdict, why: Why): string { | ||
| 18 | 19 | return `${show(c, v.value)} in the hour of ${at(v.hour)}; ${base}; ${rule}.`; |
| 19 | 20 | } |
| 20 | 21 | |
| 21 | export function bodyOf(c: Cause, v: HourVerdict, why: Why): string { | |
| 22 | export function bodyOf(c: Cause, v: HourVerdict, why: Why, now: Date = new Date()): string { | |
| 22 | 23 | return ( |
| 23 | 24 | `The efficiency watcher (duty \`efficiency-watcher\`) found **${c.label}**: ${numbers(c, v, why)}\n\n` + |
| 24 | 25 | `Code path that starts the cost: ${c.path}.\n\n` + |
| 25 | 26 | `This issue stands while the cost does: it gets a comment each hour it is still bad, and is closed with a note once it has been back under for 24 hours. ` + |
| 26 | `The hourly series is the \`efficiency\` board source.` | |
| 27 | `The hourly series is the \`efficiency\` board source.\n\n` + | |
| 28 | evidenceBlock({ where: `efficiency board: ${c.id}`, bytes: c.unit === "bytes" ? v.value : undefined, at: now.toISOString() }) | |
| 27 | 29 | ); |
| 28 | 30 | } |
| 29 | 31 |
src/efficiency/watch.ts +1 −1
| @@ -52,7 +52,7 @@ export function watch(app: App, now: number, readings: Readings = readNow(app.db | ||
| 52 | 52 | } |
| 53 | 53 | continue; |
| 54 | 54 | } |
| 55 | const filed = openIssue(app, actor, repo, { title: titleOf(c), body: bodyOf(c, stands.now, stands.why) }); | |
| 55 | const filed = openIssue(app, actor, repo, { title: titleOf(c), body: bodyOf(c, stands.now, stands.why, new Date(now)) }); | |
| 56 | 56 | if (filed.ok) { |
| 57 | 57 | putStanding(db, { cause: c.id, issueId: filed.value.id, lastBad: hour }); |
| 58 | 58 | actions.push(`filed #${filed.value.number} (${c.id})`); |
src/http/server.ts +6 −1
| @@ -2730,6 +2730,11 @@ export async function serve(app: App): Promise<Servers> { | ||
| 2730 | 2730 | const readWriter = createReadWriter(app.db, readLedger, app.log); |
| 2731 | 2731 | // The worst event-loop lag of the last 10 s, on every response (#10288). |
| 2732 | 2732 | startLoopWorst(); |
| 2733 | /** The body size the response declared, when it declared one: a stream has none. */ | |
| 2734 | const declaredBytes = (response: Response): { bytes?: number } => { | |
| 2735 | const n = Number(response.headers.get("content-length") ?? NaN); | |
| 2736 | return Number.isFinite(n) ? { bytes: n } : {}; | |
| 2737 | }; | |
| 2733 | 2738 | const counted = ( |
| 2734 | 2739 | owner: App, |
| 2735 | 2740 | request: Request, |
| @@ -2757,7 +2762,7 @@ export async function serve(app: App): Promise<Servers> { | ||
| 2757 | 2762 | const family = familyOf(request.headers.get("user-agent") ?? ""); |
| 2758 | 2763 | liveOf(app).perfRecord.add( |
| 2759 | 2764 | tag.pattern, |
| 2760 | { at: Date.now(), ms, signedIn: tag.role !== "anon" && tag.role !== "crawler", crawler: wasCrawler(request), deepStranger: wasDeepStranger(request), internal: wasInternal(request), refused: response.status === 429 || wasShielded(request), phases: parseServerTiming(header) }, | |
| 2765 | { at: Date.now(), ms, signedIn: tag.role !== "anon" && tag.role !== "crawler", crawler: wasCrawler(request), deepStranger: wasDeepStranger(request), internal: wasInternal(request), refused: response.status === 429 || wasShielded(request), status: response.status, ...declaredBytes(response), phases: parseServerTiming(header) }, | |
| 2761 | 2766 | { |
| 2762 | 2767 | address: clientAddress(request), |
| 2763 | 2768 | family, |
src/hunt/contract.ts +3 −3
| @@ -19,17 +19,17 @@ export async function contract(p: Probe): Promise<Finding[]> { | ||
| 19 | 19 | try { |
| 20 | 20 | parsed = JSON.parse(doc.body) as Doc; |
| 21 | 21 | } catch { |
| 22 | return [found("contract.openapi", "/contract/openapi.json", "the OpenAPI document is not JSON", seenLine("/contract/openapi.json", doc.status))]; | |
| 22 | return [found("contract.openapi", "/contract/openapi.json", "the OpenAPI document is not JSON", seenLine("/contract/openapi.json", doc.status), doc)]; | |
| 23 | 23 | } |
| 24 | 24 | for (const [path, ops] of Object.entries(parsed.paths ?? {})) { |
| 25 | 25 | const get = ops.get; |
| 26 | 26 | if (get === undefined || /[{}]/.test(path)) continue; |
| 27 | 27 | const r = await p.get(path); |
| 28 | if (r.status === 0 || r.status >= 500) out.push(found("contract.no-500", path, `${path} answers ${r.status}`, seenLine(path, r.status))); | |
| 28 | if (r.status === 0 || r.status >= 500) out.push(found("contract.no-500", path, `${path} answers ${r.status}`, seenLine(path, r.status), r)); | |
| 29 | 29 | else if (get.responses && Object.keys(get.responses).length > 0) { |
| 30 | 30 | const code = String(r.status); |
| 31 | 31 | if (!(code in get.responses) && !REFUSALS.has(code) && !("default" in get.responses)) |
| 32 | out.push(found("contract.documented", path, `${path} answers ${r.status}, which its entry does not document`, `${seenLine(path, r.status)}; documented: ${Object.keys(get.responses).join(", ")}`)); | |
| 32 | out.push(found("contract.documented", path, `${path} answers ${r.status}, which its entry does not document`, `${seenLine(path, r.status)}; documented: ${Object.keys(get.responses).join(", ")}`, r)); | |
| 33 | 33 | } |
| 34 | 34 | } |
| 35 | 35 | return out; |
src/hunt/door.ts +4 −4
| @@ -16,16 +16,16 @@ export async function frontDoor(p: Probe): Promise<Finding[]> { | ||
| 16 | 16 | const r = await p.get(path); |
| 17 | 17 | const sent = r.status >= 300 && r.status < 400 && /\/signin(\?|$)/.test(r.location ?? ""); |
| 18 | 18 | if (!sent) |
| 19 | out.push(found("door.signin-redirect", path, `${path} does not send an anonymous visitor to sign-in`, seenLine(path, r.status, r.location))); | |
| 19 | out.push(found("door.signin-redirect", path, `${path} does not send an anonymous visitor to sign-in`, seenLine(path, r.status, r.location), r)); | |
| 20 | 20 | else if (!(r.location ?? "").includes("next=")) |
| 21 | out.push(found("door.signin-next", path, `${path} sends to sign-in without a way back`, seenLine(path, r.status, r.location))); | |
| 21 | out.push(found("door.signin-next", path, `${path} sends to sign-in without a way back`, seenLine(path, r.status, r.location), r)); | |
| 22 | 22 | } |
| 23 | 23 | for (const path of PUBLIC_PAGES) { |
| 24 | 24 | const r = await p.get(path); |
| 25 | if (r.status !== 200) out.push(found("door.public-page", path, `${path} does not answer 200 to an anonymous visitor`, seenLine(path, r.status, r.location))); | |
| 25 | if (r.status !== 200) out.push(found("door.public-page", path, `${path} does not answer 200 to an anonymous visitor`, seenLine(path, r.status, r.location), r)); | |
| 26 | 26 | } |
| 27 | 27 | const missing = "/no-such-page-hunter"; |
| 28 | 28 | const r = await p.get(missing); |
| 29 | if (r.status !== 404) out.push(found("door.unknown-404", missing, "an unknown path does not answer 404", seenLine(missing, r.status, r.location))); | |
| 29 | if (r.status !== 404) out.push(found("door.unknown-404", missing, "an unknown path does not answer 404", seenLine(missing, r.status, r.location), r)); | |
| 30 | 30 | return out; |
| 31 | 31 | } |
src/hunt/finding.ts +12 −1
| @@ -1,5 +1,8 @@ | ||
| 1 | 1 | /** What a detector says: plain data, one issue per `key` (the sentinel's rule). */ |
| 2 | 2 | |
| 3 | import { evidenceBlock, type Evidence } from "../maintain/evidence.ts"; | |
| 4 | import type { Seen } from "./probe.ts"; | |
| 5 | ||
| 3 | 6 | /** |
| 4 | 7 | * What a finding is, so it is filed already sorted (#8856): a security hole or |
| 5 | 8 | * a route that answers broken is a `high` bug (the 4 h clock in |
| @@ -50,15 +53,23 @@ export interface Finding { | ||
| 50 | 53 | title: string; |
| 51 | 54 | /** The minimal repro: the request and what came back. */ |
| 52 | 55 | evidence: string; |
| 56 | /** What the detector read, for the evidence block of the issue (#13033). */ | |
| 57 | seen?: Evidence; | |
| 53 | 58 | } |
| 54 | 59 | |
| 55 | export const found = (detector: Detector, subject: string, title: string, evidence: string): Finding => ({ | |
| 60 | /** `r` is the answer the detector read: its path, status, bytes and ms go in the block. */ | |
| 61 | export const found = (detector: Detector, subject: string, title: string, evidence: string, r?: Seen): Finding => ({ | |
| 56 | 62 | detector, |
| 57 | 63 | kind: DETECTOR_KINDS[detector], |
| 58 | 64 | key: `${detector}:${subject}`, |
| 59 | 65 | title, |
| 60 | 66 | evidence, |
| 67 | ...(r === undefined ? {} : { seen: { where: r.path, status: r.status === 0 ? undefined : r.status, bytes: r.bytes, ms: r.ms } }), | |
| 61 | 68 | }); |
| 62 | 69 | |
| 70 | /** The text an issue is filed with: the repro, then the evidence block of what was read. */ | |
| 71 | export const evidenceOf = (f: Finding, at?: string): string => | |
| 72 | `${f.evidence}\n\n${evidenceBlock({ ...f.seen, ...(at === undefined ? {} : { at }) })}`; | |
| 73 | ||
| 63 | 74 | export const seenLine = (path: string, status: number, location?: string | null): string => |
| 64 | 75 | `GET ${path} → ${status}${location ? ` → ${location}` : ""}`; |
src/hunt/probe.ts +15 −9
| @@ -12,6 +12,9 @@ export interface Seen { | ||
| 12 | 12 | location: string | null; |
| 13 | 13 | type: string; |
| 14 | 14 | body: string; |
| 15 | /** Bytes of the body as read (bounded), and milliseconds to the answer. */ | |
| 16 | bytes: number; | |
| 17 | ms: number; | |
| 15 | 18 | } |
| 16 | 19 | |
| 17 | 20 | export interface Probe { |
| @@ -32,20 +35,21 @@ export function inside(base: string, href: string): string | null { | ||
| 32 | 35 | |
| 33 | 36 | const BODY_MAX = 400_000; |
| 34 | 37 | |
| 35 | /** At most BODY_MAX bytes of the body; the rest is never read. */ | |
| 36 | async function readBounded(res: Response): Promise<string> { | |
| 38 | /** At most BODY_MAX bytes of the body, and how many were read; the rest is never read. */ | |
| 39 | export async function readBounded(res: Response, max = BODY_MAX): Promise<{ bytes: Uint8Array; size: number }> { | |
| 37 | 40 | const reader = res.body?.getReader(); |
| 38 | if (!reader) return ""; | |
| 41 | if (!reader) return { bytes: new Uint8Array(0), size: 0 }; | |
| 39 | 42 | const chunks: Uint8Array[] = []; |
| 40 | 43 | let size = 0; |
| 41 | while (size < BODY_MAX) { | |
| 44 | while (size < max) { | |
| 42 | 45 | const { done, value } = await reader.read(); |
| 43 | 46 | if (done) break; |
| 44 | 47 | chunks.push(value); |
| 45 | 48 | size += value.length; |
| 46 | 49 | } |
| 47 | 50 | await reader.cancel().catch(() => {}); |
| 48 | return new TextDecoder().decode(Buffer.concat(chunks).subarray(0, BODY_MAX)); | |
| 51 | const bytes = Buffer.concat(chunks).subarray(0, max); | |
| 52 | return { bytes, size: bytes.length }; | |
| 49 | 53 | } |
| 50 | 54 | |
| 51 | 55 | export function probe(base: string, fetcher: Fetcher = fetch as Fetcher): Probe { |
| @@ -53,6 +57,7 @@ export function probe(base: string, fetcher: Fetcher = fetch as Fetcher): Probe | ||
| 53 | 57 | async get(path) { |
| 54 | 58 | const target = inside(base, path); |
| 55 | 59 | if (target === null) throw new Error(`refusing a path off the target: ${path}`); |
| 60 | const asked = performance.now(); | |
| 56 | 61 | try { |
| 57 | 62 | const res = await fetcher(new URL(target, base).href, { |
| 58 | 63 | method: "GET", |
| @@ -61,12 +66,13 @@ export function probe(base: string, fetcher: Fetcher = fetch as Fetcher): Probe | ||
| 61 | 66 | signal: AbortSignal.timeout(10_000), |
| 62 | 67 | }); |
| 63 | 68 | const type = res.headers.get("content-type") ?? ""; |
| 64 | const body = /text|json|xml/.test(type) ? await readBounded(res) : ""; | |
| 65 | await res.body?.cancel().catch(() => {}); | |
| 66 | return { path: target, status: res.status, location: res.headers.get("location"), type, body }; | |
| 69 | // Bytes are what was read (bounded), never the target's own Content-Length claim. | |
| 70 | const read = await readBounded(res); | |
outdated
bytes comes from the target's Content-Length header, not what was read; use the bytes actually read. |
||
| 71 | const body = /text|json|xml/.test(type) ? new TextDecoder().decode(read.bytes) : ""; | |
| 72 | return { path: target, status: res.status, location: res.headers.get("location"), type, body, bytes: read.size, ms: performance.now() - asked }; | |
| 67 | 73 | } catch { |
| 68 | 74 | // A dead or hung route is a result (status 0), not a crash that loses the pass. |
| 69 | return { path: target, status: 0, location: null, type: "", body: "" }; | |
| 75 | return { path: target, status: 0, location: null, type: "", body: "", bytes: 0, ms: performance.now() - asked }; | |
| 70 | 76 | } |
| 71 | 77 | }, |
| 72 | 78 | }; |
src/maintain/evidence.ts added +54
| @@ -0,0 +1,54 @@ | ||
| 1 | /** | |
| 2 | * The evidence block a defect carries (#13033): where the duty looked, what | |
| 3 | * answered, how many bytes, how long it took, when, and which release it ran | |
| 4 | * against. One fenced block, one line per field, so the next builder reads | |
| 5 | * what the duty saw instead of reproducing it. A field the duty does not have | |
| 6 | * is printed `n/a`, never omitted, so the shape is always the same. | |
| 7 | */ | |
| 8 | ||
| 9 | import { running } from "../kernel/version.ts"; | |
| 10 | ||
| 11 | export interface Evidence { | |
| 12 | /** The URL or path the duty read (a route, a data path); n/a when it read neither. */ | |
| 13 | readonly where?: string | undefined; | |
| 14 | readonly status?: number | undefined; | |
| 15 | readonly bytes?: number | undefined; | |
| 16 | readonly ms?: number | undefined; | |
| 17 | /** ISO time of the reading; defaults to now. */ | |
| 18 | readonly at?: string | undefined; | |
| 19 | /** The release this process runs; defaults to the stamped commit or version. */ | |
| 20 | readonly release?: string | undefined; | |
| 21 | } | |
| 22 | ||
| 23 | export const NA = "n/a"; | |
| 24 | ||
| 25 | const FIELDS = ["url", "status", "bytes", "ms", "time", "release"] as const; | |
| 26 | export type EvidenceField = (typeof FIELDS)[number]; | |
| 27 | ||
| 28 | const number = (n: number | undefined): string => (n === undefined || !Number.isFinite(n) ? NA : String(Math.round(n * 10) / 10)); | |
| 29 | /** One line of text: a newline or a fence in a value cannot break out of the block. */ | |
| 30 | const line = (s: string | undefined): string => { | |
| 31 | const t = (s ?? "").replace(/[\r\n`]+/g, " ").trim(); | |
| 32 | return t === "" ? NA : t.slice(0, 300); | |
| 33 | }; | |
| 34 | ||
| 35 | /** The release the process runs, as a short commit or the version. */ | |
| 36 | export function releaseNow(env: Readonly<Record<string, string | undefined>> = process.env): string { | |
| 37 | const r = running(env); | |
| 38 | return r.commit !== null ? r.commit.slice(0, 12) : r.version; | |
| 39 | } | |
| 40 | ||
| 41 | /** The fenced block, with all six lines (the issue's five facts; the URL or path is `url`). */ | |
| 42 | export function evidenceBlock(e: Evidence, now: () => Date = () => new Date()): string { | |
| 43 | const at = e.at !== undefined && !Number.isNaN(Date.parse(e.at)) ? new Date(Date.parse(e.at)).toISOString() : now().toISOString(); | |
| 44 | return [ | |
| 45 | "```evidence", | |
| 46 | `url: ${line(e.where)}`, | |
| 47 | `status: ${number(e.status)}`, | |
|
||
| 48 | `bytes: ${number(e.bytes)}`, | |
| 49 | `ms: ${number(e.ms)}`, | |
| 50 | `time: ${at}`, | |
| 51 | `release: ${line(e.release ?? releaseNow())}`, | |
| 52 | "```", | |
| 53 | ].join("\n"); | |
| 54 | } |
src/maintain/proven.ts +2 −1
| @@ -10,6 +10,7 @@ import { EventBus } from "../core/events.ts"; | ||
| 10 | 10 | import { addComment, createIssue, getIssueById } from "../core/issues.ts"; |
| 11 | 11 | import { line, prove, type Proof, type Result } from "../core/v1_proof.ts"; |
| 12 | 12 | import { watcherId } from "../rules/bot.ts"; |
| 13 | import { evidenceBlock } from "./evidence.ts"; | |
| 13 | 14 | |
| 14 | 15 | export const defectTitle = (r: Pick<Result, "issue" | "item">): string => `Defect of #${r.issue}: ${r.item}`; |
| 15 | 16 | |
| @@ -40,7 +41,7 @@ export function reportFailures(db: Database, repoId: string, results: readonly R | ||
| 40 | 41 | const bot = watcherId(db); |
| 41 | 42 | for (const r of results) { |
| 42 | 43 | if (r.status !== "fail") continue; |
| 43 | const quote = `> ${line(r).replace(/\r?\n/g, " ")}`; | |
| 44 | const quote = `> ${line(r).replace(/\r?\n/g, " ")}\n\n${evidenceBlock({ ...(r.seen === undefined ? {} : { where: r.seen.url, status: r.seen.status, bytes: r.seen.bytes, ms: r.seen.ms }), at: at.toISOString() })}`; | |
| 44 | 45 | const open = openDefect(db, repoId, r.issue); |
| 45 | 46 | if (open !== undefined) { |
| 46 | 47 | const c = addComment(db, { issueId: open, authorId: bot, body: `Still failing on ${at.toISOString().slice(0, 10)}:\n\n${quote}` }); |
src/perf/duty.ts +2 −1
| @@ -13,6 +13,7 @@ import { openIssue } from "../actions/issues.ts"; | ||
| 13 | 13 | import { getUserById } from "../core/users.ts"; |
| 14 | 14 | import { getRepo } from "../core/repos.ts"; |
| 15 | 15 | import { listIssues } from "../core/issues.ts"; |
| 16 | import { evidenceOf } from "../hunt/finding.ts"; | |
| 16 | 17 | import { addLabels, kindLabels } from "../hunt/label.ts"; |
| 17 | 18 | import { createGitRecord } from "./git.ts"; |
| 18 | 19 | import type { PerfRecord } from "./record.ts"; |
| @@ -50,7 +51,7 @@ export function startPerfWatcher( | ||
| 50 | 51 | const findings = [...evaluate(record, now()), ...evaluateGit(gitRecord, now())]; |
| 51 | 52 | for (const f of fresh(findings, filed)) { |
| 52 | 53 | if (open.has(f.title)) continue; |
| 53 | const made = openIssue(app, actor, repo, { title: f.title, body: f.evidence }); | |
| 54 | const made = openIssue(app, actor, repo, { title: f.title, body: evidenceOf(f, new Date(now()).toISOString()) }); | |
| 54 | 55 | if (made.ok) addLabels(app, repo.id, actor.id, made.value, kindLabels(app, repo.id, f.kind)); |
| 55 | 56 | } |
| 56 | 57 | } catch (error) { |
src/perf/record.ts +3
| @@ -35,6 +35,9 @@ export interface Sample { | ||
| 35 | 35 | readonly internal?: boolean; |
| 36 | 36 | /** Answered 429. */ |
| 37 | 37 | readonly refused?: boolean; |
| 38 | /** The status it answered and the bytes its body declared, for a filed defect (#13033). */ | |
| 39 | readonly status?: number; | |
| 40 | readonly bytes?: number; | |
| 38 | 41 | readonly phases: readonly PhaseSample[]; |
| 39 | 42 | } |
| 40 | 43 |
src/perf/watch.ts +16 −7
| @@ -6,10 +6,11 @@ | ||
| 6 | 6 | */ |
| 7 | 7 | |
| 8 | 8 | import { found, type Finding } from "../hunt/finding.ts"; |
| 9 | import type { Evidence } from "../maintain/evidence.ts"; | |
| 9 | 10 | import type { Database } from "bun:sqlite"; |
| 10 | 11 | import { dayOf } from "./day.ts"; |
| 11 | 12 | import { budgetFor, gitBudgetFor, PAGE_BUDGET } from "./budgets.ts"; |
| 12 | import { percentile, summarize, WINDOW_MS, type PerfRecord, type RouteStat } from "./record.ts"; | |
| 13 | import { percentile, summarize, WINDOW_MS, type PerfRecord, type RouteStat, type Sample } from "./record.ts"; | |
| 13 | 14 | |
| 14 | 15 | export { WINDOW_MS }; |
| 15 | 16 | export const MIN_REQUESTS = 20; |
| @@ -38,6 +39,12 @@ export function slowestRoutes(record: PerfRecord, now: number, limit = 10): Slow | ||
| 38 | 39 | return rows.sort((a, b) => b.over - a.over).slice(0, limit).map((r) => r.row); |
| 39 | 40 | } |
| 40 | 41 | |
| 42 | /** The status and bytes of the request that stood at the route's p95. */ | |
| 43 | function atP95(samples: readonly Sample[], p95: number): Pick<Evidence, "status" | "bytes" | "ms"> { | |
| 44 | const at = samples.reduce((best, s) => (Math.abs(s.ms - p95) < Math.abs(best.ms - p95) ? s : best)); | |
| 45 | return { status: at.status, bytes: at.bytes, ms: p95 }; | |
| 46 | } | |
| 47 | ||
| 41 | 48 | export function evaluate(record: PerfRecord, now: number): Finding[] { |
| 42 | 49 | const out: Finding[] = []; |
| 43 | 50 | for (const pattern of record.patterns()) { |
| @@ -63,15 +70,16 @@ export function evaluate(record: PerfRecord, now: number): Finding[] { | ||
| 63 | 70 | : `Cache: ${slow.name} hit ${slow.hits} of ${slow.n} requests (${Math.round((slow.hits / slow.n) * 100)}%).`, |
| 64 | 71 | ); |
| 65 | 72 | } |
| 66 | out.push( | |
| 67 | found( | |
| 73 | out.push({ | |
| 74 | ...found( | |
| 68 | 75 | "performance-watcher", |
| 69 | 76 | `${pattern}:${phase}`, |
| 70 | 77 | // Stable: the numbers move every tick, the title is how "already open" matches. |
| 71 | 78 | `${pattern} is over its budget: ${phase}`, |
| 72 | 79 | lines.join("\n"), |
| 73 | 80 | ), |
| 74 | ); | |
| 81 | seen: { where: pattern, ...atP95(samples, stat.p95) }, | |
| 82 | }); | |
| 75 | 83 | } |
| 76 | 84 | return out; |
| 77 | 85 | } |
| @@ -88,8 +96,8 @@ export function evaluateGit(record: PerfRecord, now: number): Finding[] { | ||
| 88 | 96 | const stat = summarize(record.traffic(key, now - WINDOW_MS)); |
| 89 | 97 | if (stat === null || stat.n < MIN_REQUESTS || stat.p95 <= budget.p95Ms) continue; |
| 90 | 98 | if (now - (record.first(key) ?? now) < WINDOW_MS * 0.9) continue; |
| 91 | out.push( | |
| 92 | found( | |
| 99 | out.push({ | |
| 100 | ...found( | |
| 93 | 101 | "performance-watcher", |
| 94 | 102 | key, |
| 95 | 103 | `${key} is over its budget`, |
| @@ -98,7 +106,8 @@ export function evaluateGit(record: PerfRecord, now: number): Finding[] { | ||
| 98 | 106 | `Subcommand: ${key}, ${stat.n} processes in 15 min: p50 ${Math.round(stat.p50)} ms, p95 ${Math.round(stat.p95)} ms, p99 ${Math.round(stat.p99)} ms.`, |
| 99 | 107 | ].join("\n"), |
| 100 | 108 | ), |
| 101 | ); | |
| 109 | seen: { where: key, ms: stat.p95 }, | |
| 110 | }); | |
| 102 | 111 | } |
| 103 | 112 | return out; |
| 104 | 113 | } |
test/helpers/evidence.ts added +19
| @@ -0,0 +1,19 @@ | ||
| 1 | import type { EvidenceField } from "../../src/maintain/evidence.ts"; | |
| 2 | ||
| 3 | const FIELDS: readonly EvidenceField[] = ["url", "status", "bytes", "ms", "time", "release"]; | |
| 4 | ||
| 5 | /** | |
| 6 | * The fields a body's evidence block names, for a test or a reader; undefined when the body has | |
| 7 | * no block. | |
| 8 | */ | |
| 9 | export function readEvidence(body: string): Record<EvidenceField, string> | undefined { | |
| 10 | const m = /```evidence\n([\s\S]*?)\n```/.exec(body); | |
| 11 | if (m === null) return undefined; | |
| 12 | const out: Partial<Record<EvidenceField, string>> = {}; | |
| 13 | for (const l of (m[1] ?? "").split("\n")) { | |
| 14 | const i = l.indexOf(": "); | |
| 15 | const k = l.slice(0, i) as EvidenceField; | |
| 16 | if (FIELDS.includes(k)) out[k] = l.slice(i + 2); | |
| 17 | } | |
| 18 | return FIELDS.every((f) => out[f] !== undefined) ? (out as Record<EvidenceField, string>) : undefined; | |
| 19 | } |
test/hunt.test.ts +9
| @@ -145,4 +145,13 @@ describe("more detectors", () => { | ||
| 145 | 145 | expect(seen.body.length).toBe(400_000); |
| 146 | 146 | expect(pulled).toBeLessThan(1_000_000); |
| 147 | 147 | }); |
| 148 | ||
| 149 | test("bytes are what was read, not what the target's Content-Length claims (#13033)", async () => { | |
| 150 | const liar: Fetcher = async () => new Response("hello", { headers: { "content-type": "text/plain", "content-length": "999999" } }); | |
| 151 | const seen = await probe(BASE, liar).get("/"); | |
| 152 | expect(seen.bytes).toBe(5); | |
| 153 | const binary: Fetcher = async () => new Response(new Uint8Array(7), { headers: { "content-type": "image/png" } }); | |
| 154 | const img = await probe(BASE, binary).get("/"); | |
| 155 | expect([img.body, img.bytes]).toEqual(["", 7]); | |
| 156 | }); | |
| 148 | 157 | }); |
test/maintain-evidence.test.ts added +184
| @@ -0,0 +1,184 @@ | ||
| 1 | /** | |
| 2 | * Every defect a duty files carries the evidence block (#13033): url, status, | |
| 3 | * bytes, ms, time and release, each printed, `n/a` where the duty has none. | |
| 4 | * One test per filing: the perf watcher (route and git), the efficiency | |
| 5 | * watcher, the sentinel (production probe, backup and disk findings file | |
| 6 | * through it) and the contract probe. | |
| 7 | */ | |
| 8 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 9 | import { mkdtempSync, rmSync } from "node:fs"; | |
| 10 | import { tmpdir } from "node:os"; | |
| 11 | import { join } from "node:path"; | |
| 12 | import { createRepo } from "../src/actions/repos.ts"; | |
| 13 | import { newApp } from "../src/actions/apps.ts"; | |
| 14 | import { createApp, type App } from "../src/app.ts"; | |
| 15 | import type { AppsRuntime } from "../src/apps/runtime.ts"; | |
| 16 | import { probeApp, probeIssue } from "../src/contract/probes.ts"; | |
| 17 | import { listIssues } from "../src/core/issues.ts"; | |
| 18 | import { getRepo } from "../src/core/repos.ts"; | |
| 19 | import { createUser, type User } from "../src/core/users.ts"; | |
| 20 | import { CAUSES } from "../src/efficiency/causes.ts"; | |
| 21 | import { bodyOf as efficiencyBody } from "../src/efficiency/finding.ts"; | |
| 22 | import { hunt as sentinelHunt } from "../src/crew/sentinel.ts"; | |
| 23 | import { huntTarget } from "../src/hunt/hunt.ts"; | |
| 24 | import { evidenceOf } from "../src/hunt/finding.ts"; | |
| 25 | import { provideDiskForTest } from "../src/kernel/disk_reading.ts"; | |
| 26 | import { record as recordDegraded } from "../src/kernel/degraded_history.ts"; | |
| 27 | import { keepReads } from "../src/snapshot/bucket.ts"; | |
| 28 | import { evidenceBlock } from "../src/maintain/evidence.ts"; | |
| 29 | import { readEvidence } from "./helpers/evidence.ts"; | |
| 30 | import { startPerfWatcher } from "../src/perf/duty.ts"; | |
| 31 | import { createGitRecord } from "../src/perf/git.ts"; | |
| 32 | import { createPerfRecord, WINDOW_MS } from "../src/perf/record.ts"; | |
| 33 | import { quietLogs } from "./helpers/globals.ts"; | |
| 34 | import { proveAll } from "./helpers/verified.ts"; | |
| 35 | ||
| 36 | quietLogs(); | |
| 37 | ||
| 38 | const NOW = Date.parse("2026-11-15T10:00:00.000Z"); | |
| 39 | const ISO = new Date(NOW).toISOString(); | |
| 40 | ||
| 41 | describe("the shared formatter", () => { | |
| 42 | test("prints all six lines, n/a for what the duty does not have", () => { | |
| 43 | const block = evidenceBlock({ where: "/explore", status: 503, bytes: 1234, ms: 88.26, at: ISO, release: "abc123def456" }); | |
| 44 | expect(block.startsWith("```evidence\n")).toBe(true); | |
| 45 | expect(block.endsWith("\n```")).toBe(true); | |
| 46 | expect(readEvidence(block)).toEqual({ url: "/explore", status: "503", bytes: "1234", ms: "88.3", time: ISO, release: "abc123def456" }); | |
| 47 | const bare = readEvidence(evidenceBlock({ at: ISO, release: "dev" })); | |
| 48 | expect(bare).toEqual({ url: "n/a", status: "n/a", bytes: "n/a", ms: "n/a", time: ISO, release: "dev" }); | |
| 49 | }); | |
| 50 | test("a value cannot close the fence or add a line", () => { | |
| 51 | const block = evidenceBlock({ where: "/x\n```\n# injected", at: ISO, release: "dev" }); | |
| 52 | expect(block.split("\n")).toHaveLength(8); | |
| 53 | expect(readEvidence(block)?.url).not.toContain("`"); | |
| 54 | }); | |
| 55 | test("the time is printed normalised, never as the caller wrote it", () => { | |
| 56 | const block = evidenceBlock({ at: "2026-11-15 10:00:00Z ```", release: "dev" }); | |
| 57 | const t = readEvidence(block)?.time ?? ""; | |
| 58 | if (!Number.isNaN(Date.parse("2026-11-15 10:00:00Z ```"))) expect(t).toBe(new Date(Date.parse("2026-11-15 10:00:00Z ```")).toISOString()); | |
| 59 | expect(t).toMatch(/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3}Z$/); | |
| 60 | expect(readEvidence(evidenceBlock({ at: "Sun, 15 Nov 2026 10:00:00 GMT", release: "dev" }))?.time).toBe(ISO); | |
| 61 | }); | |
| 62 | test("the release is the stamped commit or version by default", () => { | |
| 63 | expect(readEvidence(evidenceBlock({ at: ISO }))?.release).not.toBe("n/a"); | |
| 64 | }); | |
| 65 | }); | |
| 66 | ||
| 67 | describe("each duty's filing carries the block", () => { | |
| 68 | let root = ""; | |
| 69 | let app: App; | |
| 70 | let alice: User; | |
| 71 | const forge = () => getRepo(app.db, "alice", "forge")!; | |
| 72 | ||
| 73 | beforeAll(async () => { | |
| 74 | root = mkdtempSync(join(tmpdir(), "forge-evidence-")); | |
| 75 | app = createApp({ dataDir: root, port: 0, forgeRepo: "alice/forge" }); | |
| 76 | const u = await createUser(app.db, { | |
| 77 | origin: "seed", trusted: true, username: "alice", email: "[email protected]", | |
| 78 | password: "a-good-passphrase", adminIfFirst: true, | |
| 79 | }); | |
| 80 | if (!u.ok) throw new Error(u.error); | |
| 81 | alice = u.value; | |
| 82 | proveAll(app); | |
| 83 | const made = await createRepo(app, alice, { name: "forge" }); | |
| 84 | if (!made.ok) throw new Error(made.error); | |
| 85 | }); | |
| 86 | afterAll(() => { | |
| 87 | app.close(); | |
| 88 | rmSync(root, { recursive: true, force: true }); | |
| 89 | }); | |
| 90 | ||
| 91 | test("the performance watcher: a slow route and a slow git subcommand", async () => { | |
| 92 | const record = createPerfRecord(); | |
| 93 | const git = createGitRecord(); | |
| 94 | for (let i = 0; i < 100; i++) { | |
| 95 | const at = NOW - WINDOW_MS + (i * WINDOW_MS) / 100; | |
| 96 | record.add("/", { at, ms: 4600, status: 200, bytes: 48213, signedIn: false, phases: [{ name: "kpis", ms: 4400, hit: false }] }); | |
| 97 | git.add("git rev-parse", { at, ms: 9000, signedIn: false, phases: [] }); | |
| 98 | } | |
| 99 | const stop = startPerfWatcher(app, record, () => NOW, 5, git); | |
| 100 | await Bun.sleep(80); // real-timer: the watcher ticks on its own 5 ms interval | |
| 101 | stop(); | |
| 102 | const issues = listIssues(app.db, forge().id, { state: "open" }); | |
| 103 | const route = issues.find((i) => i.title === "/ is over its budget: kpis"); | |
| 104 | const sub = issues.find((i) => i.title.startsWith("git rev-parse")); | |
| 105 | expect(route).toBeDefined(); | |
| 106 | expect(sub).toBeDefined(); | |
| 107 | const r = readEvidence(route!.body); | |
| 108 | expect(r).toBeDefined(); | |
| 109 | expect(r!.url).toBe("/"); | |
| 110 | expect(Number(r!.ms)).toBeGreaterThan(4000); | |
| 111 | expect(r!.status).toBe("200"); | |
| 112 | expect(r!.bytes).toBe("48213"); | |
| 113 | expect(r!.time).toBe(ISO); | |
| 114 | expect(readEvidence(sub!.body)?.url).toBe("git rev-parse"); | |
| 115 | }); | |
| 116 | ||
| 117 | test("the efficiency watcher", () => { | |
| 118 | const c = CAUSES.find((x) => x.unit === "bytes")!; | |
| 119 | const body = efficiencyBody(c, { hour: NOW, value: 5 * 1024 ** 3, baseline: 1024 ** 3, doubled: true, fell: false, overCeiling: true }, "ceiling", new Date(NOW)); | |
| 120 | const e = readEvidence(body); | |
| 121 | expect(e).toBeDefined(); | |
| 122 | expect(e!.bytes).toBe(String(5 * 1024 ** 3)); | |
| 123 | expect(e!.url).toContain(c.id); | |
| 124 | expect(e!.status).toBe("n/a"); | |
| 125 | expect(e!.time).toBe(ISO); | |
| 126 | }); | |
| 127 | ||
| 128 | test("the sentinel's disk finding carries the path and the free bytes it read", () => { | |
| 129 | provideDiskForTest(() => ({ freeBytes: 123_456_789, totalBytes: 10 ** 12 })); | |
| 130 | try { | |
| 131 | recordDegraded(app.db, [{ cause: "disk-red", sentence: "The data disk is nearly full.", href: "/settings/capacity" }], NOW - 60 * 60_000); | |
| 132 | sentinelHunt(app, "alice/forge", NOW); | |
| 133 | } finally { | |
| 134 | provideDiskForTest(undefined); | |
| 135 | } | |
| 136 | const issue = listIssues(app.db, forge().id, { state: "open" }).find((i) => i.title.includes("disk-red")); | |
| 137 | expect(issue).toBeDefined(); | |
| 138 | const e = readEvidence(issue!.body); | |
| 139 | expect(e).toMatchObject({ url: "/settings/capacity", status: "n/a", bytes: "123456789", ms: "n/a", time: ISO }); | |
| 140 | expect(e!.release).not.toBe("n/a"); | |
| 141 | }); | |
| 142 | ||
| 143 | test("the sentinel's backup finding carries the path and the newest snapshot's size", () => { | |
| 144 | keepReads( | |
| 145 | root, | |
| 146 | [{ lane: "primary", where: "s3://bucket/forge", offBox: true, newestAt: "2026-11-10T00:00:00.000Z", newestName: "snap.tar", newestBytes: 7_340_032, count: 3, readAt: ISO }], | |
| 147 | NOW, | |
| 148 | ); | |
| 149 | recordDegraded(app.db, [{ cause: "backup", sentence: "The newest backup is five days old.", href: "/settings/backups" }], NOW - 60 * 60_000); | |
| 150 | sentinelHunt(app, "alice/forge", NOW); | |
| 151 | const issue = listIssues(app.db, forge().id, { state: "open" }).find((i) => i.title.includes("backup was degraded")); | |
| 152 | expect(issue).toBeDefined(); | |
| 153 | expect(readEvidence(issue!.body)).toMatchObject({ url: "/settings/backups", status: "n/a", bytes: "7340032", ms: "n/a", time: ISO }); | |
| 154 | }); | |
| 155 | ||
| 156 | test("the production probe: the hunter's finding carries the path, status, bytes and ms it saw", async () => { | |
| 157 | const fetcher = async (url: string): Promise<Response> => | |
| 158 | new URL(url).pathname === "/healthz" | |
| 159 | ? new Response("not well", { status: 503, headers: { "content-type": "text/plain" } }) | |
| 160 | : new Response("fine", { status: 200, headers: { "content-type": "text/plain" } }); | |
| 161 | const report = await huntTarget("https://prod.example", fetcher); | |
| 162 | const f = report.findings.find((x) => x.key === "door.public-page:/healthz"); | |
| 163 | expect(f).toBeDefined(); | |
| 164 | const e = readEvidence(evidenceOf(f!, ISO)); | |
| 165 | expect(e).toMatchObject({ url: "/healthz", status: "503", bytes: "8", time: ISO }); | |
| 166 | expect(Number(e!.ms)).toBeGreaterThanOrEqual(0); | |
| 167 | expect(e!.ms).not.toBe("n/a"); | |
| 168 | }); | |
| 169 | ||
| 170 | test("the contract probe", async () => { | |
| 171 | const out = await newApp(app, alice, { name: "doors", visibility: "public" }); | |
| 172 | if (!out.ok) throw new Error(String(out.error)); | |
| 173 | await probeApp(app, {} as AppsRuntime, out.value.app, async (input) => ({ | |
| 174 | repo: input.repo, commitOid: "abcdef0123456789", at: ISO, verdict: "mismatch", | |
| 175 | knocks: [{ resource: "note", verb: "delete", as: "anon", door: "DELETE /api/notes/0", cell: "no", result: { kind: "admitted", status: 404 }, ms: 12.5, bytes: 31, verdict: "mismatch" }], | |
| 176 | said: ["note.delete as anon: the table says no, the app said 404"], | |
| 177 | })); | |
| 178 | const issue = probeIssue(app, out.value.repo); | |
| 179 | expect(issue).toBeDefined(); | |
| 180 | const e = readEvidence(issue!.body); | |
| 181 | expect(e).toBeDefined(); | |
| 182 | expect(e).toEqual({ url: "/api/notes/0", status: "404", bytes: "31", ms: "12.5", time: ISO, release: "abcdef012345" }); | |
| 183 | }); | |
| 184 | }); |
test/maintain-v1-proven.test.ts +20
| @@ -12,6 +12,7 @@ import { createUser } from "../src/core/users.ts"; | ||
| 12 | 12 | import { probes, type Result } from "../src/core/v1_proof.ts"; |
| 13 | 13 | import { v1Proven } from "../src/maintain/duties/v1_proven.ts"; |
| 14 | 14 | import { duties } from "../src/maintain/registry.ts"; |
| 15 | import { readEvidence } from "./helpers/evidence.ts"; | |
| 15 | 16 | import { quietLogs } from "./helpers/globals.ts"; |
| 16 | 17 | |
| 17 | 18 | quietLogs(); |
| @@ -116,3 +117,22 @@ test("no public origin: nothing is called", async () => { | ||
| 116 | 117 | test("the duty is registered, weekly", () => { |
| 117 | 118 | expect(duties.find((d) => d.name === "v1-proven")?.intervalMs).toBe(7 * 86_400_000); |
| 118 | 119 | }); |
| 120 | ||
| 121 | test("a filed defect and its still-failing comment carry the evidence block of what the probe read (#13033)", async () => { | |
| 122 | app.db.query("UPDATE issue SET state = 'closed' WHERE repo_id = ?1 AND title LIKE 'Defect of #%'").run(repoId); | |
| 123 | const before = defects().length; | |
| 124 | const bad = probes[0]!; | |
| 125 | const seen = { url: "https://forge.example/", status: 500, bytes: 12, ms: 41.5 }; | |
| 126 | const proof = async (): Promise<Result[]> => | |
| 127 | probes.map((p) => (p.issue === bad.issue ? { issue: p.issue, item: p.item, status: "fail", detail: "/ answers 500, not 200", seen } : { issue: p.issue, item: p.item, status: "pass", detail: "ok" })); | |
| 128 | await run(undefined, undefined, proof); | |
| 129 | const made = defects().slice(before); | |
| 130 | expect(made).toHaveLength(1); | |
| 131 | const body = app.db.query<{ body: string }, [string]>("SELECT body FROM issue WHERE id = ?1").get(made[0]!.id)!.body; | |
| 132 | const ev = readEvidence(body)!; | |
| 133 | expect(ev).toMatchObject({ url: seen.url, status: "500", bytes: "12", ms: "41.5", time: NOW.toISOString() }); | |
| 134 | expect(ev.release).not.toBe(""); | |
| 135 | await run(undefined, undefined, proof); | |
| 136 | const comment = listComments(app.db, made[0]!.id).at(-1)!; | |
| 137 | expect(readEvidence(comment.body)).toMatchObject({ url: seen.url, status: "500", bytes: "12" }); | |
| 138 | }); |
test/probe.test.ts +2
| @@ -126,6 +126,8 @@ describe("the probe asks the running app whether it obeys its table", () => { | ||
| 126 | 126 | // Every yes/no cell across five doors and three identities was asked. |
| 127 | 127 | expect(asked.length).toBeGreaterThanOrEqual(10); |
| 128 | 128 | expect(asked.every((k) => k.verdict === "ok")).toBe(true); |
| 129 | // Each asked door says how long it took and how many bytes it sent (#13033). | |
| 130 | expect(asked.every((k) => typeof k.ms === "number" && k.ms >= 0 && typeof k.bytes === "number")).toBe(true); | |
| 129 | 131 | // The own cells were not: they need a row that names the actor. |
| 130 | 132 | for (const k of out.knocks.filter((k) => k.cell === "own")) |
| 131 | 133 | expect(k.result.kind).toBe("unprobed"); |
test/v1-proven.test.ts +7
| @@ -109,6 +109,13 @@ describe("v1-proven", () => { | ||
| 109 | 109 | expect(line(results.find((r) => r.issue === issue)!)).toStartWith(`fail #${issue}`); |
| 110 | 110 | }); |
| 111 | 111 | |
| 112 | test("a fail carries the last read the probe made: its URL, status, bytes and ms (#13033)", async () => { | |
| 113 | const r = (await prove("https://f.test", stub({ "/healthz": () => new Response("down", { status: 503 }) }))).find((x) => x.issue === 11539)!; | |
| 114 | expect(r.status).toBe("fail"); | |
| 115 | expect(r.seen).toMatchObject({ url: "https://f.test/healthz", status: 503, bytes: 4 }); | |
| 116 | expect(r.seen!.ms).toBeGreaterThanOrEqual(0); | |
| 117 | }); | |
| 118 | ||
| 112 | 119 | test("the schedule probe fails naming the oldest age, is unproven without the field, and a network error is a fail", async () => { |
| 113 | 120 | const late = (await prove("https://f.test", stub({ "/api/v1/health": () => health({ overdue: 2, oldestSeconds: 3 * 3600 + 720 }) }))).find((x) => x.issue === 11595)!; |
| 114 | 121 | expect(line(late)).toBe("fail #11595 no schedule is past its period: 2 schedule(s) overdue, the oldest by 3 h 12 min"); |
new above
offline
Unbounded
arrayBuffer()on a response from the probed app. Cap the read, or cancel the body and use the declared length.