Public: anyone can read this; the crew can change it.
A failing probe keeps one open defect, updated, not a new defect each week #13155
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/13134-a-failing-probe-keeps-one-open-defect-up → main 5 commits fork
5 files
changed +120
−46
docs/features/v1-proven.md +10 −5
| @@ -11,11 +11,16 @@ A weekly maintenance duty, `v1-proven` (#12989), runs the production probes of ` | ||
| 11 | 11 | `GET /api/v1/maintenance/runs` (admin token) or in the terminal's "Maintenance duty runs" screen. The web |
| 12 | 12 | `/maintenance` page is the power-off preparation and does not list duty runs. |
| 13 | 13 | - **A defect issue.** A failing probe opens one issue in the forge's own repository (`FORGE_REPO`, e.g. `owner/forge`), |
| 14 | titled `Defect of #<probe issue>: <item>`, whose body quotes the failing line and says to fix the cause and close it. | |
| 15 | With no `FORGE_REPO` nothing is filed and the run says "no issue filed". | |
| 16 | - **A repeat failure.** While that issue is open, the same probe failing again adds a comment ("Still failing on | |
| 17 | <date>") and no second issue. Two probes failing in one run open two issues. Once the issue is closed, a new failure | |
| 18 | opens a new one. | |
| 14 | titled `Defect of #<probe issue>: <item>`, whose body quotes the failing line, carries the stable line | |
| 15 | `Probe key: v1-proven/<probe issue>` and says to fix the cause and close it. With no `FORGE_REPO` nothing is filed and | |
| 16 | the run says "no issue filed". | |
| 17 | - **A repeat failure.** The open defect of a probe is found by that `Probe key:` line in the body, so renaming the title | |
| 18 | loses nothing; a defect filed before the key existed is found by its `Defect of #<probe issue>: ` title. While it is | |
| 19 | open, the same probe failing again adds a comment ("Still failing on <date>") and no second issue. Two probes failing | |
| 20 | in one run open two issues. Once the issue is closed, a new failure opens a new one. | |
| 21 | - **A pass.** A probe that passes while its defect is open adds one comment ("Passing again on <date>") and leaves the | |
| 22 | defect open: a person closes it. Repeated passes add nothing; a failure after a pass comments again, and the next pass | |
| 23 | notes again. A pass with no open defect says nothing. | |
| 19 | 24 | - **Limits.** Runs at most twice a day, weekly by default, and spends no money. The findings only land on the run record; |
| 20 | 25 | the runner files no notice of its own. |
| 21 | 26 |
src/core/event_audience.ts +20 −11
| @@ -45,7 +45,10 @@ function answersForDeleted(db: Database, viewer: User | undefined, event: ForgeE | ||
| 45 | 45 | return typeof owner === "string" && (owner === viewer.id || personOwners(db, owner).includes(viewer.id)); |
| 46 | 46 | } |
| 47 | 47 | |
| 48 | /** A `repo.deleted` names its repository; unless it was public (said so outright), the name is not for readers. */ | |
| 48 | /** | |
| 49 | * A `repo.deleted` names its repository; unless it was public (said so outright), the name is not | |
| 50 | * for readers. | |
| 51 | */ | |
| 49 | 52 | const namesHiddenRepo = (event: ForgeEvent): boolean => |
| 50 | 53 | event.kind === "repo.deleted" && |
| 51 | 54 | event.payload.repo !== undefined && |
| @@ -237,21 +240,23 @@ export function classifiedAudience(kind: string): EventAudience | undefined { | ||
| 237 | 240 | kind.startsWith("maintenance.") || |
| 238 | 241 | // The weekly model review's changes and results are the owner's (`actions/model_review.ts`). |
| 239 | 242 | kind.startsWith("crew.model_") || |
| 240 | // A child forge answering no one at its public address is the owner's to hear (`spawn/reach.ts`). | |
| 243 | // A child forge answering no one at its public address is the owner's to hear | |
| 244 | // (`spawn/reach.ts`). | |
| 241 | 245 | kind.startsWith("forge.child_") || |
| 242 | // What a hosted forge asked to reach, was refused and was allowed is the owner's to answer (`spawn/child_reach.ts`). | |
| 246 | // What a hosted forge asked to reach, was refused and was allowed is the owner's to answer | |
| 247 | // (`spawn/child_reach.ts`). | |
| 243 | 248 | kind.startsWith("forge.reach_") || |
| 244 | // A failed production probe names the forge's own defect: the admins' (`maintain/proven.ts`). | |
| 245 | kind === "probe_defect" || | |
| 246 | 249 | // A clog is the whole forge's flow, the owner's to hear (`flow/clog_tick.ts`). |
| 247 | 250 | kind.startsWith("clog.") || |
| 248 | 251 | // What the throughput keeper did to lift a brake is the owner's too (`flow/keeper.ts`). |
| 249 | 252 | kind.startsWith("keeper.") || |
| 250 | // The flow keeper's acts on pulls, lulls and forge-down are the owner's too (`maintain/keeper/`, #7556). | |
| 253 | // The flow keeper's acts on pulls, lulls and forge-down are the owner's too | |
| 254 | // (`maintain/keeper/`, #7556). | |
| 251 | 255 | kind.startsWith("keeper_") || |
| 252 | 256 | kind === "forge_down" || |
| 253 | 257 | kind === "forge_up" || |
| 254 | // Who is trusted to fold in, and the switch that stops it, are the owner's (`actions/trusted_sources.ts`). | |
| 258 | // Who is trusted to fold in, and the switch that stops it, are the owner's | |
| 259 | // (`actions/trusted_sources.ts`). | |
| 255 | 260 | kind.startsWith("trust.") || |
| 256 | 261 | kind.startsWith("foldin.") || |
| 257 | 262 | // A way-back link minted for an account the platform edge signed in (#11802). |
| @@ -291,18 +296,21 @@ export function classifiedAudience(kind: string): EventAudience | undefined { | ||
| 291 | 296 | kind === "repo.crew_outsiders" || |
| 292 | 297 | // An outside agent's token and the owner's yes to them are the admins' (#12012). |
| 293 | 298 | kind.startsWith("contribute.") || |
| 294 | // A valve turn names a repository and the owner's budget: for those who administer it (`actions/valves.ts`). | |
| 299 | // A valve turn names a repository and the owner's budget: for those who administer it | |
| 300 | // (`actions/valves.ts`). | |
| 295 | 301 | kind === "valve.turned" || |
| 296 | 302 | // A clog starting names the forge's own pile-up: the owner's to hear (#6524). |
| 297 | 303 | kind.startsWith("clog.") || |
| 298 | // The release keeper's steps name images and probe numbers: the forge owner's (`keeper/keeper.ts`, #7557). | |
| 304 | // The release keeper's steps name images and probe numbers: the forge owner's | |
| 305 | // (`keeper/keeper.ts`, #7557). | |
| 299 | 306 | kind.startsWith("release.") || |
| 300 | 307 | // A file held against new work is its admins' book-keeping (#6522). |
| 301 | 308 | kind === "repo.file_held" || |
| 302 | 309 | kind === "repo.file_released" || |
| 303 | 310 | // Which repositories New app offers is the admins' choice (`actions/app_templates.ts`). |
| 304 | 311 | kind.startsWith("app_template.") || |
| 305 | // What a parent lends a child's crew, by name and dollar, is the admins' business (`actions/child_credit.ts`, #10484). | |
| 312 | // What a parent lends a child's crew, by name and dollar, is the admins' business | |
| 313 | // (`actions/child_credit.ts`, #10484). | |
| 306 | 314 | kind === "forge.credit_set" || |
| 307 | 315 | // Which child forge was moved out of this forge's process, left there or put back, and why: |
| 308 | 316 | // what its |
| @@ -331,7 +339,8 @@ export function classifiedAudience(kind: string): EventAudience | undefined { | ||
| 331 | 339 | // A person's pinned panels changed (`actions/board-pins.ts`): theirs alone. |
| 332 | 340 | kind === "board.pins" || |
| 333 | 341 | kind.startsWith("fleet.") || |
| 334 | // A person's own ask to the forge that hosts this one, and how it was answered (`actions/ask_host.ts`). | |
| 342 | // A person's own ask to the forge that hosts this one, and how it was answered | |
| 343 | // (`actions/ask_host.ts`). | |
| 335 | 344 | kind.startsWith("reach.") || |
| 336 | 345 | kind.startsWith("thought.") |
| 337 | 346 | ) |
src/maintain/duties/v1_proven.ts +3 −5
| @@ -7,7 +7,7 @@ import { getRepo } from "../../core/repos.ts"; | ||
| 7 | 7 | import type { Proof } from "../../core/v1_proof.ts"; |
| 8 | 8 | import { publicOrigin } from "../../core/public_origin.ts"; |
| 9 | 9 | import type { Duty, Finding } from "../duty.ts"; |
| 10 | import { recordLines, reportFailures, weeklyProof } from "../proven.ts"; | |
| 10 | import { recordLines, reportResults, weeklyProof } from "../proven.ts"; | |
| 11 | 11 | |
| 12 | 12 | export function v1Proven(proof: Proof = weeklyProof): Duty { |
| 13 | 13 | return { |
| @@ -27,10 +27,8 @@ export function v1Proven(proof: Proof = weeklyProof): Duty { | ||
| 27 | 27 | const actions = recordLines(results, now); |
| 28 | 28 | const [owner, name] = (app?.config.forgeRepo ?? "").split("/"); |
| 29 | 29 | const repo = owner && name ? getRepo(db, owner, name) : undefined; |
| 30 | if (findings.length > 0) { | |
| 31 | if (repo) actions.push(...reportFailures(db, repo.id, results, now)); | |
| 32 | else actions.push("no issue filed: the forge names no repository of its own"); | |
| 33 | } | |
| 30 | if (repo) actions.push(...reportResults(db, repo.id, results, now)); | |
| 31 | else if (findings.length > 0) actions.push("no issue filed: the forge names no repository of its own"); | |
| 34 | 32 | return { findings, actions }; |
| 35 | 33 | }, |
| 36 | 34 | }; |
src/maintain/proven.ts +29 −24
| @@ -6,8 +6,7 @@ | ||
| 6 | 6 | */ |
| 7 | 7 | |
| 8 | 8 | import type { Database } from "bun:sqlite"; |
| 9 | import { EventBus } from "../core/events.ts"; | |
| 10 | import { addComment, createIssue, getIssueById } from "../core/issues.ts"; | |
| 9 | import { addComment, createIssue, listComments } from "../core/issues.ts"; | |
| 11 | 10 | import { line, prove, type Proof, type Result } from "../core/v1_proof.ts"; |
| 12 | 11 | import { watcherId } from "../rules/bot.ts"; |
| 13 | 12 | import { evidenceBlock } from "./evidence.ts"; |
| @@ -17,34 +16,41 @@ export const defectTitle = (r: Pick<Result, "issue" | "item">): string => `Defec | ||
| 17 | 16 | /** The proof a week runs: the probes over fetch, at the forge's own origin and no other host. */ |
| 18 | 17 | export const weeklyProof: Proof = (base) => prove(base, fetch); |
| 19 | 18 | |
| 20 | /** The open defect issue for a probe: the newest one noted that has not been closed since. */ | |
| 19 | /** The stable key a defect carries in its body; the title may be edited, this line is not. */ | |
| 20 | export const probeKey = (probeIssue: number): string => `Probe key: v1-proven/${probeIssue}`; | |
| 21 | ||
| 22 | /** The open defect issue for a probe: the newest open issue of the repository whose body | |
| 23 | * carries its key, or, for a defect filed before the key existed, whose title is the one this | |
| 24 | * duty gave it (`Defect of #N: `). */ | |
| 21 | 25 | function openDefect(db: Database, repoId: string, probeIssue: number): string | undefined { |
| 22 | const rows = db | |
| 23 | .query<{ payload: string }, [string]>("SELECT payload FROM event WHERE repo_id = ?1 AND kind = 'probe_defect' ORDER BY seq DESC LIMIT 200") | |
| 24 | .all(repoId); | |
| 25 | for (const row of rows) { | |
| 26 | try { | |
| 27 | const p = JSON.parse(row.payload) as { probe?: unknown; issue?: unknown }; | |
| 28 | if (p.probe !== probeIssue || typeof p.issue !== "string") continue; | |
| 29 | const issue = getIssueById(db, p.issue); | |
| 30 | return issue?.state === "open" ? issue.id : undefined; | |
| 31 | } catch { | |
| 32 | // An unreadable note names no issue. | |
| 33 | } | |
| 34 | } | |
| 35 | return undefined; | |
| 26 | const row = db | |
| 27 | .query<{ id: string }, [string, string, string]>("SELECT id FROM issue WHERE repo_id = ?1 AND state = 'open' AND (instr(body, ?2) > 0 OR substr(title, 1, length(?3)) = ?3) ORDER BY created_at DESC, id DESC LIMIT 1") | |
| 28 | .get(repoId, probeKey(probeIssue), `Defect of #${probeIssue}: `); | |
| 29 | return row?.id; | |
| 36 | 30 | } |
| 37 | 31 | |
| 38 | /** Files or comments on each failing result; returns one sentence per thing done. */ | |
| 39 | export function reportFailures(db: Database, repoId: string, results: readonly Result[], at: Date): string[] { | |
| 32 | const passingNote = (day: string): string => `Passing again on ${day}. Close this defect when you are satisfied; the probe will not close it.`; | |
| 33 | ||
| 34 | /** Files, comments or notes passing on each result with an open defect; returns one sentence | |
| 35 | * per thing done. */ | |
| 36 | export function reportResults(db: Database, repoId: string, results: readonly Result[], at: Date): string[] { | |
| 40 | 37 | const said: string[] = []; |
| 41 | 38 | const bot = watcherId(db); |
| 39 | const day = at.toISOString().slice(0, 10); | |
| 42 | 40 | for (const r of results) { |
| 43 | if (r.status !== "fail") continue; | |
| 44 | const quote = `> ${line(r).replace(/\r?\n/g, " ")}\n\n${evidenceBlock({ ...(r.seen === undefined ? {} : { where: r.seen.url, status: r.seen.status, bytes: r.seen.bytes, ms: r.seen.ms }), at: at.toISOString() })}`; | |
| 41 | if (r.status === "unproven") continue; | |
| 45 | 42 | const open = openDefect(db, repoId, r.issue); |
| 43 | if (r.status === "pass") { | |
| 44 | if (open === undefined) continue; | |
| 45 | const last = listComments(db, open).at(-1); | |
| 46 | if (last?.body.startsWith("Passing again on ")) continue; // said once | |
| 47 | const c = addComment(db, { issueId: open, authorId: bot, body: passingNote(day) }); | |
| 48 | if (c.ok) said.push(`commented on the open defect of #${r.issue}: passing again`); | |
| 49 | continue; | |
| 50 | } | |
| 51 | const quote = `> ${line(r).replace(/\r?\n/g, " ")}\n\n${evidenceBlock({ ...(r.seen === undefined ? {} : { where: r.seen.url, status: r.seen.status, bytes: r.seen.bytes, ms: r.seen.ms }), at: at.toISOString() })}`; | |
| 46 | 52 | if (open !== undefined) { |
| 47 | const c = addComment(db, { issueId: open, authorId: bot, body: `Still failing on ${at.toISOString().slice(0, 10)}:\n\n${quote}` }); | |
| 53 | const c = addComment(db, { issueId: open, authorId: bot, body: `Still failing on ${day}:\n\n${quote}` }); | |
| 48 | 54 | if (c.ok) said.push(`commented on the open defect of #${r.issue}: still failing`); |
| 49 | 55 | continue; |
| 50 | 56 | } |
| @@ -52,10 +58,9 @@ export function reportFailures(db: Database, repoId: string, results: readonly R | ||
| 52 | 58 | repoId, |
| 53 | 59 | authorId: bot, |
| 54 | 60 | title: defectTitle(r), |
| 55 | body: `The weekly production probe (\`scripts/v1-proven.ts\`) failed on ${at.toISOString().slice(0, 10)}:\n\n${quote}\n\nFix the cause, then close this issue; a probe that fails again meanwhile is added here as a comment.`, | |
| 61 | body: `The weekly production probe (\`scripts/v1-proven.ts\`) failed on ${day}:\n\n${quote}\n\n${probeKey(r.issue)}\n\nFix the cause, then close this issue; a probe that fails again meanwhile is added here as a comment.`, | |
| 56 | 62 | }); |
| 57 | 63 | if (!made.ok) continue; |
| 58 | new EventBus(db).append({ kind: "probe_defect", repoId, payload: { probe: r.issue, issue: made.value.id } }); | |
| 59 | 64 | said.push(`opened issue #${made.value.number}: ${defectTitle(r)}`); |
| 60 | 65 | } |
| 61 | 66 | return said; |
test/maintain-v1-proven.test.ts +58 −1
| @@ -6,11 +6,12 @@ import { mkdtemp, rm } from "node:fs/promises"; | ||
| 6 | 6 | import { tmpdir } from "node:os"; |
| 7 | 7 | import { join } from "node:path"; |
| 8 | 8 | import { createApp, type App } from "../src/app.ts"; |
| 9 | import { listComments } from "../src/core/issues.ts"; | |
| 9 | import { createIssue, listComments } from "../src/core/issues.ts"; | |
| 10 | 10 | import { createRepo } from "../src/core/repos.ts"; |
| 11 | 11 | import { createUser } from "../src/core/users.ts"; |
| 12 | 12 | import { probes, type Result } from "../src/core/v1_proof.ts"; |
| 13 | 13 | import { v1Proven } from "../src/maintain/duties/v1_proven.ts"; |
| 14 | import { watcherId } from "../src/rules/bot.ts"; | |
| 14 | 15 | import { duties } from "../src/maintain/registry.ts"; |
| 15 | 16 | import { readEvidence } from "./helpers/evidence.ts"; |
| 16 | 17 | import { quietLogs } from "./helpers/globals.ts"; |
| @@ -118,6 +119,26 @@ test("the duty is registered, weekly", () => { | ||
| 118 | 119 | expect(duties.find((d) => d.name === "v1-proven")?.intervalMs).toBe(7 * 86_400_000); |
| 119 | 120 | }); |
| 120 | 121 | |
| 122 | test("a passing probe notes it once on its open defect and leaves it open; the defect is found by its body key, not its title", async () => { | |
| 123 | app.db.query("UPDATE issue SET state = 'closed' WHERE repo_id = ?1 AND title LIKE 'Defect of #%'").run(repoId); | |
| 124 | const bad = probes[2]!; | |
| 125 | await run(undefined, undefined, mixed({ [bad.issue]: "fail" })); | |
| 126 | const open = defects().at(-1)!; | |
| 127 | app.db.query("UPDATE issue SET title = 'renamed by a person' WHERE id = ?1").run(open.id); | |
| 128 | const total = () => app.db.query<{ c: number }, [string]>("SELECT count(*) AS c FROM issue WHERE repo_id = ?1").get(repoId)!.c; | |
| 129 | const before = total(); | |
| 130 | await run(undefined, undefined, mixed({ [bad.issue]: "fail" })); | |
| 131 | expect(total()).toBe(before); | |
| 132 | expect(listComments(app.db, open.id).map((c) => c.body.split(" ")[0])).toEqual(["Still"]); | |
| 133 | ||
| 134 | await run(); | |
| 135 | await run(); | |
| 136 | const notes = listComments(app.db, open.id).filter((c) => c.body.startsWith("Passing again")); | |
| 137 | expect(notes).toHaveLength(1); | |
| 138 | const state = app.db.query<{ state: string }, [string]>("SELECT state FROM issue WHERE id = ?1").get(open.id)!; | |
| 139 | expect(state.state).toBe("open"); | |
| 140 | }); | |
| 141 | ||
| 121 | 142 | test("a filed defect and its still-failing comment carry the evidence block of what the probe read (#13033)", async () => { |
| 122 | 143 | app.db.query("UPDATE issue SET state = 'closed' WHERE repo_id = ?1 AND title LIKE 'Defect of #%'").run(repoId); |
| 123 | 144 | const before = defects().length; |
| @@ -136,3 +157,39 @@ test("a filed defect and its still-failing comment carry the evidence block of w | ||
| 136 | 157 | const comment = listComments(app.db, made[0]!.id).at(-1)!; |
| 137 | 158 | expect(readEvidence(comment.body)).toMatchObject({ url: seen.url, status: "500", bytes: "12" }); |
| 138 | 159 | }); |
| 160 | ||
| 161 | const closeDefects = () => app.db.query("UPDATE issue SET state = 'closed' WHERE repo_id = ?1 AND title LIKE 'Defect of #%'").run(repoId); | |
| 162 | const total = () => app.db.query<{ c: number }, [string]>("SELECT count(*) AS c FROM issue WHERE repo_id = ?1").get(repoId)!.c; | |
| 163 | ||
| 164 | test("a pass with no open defect comments nothing and files nothing", async () => { | |
| 165 | closeDefects(); | |
| 166 | const before = total(); | |
| 167 | const comments = app.db.query<{ c: number }, []>("SELECT count(*) AS c FROM comment").get()!.c; | |
| 168 | await run(undefined, undefined, mixed({})); | |
| 169 | expect(total()).toBe(before); | |
| 170 | expect(app.db.query<{ c: number }, []>("SELECT count(*) AS c FROM comment").get()!.c).toBe(comments); | |
| 171 | }); | |
| 172 | ||
| 173 | test("fail, pass, fail again, pass: the passing note is said again after a new failure comment", async () => { | |
| 174 | closeDefects(); | |
| 175 | const bad = probes[3]!; | |
| 176 | await run(undefined, undefined, mixed({ [bad.issue]: "fail" })); | |
| 177 | const open = defects().at(-1)!; | |
| 178 | const ok = mixed({}); | |
| 179 | await run(undefined, undefined, ok); | |
| 180 | await run(undefined, undefined, mixed({ [bad.issue]: "fail" })); | |
| 181 | await run(undefined, undefined, ok); | |
| 182 | await run(undefined, undefined, ok); | |
| 183 | expect(listComments(app.db, open.id).map((c) => c.body.split(" ")[0])).toEqual(["Passing", "Still", "Passing"]); | |
| 184 | }); | |
| 185 | ||
| 186 | test("a legacy open defect with no body key is found by its title: a failure comments, files no duplicate", async () => { | |
| 187 | closeDefects(); | |
| 188 | const bad = probes[4]!; | |
| 189 | const legacy = createIssue(app.db, { repoId, authorId: watcherId(app.db), title: `Defect of #${bad.issue}: ${bad.item}`, body: "Filed by the old duty, no key." }); | |
| 190 | if (!legacy.ok) throw new Error("issue"); | |
| 191 | const before = total(); | |
| 192 | await run(undefined, undefined, mixed({ [bad.issue]: "fail" })); | |
| 193 | expect(total()).toBe(before); | |
| 194 | expect(listComments(app.db, legacy.value.id).map((c) => c.body.split(" ")[0])).toEqual(["Still"]); | |
| 195 | }); |
new above
offline