Public: anyone can read this; the crew can change it.
A maintenance duty files one issue naming open pulls whose check has been red for over 24 hours with no push #13554
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/13530-a-maintenance-duty-files-one-issue-namin → main 4 commits fork 3 threads · 3 unresolved
6 files
changed +199
−0
docs/features/README.md +1
| @@ -18,6 +18,7 @@ Read with the guide page [What a forge is](../guide-what-a-forge-is.md). | ||
| 18 | 18 | |
| 19 | 19 | - [](started-from.md) |
| 20 | 20 | - [](feed-look.md) |
| 21 | - [](red-pulls.md) | |
| 21 | 22 | - [](empty-words.md) |
| 22 | 23 | - [](readme-description.md) |
| 23 | 24 | - [](arcade-carts.md) |
docs/features/red-pulls.md added +18
| @@ -0,0 +1,18 @@ | ||
| 1 | # Open pulls red for over a day | |
| 2 | ||
| 3 | A background duty looks hourly at every open pull of the forge's own repository. | |
| 4 | A pull is listed when the check on its current head has failed and the newest | |
| 5 | failed run finished more than 24 hours ago. | |
| 6 | ||
| 7 | - **One issue.** The duty files a single issue titled "Open pulls whose check | |
| 8 | has been red for over 24 hours", listing each pull with its age in whole days. | |
| 9 | It never files a second while that one is open. | |
| 10 | - **Edited in place, only on change.** Each pass compares the new list with the | |
| 11 | issue's body and writes only when they differ, so watchers are not notified | |
| 12 | hourly. Ages are whole days for that reason. | |
| 13 | - **The clock.** The age counts from the newest failed run, not from when the | |
| 14 | pull first went red: a rerun that fails again on the same head restarts it. | |
| 15 | A push (a new head) or a green run drops the pull off the list. | |
| 16 | - **A person closes it.** The duty never closes the issue. When nothing is red | |
| 17 | it edits the body to say so; if the issue stays open it is reused, and once a | |
| 18 | person closes it, the next red pull files a new one. |
src/engine/duties.ts +5
| @@ -32,6 +32,7 @@ import { recordAskLedger } from "../web/studio/asks.ts"; | ||
| 32 | 32 | import { noteDbSize } from "../system/db_size.ts"; |
| 33 | 33 | import { adoptObserved } from "./observed.ts"; |
| 34 | 34 | import { startRetention } from "../system/retention.ts"; |
| 35 | import { reportRedPulls } from "./red_pulls.ts"; | |
| 35 | 36 | import { pruneBroker } from "../spawn/origin_broker.ts"; |
| 36 | 37 | |
| 37 | 38 | const HOUR_MS = 3_600_000; |
| @@ -69,6 +70,10 @@ export function startDuties(app: App, options: { onWorker: boolean }): () => voi | ||
| 69 | 70 | // (`spawn/origin_broker.ts`), and a timer dies with its process: what a restart left past its |
| 70 | 71 | // minute goes here, on boot. |
| 71 | 72 | during("origin-broker-sweep", () => pruneBroker(app), undefined, app); |
| 73 | // Open pulls red for over a day with no push are named in one issue, edited in place (#13530). | |
| 74 | const redPulls = (): void => reportRedPulls(app); | |
| 75 | timers.push(setInterval(() => during("red-pulls", redPulls, undefined, app), HOUR_MS)); | |
| 76 | during("red-pulls", redPulls, undefined, app); | |
| 72 | 77 | // An agent's hold that has stood 24 h is released each minute (#6918). |
| 73 | 78 | timers.push( |
| 74 | 79 | setInterval( |
src/engine/red_pulls.ts added +70
| @@ -0,0 +1,70 @@ | ||
| 1 | /** | |
| 2 | * The red-pulls duty (#13530): an open pull whose check on its current head has been red for | |
| 3 | * over 24 hours is a quiet clog. One issue in the forge's own repository names each with the | |
| 4 | * age of its red check; the next pass edits that issue rather than filing again. The state is | |
| 5 | * the database's: a pull's newest check or deploy of its current head. A push makes a new head | |
| 6 | * with no failed run, and a green run is not red, so either drops the pull off the list. | |
| 7 | */ | |
| 8 | ||
| 9 | import type { App } from "../app.ts"; | |
| 10 | import { editIssue, openIssue } from "../actions/issues.ts"; | |
| 11 | import { listIssues } from "../core/issues.ts"; | |
| 12 | import { listOpenPulls } from "../core/pulls.ts"; | |
| 13 | import { getRepo } from "../core/repos.ts"; | |
| 14 | import { checkSummary } from "../core/runs.ts"; | |
| 15 | import { getUserById } from "../core/users.ts"; | |
| 16 | ||
| 17 | export const RED_TITLE = "Open pulls whose check has been red for over 24 hours"; | |
| 18 | export const RED_AFTER_MS = 24 * 3_600_000; | |
| 19 | ||
| 20 | export interface RedPull { | |
| 21 | readonly number: number; | |
| 22 | readonly title: string; | |
| 23 | readonly redMs: number; | |
| 24 | } | |
| 25 | ||
| 26 | /** Open pulls of one repository whose current head's check is red and has been for over a day. */ | |
| 27 | export function redPulls(app: App, repoId: string, now: number): RedPull[] { | |
| 28 | const red: RedPull[] = []; | |
| 29 | for (const pull of listOpenPulls(app.db, repoId)) { | |
| 30 | const check = checkSummary(app.db, pull.id, pull.headOid); | |
| 31 | if (check.state !== "failed" || !check.run) continue; | |
| 32 | const since = Date.parse(check.run.finishedAt ?? check.run.createdAt); | |
| 33 | if (Number.isNaN(since) || now - since <= RED_AFTER_MS) continue; | |
| 34 | red.push({ number: pull.number, title: pull.title, redMs: now - since }); | |
| 35 | } | |
| 36 | return red.sort((a, b) => b.redMs - a.redMs); | |
| 37 | } | |
| 38 | ||
| 39 | export function age(ms: number): string { | |
| 40 | // Whole days, so the body is the same all day and a pass has nothing to write. | |
| 41 | const days = Math.max(1, Math.floor(ms / 86_400_000)); | |
| 42 | return days === 1 ? "1 day" : `${days} days`; | |
| 43 | } | |
| 44 | ||
| 45 | export function reportOf(list: readonly RedPull[]): string { | |
| 46 | if (list.length === 0) return "No open pull has had a red check for over 24 hours."; | |
| 47 | return [ | |
| 48 | "These open pulls have had a red check for over 24 hours with no push:", | |
| 49 | "", | |
| 50 | ...list.map((p) => `- #${p.number} ${p.title.replace(/\s+/g, " ").slice(0, 120)}: red for ${age(p.redMs)}`), | |
| 51 | ].join("\n"); | |
| 52 | } | |
| 53 | ||
| 54 | /** One pass: file the issue, or edit the open one; nothing red and nothing open files nothing. */ | |
| 55 | export function reportRedPulls(app: App, now: number = Date.now()): void { | |
| 56 | const slug = app.config.forgeRepo; | |
| 57 | if (slug === undefined) return; | |
| 58 | const [owner, name] = slug.split("/") as [string, string]; | |
| 59 | const repo = getRepo(app.db, owner, name); | |
| 60 | const actor = repo && getUserById(app.db, repo.ownerId); | |
| 61 | if (!repo || !actor) return; | |
| 62 | const list = redPulls(app, repo.id, now); | |
| 63 | const body = reportOf(list); | |
| 64 | const existing = listIssues(app.db, repo.id, { state: "open", kind: "issue" }).find((i) => i.title === RED_TITLE); | |
| 65 | if (existing) { | |
outdated
Edits every hour even when body is unchanged, and age in hours makes the body differ each pass. Skip when existing.body === body, and consider days granularity. |
||
| 66 | if (existing.body !== body) editIssue(app, actor, repo, existing, { body }); | |
| 67 | return; | |
| 68 | } | |
| 69 | if (list.length > 0) openIssue(app, actor, repo, { title: RED_TITLE, body }); | |
| 70 | } |
test/duty-red-pulls.test.ts added +103
| @@ -0,0 +1,103 @@ | ||
| 1 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 2 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 3 | import { tmpdir } from "node:os"; | |
| 4 | import { join } from "node:path"; | |
| 5 | import { createApp, type App } from "../src/app.ts"; | |
| 6 | import { commitToBranch } from "../src/actions/repos.ts"; | |
| 7 | import { listIssues } from "../src/core/issues.ts"; | |
| 8 | import { createPull, listOpenPulls } from "../src/core/pulls.ts"; | |
| 9 | import { createRepo, type Repo } from "../src/core/repos.ts"; | |
| 10 | import { createRun } from "../src/core/runs.ts"; | |
| 11 | import { createUser } from "../src/core/users.ts"; | |
| 12 | import { repoDir } from "../src/git/repos.ts"; | |
| 13 | import { RED_TITLE, reportRedPulls } from "../src/engine/red_pulls.ts"; | |
| 14 | import { quietLogs } from "./helpers/globals.ts"; | |
| 15 | ||
| 16 | quietLogs(); | |
| 17 | ||
| 18 | const ok = <T>(r: { ok: true; value: T } | { ok: false; error: unknown }): T => { | |
| 19 | if (!r.ok) throw new Error(JSON.stringify(r.error)); | |
| 20 | return r.value; | |
| 21 | }; | |
| 22 | let root = ""; | |
| 23 | let app: App; | |
| 24 | let repo: Repo; | |
| 25 | let dir = ""; | |
| 26 | let ada: Awaited<ReturnType<typeof mk>>; | |
| 27 | const mk = async () => | |
| 28 | ok(await createUser(app.db, { origin: "seed", trusted: true, username: "ada", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true })); | |
| 29 | const HOUR = 3_600_000; | |
| 30 | const now = Date.now(); | |
| 31 | ||
| 32 | beforeAll(async () => { | |
| 33 | root = await mkdtemp(join(tmpdir(), "forge-red-pulls-")); | |
| 34 | app = createApp({ dataDir: root, port: 0, forgeRepo: "ada/forge" }); | |
| 35 | ada = await mk(); | |
| 36 | repo = ok(await createRepo(app.db, app.reposRoot, { ownerId: ada.id, name: "forge", visibility: "public" })); | |
| 37 | dir = repoDir(app.reposRoot, ada.username, repo.name); | |
| 38 | ok(await commitToBranch(app, ada, repo, { message: "seed", files: [{ path: "a.txt", content: "a\n" }] })); | |
| 39 | }); | |
| 40 | afterAll(async () => { | |
| 41 | app.close(); | |
| 42 | await rm(root, { recursive: true, force: true }); | |
| 43 | }); | |
| 44 | ||
| 45 | const pullOn = async (branch: string) => { | |
| 46 | ok(await commitToBranch(app, ada, repo, { message: branch, branch, files: [{ path: `${branch}.txt`, content: "x\n" }] } as any)); | |
| 47 | return ok(await createPull(app.db, dir, { repoId: repo.id, authorId: ada.id, title: `pull ${branch}`, baseBranch: "main", headBranch: branch })); | |
| 48 | }; | |
| 49 | const check = (pull: { id: string; headOid: string }, state: "ok" | "failed", hoursAgo: number) => { | |
| 50 | const run = createRun(app.db, { repoId: repo.id, kind: "check", issueId: pull.id, commitOid: pull.headOid }); | |
| 51 | app.db.query("UPDATE run SET state = ?1, summary = ?1, finished_at = ?2 WHERE id = ?3").run(state, new Date(now - hoursAgo * HOUR).toISOString(), run.id); | |
| 52 | }; | |
| 53 | const reports = () => listIssues(app.db, repo.id, { state: "open", kind: "issue" }).filter((i) => i.title === RED_TITLE); | |
| 54 | ||
| 55 | test("nothing red files nothing", () => { | |
| 56 | reportRedPulls(app, now); | |
| 57 | expect(reports()).toHaveLength(0); | |
| 58 | }); | |
| 59 | ||
| 60 | test("one issue names each pull red over 24 h with its age; a rerun edits it, a fresh red and a green stay off", async () => { | |
| 61 | const old = await pullOn("old"); | |
| 62 | const fresh = await pullOn("fresh"); | |
Add a case: second pass with the same list leaves the issue untouched. |
||
| 63 | const green = await pullOn("green"); | |
| 64 | check(old, "failed", 50); | |
| 65 | check(fresh, "failed", 2); | |
| 66 | check(green, "failed", 80); | |
| 67 | check(green, "ok", 1); | |
| 68 | reportRedPulls(app, now); | |
| 69 | reportRedPulls(app, now); | |
| 70 | const [issue, ...rest] = reports(); | |
| 71 | expect(rest).toHaveLength(0); | |
| 72 | expect(issue!.body).toContain(`#${old.number} pull old: red for 2 days`); | |
| 73 | expect(issue!.body).not.toContain(`#${fresh.number} `); | |
| 74 | expect(issue!.body).not.toContain(`#${green.number} `); | |
| 75 | }); | |
| 76 | ||
| 77 | test("a second pass with the same list, even hours later, leaves the issue untouched", () => { | |
| 78 | const before = reports()[0]!; | |
| 79 | reportRedPulls(app, now + 3 * HOUR); | |
| 80 | const after = reports()[0]!; | |
| 81 | expect(after.body).toBe(before.body); | |
| 82 | expect(after.updatedAt).toBe(before.updatedAt); | |
| 83 | }); | |
| 84 | ||
| 85 | test("a failed rerun on the same head restarts the clock: the age counts from the newest failed run", async () => { | |
| 86 | const rerun = await pullOn("rerun"); | |
| 87 | check(rerun, "failed", 100); | |
| 88 | reportRedPulls(app, now); | |
| 89 | expect(reports()[0]!.body).toContain(`#${rerun.number} pull rerun: red for 4 days`); | |
| 90 | check(rerun, "failed", 2); | |
| 91 | reportRedPulls(app, now); | |
| 92 | expect(reports()[0]!.body).not.toContain(`#${rerun.number} `); | |
| 93 | }); | |
| 94 | ||
| 95 | test("a push drops the pull off the list, and a pull that is no longer red leaves it empty, not refiled", async () => { | |
| 96 | const old = listOpenPulls(app.db, repo.id).find((p) => p.title === "pull old")!; | |
| 97 | // A push is a new head, and a new head has no failed run of its own. | |
| 98 | app.db.query("UPDATE pull SET head_oid = ?1 WHERE issue_id = ?2").run("a".repeat(40), old.id); | |
| 99 | reportRedPulls(app, now); | |
| 100 | expect(reports()).toHaveLength(1); | |
| 101 | expect(reports()[0]!.body).not.toContain("pull old"); | |
| 102 | expect(reports()[0]!.body).toContain("No open pull"); | |
| 103 | }); |
test/job-slices.test.ts +2
| @@ -23,6 +23,7 @@ import { bigForge } from "./helpers/big-forge.ts"; | ||
| 23 | 23 | import { quietLogs } from "./helpers/globals.ts"; |
| 24 | 24 | import { startSampler } from "../src/boards/sampler.ts"; |
| 25 | 25 | import { startMemorySampler } from "../src/boards/memory.ts"; |
| 26 | import { reportRedPulls } from "../src/engine/red_pulls.ts"; | |
| 26 | 27 | import { tickKeeper } from "../src/flow/keeper.ts"; |
| 27 | 28 | import { keepTick } from "../src/flow/keep.ts"; |
| 28 | 29 | import { tickClog } from "../src/flow/clog_tick.ts"; |
| @@ -154,6 +155,7 @@ function jobs(): Record<string, () => unknown> { | ||
| 154 | 155 | "memory-sampler": () => memory.tick(), |
| 155 | 156 | keeper: () => tickKeeper(app, new Date(), () => {}), |
| 156 | 157 | "gate-snooze": () => returnSnoozed(app), |
| 158 | "red-pulls": () => reportRedPulls(app), | |
| 157 | 159 | // The pictures' keeper (`flow/keep.ts`): one look at what is kept; a due reading is started on |
| 158 | 160 | // a worker, never read here. |
| 159 | 161 | pictures: () => keepTick(app.db), |
new above
offline
during() already isolates failures; this try/catch is a second layer. Keep one.