Public: anyone can read this; the crew can change it.
A weekly maintenance duty lists the five slowest test files in the last check #13980
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/13966-a-weekly-maintenance-duty-lists-the-five → main 11 commits fork 1 thread · 1 unresolved
6 files
changed +167
−4
docs/features/maintenance.md +11
| @@ -137,3 +137,14 @@ closes it, or the moment it entered the open chapter (or the chapter was opened) | ||
| 137 | 137 | issue, labelled `bug`, in its repository, titled "#N in chapter X has had no run and no pull for 24 hours"; |
| 138 | 138 | while one is open no second is filed. Draft and closed chapters, and repositories with no chapter, are left |
| 139 | 139 | alone. Shows as the `chapter-stall` row on `/maintenance`. Test: `test/maintenance-chapter-stall.test.ts`. |
| 140 | ||
| 141 | ## Slow tests (#13966) | |
| 142 | ||
| 143 | The `slow-tests` duty runs weekly and names the five test files that took longest in the forge's own last | |
| 144 | finished check (state `ok` or `failed`). It reads that check's stored run log, never runs a test: a file's | |
| 145 | seconds is the sum of its tests' own `[12.3ms]` times, the tests' time and not the file's wall clock, so | |
| 146 | load and teardown are not in it. It files one finding with the five names and seconds, and a measurement | |
| 147 | of the slowest file's seconds; it changes nothing. Only the last 512 KB of a log is stored, so when the | |
| 148 | log was cut the finding says earlier files are missing; where the log has no per-test times, or no | |
| 149 | finished check is stored, the run says so and names nothing it did not read. It shows as the `slow-tests` | |
| 150 | row on `/maintenance`. Test: `test/maintain-slow-tests.test.ts`. |
src/maintain/duties/slow_tests.ts added +80
| @@ -0,0 +1,80 @@ | ||
| 1 | /** | |
| 2 | * Slow tests (#13966): once a week, the five test files that took longest in the forge's own last | |
| 3 | * finished check, with seconds. Read from the check's stored log (`runs/log.ts`), not by running | |
| 4 | * anything: `bun test` prints `test/x.test.ts:` and then `(pass) name [12.3ms]` per test, so a | |
| 5 | * file's seconds is the sum of its tests' own times (the tests' time, not the file's wall clock: | |
| 6 | * load and teardown are not in it). Where the log lacks timings, or was cut to its last 512 KB, | |
| 7 | * the run says what is missing and names nothing it did not read. It changes nothing. | |
| 8 | */ | |
| 9 | ||
| 10 | import { getRepo } from "../../core/repos.ts"; | |
| 11 | import { ownRepoPath } from "../../kernel/own_repo.ts"; | |
| 12 | import { readRunLog } from "../../runs/log.ts"; | |
| 13 | import type { Duty, Finding } from "../duty.ts"; | |
| 14 | ||
| 15 | export const TOP = 5; | |
| 16 | const HEADER = /^(\S+\.test\.[cm]?[jt]sx?):$/; | |
| 17 | const TIMED = /^\((?:pass|fail)\) .*\[([0-9]+(?:\.[0-9]+)?)(m?s)\]/; | |
| 18 | ||
| 19 | export interface SlowFile { | |
| 20 | readonly file: string; | |
| 21 | readonly seconds: number; | |
| 22 | readonly tests: number; | |
| 23 | } | |
| 24 | ||
| 25 | /** | |
| 26 | * Test files by summed test time, slowest first (ties by name); `timed` is how many tests had a | |
| 27 | * time. | |
| 28 | */ | |
| 29 | export function slowestFiles(lines: readonly string[], top = TOP): { files: SlowFile[]; timed: number } { | |
| 30 | const byFile = new Map<string, { ms: number; tests: number }>(); | |
| 31 | let file: string | undefined; | |
| 32 | let timed = 0; | |
| 33 | for (const line of lines) { | |
| 34 | const h = HEADER.exec(line); | |
| 35 | if (h) { | |
| 36 | file = h[1]; | |
| 37 | continue; | |
| 38 | } | |
| 39 | const t = TIMED.exec(line); | |
| 40 | if (t === null || file === undefined) continue; | |
| 41 | const ms = Number(t[1]) * (t[2] === "s" ? 1000 : 1); | |
| 42 | const cur = byFile.get(file) ?? { ms: 0, tests: 0 }; | |
| 43 | byFile.set(file, { ms: cur.ms + ms, tests: cur.tests + 1 }); | |
| 44 | timed++; | |
| 45 | } | |
| 46 | const files = [...byFile.entries()] | |
| 47 | .map(([f, v]) => ({ file: f, seconds: Math.round(v.ms) / 1000, tests: v.tests })) | |
| 48 | .sort((a, b) => b.seconds - a.seconds || a.file.localeCompare(b.file)) | |
| 49 | .slice(0, top); | |
| 50 | return { files, timed }; | |
| 51 | } | |
| 52 | ||
| 53 | export const duty: Duty = { | |
| 54 | name: "slow-tests", | |
| 55 | intervalMs: 7 * 24 * 3_600_000, | |
| 56 | budget: { runsPerDay: 2, dollarsPerDay: 1, reason: "once a week; the rest is room for a catch-up, and it spends no money" }, | |
| 57 | run({ db, app, now }) { | |
| 58 | if (app === undefined) return { findings: [], actions: [] }; | |
| 59 | const [owner = "", name = ""] = ownRepoPath().split("/"); | |
| 60 | const repo = getRepo(db, owner, name); | |
| 61 | if (repo === undefined) return { findings: [], actions: ["skipped: this forge's own repository is not known"] }; | |
| 62 | const row = db | |
| 63 | .query<{ id: string }, [string]>( | |
| 64 | "SELECT id FROM run WHERE repo_id = ?1 AND kind = 'check' AND state IN ('ok','failed') ORDER BY id DESC LIMIT 1", | |
| 65 | ) | |
| 66 | .get(repo.id); | |
| 67 | if (row === null) return { findings: [], actions: ["no finished check is stored, so no test file was timed"] }; | |
| 68 | const { lines, truncated } = readRunLog(app.config.dataDir, row.id); | |
| 69 | const { files, timed } = slowestFiles(lines); | |
| 70 | if (timed === 0) | |
| 71 | return { findings: [], actions: [`check ${row.id} has no stored output with per-test times (${lines.length} lines), so no test file was timed`] }; | |
| 72 | const cut = truncated ? " Only the last 512 KB of the log is stored, so earlier files are missing." : ""; | |
| 73 | const summary = `Slowest test files in check ${row.id}: ${files.map((f) => `${f.file} ${f.seconds.toFixed(1)} s`).join(", ")}.${cut}`; | |
| 74 | const finding: Finding = { | |
| 75 | summary, | |
| 76 | probe: { kind: "measurement", what: `seconds of test time (sum of per-test times) in the slowest file of check ${row.id}`, number: files[0]!.seconds, at: now.toISOString() }, | |
| 77 | }; | |
| 78 | return { findings: [finding], actions: [] }; | |
| 79 | }, | |
| 80 | }; |
src/maintain/registry.ts +2 −1
| @@ -14,6 +14,7 @@ import { duty as previewDirs } from "./previews.ts"; | ||
| 14 | 14 | import { duty as diskFree } from "./duties/disk_free.ts"; |
| 15 | 15 | import { duty as staleIssues } from "./duties/stale_issues.ts"; |
| 16 | 16 | import { duty as waitSplit } from "./duties/wait_split.ts"; |
| 17 | import { duty as slowTests } from "./duties/slow_tests.ts"; | |
| 17 | 18 | import { duty as deadDocPaths } from "./duties/dead_doc_paths.ts"; |
| 18 | 19 | import { duty as redApps } from "./duties/red_apps.ts"; |
| 19 | 20 | import { duty as flakyTests } from "./duties/flaky_tests.ts"; |
| @@ -21,7 +22,7 @@ import { duty as untestedSources } from "./duties/untested_sources.ts"; | ||
| 21 | 22 | import { duty as chapterStall } from "./duties/chapter_stall.ts"; |
| 22 | 23 | import { dutyIdle } from "./duties/duty_idle.ts"; |
| 23 | 24 | |
| 24 | const others: readonly Duty[] = [pullsWaiting, ruleWatcher, efficiencyWatcher, flowKeeper, visitorRetention, backupGrowth, backupAge, v1Proven, diskFree, mergedBranches, previewDirs, staleIssues, waitSplit, flakyTests, redApps, chapterStall, deadDocPaths, untestedSources]; | |
| 25 | const others: readonly Duty[] = [pullsWaiting, ruleWatcher, efficiencyWatcher, flowKeeper, visitorRetention, backupGrowth, backupAge, v1Proven, diskFree, mergedBranches, previewDirs, staleIssues, waitSplit, slowTests, flakyTests, redApps, chapterStall, deadDocPaths, untestedSources]; | |
| 25 | 26 | |
| 26 | 27 | /** The duties plus the one that watches them for silence. */ |
| 27 | 28 | export const duties: readonly Duty[] = [...others, dutyIdle(others)]; |
src/perf/page_budget.ts +2 −2
| @@ -44,9 +44,9 @@ export const PAGE_BUDGETS: readonly PageBudget[] = [ | ||
| 44 | 44 | // warm (53, with main's duties) plus nine: each of the eight duties reads its own state and |
| 45 | 45 | // runs, so the count moves with duty state and background ticks; an N+1 over the duties would |
| 46 | 46 | // still show. Four more duties (flaky-tests #13937, red apps #13938, chapter stall #14057, |
| 47 | // dead-doc-paths #13977, untested-sources #14125) each | |
| 47 | // dead-doc-paths #13977, untested-sources #14125, slow-tests #13966) each | |
| 48 | 48 | // read their own state. One query reads every dismissal (#13967). |
| 49 | { page: "maintenance", pattern: "/maintenance", medianMs: 50, statements: 75 }, | |
| 49 | { page: "maintenance", pattern: "/maintenance", medianMs: 50, statements: 78 }, | |
| 50 | 50 | // A stranger's way to a forge (#11410): walked signed out. `/claim` is |
| 51 | 51 | // measured on a spawned forge with a live link, as the child's owner sees it. |
| 52 | 52 | { page: "pricing", pattern: "/pricing", medianMs: 50, statements: 10, signedOut: true }, |
test/maintain-slow-tests.test.ts added +70
| @@ -0,0 +1,70 @@ | ||
| 1 | /** | |
| 2 | * Slow tests (#13966): check rows with stored logs give the five slowest files, in order; no | |
| 3 | * timings says so. | |
| 4 | */ | |
| 5 | import { expect, test } from "bun:test"; | |
| 6 | import { rulesWorld } from "./helpers/rules-app.ts"; | |
| 7 | import { duty, slowestFiles } from "../src/maintain/duties/slow_tests.ts"; | |
| 8 | import { duties } from "../src/maintain/registry.ts"; | |
| 9 | import { dutyStates } from "../src/maintain/pause.ts"; | |
| 10 | import { appendRunLog, flushRunLogs } from "../src/runs/log.ts"; | |
| 11 | import { id } from "../src/kernel/id.ts"; | |
| 12 | ||
| 13 | process.env.FORGE_SENTINEL_REPO = "alice/rules"; | |
| 14 | const w = rulesWorld(); | |
| 15 | const NOW = new Date("2026-11-16T12:00:00Z"); | |
| 16 | const run = async () => await duty.run({ db: w.app.db, bus: w.app.bus, app: w.app, now: NOW }); | |
| 17 | function check(lines: string[], state = "ok"): string { | |
| 18 | const i = id(); | |
| 19 | w.app.db.query("INSERT INTO run (id, repo_id, kind, state) VALUES (?1, ?2, 'check', ?3)").run(i, w.repoId, state); | |
| 20 | for (const l of lines) appendRunLog(w.app.config.dataDir, i, l); | |
| 21 | flushRunLogs(); | |
| 22 | return i; | |
| 23 | } | |
| 24 | ||
| 25 | test("hand calculation: per-test times sum to seconds per file, slowest five in order", () => { | |
| 26 | const lines = [ | |
| 27 | "test/a.test.ts:", "(pass) one [500.00ms]", "(pass) two [1.5s]", // 2.0 s | |
| 28 | "test/b.test.ts:", "(pass) x [3000.4ms]", // 3.0 s | |
| 29 | "test/c.test.ts:", "(pass) x [100ms]", | |
| 30 | "test/d.test.ts:", "(fail) x [4000ms]", | |
| 31 | "test/e.test.ts:", "(pass) x [250ms]", "(pass) y [250ms]", | |
| 32 | "test/f.test.ts:", "(pass) x [10ms]", | |
| 33 | "test/g.test.ts:", "(pass) no time here", | |
| 34 | ]; | |
| 35 | const { files, timed } = slowestFiles(lines); | |
| 36 | expect(files.map((f) => [f.file, f.seconds])).toEqual([["test/d.test.ts", 4], ["test/b.test.ts", 3], ["test/a.test.ts", 2], ["test/e.test.ts", 0.5], ["test/c.test.ts", 0.1]]); | |
| 37 | expect(timed).toBe(8); | |
| 38 | }); | |
| 39 | ||
| 40 | test("the duty reads the newest finished check's stored log and names five", async () => { | |
| 41 | expect((await run()).actions[0]).toContain("no finished check"); | |
| 42 | check(["test/old.test.ts:", "(pass) x [99000ms]"]); | |
| 43 | const lines = ["test/f0.test.ts:"]; | |
| 44 | for (let n = 1; n <= 7; n++) lines.push(`test/f${n}.test.ts:`, `(pass) t [${n * 1000}ms]`); | |
| 45 | const last = check(lines); | |
| 46 | const out = await run(); | |
| 47 | expect(out.findings).toHaveLength(1); | |
| 48 | const s = out.findings[0]!.summary; | |
| 49 | expect(s).toContain(last); | |
| 50 | expect(s).not.toContain("old.test.ts"); | |
| 51 | expect(s).toContain("f7.test.ts 7.0 s, test/f6.test.ts 6.0 s, test/f5.test.ts 5.0 s, test/f4.test.ts 4.0 s, test/f3.test.ts 3.0 s."); | |
| 52 | expect(s).not.toContain("f2.test.ts"); | |
| 53 | expect(out.findings[0]!.probe!.number).toBe(7); | |
| 54 | }); | |
| 55 | ||
| 56 | test("a stored log without timings is said, not guessed", async () => { | |
| 57 | const bare = check(["test/a.test.ts:", "(pass) x", "3 pass"]); | |
| 58 | const out = await run(); | |
| 59 | expect(out.findings).toEqual([]); | |
| 60 | expect(out.actions[0]).toContain(bare); | |
| 61 | expect(out.actions[0]).toContain("no stored output with per-test times"); | |
| 62 | }); | |
| 63 | ||
| 64 | test("it is registered", () => { | |
| 65 | expect(duties.map((d) => d.name)).toContain("slow-tests"); | |
| 66 | }); | |
| 67 | ||
| 68 | test("the slow-tests duty is one of the rows /maintenance lists, on, from the same registry", () => { | |
| 69 | expect(dutyStates(w.app.db).find((d) => d.name === "slow-tests")?.says).toBe("slow-tests: on."); | |
| 70 | }); |
test/pages-budget.test.ts +2 −1
| @@ -56,7 +56,8 @@ const HEAVIER: Record<string, { statements?: number; git?: number }> = { | ||
| 56 | 56 | "/api/v1/asks": { statements: 61 }, // +1: the stopped apps someone asked for, one read of the viewer's archived repositories (#12277) |
| 57 | 57 | "/board": { statements: 72 }, // +1: the forge-copy read for the tell box (#12037); +1: the vitals read the database's growth (#12025); +6: the money line (#510) — one settings read, then the month's and the 30 days' meter, storage and ledger reads; constant in the number of apps |
| 58 | 58 | "/settings/capacity": { statements: 70 }, |
| 59 | "/maintenance": { statements: 75 }, // +3: untested-sources (#14125) reads its own state and runs, and the dismissal query (#13967) is one more; +4: the flaky-tests (#13937), red apps (#13938) and chapter stall (#14057) and dead-doc-paths (#13977) duties each read their own state and run | |
| 59 | "/maintenance": { statements: 78 }, // +5: the slow-tests duty (#13966) reads its own state, runs and is listed; +3: untested-sources (#14125) reads its own state and runs, and the dismissal query (#13967) is one more; +4: the flaky-tests (#13937), red apps (#13938) and chapter stall (#14057) and dead-doc-paths (#13977) duties each read their own state and run | |
| 60 | "/api/v1/maintenance/duties": { statements: 63 }, // +3: the slow-tests duty (#13966) reads its own state and runs, as on /maintenance | |
| 60 | 61 | }; |
| 61 | 62 | |
| 62 | 63 | /** Patterns with no fixture today: lower this as fixtures are added, never raise it silently. */ |
new above
offline
Doc missing: add the slow-tests section to docs/features/maintenance.md (see untested-sources there).