trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRuns12g rEventsFlows

Check time is predictable: test-times.json measured for every gate file, tsc cached, the gate-list audit before tsc, and pull checks pause while main is red #1836

Merged torquecrew opened this in answer to trevatoowner

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

torque/forge:torque/1797-check-time-is-predictable-test-times-jso → main 4 commits fork

  • Merged. torque/1797-check-time-is-predictable-test-times-jso was merged into main as 095ee34.

Review

Sign in to review this change.
15 files changed +933 −70
.gitignore +3
@@ -19,3 +19,6 @@ typesafe-api-key
1919 /forge.db*
2020 /ssh/
2121 /*.sock
22
23 # what a check keeps for the next one (src/checks/cache.ts)
24 /.forge-cache/
README.md +11
@@ -659,6 +659,15 @@ shard's output is written whole as it finishes; the log ends with every
659659 failing test from every shard, or on green with one line — how many tests,
660660 across how many files, in how long, then the slowest and fastest shard.
661661
662 The check's first step is the gate-list audit (`bun scripts/gate.ts`), so a
663 test file listed in neither `GATE` nor `EXCLUDED` fails in under a second,
664 before install-heavy work. `tsc` then runs `--incremental` with a 2 GB heap,
665 its build info in `.forge-cache/`: the forge restores that directory from the
666 base's own green check and keeps it only from green checks of the default
667 branch (`src/checks/cache.ts`), so a pull's code never fills a cache another
668 check reads. The full gate also writes every file's time there, and the next
669 split weighs every file by measurement rather than the 41 the table named.
670
662671 The fast tier picks by measurement. The full gate runs each file in a
663672 `bun test` of its own under Bun's coverage (`--impact`; coverage is per
664673 process, so a shard of many files would blur theirs together) and writes
@@ -896,6 +905,8 @@ Invariants are proven by faults, not by hope. No dependencies were added.
896905
897906 In the forge itself: a pull's check runs `fast` when the manifest has one, in a clone holding the fork point and the head as `FORGE_CHECK_BASE` and `HEAD` (`origin/main` is not a ref there), with `FORGE_CHECK_LEARNED` naming tests earlier land attempts taught the pull (`src/checks/history.ts`, `learned.ts`). Landing (`landIfGreen`) runs `full` once on the merge with its base before merging (`src/checks/landing.ts`); red comments on the pull with the tests named, is remembered per head and base, and the files ride the pull's next fast run. The pull page's checks line shows the tier words.
898907
908 While the default branch's own last check is red, the forge holds its pulls' checks queued rather than fail each one on the same test (`src/checks/red_main.ts`): the vitals carry one card naming the failing test and how many checks wait, and main going green releases them. A check a person asks for again runs at once, and no check is held more than 20 minutes, so the pull that fixes main can always show green.
909
899910 ## Limits
900911
901912 Every limit the owner can hit is declared in `src/settings/limits.ts` with its default, bounds and reason, and changed live by a forge admin at `/admin/limits`, `PUT /api/v1/admin/limits/:key` or the `limits` MCP tool (never by an agent). Each change is kept with who made it.
forge.json +2 −2
@@ -14,8 +14,8 @@
1414 }
1515 },
1616 "check": {
17 "fast": "node --max-old-space-size=1024 ./node_modules/typescript/lib/tsc.js --noEmit && bun scripts/test-shards.ts --timeout 60000 --tier fast --diff $FORGE_CHECK_BASE..HEAD --learned \"$FORGE_CHECK_LEARNED\"",
18 "run": "node --max-old-space-size=1024 ./node_modules/typescript/lib/tsc.js --noEmit && bun scripts/test-shards.ts --timeout 60000 --gate --impact",
17 "fast": "bun scripts/gate.ts && node --max-old-space-size=2048 ./node_modules/typescript/lib/tsc.js --noEmit --incremental --tsBuildInfoFile .forge-cache/tsc.tsbuildinfo && bun scripts/test-shards.ts --timeout 60000 --tier fast --diff $FORGE_CHECK_BASE..HEAD --learned \"$FORGE_CHECK_LEARNED\"",
18 "run": "bun scripts/gate.ts && node --max-old-space-size=2048 ./node_modules/typescript/lib/tsc.js --noEmit --incremental --tsBuildInfoFile .forge-cache/tsc.tsbuildinfo && bun scripts/test-shards.ts --timeout 60000 --gate --impact",
1919 "timeout": 7200,
2020 "limits": {
2121 "memory": 8192,
scripts/gate.ts +39
@@ -15,6 +15,8 @@
1515 * issue the nightly hunt filed (`.forgejo/workflows/flake-hunt.yml`).
1616 */
1717
18 import { readdirSync } from "node:fs";
19
1820 export type Exclusion = {
1921 /** Repository-relative path, `test/….test.ts`. */
2022 file: string;
@@ -163,6 +165,7 @@ export const GATE: readonly string[] = [
163165 "test/schema-migrations.test.ts",
164166 "test/vitals.test.ts",
165167 "test/vitals-degraded.test.ts",
168 "test/red-main.test.ts",
166169 "test/sentinel.test.ts",
167170 "test/tui-sentinel.test.ts",
168171 "test/version.test.ts",
@@ -448,3 +451,39 @@ export function auditGate(
448451 problems.push(`${file} is in neither GATE nor EXCLUDED`);
449452 return problems;
450453 }
454
455 /** The `test/*.test.ts` files under `root`, as the gate names them. */
456 export function presentTests(root = "."): string[] {
457 try {
458 return readdirSync(`${root}/test`)
459 .filter((name) => name.endsWith(".test.ts"))
460 .map((name) => `test/${name}`)
461 .sort();
462 } catch {
463 return [];
464 }
465 }
466
467 /**
468 * What the check prints when the list is wrong: every problem, then one
469 * last line naming the files to list. Empty when the list is right.
470 */
471 export function gateRefusal(present: readonly string[]): string[] {
472 const problems = auditGate(present, GATE, EXCLUDED);
473 if (problems.length === 0) return [];
474 const named = [
475 ...new Set(problems.map((p) => p.match(/^(test\/\S+\.test\.ts) /)?.[1])),
476 ].filter((f): f is string => f !== undefined);
477 return [
478 ...problems,
479 `${named.join(", ")}: list it in GATE or EXCLUDED in scripts/gate.ts`,
480 ];
481 }
482
483 // `bun scripts/gate.ts`: the audit alone, first in `forge.json`'s check, so
484 // an unlisted file fails in under a second — not after install and tsc.
485 if (import.meta.main) {
486 const refusal = gateRefusal(presentTests());
487 for (const line of refusal) console.error(`gate: ${line}`);
488 process.exit(refusal.length === 0 ? 0 : 2);
489 }
scripts/shards.ts +40
@@ -206,6 +206,46 @@ export function formatTimes(times: ReadonlyMap<string, number>): string {
206206 return lines.length === 0 ? "{}\n" : `{\n${lines.join(",\n")}\n}\n`;
207207 }
208208
209 /**
210 * The order a pool of one-file workers takes files in: heaviest first (LPT
211 * on a work queue: the wall is at most the heaviest file plus an even
212 * share of the rest), ties in the list's own order.
213 */
214 export function heaviestFirst(
215 files: readonly string[],
216 weight: (file: string) => number,
217 ): string[] {
218 return files
219 .map((file, index) => ({ file, index, weight: weight(file) }))
220 .sort((a, b) => b.weight - a.weight || a.index - b.index)
221 .map((item) => item.file);
222 }
223
224 /** `over`'s times where it has them, `base`'s for every other file. */
225 export function overlayTimes(
226 base: ReadonlyMap<string, number>,
227 over: ReadonlyMap<string, number>,
228 ): Map<string, number> {
229 const merged = new Map(base);
230 for (const [file, seconds] of over) merged.set(file, seconds);
231 return merged;
232 }
233
234 /**
235 * The table after a green run: every file it ran at the time it took, every
236 * other file it still has at its old time, a file no longer on disk gone.
237 */
238 export function refreshTimes(
239 old: ReadonlyMap<string, number>,
240 ran: readonly { file: string; seconds: number }[],
241 exists: (file: string) => boolean,
242 ): Map<string, number> {
243 const fresh = new Map<string, number>();
244 for (const [file, seconds] of old) if (exists(file)) fresh.set(file, seconds);
245 for (const { file, seconds } of ran) fresh.set(normalise(file), seconds);
246 return fresh;
247 }
248
209249 /** One shard, finished: what it ran, how it exited, everything it printed. */
210250 export interface ShardResult {
211251 readonly files: readonly string[];
scripts/test-shards.ts +78 −60
@@ -24,18 +24,28 @@
2424 * the rest (fail-fast) so a known fail does not burn the remaining wall
2525 * clock. Exits non-zero if any shard does.
2626 *
27 * The whole gate, unsliced (`--gate` on the full tier, as `check.run`),
28 * runs one file per `bun test`, heaviest first into whichever worker is
29 * free, so every file is timed alone; when it is green it writes those
30 * times to `.forge-cache/test-times.json`, which the forge keeps for the
31 * next check (`src/checks/cache.ts`).
32 *
2733 * Files are weighed by how long they took, from `scripts/test-times.json`
28 * (a file it does not name, or a table it refuses, falls back to bytes).
34 * overlaid by `.forge-cache/test-times.json` (a file neither names, or a
35 * table it refuses, falls back to bytes).
2936 * `--record` refreshes that table: the same number of workers, but each
3037 * file is its own `bun test`, timed alone from spawn to exit, and the
3138 * table is rewritten from those times when every file is green.
3239 */
3340
3441 import { availableParallelism, tmpdir } from "node:os";
35 import { mkdtempSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
42 import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
3643 import { join } from "node:path";
3744 import {
3845 formatTimes,
46 heaviestFirst,
47 overlayTimes,
48 refreshTimes,
3949 parseShardSpec,
4050 parseTimes,
4151 shardCount,
@@ -46,9 +56,10 @@ import {
4656 weigher,
4757 type ShardResult,
4858 } from "./shards.ts";
49 import { auditGate, EXCLUDED, GATE } from "./gate.ts";
59 import { GATE, gateRefusal, presentTests } from "./gate.ts";
5060 import { executedFiles, formatImpact, handedImpact, mergeImpact, parseImpact } from "../src/checks/impact.ts";
5161 import { fastSet, importsOf, duration } from "../src/checks/tiers.ts";
62 import { CHECK_CACHE } from "../src/checks/cache.ts";
5263
5364 const TIMES = new URL("./test-times.json", import.meta.url).pathname;
5465 // The forge hands the map over in the checkout (`src/checks/impact-store.ts`):
@@ -57,12 +68,41 @@ const TIMES = new URL("./test-times.json", import.meta.url).pathname;
5768 const handed = handedImpact(process.env["FORGE_CHECK_IMPACT"]);
5869 const IMPACT = handed ?? new URL("./test-impact.json", import.meta.url).pathname;
5970 const IMPACT_NAME = handed ?? "scripts/test-impact.json";
71 // What the forge keeps between checks (`src/checks/cache.ts`): the full
72 // gate's own per-file times, fresher than the committed table.
73 const CACHED_TIMES = `${CHECK_CACHE}/test-times.json`;
6074
6175 const argv = process.argv.slice(2);
6276 const record = argv.includes("--record");
6377 const gate = argv.includes("--gate");
6478 const measure = argv.includes("--impact");
6579
80 /**
81 * Measured seconds where a table has them, bytes scaled to seconds where
82 * it does not: the committed table, overlaid by the times the last green
83 * full gate left in the forge's cache. A missing or refused table is said
84 * out loud and the gate still runs, on what is left.
85 */
86 function loadTimes(): Map<string, number> {
87 const read = (path: string, name: string, quiet: boolean): Map<string, number> => {
88 let text: string;
89 try {
90 text = readFileSync(path, "utf8");
91 } catch {
92 if (!quiet) console.log(`test-shards: no ${name}; weighing by bytes`);
93 return new Map();
94 }
95 const parsed = parseTimes(text);
96 if (parsed.ok) return parsed.value;
97 console.log(`test-shards: ${name} refused (${parsed.error}); weighing by bytes`);
98 return new Map();
99 };
100 return overlayTimes(
101 read(TIMES, "scripts/test-times.json", false),
102 read(CACHED_TIMES, CACHED_TIMES, true),
103 );
104 }
105
66106 /** Pull `--shard N/M` or `--shard=N/M` out of argv so it never reaches bun test. */
67107 function takeShardArg(raw: string[]): {
68108 rest: string[];
@@ -182,22 +222,9 @@ const flags = split.flags;
182222 // A test file in neither GATE nor EXCLUDED costs a whole gate run to find
183223 // out; say so before anything spawns. Explicit files alone are not audited.
184224 if (gate || tier === "fast") {
185 let present: string[] = [];
186 try {
187 present = readdirSync("test")
188 .filter((name) => name.endsWith(".test.ts"))
189 .map((name) => `test/${name}`)
190 .sort();
191 } catch {}
192 const problems = auditGate(present, GATE, EXCLUDED);
193 if (problems.length > 0) {
194 for (const problem of problems) console.error(`test-shards: ${problem}`);
195 const named = [
196 ...new Set(problems.map((p) => p.match(/^(test\/\S+\.test\.ts) /)?.[1])),
197 ].filter((f): f is string => f !== undefined);
198 console.error(
199 `test-shards: ${named.join(", ")}: list it in GATE or EXCLUDED in scripts/gate.ts`,
200 );
225 const refusal = gateRefusal(presentTests());
226 if (refusal.length > 0) {
227 for (const line of refusal) console.error(`test-shards: ${line}`);
201228 process.exit(2);
202229 }
203230 }
@@ -212,15 +239,6 @@ if (tier === "fast") {
212239 process.exit(2);
213240 }
214241 const changed = diff.stdout.toString().split("\n").filter((f) => f !== "");
215 const timesTable = parseTimes(
216 (() => {
217 try {
218 return readFileSync(TIMES, "utf8");
219 } catch {
220 return "";
221 }
222 })(),
223 );
224242 const cache = new Map<string, string[]>();
225243 const imports = (file: string): string[] => {
226244 let found = cache.get(file);
@@ -249,7 +267,7 @@ if (tier === "fast") {
249267 changed,
250268 tests: files,
251269 imports,
252 times: timesTable.ok ? timesTable.value : new Map(),
270 times: loadTimes(),
253271 learned: (learnedArg.value ?? "").split(",").filter((f) => f !== ""),
254272 });
255273 console.log(
@@ -314,24 +332,7 @@ if (!count.ok) {
314332 delete process.env["FORGE_CHECK_SHARDS"];
315333 delete process.env["FORGE_CHECK_SHARD_INDEX"];
316334 delete process.env["FORGE_CHECK_SHARD_TOTAL"];
317 // Measured seconds where the table has them, bytes scaled to seconds
318 // where it does not. A missing or refused table is said out loud and the
319 // gate still runs, on bytes alone.
320 let times = new Map<string, number>();
321 let table = "";
322 try {
323 table = readFileSync(TIMES, "utf8");
324 } catch {
325 console.log("test-shards: no scripts/test-times.json; weighing by bytes");
326 }
327 if (table !== "") {
328 const parsed = parseTimes(table);
329 if (parsed.ok) times = parsed.value;
330 else
331 console.log(
332 `test-shards: scripts/test-times.json refused (${parsed.error}); weighing by bytes`,
333 );
334 }
335 const times = loadTimes();
335336 const weight = weigher(files, size, times);
336337 // Recording and measuring run one file per process, so every file is its
337338 // own slice: Bun's coverage is per process, and a shard of many files would
@@ -340,11 +341,17 @@ const weight = weigher(files, size, times);
340341 // bun test held the whole gate (#386/#388, exit 137 ~15–17m). Same count
341342 // with one-file-per-process, serial, keeps peak RSS to a single suite.
342343 const serialFiles = !record && count.value === 1;
343 const perFile = record || measure || serialFiles;
344 // The whole gate, unsliced, runs one file per process too, heaviest first
345 // into whichever worker frees: no shard waits on a slow neighbour it was
346 // statically paired with, and every file is timed alone, so a green gate
347 // refreshes the time of every file it ran (#1797).
348 const timed = !record && !serialFiles && gate && tier === "full" && "none" in shardSpec;
349 const perFile = record || measure || serialFiles || timed;
344350 const slices =
345351 perFile
346 ? files.map((file) => [file])
352 ? heaviestFirst(files, weight).map((file) => [file])
347353 : splitFiles(files, count.value, weight);
354 const work = files.reduce((sum, file) => sum + weight(file), 0);
348355
349356 const banner = record
350357 ? `test-shards: recording ${files.length} files, ${count.value} at a time`
@@ -352,9 +359,12 @@ const banner = record
352359 ? `test-shards: measuring ${files.length} files under coverage, one bun test each, ${count.value} at a time`
353360 : serialFiles
354361 ? `test-shards: ${files.length} files serial (one bun test each; FORGE_CHECK_SHARDS=1), ` +
355 `expected ${files.reduce((sum, file) => sum + weight(file), 0).toFixed(0)}s`
356 : `test-shards: ${files.length} files in ${slices.length} shard${slices.length === 1 ? "" : "s"}, ` +
357 `expected ${slices.map((slice) => slice.reduce((sum, file) => sum + weight(file), 0).toFixed(0) + "s").join(" ")}`;
362 `expected ${work.toFixed(0)}s`
363 : timed
364 ? `test-shards: ${files.length} files, one bun test each, ${count.value} at a time, ` +
365 `expected ${(work / count.value).toFixed(0)}s (${work.toFixed(0)}s of work)`
366 : `test-shards: ${files.length} files in ${slices.length} shard${slices.length === 1 ? "" : "s"}, ` +
367 `expected ${slices.map((slice) => slice.reduce((sum, file) => sum + weight(file), 0).toFixed(0) + "s").join(" ")}`;
358368 console.log(banner);
359369 process.stderr.write(banner + "\n");
360370
@@ -496,7 +506,7 @@ const started = performance.now();
496506 // Serial one-file processes when SHARDS=1; record keeps its worker width;
497507 // multi-shard still runs every slice side by side.
498508 const results = await pool(
499 serialFiles ? 1 : record || measure ? count.value : slices.length,
509 serialFiles ? 1 : record || measure || timed ? count.value : slices.length,
500510 );
501511 if (results.length < slices.length) {
502512 const skipped = slices.length - results.length;
@@ -507,23 +517,31 @@ if (results.length < slices.length) {
507517 const ending = tally(results, (performance.now() - started) / 1000);
508518 const spread = results.map((result) => result.seconds);
509519 console.log(`\n${ending.lines.join("\n")}`);
510 if (!record && results.length > 1)
520 if (!record && !timed && results.length > 1)
511521 console.log(
512522 `Slowest shard ${Math.max(...spread).toFixed(1)}s, fastest ${Math.min(...spread).toFixed(1)}s.`,
513523 );
514 if (record) {
524 if (record || timed) {
515525 if (ending.code !== 0) {
516526 console.log(
517527 "test-shards: not recording: a file is red, so its time is not its time",
518528 );
519529 } else {
520530 // Files this run did not name keep their old times: a partial run
521 // refreshes what it ran and forgets nothing else.
522 const fresh = new Map(times);
523 for (const result of results) fresh.set(result.files[0]!, result.seconds);
524 writeFileSync(TIMES, formatTimes(fresh));
531 // refreshes what it ran and forgets nothing else. A file gone from
532 // the tree is forgotten.
533 const fresh = refreshTimes(
534 times,
535 results.map((result) => ({ file: result.files[0]!, seconds: result.seconds })),
536 (file) => existsSync(file),
537 );
538 // `--record` rewrites the committed table; the check's full gate
539 // writes the cache the forge keeps for the next check.
540 const target = record ? TIMES : CACHED_TIMES;
541 if (!record) mkdirSync(CHECK_CACHE, { recursive: true });
542 writeFileSync(target, formatTimes(fresh));
525543 console.log(
526 `test-shards: wrote ${results.length} times to scripts/test-times.json`,
544 `test-shards: wrote ${results.length} times to ${record ? "scripts/test-times.json" : CACHED_TIMES}`,
527545 );
528546 }
529547 }
src/checks/cache.ts added +129
@@ -0,0 +1,129 @@
1 /**
2 * What a check keeps for the next one (#1797). A check's checkout is a
3 * fresh export every time, so `tsc` ran cold in every check and the gate's
4 * per-file times died with the checkout. `.forge-cache/` in the checkout is
5 * the one place that outlives it: after a green check of the default
6 * branch its regular files are copied under the data directory, keyed by
7 * that commit, and a later check of the repository starts with them — a
8 * pull's check with the copy its base's own check left, else the newest.
9 *
10 * Only the default branch writes: a pull's code never fills a cache another
11 * check reads (a forged `.tsbuildinfo` could tell `tsc` a file has no
12 * errors). Only regular files, flat, by a safe name and bounded in count
13 * and bytes, are copied either way: a symlink planted in the checkout is
14 * never followed out of it.
15 */
16
17 import {
18 copyFileSync,
19 lstatSync,
20 mkdirSync,
21 readdirSync,
22 renameSync,
23 rmSync,
24 statSync,
25 } from "node:fs";
26 import { join } from "node:path";
27 import { isSafeOid } from "../git/guard.ts";
28
29 /** The directory, relative to the checkout, the check command writes to. */
30 export const CHECK_CACHE = ".forge-cache";
31
32 /** How many default-branch commits' caches a repository keeps. */
33 export const CACHE_KEEP = 4;
34 /** At most this many files, and this many bytes, are kept per commit. */
35 export const CACHE_FILES = 32;
36 export const CACHE_BYTES = 64 * 1024 * 1024;
37
38 const NAME = /^[A-Za-z0-9_][A-Za-z0-9_.-]{0,99}$/;
39
40 const repoCache = (dataDir: string, repoId: string): string =>
41 join(dataDir, "check-cache", repoId.replace(/[^A-Za-z0-9_-]/g, "_"));
42
43 /** The regular files of `dir` worth copying, or none if it breaks a bound. */
44 function keepable(dir: string): string[] {
45 // The directory itself is not followed either: a link to elsewhere is no cache.
46 if (!lstatSync(dir, { throwIfNoEntry: false })?.isDirectory()) return [];
47 let names: string[];
48 try {
49 names = readdirSync(dir);
50 } catch {
51 return [];
52 }
53 const files: string[] = [];
54 let bytes = 0;
55 for (const name of names.sort()) {
56 if (!NAME.test(name)) continue;
57 const stat = lstatSync(join(dir, name), { throwIfNoEntry: false });
58 if (!stat?.isFile()) continue;
59 bytes += stat.size;
60 files.push(name);
61 }
62 return files.length > CACHE_FILES || bytes > CACHE_BYTES ? [] : files;
63 }
64
65 /**
66 * Seed `checkout/.forge-cache` from the first of `keys` (commit ids) that
67 * has a cache, else the newest one. Answers the key used, if any.
68 */
69 export function restoreCache(
70 dataDir: string,
71 repoId: string,
72 keys: readonly string[],
73 checkout: string,
74 ): string | undefined {
75 const root = repoCache(dataDir, repoId);
76 const into = join(checkout, CHECK_CACHE);
77 // Whatever the tree carried under the name is not a cache.
78 rmSync(into, { recursive: true, force: true });
79 mkdirSync(into, { recursive: true });
80 const key =
81 keys.find((key) => isSafeOid(key) && statSync(join(root, key), { throwIfNoEntry: false })?.isDirectory()) ??
82 newest(root);
83 if (key === undefined) return undefined;
84 for (const name of keepable(join(root, key)))
85 copyFileSync(join(root, key, name), join(into, name));
86 return key;
87 }
88
89 /**
90 * Keep `checkout/.forge-cache` as the cache of `commit`, replacing any copy
91 * under that key whole, and drop all but the newest {@link CACHE_KEEP}.
92 */
93 export function keepCache(
94 dataDir: string,
95 repoId: string,
96 commit: string,
97 checkout: string,
98 ): boolean {
99 if (!isSafeOid(commit)) return false;
100 const from = join(checkout, CHECK_CACHE);
101 const files = keepable(from);
102 if (files.length === 0) return false;
103 const root = repoCache(dataDir, repoId);
104 const staging = join(root, `.${commit}.${process.pid}`);
105 rmSync(staging, { recursive: true, force: true });
106 mkdirSync(staging, { recursive: true });
107 for (const name of files) copyFileSync(join(from, name), join(staging, name));
108 rmSync(join(root, commit), { recursive: true, force: true });
109 renameSync(staging, join(root, commit));
110 for (const old of byAge(root).slice(CACHE_KEEP))
111 rmSync(join(root, old), { recursive: true, force: true });
112 return true;
113 }
114
115 /** The commits with a cache, newest first. */
116 function byAge(root: string): string[] {
117 let names: string[];
118 try {
119 names = readdirSync(root).filter(isSafeOid);
120 } catch {
121 return [];
122 }
123 return names
124 .map((name) => ({ name, at: statSync(join(root, name), { throwIfNoEntry: false })?.mtimeMs ?? 0 }))
125 .sort((a, b) => b.at - a.at || (a.name < b.name ? -1 : 1))
126 .map((entry) => entry.name);
127 }
128
129 const newest = (root: string): string | undefined => byAge(root)[0];
src/checks/red_main.ts added +175
@@ -0,0 +1,175 @@
1 /**
2 * A red main pauses its pulls' checks (#1797). While the default branch's
3 * own last check failed, every pull's check fails the same case for a reason
4 * none of them caused — 21 heads failed one test in 0.3 h, each sent back to
5 * its agent. So a pull's check of that repository waits, queued, until main
6 * is green again; main's own checks keep running, and the vitals carry one
7 * card naming the failing test in place of a failure per pull.
8 *
9 * The fix for a red main comes through a pull, and that pull must be able
10 * to show green, so the hold is bounded twice: a pull a person asks to
11 * check (`checks/again.ts`, reason `asked`) is not held, and no check is
12 * held past {@link RED_HOLD_MS} after it was queued. The service holds only
13 * when told how long (`ChecksOptions.redHoldMs`); the forge's `main.ts`
14 * says this.
15 *
16 * Plain data: the last finished check of each default branch, read by the
17 * `run_repo` index, one query whoever asks.
18 */
19
20 import type { Database } from "bun:sqlite";
21 import type { Run } from "../core/runs.ts";
22
23 export interface RedMain {
24 readonly repoId: string;
25 /** `owner/name`, for the card. */
26 readonly repo: string;
27 readonly runId: string;
28 readonly commitOid: string;
29 /** The failing tests' names the check recorded, else its summary. */
30 readonly failing: readonly string[];
31 readonly summary: string;
32 readonly since: string;
33 }
34
35 /** The reason a person's own ask carries; it runs while main is red. */
36 export const ASKED = "asked";
37
38 /** The longest a red main holds a pull's check before it runs anyway. */
39 export const RED_HOLD_MS = 20 * 60_000;
40
41 interface Row {
42 id: string;
43 repo_id: string;
44 repo: string;
45 commit_oid: string;
46 state: string;
47 summary: string;
48 check_detail: string;
49 finished_at: string | null;
50 created_at: string;
51 }
52
53 /**
54 * The repository's red main, if its default branch's last finished check
55 * failed. A check cancelled or still running says nothing either way.
56 */
57 export function redMain(db: Database, repoId: string): RedMain | undefined {
58 const row = db.query<Row, [string]>(lastMain("SELECT ?1 AS repo_id")).get(repoId);
59 return row === null ? undefined : toRed(row);
60 }
61
62 /**
63 * The last finished check of the default branch of each repository `from`
64 * names (a query answering `repo_id`),
65 * joined to its owner for the card. A constant of this module, never input.
66 */
67 const lastMain = (from: string): string => `
68 SELECT run.id, run.repo_id, run.commit_oid, run.state, run.summary,
69 run.check_detail, run.finished_at, run.created_at,
70 u.username || '/' || repo.name AS repo
71 FROM (${from}) l
72 JOIN repo ON repo.id = l.repo_id
73 JOIN user u ON u.id = repo.owner_id
74 JOIN run ON run.id = (
75 SELECT r.id FROM run r
76 WHERE r.repo_id = l.repo_id AND r.kind = 'check'
77 AND r.issue_id IS NULL AND r.env = repo.default_branch
78 AND r.state IN ('ok', 'failed')
79 ORDER BY r.id DESC LIMIT 1)`;
80
81 function toRed(row: Row): RedMain | undefined {
82 if (row.state !== "failed") return undefined;
83 let failing: string[] = [];
84 try {
85 const detail = row.check_detail ? (JSON.parse(row.check_detail) as { names?: unknown }) : {};
86 if (Array.isArray(detail.names))
87 failing = detail.names.filter((n): n is string => typeof n === "string");
88 } catch {
89 failing = [];
90 }
91 return {
92 repoId: row.repo_id,
93 repo: row.repo,
94 runId: row.id,
95 commitOid: row.commit_oid,
96 failing,
97 summary: row.summary,
98 since: row.finished_at ?? row.created_at,
99 };
100 }
101
102 /** Whether a red main still holds `run`: a pull's check, not asked for, queued under `holdMs` ago. */
103 export function heldByRed(
104 run: Pick<Run, "kind" | "issueId" | "reason" | "createdAt">,
105 red: RedMain | undefined,
106 holdMs: number,
107 now: number,
108 ): boolean {
109 return (
110 red !== undefined &&
111 run.kind === "check" &&
112 run.issueId !== null &&
113 run.reason !== ASKED &&
114 now - Date.parse(run.createdAt) < holdMs
115 );
116 }
117
118 /**
119 * Which queued checks may start: all but the pull checks a red main holds.
120 * `red` is asked once per repository among `queued`, and not at all when
121 * nothing is held (`holdMs` 0).
122 */
123 export function unheld(
124 queued: readonly Run[],
125 red: (repoId: string) => RedMain | undefined,
126 holdMs: number,
127 now: number = Date.now(),
128 ): Run[] {
129 if (holdMs <= 0) return [...queued];
130 const seen = new Map<string, RedMain | undefined>();
131 return queued.filter((run) => {
132 if (run.issueId === null || run.reason === ASKED) return true;
133 if (!seen.has(run.repoId)) seen.set(run.repoId, red(run.repoId));
134 return !heldByRed(run, seen.get(run.repoId), holdMs, now);
135 });
136 }
137
138 /** Pull checks queued, not asked for, per repository: `n` of them. Read by `run_queue`. */
139 const HELD = `
140 SELECT repo_id, COUNT(*) AS n FROM run
141 WHERE state = 'queued' AND kind = 'check'
142 AND issue_id IS NOT NULL AND reason <> '${ASKED}'
143 GROUP BY repo_id`;
144
145 /**
146 * Each red main with pull checks waiting on it, and how many wait: one
147 * query. A check waiting past the hold is still counted until it runs.
148 */
149 export function heldChecks(db: Database): { red: RedMain; held: number }[] {
150 const rows = db
151 .query<Row & { n: number }, []>(
152 `SELECT last.*, h.n FROM (${lastMain(HELD)}) last
153 JOIN (${HELD}) h ON h.repo_id = last.repo_id
154 WHERE last.state = 'failed'`,
155 )
156 .all();
157 const out: { red: RedMain; held: number }[] = [];
158 for (const row of rows) {
159 const red = toRed(row);
160 if (red !== undefined) out.push({ red, held: row.n });
161 }
162 return out;
163 }
164
165 /** The card's sentence: what is red, by which test, and what waits. */
166 export function redSentence(red: RedMain, held: number): string {
167 const first = red.failing[0];
168 const what =
169 first === undefined
170 ? red.summary.replace(/^check failed: /, "")
171 : red.failing.length === 1
172 ? `"${first}" fails`
173 : `"${first}" and ${red.failing.length - 1} more fail`;
174 return `${red.repo}: main is red at ${red.commitOid.slice(0, 7)} — ${what}; ${held} pull check${held === 1 ? "" : "s"} wait for it to pass.`;
175 }
src/checks/service.ts +38 −3
@@ -42,6 +42,8 @@ import { resolveOid } from "../git/merge.ts";
4242 import { mergeForCheck } from "./merged.ts";
4343 import { giveHistory } from "./history.ts";
4444 import { readLearned } from "./learned.ts";
45 import { keepCache, restoreCache } from "./cache.ts";
46 import { ASKED, redMain, unheld } from "./red_main.ts";
4547 import { IMPACT_FILE, keepImpact, seedImpact } from "./impact-store.ts";
4648 import { getIssueById } from "../core/issues.ts";
4749 import { repoDir } from "../git/repos.ts";
@@ -79,6 +81,11 @@ export interface ChecksOptions {
7981 readonly forgeUrl: string;
8082 /** How long a check may run before it is killed. @default BUDGETS.gate */
8183 readonly budgetMs?: number | undefined;
84 /**
85 * How long a red main holds a pull's check (`checks/red_main.ts`). @default 0,
86 * nothing held; the forge says `RED_HOLD_MS`.
87 */
88 readonly redHoldMs?: number | undefined;
8289 /**
8390 * How long `stop` lets checks already running finish before it cuts them
8491 * off and queues them again (#1796). Nothing new starts meanwhile.
@@ -237,7 +244,12 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
237244 const { run, fresh } = transact(app.db, () => {
238245 for (const active of listActive(app.db, "check")) {
239246 if (active.repoId !== repo.id || active.env !== env) continue;
240 if (active.commitOid === commitOid)
247 // A person asking for a check a red main holds queues it as theirs,
248 // which runs (`checks/red_main.ts`); any other repeat is the same check.
249 if (
250 active.commitOid === commitOid &&
251 !(reason === ASKED && active.state === "queued" && active.reason !== ASKED)
252 )
241253 return { run: active, fresh: false };
242254 // A pull's running check is for a head nobody will land either:
243255 // it is stopped too, below, once the row says so.
@@ -302,8 +314,13 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
302314 const limit = capacity();
303315 gate.resize(limit);
304316 while (inFlight.size < limit) {
305 const queued = listActive(app.db, "check").filter(
306 (run) => run.state === "queued" && !inFlight.has(run.id),
317 // A pull's check waits while its main is red (`checks/red_main.ts`).
318 const queued = unheld(
319 listActive(app.db, "check").filter(
320 (run) => run.state === "queued" && !inFlight.has(run.id),
321 ),
322 (repoId) => redMain(app.db, repoId),
323 options.redHoldMs ?? 0,
307324 );
308325 const next = pickNext(queued);
309326 if (!next) return;
@@ -472,6 +489,7 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
472489 commitOid: run.commitOid,
473490 pull,
474491 tier: pull === undefined ? "run" : "fast",
492 keep: run.issueId === null && run.env === repo.defaultBranch,
475493 keepImpact: pull === undefined && run.env === repo.defaultBranch,
476494 write,
477495 signal,
@@ -489,6 +507,8 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
489507 readonly pull?: Pull | undefined;
490508 /** `run`: a push to a branch; `fast`: a pull's push; `full`: on land. */
491509 readonly tier: "run" | "fast" | "full";
510 /** A check of the default branch's own push: its cache is kept. */
511 readonly keep?: boolean;
492512 /** A green run keeps the impact map it measured; `full` always does. */
493513 readonly keepImpact?: boolean;
494514 readonly write: (line: string) => void;
@@ -591,6 +611,15 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
591611 return { ok: false, error: given.error };
592612 }
593613 }
614 // What the base's own check left (`checks/cache.ts`): tsc's build info
615 // and the gate's per-file times, keyed by the commit this one builds on.
616 const cached = restoreCache(
617 app.config.dataDir,
618 repo.id,
619 tip === undefined ? [] : [tip],
620 dir,
621 );
622 if (cached !== undefined) write(`cache from ${cached.slice(0, 7)}`);
594623 // Every check starts from the impact map the last green default-branch
595624 // run measured; the fast tier selects by it.
596625 try {
@@ -686,6 +715,12 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
686715 );
687716 return { ok: false, error: summary, detail };
688717 }
718 // Only a green check of the default branch is kept for the next.
719 if (
720 x.keep === true &&
721 keepCache(app.config.dataDir, repo.id, run.commitOid, dir)
722 )
723 write(`cache kept for ${run.commitOid.slice(0, 7)}`);
689724 if (x.tier === "full" || x.keepImpact === true) {
690725 try {
691726 const kept = keepImpact(app.config.dataDir, repo.id, dir);
src/kernel/vitals.ts +11 −1
@@ -22,6 +22,7 @@ import {
2222 import { STALE_AFTER_MS } from "../runs/lease.ts";
2323 import { BUDGETS } from "./budgets.ts";
2424 import { failedSentence, lastVerdict } from "../snapshot/verdict.ts";
25 import { heldChecks, redSentence } from "../checks/red_main.ts";
2526
2627 /** Free disk under this share of the volume is something to say. */
2728 export const DISK_FREE_MIN = 0.1;
@@ -40,7 +41,7 @@ export interface EnvCount {
4041 /**
4142 * One cause of the forge being off: the `needs` sentence, a `cause` key that
4243 * holds for as long as its condition does (`disk`, `errors`, `internal`,
43 * `queue:<kind>`, `stuck:<runId>`), when it began by the forge's own evidence
44 * `queue:<kind>`, `stuck:<runId>`, `red:<repoId>`), when it began by the forge's own evidence
4445 * (the oldest evidence the forge holds, else the first time this process saw
4546 * it, held until the cause clears; `at` is never the answer to a repeat
4647 * call), and where to look.
@@ -302,6 +303,15 @@ export function vitals(input: VitalsInput): Vitals {
302303 "/settings/capacity",
303304 );
304305
306 // A red main holding its pulls' checks is one card, naming the test.
307 for (const { red, held } of heldChecks(input.db))
308 need(
309 `red:${red.repoId}`,
310 redSentence(red, held),
311 red.since,
312 `/${red.repo}/runs/${red.runId}`,
313 );
314
305315 // A failed restore drill is a card until a drill passes.
306316 const drill = lastVerdict(input.dataDir);
307317 const failed = failedSentence(drill);
src/main.ts +2
@@ -24,6 +24,7 @@ import { startForgeSsh } from "./tui/ssh.ts";
2424 import { AppsRuntime } from "./apps/runtime.ts";
2525 import { resolveAppDomain } from "./apps/host.ts";
2626 import { requeueAbandonedChecks, startChecks } from "./checks/service.ts";
27 import { RED_HOLD_MS } from "./checks/red_main.ts";
2728 import { startLooks } from "./looks/service.ts";
2829 import { startProbes } from "./contract/probes.ts";
2930 import { startSnapshots } from "./snapshot/service.ts";
@@ -278,6 +279,7 @@ const checks =
278279 : startChecks(app, {
279280 forgeUrl,
280281 concurrency: positive(process.env.FORGE_CHECKS_CONCURRENCY, 1),
282 redHoldMs: RED_HOLD_MS,
281283 // A roll lets running checks finish this long before cutting them off.
282284 drainMs: positive(process.env.FORGE_CHECKS_DRAIN_S, 300) * 1000,
283285 });
test/check-tiers.test.ts +210 −2
@@ -1,5 +1,7 @@
11 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
22 import { mkdtemp, rm } from "node:fs/promises";
3 import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, symlinkSync, utimesSync, writeFileSync } from "node:fs";
4 import { CACHE_KEEP, CHECK_CACHE, keepCache, restoreCache } from "../src/checks/cache.ts";
35 import { tmpdir } from "node:os";
46 import { join } from "node:path";
57 import { createApp, type App } from "../src/app.ts";
@@ -34,7 +36,6 @@ import {
3436 } from "../src/checks/impact.ts";
3537 import { parseCheck } from "../src/apps/manifest.ts";
3638 import { IMPACT_FILE, keepImpact, seedImpact } from "../src/checks/impact-store.ts";
37 import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
3839
3940 const tests = [
4041 "test/snapshot-store.test.ts",
@@ -589,7 +590,7 @@ describe("a pull's fast check is of the merge", () => {
589590 process.env["FORGE_CELLS"] = "off";
590591 dataDir = await mkdtemp(join(tmpdir(), "forge-merged-"));
591592 app = createApp({ dataDir, port: 0 });
592 checks = startChecks(app, { forgeUrl: "http://127.0.0.1:1", concurrency: 1 });
593 checks = startChecks(app, { forgeUrl: "http://127.0.0.1:1", concurrency: 1, redHoldMs: 60_000 });
593594 const a = await createUser(app.db, {
594595 trusted: true,
595596 username: "alice",
@@ -732,6 +733,213 @@ describe("a pull's fast check is of the merge", () => {
732733 expect(asked.run.reason).toBe("asked");
733734 expect(asked.run.commitOid).toBe(pull.headOid);
734735 });
736
737 test("a red main holds pulls' checks; a person's ask runs; green main releases the rest", async () => {
738 const mainRun = async (commitOid: string): Promise<Run> => {
739 const deadline = Date.now() + 30_000;
740 while (Date.now() < deadline) {
741 const run = listRuns(app.db, repo.id, { kind: "check", limit: 50 }).find(
742 (r) => r.commitOid === commitOid && r.issueId === null,
743 );
744 if (run && (run.state === "ok" || run.state === "failed")) return run;
745 await Bun.sleep(50);
746 }
747 throw new Error("main's check did not finish");
748 };
749 // Settle what earlier tests left in the lane first.
750 await askCheck(app, alice, repo, listPulls(app.db, repo.id, { state: "open", limit: 5 }).find((p) => p.headBranch === "late")!);
751 await settled((await resolveOid(repoDir(app.reposRoot, "alice", "merged"), "refs/heads/late"))!);
752 const red = await commit(undefined, [
753 { path: "forge.json", content: JSON.stringify({ check: { run: "exit 1", fast: "true" } }) },
754 ]);
755 expect((await mainRun(red)).state).toBe("failed");
756 const heads: Record<string, string> = {};
757 for (const branch of ["held", "asked"]) {
758 heads[branch] = await commit(branch, [{ path: `${branch}.txt`, content: "x\n" }]);
759 const opened = await openPull(app, alice, repo, { title: branch, baseBranch: "main", headBranch: branch });
760 if (!opened.ok) throw new Error(String(opened.error));
761 }
762 await Bun.sleep(500);
763 const queuedFor = (oid: string) =>
764 listRuns(app.db, repo.id, { kind: "check", limit: 50 }).find((r) => r.commitOid === oid && r.issueId !== null);
765 expect(queuedFor(heads["held"]!)?.state).toBe("queued");
766 expect(queuedFor(heads["asked"]!)?.state).toBe("queued");
767
768 const pull = listPulls(app.db, repo.id, { state: "open", limit: 10 }).find((p) => p.headBranch === "asked")!;
769 const asked = await askCheck(app, alice, repo, pull);
770 if (!asked.ok) throw new Error(asked.message);
771 expect(asked.run.reason).toBe("asked");
772 expect((await settled(heads["asked"]!)).state).toBe("ok");
773 expect(queuedFor(heads["held"]!)?.state).toBe("queued");
774
775 const green = await commit(undefined, [
776 { path: "forge.json", content: JSON.stringify({ check: { run: "true", fast: "true" } }) },
777 ]);
778 expect((await mainRun(green)).state).toBe("ok");
779 expect((await settled(heads["held"]!)).state).toBe("ok");
780 }, 90_000);
781 });
782
783 describe("what a check keeps for the next one", () => {
784 const dirs: string[] = [];
785 afterAll(async () => {
786 for (const dir of dirs) await rm(dir, { recursive: true, force: true });
787 });
788 const fresh = async (): Promise<string> => {
789 const dir = await mkdtemp(join(tmpdir(), "forge-cache-"));
790 dirs.push(dir);
791 return dir;
792 };
793 const oid = (c: string): string => c.repeat(40);
794
795 test("a kept cache seeds the next checkout: by its key, else the newest", async () => {
796 const data = await fresh();
797 const one = await fresh();
798 mkdirSync(join(one, CHECK_CACHE));
799 writeFileSync(join(one, CHECK_CACHE, "tsc.tsbuildinfo"), "A");
800 expect(keepCache(data, "r1", oid("a"), one)).toBe(true);
801 writeFileSync(join(one, CHECK_CACHE, "tsc.tsbuildinfo"), "B");
802 expect(keepCache(data, "r1", oid("b"), one)).toBe(true);
803 utimesSync(join(data, "check-cache", "r1", oid("a")), 1, 1);
804
805 const next = await fresh();
806 expect(restoreCache(data, "r1", [oid("a")], next)).toBe(oid("a"));
807 expect(readFileSync(join(next, CHECK_CACHE, "tsc.tsbuildinfo"), "utf8")).toBe("A");
808 expect(restoreCache(data, "r1", [oid("c")], next)).toBe(oid("b"));
809 expect(readFileSync(join(next, CHECK_CACHE, "tsc.tsbuildinfo"), "utf8")).toBe("B");
810 // Another repository's cache is not this one's.
811 const other = await fresh();
812 expect(restoreCache(data, "r2", [oid("a")], other)).toBeUndefined();
813 expect(readdirSync(join(other, CHECK_CACHE))).toEqual([]);
814 });
815
816 test("a symlink is never followed, in or out, and a key is a commit id", async () => {
817 const data = await fresh();
818 const outside = await fresh();
819 writeFileSync(join(outside, "secret"), "s3cret");
820 const linked = await fresh();
821 symlinkSync(outside, join(linked, CHECK_CACHE));
822 expect(keepCache(data, "r", oid("a"), linked)).toBe(false);
823 const planted = await fresh();
824 mkdirSync(join(planted, CHECK_CACHE));
825 symlinkSync(join(outside, "secret"), join(planted, CHECK_CACHE, "secret"));
826 writeFileSync(join(planted, CHECK_CACHE, "ok"), "fine");
827 expect(keepCache(data, "r", oid("a"), planted)).toBe(true);
828 expect(readdirSync(join(data, "check-cache", "r", oid("a")))).toEqual(["ok"]);
829 expect(keepCache(data, "r", "../../x", planted)).toBe(false);
830 // The tree's own `.forge-cache` link is replaced, not written through.
831 const next = await fresh();
832 symlinkSync(outside, join(next, CHECK_CACHE));
833 restoreCache(data, "r", [oid("a")], next);
834 expect(readdirSync(outside)).toEqual(["secret"]);
835 expect(readdirSync(join(next, CHECK_CACHE))).toEqual(["ok"]);
836 });
837
838 test("only the newest few commits are kept", async () => {
839 const data = await fresh();
840 const dir = await fresh();
841 mkdirSync(join(dir, CHECK_CACHE));
842 writeFileSync(join(dir, CHECK_CACHE, "t"), "x");
843 const keys = "abcdef".split("").map(oid);
844 keys.forEach((key, i) => {
845 keepCache(data, "r", key, dir);
846 utimesSync(join(data, "check-cache", "r", key), 1000 + i, 1000 + i);
847 });
848 expect(readdirSync(join(data, "check-cache", "r")).sort()).toEqual(keys.slice(-CACHE_KEEP).sort());
849 });
850 });
851
852 describe("the cache between the forge's own checks", () => {
853 let dataDir: string;
854 let app: App;
855 let checks: ChecksHandle;
856 let alice: User;
857 let repo: Repo;
858 const cells = process.env["FORGE_CELLS"];
859
860 beforeAll(async () => {
861 process.env["FORGE_CELLS"] = "off";
862 dataDir = await mkdtemp(join(tmpdir(), "forge-kept-"));
863 app = createApp({ dataDir, port: 0 });
864 checks = startChecks(app, { forgeUrl: "http://127.0.0.1:1", concurrency: 1 });
865 const a = await createUser(app.db, {
866 trusted: true,
867 username: "alice",
868 email: "[email protected]",
869 password: "a-good-passphrase",
870 adminIfFirst: true,
871 });
872 if (!a.ok) throw new Error(a.error);
873 alice = a.value;
874 const made = await createRepo(app, alice, { name: "kept", visibility: "public" });
875 if (!made.ok) throw new Error(String(made.error));
876 repo = made.value;
877 });
878
879 afterAll(async () => {
880 await checks.stop();
881 app.close();
882 if (cells === undefined) delete process.env["FORGE_CELLS"];
883 else process.env["FORGE_CELLS"] = cells;
884 await rm(dataDir, { recursive: true, force: true });
885 });
886
887 const settled = async (commit: string): Promise<Run> => {
888 const deadline = Date.now() + 30_000;
889 while (Date.now() < deadline) {
890 const run = listRuns(app.db, repo.id, { kind: "check", limit: 50 }).find(
891 (r) => r.commitOid === commit && (r.state === "ok" || r.state === "failed"),
892 );
893 if (run) return run;
894 await Bun.sleep(50);
895 }
896 throw new Error(`no check settled for ${commit}`);
897 };
898
899 test("main's green check is what the next check starts from; a pull's writes are not kept", async () => {
900 const manifest = JSON.stringify({
901 check: {
902 run: 'echo "MAIN-SAW $(cat .forge-cache/note 2>/dev/null)"; echo "$FORGE_COMMIT" > .forge-cache/note',
903 fast: 'echo "PULL-SAW $(cat .forge-cache/note 2>/dev/null)"; echo pull > .forge-cache/note',
904 },
905 });
906 const first = await commitToBranch(app, alice, repo, {
907 message: "Start",
908 files: [{ path: "forge.json", content: manifest }],
909 });
910 if (!first.ok) throw new Error(JSON.stringify(first.error));
911 const a = first.value.commitOid;
912 const ranA = await settled(a);
913 expect(ranA.state).toBe("ok");
914 expect(readRunLog(dataDir, ranA.id).lines.join("\n")).toContain("MAIN-SAW \n");
915
916 const branched = await commitToBranch(app, alice, repo, {
917 branch: "feature",
918 message: "Feature",
919 files: [{ path: "f.txt", content: "f\n" }],
920 });
921 if (!branched.ok) throw new Error(JSON.stringify(branched.error));
922 const opened = await openPull(app, alice, repo, {
923 title: "Feature",
924 baseBranch: "main",
925 headBranch: "feature",
926 });
927 if (!opened.ok) throw new Error(String(opened.error));
928 const pulled = await settled(branched.value.commitOid);
929 const pullLog = readRunLog(dataDir, pulled.id).lines.join("\n");
930 expect(pullLog).toContain(`PULL-SAW ${a}`);
931 expect(pullLog).toContain(`cache from ${a.slice(0, 7)}`);
932
933 const moved = await commitToBranch(app, alice, repo, {
934 message: "Main moves",
935 files: [{ path: "m.txt", content: "m\n" }],
936 });
937 if (!moved.ok) throw new Error(JSON.stringify(moved.error));
938 const ranB = await settled(moved.value.commitOid);
939 const mainLog = readRunLog(dataDir, ranB.id).lines.join("\n");
940 expect(mainLog).toContain(`MAIN-SAW ${a}`);
941 expect(mainLog).not.toContain("MAIN-SAW pull");
942 });
735943 });
736944
737945 describe("the impact map kept beside the forge", () => {
test/red-main.test.ts added +122
@@ -0,0 +1,122 @@
1 /**
2 * A red main holds its pulls' checks (#1797): on in-memory SQLite, the
3 * last finished check of the default branch decides, a person's ask is not
4 * held, and the vitals carry one card naming the failing test.
5 */
6 import { beforeEach, describe, expect, test } from "bun:test";
7 import type { Database } from "bun:sqlite";
8 import { tmpdir } from "node:os";
9 import { migrations } from "../src/core/schema.ts";
10 import { openDb } from "../src/kernel/db.ts";
11 import { claimRun, createRun, finishRun, listActive, type Run } from "../src/core/runs.ts";
12 import { RED_HOLD_MS, heldChecks, redMain, redSentence, unheld } from "../src/checks/red_main.ts";
13 import { vitals } from "../src/kernel/vitals.ts";
14 import { ErrorLedger } from "../src/kernel/log.ts";
15 import { RefusedLedger } from "../src/kernel/internal_refused.ts";
16
17 const OID = (c: string) => c.repeat(40);
18 const HOLD = RED_HOLD_MS;
19
20 let db: Database;
21 let pull: string;
22 beforeEach(() => {
23 db = openDb(":memory:", migrations);
24 db.run(
25 "INSERT INTO user (id, username, email, password_hash) VALUES ('u1', 'trevato', 't@x', '')",
26 );
27 db.run("INSERT INTO repo (id, owner_id, name) VALUES ('r1', 'u1', 'forge')");
28 db.run(
29 "INSERT INTO issue (id, repo_id, number, kind, title, author_id) VALUES ('i1', 'r1', 7, 'pull', 'p', 'u1')",
30 );
31 pull = "i1";
32 });
33
34 const main = (commit: string, state: "ok" | "failed", names: string[] = []): Run => {
35 const run = createRun(db, { repoId: "r1", kind: "check", env: "main", commitOid: OID(commit), reason: "push" });
36 claimRun(db, run.id);
37 return finishRun(db, run.id, {
38 state,
39 summary: state === "ok" ? "check passed in 9s" : "check failed: 1 test failed",
40 ...(state === "failed" ? { checkDetail: { passed: 10, failed: names.length, names } } : {}),
41 })!;
42 };
43 const pullCheck = (commit: string, reason = "push"): Run =>
44 createRun(db, { repoId: "r1", kind: "check", env: "pull-7", issueId: pull, commitOid: OID(commit), reason });
45 const queued = () => listActive(db, "check").filter((r) => r.state === "queued");
46 const red = (id: string) => redMain(db, id);
47
48 describe("red main", () => {
49 test("no check of main yet, or a green one, holds nothing", () => {
50 pullCheck("a");
51 expect(redMain(db, "r1")).toBeUndefined();
52 main("b", "ok");
53 expect(unheld(queued(), red, HOLD)).toHaveLength(1);
54 });
55
56 test("a failed main holds a pull's pushed check, names the test, and lets main's own check run", () => {
57 main("b", "failed", ["acting on many asks at once"]);
58 const held = pullCheck("a");
59 const own = createRun(db, { repoId: "r1", kind: "check", env: "main", commitOid: OID("c"), reason: "push" });
60 const r = redMain(db, "r1")!;
61 expect(r).toMatchObject({ repo: "trevato/forge", commitOid: OID("b"), failing: ["acting on many asks at once"] });
62 expect(unheld(queued(), red, HOLD).map((x) => x.id)).toEqual([own.id]);
63 expect(unheld(queued(), red, HOLD).map((x) => x.id)).not.toContain(held.id);
64 });
65
66 test("a person's ask runs while main is red", () => {
67 main("b", "failed", ["x"]);
68 const asked = pullCheck("a", "asked");
69 expect(unheld(queued(), red, HOLD).map((x) => x.id)).toEqual([asked.id]);
70 });
71
72 test("the latest finished check decides: green after red releases", () => {
73 main("b", "failed", ["x"]);
74 pullCheck("a");
75 expect(unheld(queued(), red, HOLD)).toHaveLength(0);
76 main("c", "ok");
77 expect(unheld(queued(), red, HOLD)).toHaveLength(1);
78 });
79
80 test("the hold is bounded: a check queued RED_HOLD_MS ago runs anyway, and 0 holds nothing", () => {
81 main("b", "failed", ["x"]);
82 const p = pullCheck("a");
83 const at = Date.parse(p.createdAt);
84 expect(unheld(queued(), red, HOLD, at + HOLD - 1)).toHaveLength(0);
85 expect(unheld(queued(), red, HOLD, at + HOLD).map((x) => x.id)).toEqual([p.id]);
86 expect(unheld(queued(), red, 0, at)).toHaveLength(1);
87 });
88
89 test("a failed pull check is not a red main", () => {
90 const p = pullCheck("a");
91 claimRun(db, p.id);
92 finishRun(db, p.id, { state: "failed", summary: "check failed" });
93 pullCheck("d");
94 expect(redMain(db, "r1")).toBeUndefined();
95 });
96
97 test("the sentence names the first failing test and counts what waits", () => {
98 main("b", "failed", ["one", "two"]);
99 pullCheck("a");
100 const [card] = heldChecks(db);
101 expect(card!.held).toBe(1);
102 expect(redSentence(card!.red, card!.held)).toBe(
103 'trevato/forge: main is red at bbbbbbb — "one" and 1 more fail; 1 pull check wait for it to pass.',
104 );
105 });
106
107 test("the vitals carry one card for it, linked to main's failed run", () => {
108 const failed = main("b", "failed", ["acting on many asks at once"]);
109 pullCheck("a");
110 db.run("INSERT INTO issue (id, repo_id, number, kind, title, author_id) VALUES ('i2', 'r1', 8, 'pull', 'q', 'u1')");
111 createRun(db, { repoId: "r1", kind: "check", env: "pull-8", issueId: "i2", commitOid: OID("e"), reason: "push" });
112 const v = vitals({
113 db,
114 dataDir: tmpdir(),
115 signals: { errors: new ErrorLedger(), refused: new RefusedLedger() },
116 });
117 const cards = v.degraded.filter((d) => d.cause.startsWith("red:"));
118 expect(cards).toHaveLength(1);
119 expect(cards[0]!.sentence).toContain('"acting on many asks at once" fails; 2 pull checks wait');
120 expect(cards[0]!.href).toBe(`/trevato/forge/runs/${failed.id}`);
121 });
122 });
test/shards.test.ts +71
@@ -11,7 +11,10 @@ import {
1111 SECONDS_PER_BYTE_DEFAULT,
1212 SHARDS_MAX,
1313 formatTimes,
14 heaviestFirst,
15 overlayTimes,
1416 parseTimes,
17 refreshTimes,
1518 parseShardSpec,
1619 shardCount,
1720 sliceFiles,
@@ -23,6 +26,7 @@ import {
2326 } from "../scripts/shards.ts";
2427 import { parseCheck } from "../src/apps/manifest.ts";
2528 import { EXCLUDED, GATE } from "../scripts/gate.ts";
29 import { CHECK_CACHE } from "../src/checks/cache.ts";
2630
2731 const files = Array.from({ length: 23 }, (_, i) => `test/f${i}.test.ts`);
2832
@@ -478,3 +482,70 @@ describe("the gate audit before any shard", () => {
478482 expect(explicit.err).not.toContain("list it in GATE");
479483 }, 30_000);
480484 });
485
486 describe("the gate audit alone, first in the check", () => {
487 const script = new URL("../scripts/gate.ts", import.meta.url).pathname;
488 const listed = [...GATE, ...EXCLUDED.map((e) => e.file)];
489 const run = async (extra: string[]) => {
490 const dir = mkdtempSync(join(tmpdir(), "forge-gate-"));
491 mkdirSync(join(dir, "test"));
492 for (const file of [...listed, ...extra]) writeFileSync(join(dir, file), "");
493 const child = Bun.spawn(["bun", script], { cwd: dir, stdin: "ignore", stdout: "pipe", stderr: "pipe" });
494 const err = await new Response(child.stderr).text();
495 const code = await child.exited;
496 rmSync(dir, { recursive: true, force: true });
497 return { code, err };
498 };
499
500 test("an unlisted file fails it by name; a listed set passes", async () => {
501 const red = await run(["test/zz-new.test.ts"]);
502 expect(red.code).toBe(2);
503 expect(red.err.trim().split("\n").pop()).toContain(
504 "test/zz-new.test.ts: list it in GATE or EXCLUDED in scripts/gate.ts",
505 );
506 const green = await run([]);
507 expect(green.code).toBe(0);
508 expect(green.err).toBe("");
509 }, 30_000);
510
511 test("forge.json audits before tsc, and tsc runs incremental from the check's cache", async () => {
512 const check = parseCheck(readFileSync(new URL("../forge.json", import.meta.url), "utf8"), undefined);
513 if (!check.ok || check.value === undefined) throw new Error("no check");
514 for (const command of [check.value.run, check.value.fast ?? ""]) {
515 expect(command.startsWith("bun scripts/gate.ts && ")).toBe(true);
516 expect(command.indexOf("scripts/gate.ts")).toBeLessThan(command.indexOf("tsc.js"));
517 expect(command).toContain("--max-old-space-size=2048");
518 expect(command).toContain(`--incremental --tsBuildInfoFile ${CHECK_CACHE}/tsc.tsbuildinfo`);
519 }
520 });
521 });
522
523 describe("one file per worker, timed", () => {
524 test("heaviest first, ties in list order", () => {
525 const w = new Map([["a", 1], ["b", 9], ["c", 1], ["d", 5]]);
526 expect(heaviestFirst(["a", "b", "c", "d"], (f) => w.get(f)!)).toEqual(["b", "d", "a", "c"]);
527 });
528
529 test("the cache's times win over the committed table, file by file", () => {
530 const merged = overlayTimes(
531 new Map([["test/a.test.ts", 5], ["test/b.test.ts", 7]]),
532 new Map([["test/b.test.ts", 2], ["test/c.test.ts", 3]]),
533 );
534 expect([...merged].sort()).toEqual([
535 ["test/a.test.ts", 5],
536 ["test/b.test.ts", 2],
537 ["test/c.test.ts", 3],
538 ]);
539 });
540
541 test("a green run refreshes what it ran, keeps the rest, forgets what is gone", () => {
542 const fresh = refreshTimes(
543 new Map([["test/a.test.ts", 5], ["test/b.test.ts", 7], ["test/gone.test.ts", 1]]),
544 [{ file: "./test/b.test.ts", seconds: 2.04 }, { file: "test/new.test.ts", seconds: 3 }],
545 (file) => file !== "test/gone.test.ts",
546 );
547 expect(formatTimes(fresh)).toBe(
548 '{\n "test/a.test.ts": 5,\n "test/b.test.ts": 2,\n "test/new.test.ts": 3\n}\n',
549 );
550 });
551 });
test/vitals.test.ts +2 −2
@@ -153,8 +153,8 @@ test("the number of queries does not grow with runs", () => {
153153 if (i % 2) app.db.query("UPDATE run SET state='running' WHERE id=?1").run(r.id);
154154 }
155155 expect(count()).toBe(before);
156 // The queue, the running runs, the runs killed twice for their budget, and the lane's depth and finished checks.
157 expect(before).toBe(5);
156 // The queue, the running runs, the runs killed twice for their budget, the lane's depth and finished checks, and the red mains holding pulls' checks.
157 expect(before).toBe(6);
158158 });
159159
160160 test("errors are counted, bounded, redacted, and a burst is said", async () => {

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs