Public: anyone can read this; the crew can change it.

Faster pull request checks without weaker ones: measured selection, result cache, measured shards #14335

opened this in answer to

Check · Review · Land · Live — Its check is red. Next: builder, its check is red.

Ready to land

torque/forge:torque/14322-faster-pull-request-checks-without-weake → main 2 commits fork 2 threads · 2 unresolved

Review

Sign in to review this change.
13 files changed +539 −86
.gitignore +2 −1
@@ -24,9 +24,10 @@ typesafe-api-key
2424 # what a check keeps for the next one (src/checks/cache.ts)
2525 /.forge-cache/
2626
27 # what a check hands between runs: the impact map and the kept results, never committed
27 # what a check hands between runs: the impact map, the kept results and the test times, never committed
2828 /.forge-impact.json
2929 /.forge-kept.json
30 /.forge-times.json
3031
3132 # empty stubs an agent sandbox mounts over home-directory config; never the
3233 # forge's own files, and never committed (#1148)
docs/DEVELOP.md +7 −2
@@ -180,8 +180,13 @@ stderr as `full tier: … changed`), or the arguments or git failed. The
180180
181181 Files go to shards by how long they take, not how big they are: longest
182182 first onto the least-loaded shard, weighed from `scripts/test-times.json`
183 (file → seconds, from one recorded run). A file the table does not name
184 weighs its bytes at the table's own seconds-per-byte. If the table is
183 (file → seconds, from one recorded run). In a forge check, a file the table
184 does not name weighs the seconds the default branch's last whole
185 verification measured for it, rescaled to the table's clock (the forge
186 keeps them in `checks/times/`, `src/checks/times-store.ts`, and hands them
187 over as `.forge-times.json`); only a file neither names weighs its bytes at
188 the table's own seconds-per-byte. Times only weigh shards: they never
189 decide which test runs. If the table is
185190 malformed, the log says it was refused and the split falls back to bytes.
186191 A stale table only makes the split less even; it never drops a file.
187192 Refresh it with one command after adding a slow test file, or when the
scripts/shards.ts +53 −66
@@ -9,6 +9,10 @@
99
1010 import { auditLine } from "../src/checks/kept.ts";
1111 import { isSummaryLine, nameUnnamed } from "../src/checks/bun-test.ts";
12 import { TEST_FILE, formatTimes, normaliseTest } from "../src/checks/test_times.ts";
13
14 // The times table is the forge's too (`times-store.ts` validates what a check hands back).
15 export { TEST_FILE, fillTimes, formatTimes, parseTimes } from "../src/checks/test_times.ts";
1216
1317 /**
1418 * No more than this many `bun test` processes at once, whatever the host:
@@ -189,9 +193,6 @@ export function splitCapped(
189193 );
190194 }
191195
192 /** How a test file is named: the same rule `scripts/test-shards.ts` uses. */
193 export const TEST_FILE = /\.test\.[cm]?[jt]sx?$/;
194
195196 /** A test's short name after `--`: `crew-wrapup` is `test/crew-wrapup.test.ts`. */
196197 const TEST_NAME = /^[a-z0-9][a-z0-9-]{0,99}$/;
197198
@@ -220,9 +221,6 @@ export function splitArgs(
220221 return { ok: true, flags, files };
221222 }
222223
223 /** A sane bound on one entry of the times table: a day, in seconds. */
224 const TIME_MAX = 86_400;
225
226224 /**
227225 * Seconds per byte when the table says nothing about any file in the list:
228226 * about what this suite showed when the table was first recorded (a 40 kB
@@ -233,48 +231,6 @@ const TIME_MAX = 86_400;
233231 */
234232 export const SECONDS_PER_BYTE_DEFAULT = 1 / 2_000;
235233
236 /**
237 * Reads `scripts/test-times.json`: one object, test file path to the
238 * seconds it took alone, e.g. `{ "test/crew.test.ts": 50.1 }`. Anything
239 * else — not JSON, not an object, a key that is not a test file, a value
240 * that is not a finite number from 0 to a day — refuses the whole table,
241 * so a half-trusted table never steers the split.
242 */
243 export function parseTimes(
244 text: string,
245 ): { ok: true; value: Map<string, number> } | { ok: false; error: string } {
246 let data: unknown;
247 try {
248 data = JSON.parse(text);
249 } catch {
250 return { ok: false, error: "not JSON" };
251 }
252 if (typeof data !== "object" || data === null || Array.isArray(data))
253 return { ok: false, error: "not an object of file to seconds" };
254 const times = new Map<string, number>();
255 for (const [file, seconds] of Object.entries(data)) {
256 if (!TEST_FILE.test(file) || file.length > 512)
257 return { ok: false, error: `${JSON.stringify(file)} is not a test file` };
258 if (
259 typeof seconds !== "number" ||
260 !Number.isFinite(seconds) ||
261 seconds < 0 ||
262 seconds > TIME_MAX
263 )
264 return {
265 ok: false,
266 error: `${file}: ${JSON.stringify(seconds)} is not seconds from 0 to ${TIME_MAX}`,
267 };
268 times.set(normalise(file), seconds);
269 }
270 return { ok: true, value: times };
271 }
272
273 /** `./test/a.test.ts` and `test/a.test.ts` are the same file. */
274 function normalise(file: string): string {
275 return file.startsWith("./") ? file.slice(2) : file;
276 }
277
278234 /**
279235 * How long each file is expected to take, in seconds: its measured time
280236 * from the table when there is one, else its size in bytes scaled to
@@ -292,7 +248,7 @@ export function weigher(
292248 let seconds = 0;
293249 let size = 0;
294250 for (const file of files) {
295 const measured = times.get(normalise(file));
251 const measured = times.get(normaliseTest(file));
296252 const b = bytes.get(file);
297253 if (measured === undefined || b === undefined) continue;
298254 seconds += measured;
@@ -301,22 +257,7 @@ export function weigher(
301257 const perByte =
302258 seconds > 0 && size > 0 ? seconds / size : SECONDS_PER_BYTE_DEFAULT;
303259 return (file) =>
304 times.get(normalise(file)) ?? (bytes.get(file) ?? 0) * perByte;
305 }
306
307 /**
308 * The table as it is checked in: keys sorted, seconds to one decimal, one
309 * file per line, so a refresh reads as a small diff.
310 */
311 export function formatTimes(times: ReadonlyMap<string, number>): string {
312 const entries = [...times]
313 .map(([file, seconds]) => [normalise(file), seconds] as const)
314 .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
315 const lines = entries.map(
316 ([file, seconds]) =>
317 ` ${JSON.stringify(file)}: ${Math.round(seconds * 10) / 10}`,
318 );
319 return lines.length === 0 ? "{}\n" : `{\n${lines.join(",\n")}\n}\n`;
260 times.get(normaliseTest(file)) ?? (bytes.get(file) ?? 0) * perByte;
320261 }
321262
322263 /**
@@ -355,10 +296,56 @@ export function refreshTimes(
355296 ): Map<string, number> {
356297 const fresh = new Map<string, number>();
357298 for (const [file, seconds] of old) if (exists(file)) fresh.set(file, seconds);
358 for (const { file, seconds } of ran) fresh.set(normalise(file), seconds);
299 for (const { file, seconds } of ran) fresh.set(normaliseTest(file), seconds);
359300 return fresh;
360301 }
361302
303 /**
304 * The time of every one-file shard that ran green, for the times table: a
305 * red file's time is not its time, and a shard of many files times none of
306 * them alone.
307 */
308 export function greenTimes(
309 results: readonly { readonly files: readonly string[]; readonly code: number; readonly seconds?: number }[],
310 ): { file: string; seconds: number }[] {
311 return results.flatMap((result) =>
312 result.code === 0 && result.files.length === 1 && result.seconds !== undefined
313 ? [{ file: normaliseTest(result.files[0]!), seconds: result.seconds }]
314 : [],
315 );
316 }
317
318 /**
319 * Whether this run writes the handed times back (#14322): the whole gate of
320 * the full tier, unsliced and not a bisection step, measured one file per
321 * process (`--impact`), and only when the forge handed a table over.
322 */
323 export function writesHandedTimes(mode: {
324 readonly measure: boolean;
325 readonly gate: boolean;
326 readonly tier: string;
327 readonly unsliced: boolean;
328 readonly only: boolean;
329 readonly handed: boolean;
330 }): boolean {
331 return mode.measure && mode.gate && mode.tier === "full" && mode.unsliced && !mode.only && mode.handed;
332 }
333
334 /**
335 * The handed table after a whole verification: the old one refreshed by
336 * every file that ran green alone (`greenTimes`), or `undefined` when none
337 * did, so a run with nothing timed leaves the table as it was.
338 */
339 export function handBackTimes(
340 results: Parameters<typeof greenTimes>[0],
341 old: ReadonlyMap<string, number>,
342 exists: (file: string) => boolean,
343 ): { text: string; files: number } | undefined {
344 const ran = greenTimes(results);
345 if (ran.length === 0) return undefined;
346 return { text: formatTimes(refreshTimes(old, ran, exists)), files: ran.length };
347 }
348
362349 /** One shard, finished: what it ran, how it exited, everything it printed. */
363350 export interface ShardResult {
364351 readonly files: readonly string[];
scripts/test-shards.ts +45 −2
@@ -61,7 +61,12 @@
6161 *
6262 * Files are weighed by how long they took, from `scripts/test-times.json`
6363 * overlaid by `.forge-cache/test-times.json` (a file neither names, or a
64 * table it refuses, falls back to bytes).
64 * table it refuses, falls back to bytes). Packing also reads the times the
65 * default branch's whole verification measured (`FORGE_CHECK_TIMES`,
66 * `src/checks/times-store.ts`), rescaled to that table's clock, for every
67 * file the table lacks (`fillTimes`); that verification writes the time of
68 * each file it ran green back to the same file (#14322). Times only weigh
69 * shards: which tests run is never theirs to decide.
6570 * `--record` refreshes that table: the same number of workers, but each
6671 * file is its own `bun test`, timed alone from spawn to exit, and the
6772 * table is rewritten from those times when every file is green.
@@ -72,12 +77,15 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, s
7277 import { join, resolve } from "node:path";
7378 import { resolveLimits } from "../src/apps/contain.ts";
7479 import {
80 fillTimes,
7581 formatTimes,
82 handBackTimes,
7683 heaviestFirst,
7784 MEASURE_WORKER_MEMORY_BYTES,
7885 memoryLimit,
7986 overlayTimes,
8087 refreshTimes,
88 writesHandedTimes,
8189 parseShardSpec,
8290 parseTimes,
8391 SHARDS_MAX,
@@ -125,6 +133,10 @@ const IMPACT = handed ?? new URL("./test-impact.json", import.meta.url).pathname
125133 const IMPACT_NAME = handed ?? "scripts/test-impact.json";
126134 // Test results kept by what each reads (`src/checks/kept.ts`): handed over like the map, by name.
127135 const KEPT = handedImpact(process.env["FORGE_CHECK_KEPT"]);
136 // Each file's seconds from the default branch's last whole verification
137 // (`src/checks/times-store.ts`), handed over like the map: they weigh the
138 // shards, never which test runs, and that verification writes them back.
139 const HANDED_TIMES = handedImpact(process.env["FORGE_CHECK_TIMES"]);
128140 // What the forge keeps between checks (`src/checks/cache.ts`): the full
129141 // gate's own per-file times, fresher than the committed table.
130142 const CACHED_TIMES = `${CHECK_CACHE}/test-times.json`;
@@ -171,6 +183,21 @@ function loadTimes(): Map<string, number> {
171183 );
172184 }
173185
186 /** The verification's handed times; none when the forge handed none or they are refused. */
187 function handedTimes(): Map<string, number> {
188 if (HANDED_TIMES === undefined) return new Map();
189 let text: string;
190 try {
191 text = readFileSync(HANDED_TIMES, "utf8");
192 } catch {
193 return new Map();
194 }
195 const parsed = parseTimes(text);
196 if (parsed.ok) return parsed.value;
197 console.log(`test-shards: ${HANDED_TIMES} refused (${parsed.error}); weighing those files by bytes`);
198 return new Map();
199 }
200
174201 /** Pull `--shard N/M` or `--shard=N/M` out of argv so it never reaches bun test. */
175202 function takeShardArg(raw: string[]): {
176203 rest: string[];
@@ -478,7 +505,10 @@ delete process.env["FORGE_CHECK_SHARDS"];
478505 delete process.env["FORGE_CHECK_SHARD_INDEX"];
479506 delete process.env["FORGE_CHECK_SHARD_TOTAL"];
480507 const times = loadTimes();
481 const weight = weigher(files, size, times);
508 // Packing alone also reads the verification's times, rescaled to the
509 // table's clock: a file the table never timed is weighed by seconds, not bytes.
510 const weighed = fillTimes(times, handedTimes());
511 const weight = weigher(files, size, weighed);
482512 // Recording and measuring run one file per process, so every file is its
483513 // own slice: Bun's coverage is per process, and a shard of many files would
484514 // merge theirs into one (#1794).
@@ -1089,6 +1119,19 @@ if (measure) {
10891119 }
10901120 }
10911121 }
1122 // The whole verification times every file it runs, one process each: the
1123 // files that ran green refresh the handed table the forge keeps (#14322).
outdated

The write-back has no test that fails without it. Add one that runs the recorded whole mode and asserts the handed file is written for green files and not written when none ran green.

1124 if (
1125 HANDED_TIMES !== undefined &&
1126 writesHandedTimes({ measure, gate, tier, unsliced: "none" in shardSpec, only: only !== undefined, handed: true })
1127 ) {
1128 const back = handBackTimes(results, handedTimes(), (file) => existsSync(file));
1129 if (back === undefined) console.log(`test-shards: not writing ${HANDED_TIMES}: no file ran green`);
1130 else {
1131 writeFileSync(HANDED_TIMES, back.text);
1132 console.log(`test-shards: wrote ${back.files} times to ${HANDED_TIMES}`);
1133 }
1134 }
10921135 // The timed full gate is the check on the cell itself: its log ends with
10931136 // what the cell did, so nobody has to measure it by hand (#2065).
10941137 if (timed)
src/checks/handover.ts added +30
@@ -0,0 +1,30 @@
1 /**
2 * What crosses between the forge and a check's checkout, both ways (#14322):
3 * the files the forge seeds a check with, named in its environment, and which
4 * run may hand the shared times table back. One place, so the local path
5 * (`run_tier.ts`) and a runner's (`tier_runner.ts`) cannot drift apart.
6 */
7
8 import { IMPACT_FILE } from "./impact-store.ts";
9 import { KEPT_FILE } from "./kept-store.ts";
10 import { TIMES_FILE } from "./times-store.ts";
11 import type { Execution } from "./local_runner.ts";
12
13 /** The seeded files' names, as every check's environment carries them. */
14 export function handoverEnv(): Record<"FORGE_CHECK_IMPACT" | "FORGE_CHECK_KEPT" | "FORGE_CHECK_TIMES", string> {
15 return {
16 FORGE_CHECK_IMPACT: IMPACT_FILE,
17 FORGE_CHECK_KEPT: KEPT_FILE,
18 FORGE_CHECK_TIMES: TIMES_FILE,
19 };
20 }
21
22 /**
23 * Whether this run's times may replace the table every check is packed by:
24 * the default branch's own (`keepImpact`) verification of the whole gate,
25 * never a bisection step (`only`) and never a pull's, whose tree could write
26 * any table. Green or red alike: a red verification's green files were timed.
27 */
28 export function handsTimesBack(x: Pick<Execution, "tier" | "only" | "keepImpact">): boolean {
29 return x.tier === "verify" && x.only === undefined && x.keepImpact === true;
30 }
src/checks/run_tier.ts +17 −4
@@ -11,8 +11,10 @@ import { readLearned } from "./learned.ts";
1111 import { keepCache, restoreCache } from "./cache.ts";
1212 import { clearedIn } from "./base_red.ts";
1313 import { scratchEnv } from "./disk_full.ts";
14 import { IMPACT_FILE, keepImpact, seedImpact } from "./impact-store.ts";
15 import { KEPT_FILE, keepKept, seedKept } from "./kept-store.ts";
14 import { keepImpact, seedImpact } from "./impact-store.ts";
15 import { keepKept, seedKept } from "./kept-store.ts";
16 import { keepTimes, seedTimes } from "./times-store.ts";
17 import { handoverEnv, handsTimesBack } from "./handover.ts";
1618 import { redDetail } from "./report.ts";
1719 import { rerunFileCommand, rerunRed, withFlaky } from "./rerun.ts";
1820 import { failedFiles, onlyEnv, passedFiles } from "./verify.ts";
@@ -142,6 +144,13 @@ export async function runTier(lane: LaneState, x: Execution): Promise<CheckOutco
142144 } catch (error) {
143145 write(`kept results not seeded: ${String(error)}`);
144146 }
147 // Each test file's seconds from the last whole verification: the shards are packed by them.
148 try {
149 const seeded = seedTimes(app.config.dataDir, repo.id, dir);
150 if (seeded > 0) write(`test times: ${seeded} files, from ${repo.defaultBranch}'s verification`);
151 } catch (error) {
152 write(`test times not seeded: ${String(error)}`);
153 }
145154 // The check is repository code: it runs in a cell of its own
146155 // (`apps/contain.ts`), its checkout and a home its only places to write,
147156 // the forge, the registry and what it declared its only way out.
@@ -206,8 +215,7 @@ export async function runTier(lane: LaneState, x: Execution): Promise<CheckOutco
206215 FORGE_URL: options.forgeUrl,
207216 FORGE_CHECK: run.id,
208217 FORGE_COMMIT: x.commitOid,
209 FORGE_CHECK_IMPACT: IMPACT_FILE,
210 FORGE_CHECK_KEPT: KEPT_FILE,
218 ...handoverEnv(),
211219 ...(x.only === undefined ? {} : { FORGE_CHECK_ONLY: onlyEnv(x.only) }),
212220 // The cell's CPU limit, which `availableParallelism()` may not
213221 // report inside it; the shard count follows this (#2257).
@@ -331,6 +339,9 @@ export async function runTier(lane: LaneState, x: Execution): Promise<CheckOutco
331339 if (kept !== undefined) write(kept);
332340 const results = keepKept(app.config.dataDir, repo.id, dir, true);
333341 if (results !== undefined) write(results);
342 // A red verification's green files were timed all the same.
343 const times = keepTimes(app.config.dataDir, repo.id, dir, handsTimesBack(x));
344 if (times !== undefined) write(times);
outdated

Guard differs from the green branch (wholeVerify && x.keepImpact === true). Use the same guard here so only the default branch hands the table back, and test it.

334345 } catch (error) {
335346 write(`impact map not kept: ${String(error)}`);
336347 }
@@ -358,6 +369,8 @@ export async function runTier(lane: LaneState, x: Execution): Promise<CheckOutco
358369 // Only the default branch's own run hands results back: a pull's tree could write any.
359370 const results = keepKept(app.config.dataDir, repo.id, dir, x.keepImpact === true);
360371 if (results !== undefined) write(results);
372 const times = keepTimes(app.config.dataDir, repo.id, dir, handsTimesBack(x));
373 if (times !== undefined) write(times);
361374 } catch (error) {
362375 write(`impact map not kept: ${String(error)}`);
363376 }
src/checks/sweep.ts +6 −6
@@ -8,6 +8,7 @@ import { gateHolding } from "./gate_record.ts";
88 import { diskRecovered } from "./disk_red.ts";
99 import { IMPACT_DIR } from "./impact-store.ts";
1010 import { KEPT_DIR } from "./kept-store.ts";
11 import { TIMES_DIR } from "./times-store.ts";
1112 import { createLog } from "../kernel/log.ts";
1213 import { STALE_AFTER_MS } from "../runs/lease.ts";
1314 import { CHECK_TIMEOUT_MAX } from "../apps/manifest.ts";
@@ -70,13 +71,12 @@ export const checksRoot = (dataDir: string): string => join(dataDir, "checks");
7071 /**
7172 * What is kept under the checks' root that is no checkout: every
7273 * repository's impact map (`impact-store.ts`) and kept results
73 * (`kept-store.ts`), the tests a red gate taught each pull (`learned.ts`),
74 * the gates' logs (`gate_log.ts`) and the last gate's results
75 * (`gate_last.ts`). The sweep took each for a stranger's checkout and
76 * removed it at every boot and every 30 seconds after: a map lived half a
77 * minute, so no check was ever seeded with one (#12178).
74 * (`kept-store.ts`) and test times (`times-store.ts`), the tests a red gate taught each pull
75 * (`learned.ts`), the gates' logs (`gate_log.ts`) and the last gate's results (`gate_last.ts`). The
76 * sweep took each for a stranger's checkout and removed it at every boot and every 30 seconds
77 * after: a map lived half a minute, so no check was ever seeded with one (#12178).
7878 */
79 const STORES: ReadonlySet<string> = new Set([IMPACT_DIR, KEPT_DIR, "learned", "gates", "gate-last.json"]);
79 const STORES: ReadonlySet<string> = new Set([IMPACT_DIR, KEPT_DIR, TIMES_DIR, "learned", "gates", "gate-last.json"]);
8080
8181 /** A gate's checkout (`gateRunId`): it has no run, only its `check_gate` row. */
8282 const GATE_DIR = /^gate-([0-9a-f]{12})$/;
src/checks/test_times.ts added +101
@@ -0,0 +1,101 @@
1 /**
2 * The times table: test file to the seconds it took (#1797, #14322). The
3 * committed `scripts/test-times.json`, the full gate's cache and the times a
4 * default-branch verification measured (`times-store.ts`) all share this
5 * form, so the forge can validate what a check hands back before keeping
6 * it, and the gate's script can read what the forge seeds.
7 *
8 * Plain data in, plain data out.
9 */
10
11 /** How a test file is named: the same rule `scripts/test-shards.ts` uses. */
12 export const TEST_FILE = /\.test\.[cm]?[jt]sx?$/;
13
14 /** A sane bound on one entry of the times table: a day, in seconds. */
15 const TIME_MAX = 86_400;
16
17 /** `./test/a.test.ts` and `test/a.test.ts` are the same file. */
18 export function normaliseTest(file: string): string {
19 return file.startsWith("./") ? file.slice(2) : file;
20 }
21
22 /**
23 * Reads a times table: one object, test file path to the seconds it took,
24 * e.g. `{ "test/crew.test.ts": 50.1 }`. Anything else — not JSON, not an
25 * object, a key that is not a test file, a value that is not a finite
26 * number from 0 to a day — refuses the whole table, so a half-trusted
27 * table never steers the split.
28 */
29 export function parseTimes(
30 text: string,
31 ): { ok: true; value: Map<string, number> } | { ok: false; error: string } {
32 let data: unknown;
33 try {
34 data = JSON.parse(text);
35 } catch {
36 return { ok: false, error: "not JSON" };
37 }
38 if (typeof data !== "object" || data === null || Array.isArray(data))
39 return { ok: false, error: "not an object of file to seconds" };
40 const times = new Map<string, number>();
41 for (const [file, seconds] of Object.entries(data)) {
42 if (!TEST_FILE.test(file) || file.length > 512)
43 return { ok: false, error: `${JSON.stringify(file)} is not a test file` };
44 if (
45 typeof seconds !== "number" ||
46 !Number.isFinite(seconds) ||
47 seconds < 0 ||
48 seconds > TIME_MAX
49 )
50 return {
51 ok: false,
52 error: `${file}: ${JSON.stringify(seconds)} is not seconds from 0 to ${TIME_MAX}`,
53 };
54 times.set(normaliseTest(file), seconds);
55 }
56 return { ok: true, value: times };
57 }
58
59 /**
60 * The table as it is checked in: keys sorted, seconds to one decimal, one
61 * file per line, so a refresh reads as a small diff.
62 */
63 export function formatTimes(times: ReadonlyMap<string, number>): string {
64 const entries = [...times]
65 .map(([file, seconds]) => [normaliseTest(file), seconds] as const)
66 .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
67 const lines = entries.map(
68 ([file, seconds]) =>
69 ` ${JSON.stringify(file)}: ${Math.round(seconds * 10) / 10}`,
70 );
71 return lines.length === 0 ? "{}\n" : `{\n${lines.join(",\n")}\n}\n`;
72 }
73
74 /**
75 * `base` with every file only `measured` names added, rescaled to `base`'s
76 * clock. The verification times each file under coverage beside fifteen
77 * others, which is slower than the committed table's file timed alone; only
78 * the ratio between files steers a split, so the measured times are scaled
79 * by k = Σ base ÷ Σ measured over the files both name (k = 1 when they share
80 * none, or either sum is 0). A file `base` names keeps its own time.
81 *
82 * Example: base {a: 10}, measured {a: 20, b: 4} → k = 10 ÷ 20 = 0.5, so
83 * {a: 10, b: 2}.
84 */
85 export function fillTimes(
86 base: ReadonlyMap<string, number>,
87 measured: ReadonlyMap<string, number>,
88 ): Map<string, number> {
89 let baseSum = 0;
90 let measuredSum = 0;
91 for (const [file, seconds] of measured) {
92 const own = base.get(file);
93 if (own === undefined) continue;
94 baseSum += own;
95 measuredSum += seconds;
96 }
97 const k = baseSum > 0 && measuredSum > 0 ? baseSum / measuredSum : 1;
98 const filled = new Map(base);
99 for (const [file, seconds] of measured) if (!filled.has(file)) filled.set(file, seconds * k);
100 return filled;
101 }
src/checks/tier_runner.ts +2 −4
@@ -6,8 +6,7 @@ import { recordProven } from "./proven.ts";
66 import { headDepth } from "./history.ts";
77 import { clearedIn } from "./base_red.ts";
88 import { recordCrashFlakes } from "./flake.ts";
9 import { IMPACT_FILE } from "./impact-store.ts";
10 import { KEPT_FILE } from "./kept-store.ts";
9 import { handoverEnv } from "./handover.ts";
1110 import { egressSetting } from "../core/capacity.ts";
1211 import { STEP_HOSTS } from "../apps/contain.ts";
1312 import { runRunnerStep, runnerTakes, runnerVerdict } from "./runner_step.ts";
@@ -66,8 +65,7 @@ export async function tryRunner(lane: LaneState, x: Execution, c: RunnerCheck):
6665 FORGE_URL: options.forgeUrl,
6766 FORGE_CHECK: run.id,
6867 FORGE_COMMIT: x.commitOid,
69 FORGE_CHECK_IMPACT: IMPACT_FILE,
70 FORGE_CHECK_KEPT: KEPT_FILE,
68 ...handoverEnv(),
7169 ...(base === undefined ? {} : { FORGE_CHECK_BASE: base, FORGE_CHECK_LEARNED: learned }),
7270 FORGE_CELLS: "off",
7371 },
src/checks/times-store.ts added +83
@@ -0,0 +1,83 @@
1 /**
2 * The test times a verification measured, kept beside the forge (#14322),
3 * as the impact map is (`impact-store.ts`). The whole gate runs only as the
4 * default branch's verification now, one file per process; its times died
5 * with its checkout, so the committed table named 66 of ~3,760 files and a
6 * pull's shards weighed every other file by its bytes. Every check's
7 * checkout is seeded with the kept times; only the default branch's whole
8 * verification hands them back, so a pull's code never fills what another
9 * check reads. What comes back is repository code's output: size-capped and
10 * parsed by `parseTimes` before it is kept.
11 *
12 * Times only weigh shards: they never decide which test runs.
13 */
14
15 import { lstatSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
16 import { join } from "node:path";
17 import { formatTimes, parseTimes } from "./test_times.ts";
18
19 /** Where in the checkout the times are handed over, both ways. */
20 export const TIMES_FILE = ".forge-times.json";
21
22 /** ~4,000 files × ~40 bytes a line is ~160 kB; this is room and a bound. */
23 const MAX_BYTES = 1024 * 1024;
24
25 /** The directory under `<data>/checks/` the times are kept in: beside the checkouts, not one. */
26 export const TIMES_DIR = "times";
27
28 const storeOf = (dataDir: string, repoId: string): string =>
29 join(dataDir, "checks", TIMES_DIR, `${repoId.replace(/[^A-Za-z0-9_-]/g, "_")}.json`);
30
31 function readValid(path: string): Map<string, number> | undefined {
32 try {
33 if (statSync(path).size > MAX_BYTES) return undefined;
34 const parsed = parseTimes(readFileSync(path, "utf8"));
35 return parsed.ok && parsed.value.size > 0 ? parsed.value : undefined;
36 } catch {
37 return undefined;
38 }
39 }
40
41 /** Copies the kept times into a checkout; how many files they time, or 0 for none. */
42 export function seedTimes(dataDir: string, repoId: string, checkout: string): number {
43 const times = readValid(storeOf(dataDir, repoId));
44 if (times === undefined) return 0;
45 // Whatever the checkout put at this name (a symlink out, say) goes.
46 const path = join(checkout, TIMES_FILE);
47 rmSync(path, { force: true, recursive: true });
48 writeFileSync(path, formatTimes(times), { flag: "wx" });
49 return times.size;
50 }
51
52 /**
53 * Keeps the times a default-branch verification left in its checkout; the
54 * log line, or `undefined` when it left none or the run is not one
55 * (`byDefaultBranch`).
56 */
57 export function keepTimes(
58 dataDir: string,
59 repoId: string,
60 checkout: string,
61 byDefaultBranch: boolean,
62 ): string | undefined {
63 if (!byDefaultBranch) return undefined;
64 const path = join(checkout, TIMES_FILE);
65 let size: number;
66 try {
67 const stat = lstatSync(path);
68 if (!stat.isFile()) return "test times not kept: not a plain file";
69 size = stat.size;
70 } catch {
71 return undefined;
72 }
73 if (size > MAX_BYTES) return `test times not kept: ${size} bytes is over ${MAX_BYTES}`;
74 const parsed = parseTimes(readFileSync(path, "utf8"));
75 if (!parsed.ok) return `test times not kept: ${parsed.error}`;
76 if (parsed.value.size === 0) return undefined;
77 mkdirSync(join(dataDir, "checks", TIMES_DIR), { recursive: true });
78 // Whole or not at all: a check seeding mid-write reads the old table.
79 const partial = `${storeOf(dataDir, repoId)}.${process.pid}.tmp`;
80 writeFileSync(partial, formatTimes(parsed.value));
81 renameSync(partial, storeOf(dataDir, repoId));
82 return `test times kept: ${parsed.value.size} files`;
83 }
src/kernel/destroy_reach.ts +1
@@ -74,6 +74,7 @@ export const ALLOWED_REMOVALS: readonly { unit: string; why: string }[] = [
7474 "src/checks/cache.ts:keepCache",
7575 "src/checks/impact-store.ts:seedImpact",
7676 "src/checks/kept-store.ts:seedKept",
77 "src/checks/times-store.ts:seedTimes",
7778 "src/checks/install.ts:prefillInstallCache",
7879 "src/checks/install.ts:holdFile",
7980 "src/checks/install_evict.ts:evictInstallCache",
test/check-times.test.ts added +187
@@ -0,0 +1,187 @@
1 /**
2 * Every file weighed by seconds (#14322): the default branch's whole
3 * verification times each file it runs, the forge keeps that table beside
4 * the impact map (`src/checks/times-store.ts`), and every check's shards are
5 * packed by it, rescaled to the committed table's clock (`fillTimes`).
6 *
7 * @not-a-tree-scan: it lists only its own temporary directories.
8 */
9 import { describe, expect, test } from "bun:test";
10 import { mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
11 import { tmpdir } from "node:os";
12 import { join } from "node:path";
13 import { fillTimes, formatTimes, parseTimes } from "../src/checks/test_times.ts";
14 import { TIMES_FILE, keepTimes, seedTimes } from "../src/checks/times-store.ts";
15 import { greenTimes, handBackTimes, writesHandedTimes } from "../scripts/shards.ts";
16 import { handoverEnv, handsTimesBack } from "../src/checks/handover.ts";
17
18 describe("fillTimes: the verification's times on the table's clock", () => {
19 test("by hand: k = Σ table ÷ Σ measured over the shared files", () => {
20 // Shared: a (10 vs 20) and b (6 vs 12): k = 16 ÷ 32 = 0.5. c is only measured: 9 × 0.5 = 4.5.
21 const filled = fillTimes(
22 new Map([["test/a.test.ts", 10], ["test/b.test.ts", 6]]),
23 new Map([["test/a.test.ts", 20], ["test/b.test.ts", 12], ["test/c.test.ts", 9]]),
24 );
25 expect([...filled]).toEqual([
26 ["test/a.test.ts", 10],
27 ["test/b.test.ts", 6],
28 ["test/c.test.ts", 4.5],
29 ]);
30 });
31
32 test("nothing shared is k = 1; a file the table names keeps its own time", () => {
33 expect(fillTimes(new Map([["test/a.test.ts", 3]]), new Map([["test/c.test.ts", 9]])).get("test/c.test.ts")).toBe(9);
34 expect(fillTimes(new Map(), new Map([["test/c.test.ts", 9]])).get("test/c.test.ts")).toBe(9);
35 // A shared file at 0 measured seconds cannot divide by zero.
36 const zero = fillTimes(new Map([["test/a.test.ts", 2]]), new Map([["test/a.test.ts", 0], ["test/c.test.ts", 4]]));
37 expect(zero.get("test/a.test.ts")).toBe(2);
38 expect(zero.get("test/c.test.ts")).toBe(4);
39 });
40 });
41
42 describe("greenTimes: only a green file timed alone is a time", () => {
43 test("a red file and a shard of many are left out", () => {
44 expect(
45 greenTimes([
46 { files: ["./test/a.test.ts"], code: 0, seconds: 1.5 },
47 { files: ["test/b.test.ts"], code: 1, seconds: 9 },
48 { files: ["test/c.test.ts", "test/d.test.ts"], code: 0, seconds: 4 },
49 { files: ["test/e.test.ts"], code: 0 },
50 ]),
51 ).toEqual([{ file: "test/a.test.ts", seconds: 1.5 }]);
52 });
53 });
54
55 describe("the test times beside the forge", () => {
56 test("the file handed between checks is never committed", () => {
57 const ignored = readFileSync(join(import.meta.dir, "..", ".gitignore"), "utf8").split("\n");
58 expect(ignored).toContain(`/${TIMES_FILE}`);
59 });
60
61 test("seed what was kept; refuse what is not a table; only the default branch's verification hands back", () => {
62 const data = mkdtempSync(join(tmpdir(), "times-data-"));
63 const first = mkdtempSync(join(tmpdir(), "times-co-"));
64 const second = mkdtempSync(join(tmpdir(), "times-co-"));
65 try {
66 expect(seedTimes(data, "r1", first)).toBe(0);
67 expect(keepTimes(data, "r1", first, true)).toBeUndefined();
68 writeFileSync(join(first, TIMES_FILE), JSON.stringify({ "src/a.ts": 1 }));
69 expect(keepTimes(data, "r1", first, true)).toContain("not kept");
70 writeFileSync(join(first, TIMES_FILE), formatTimes(new Map([["test/a.test.ts", 2.5], ["test/b.test.ts", 40]])));
71 expect(keepTimes(data, "r1", first, false)).toBeUndefined();
72 expect(seedTimes(data, "r1", second)).toBe(0);
73 expect(keepTimes(data, "r1", first, true)).toBe("test times kept: 2 files");
74 // A planted link at the seeded name is replaced, never followed.
75 const outside = join(data, "outside.txt");
76 writeFileSync(outside, "mine");
77 symlinkSync(outside, join(second, TIMES_FILE));
78 expect(seedTimes(data, "r1", second)).toBe(2);
79 expect(readFileSync(outside, "utf8")).toBe("mine");
80 const seeded = parseTimes(readFileSync(join(second, TIMES_FILE), "utf8"));
81 expect(seeded.ok && [...seeded.value]).toEqual([["test/a.test.ts", 2.5], ["test/b.test.ts", 40]]);
82 expect(seedTimes(data, "r2", second)).toBe(0);
83 // A link handed back is no table.
84 rmSync(join(first, TIMES_FILE));
85 symlinkSync(outside, join(first, TIMES_FILE));
86 expect(keepTimes(data, "r1", first, true)).toBe("test times not kept: not a plain file");
87 } finally {
88 for (const dir of [data, first, second]) rmSync(dir, { recursive: true, force: true });
89 }
90 });
91 });
92
93 describe("the runner packs by the handed times", () => {
94 test("two files the committed table never timed are weighed by the verification's seconds, not their bytes", async () => {
95 const dir = mkdtempSync(join(tmpdir(), "shards-times-"));
96 try {
97 for (const name of ["a", "b"])
98 writeFileSync(join(dir, `${name}.test.ts`), `import { test } from "bun:test";\ntest("${name}", () => {});\n`);
99 writeFileSync(join(dir, TIMES_FILE), formatTimes(new Map([["a.test.ts", 30], ["b.test.ts", 7]])));
100 const script = new URL("../scripts/test-shards.ts", import.meta.url).pathname;
101 const run = async (env: Record<string, string>): Promise<string> => {
102 const child = Bun.spawn(["bun", script, "a.test.ts", "b.test.ts"], {
103 cwd: dir,
104 env: { ...process.env, FORGE_CHECK_SHARDS: "2", ...env },
105 stdin: "ignore",
106 stdout: "pipe",
107 stderr: "pipe",
108 });
109 const [out] = await Promise.all([new Response(child.stdout).text(), new Response(child.stderr).text()]);
110 expect(await child.exited).toBe(0);
111 return out;
112 };
113 expect(await run({ FORGE_CHECK_TIMES: TIMES_FILE })).toContain("2 files in 2 shards, expected 30s 7s");
114 // Without them, ~60 bytes each weigh next to nothing.
115 expect(await run({})).toContain("2 files in 2 shards, expected 0s 0s");
116 } finally {
117 rmSync(dir, { recursive: true, force: true });
118 }
119 }, 30_000);
120 });
121
122 describe("the whole verification writes the handed table back", () => {
123 const whole = { measure: true, gate: true, tier: "full", unsliced: true, only: false, handed: true };
124
125 test("only the measured, unsliced whole gate of the full tier, handed a table, writes it", () => {
126 expect(writesHandedTimes(whole)).toBe(true);
127 for (const off of [
128 { measure: false },
129 { gate: false },
130 { tier: "fast" },
131 { unsliced: false },
132 { only: true },
133 { handed: false },
134 ])
135 expect(writesHandedTimes({ ...whole, ...off })).toBe(false);
136 });
137
138 test("green files alone refresh the table at their seconds; a red file keeps its old time; none green writes nothing", () => {
139 const old = new Map([["test/a.test.ts", 5], ["test/b.test.ts", 8], ["test/gone.test.ts", 3]]);
140 const exists = (file: string): boolean => file !== "test/gone.test.ts";
141 const back = handBackTimes(
142 [
143 { files: ["test/a.test.ts"], code: 0, seconds: 1.2 },
144 { files: ["test/b.test.ts"], code: 1, seconds: 90 },
145 { files: ["test/c.test.ts"], code: 0, seconds: 4 },
146 ],
147 old,
148 exists,
149 );
150 expect(back?.files).toBe(2);
151 const parsed = parseTimes(back?.text ?? "");
152 // a: 1.2 measured; b: red, its 8 kept, never its 90; c: new at 4; gone: no longer on disk.
153 expect(parsed.ok && [...parsed.value]).toEqual([["test/a.test.ts", 1.2], ["test/b.test.ts", 8], ["test/c.test.ts", 4]]);
154 expect(handBackTimes([{ files: ["test/a.test.ts"], code: 1, seconds: 2 }], old, exists)).toBeUndefined();
155 });
156
157 test("the script writes through these: its whole mode names both", () => {
158 // The gate itself cannot run on a temporary tree (its exclusions audit
159 // the repository), so this holds the script to the two tested functions.
160 const script = readFileSync(join(import.meta.dir, "..", "scripts", "test-shards.ts"), "utf8");
161 expect(script).toMatch(/writesHandedTimes\(\{ measure, gate, tier, unsliced: "none" in shardSpec, only: only !== undefined/);
162 expect(script).toMatch(/handBackTimes\(results, handedTimes\(\)/);
163 expect(script).toMatch(/writeFileSync\(HANDED_TIMES, back\.text\)/);
164 });
165 });
166
167 describe("the forge's side of the handover", () => {
168 test("every check is told where the times are", () => {
169 expect(handoverEnv().FORGE_CHECK_TIMES).toBe(TIMES_FILE);
170 });
171
172 test("only the default branch's whole verification hands the table back, green or red", () => {
173 expect(handsTimesBack({ tier: "verify", keepImpact: true })).toBe(true);
174 // A pull's own verification, a bisection step, any other tier: never.
175 expect(handsTimesBack({ tier: "verify" })).toBe(false);
176 expect(handsTimesBack({ tier: "verify", keepImpact: false })).toBe(false);
177 expect(handsTimesBack({ tier: "verify", keepImpact: true, only: ["test/a.test.ts"] })).toBe(false);
178 expect(handsTimesBack({ tier: "full", keepImpact: true })).toBe(false);
179 });
180
181 test("both of run_tier's keeps ask the same guard", () => {
182 const source = readFileSync(join(import.meta.dir, "..", "src", "checks", "run_tier.ts"), "utf8");
183 const keeps = source.match(/keepTimes\([^;]*\);/g) ?? [];
184 expect(keeps).toHaveLength(2);
185 for (const keep of keeps) expect(keep).toContain("handsTimesBack(x)");
186 });
187 });
test/checks-home-sweep.test.ts +5 −1
@@ -19,6 +19,7 @@ import { formatImpact, packImpact } from "../src/checks/impact.ts";
1919 import { IMPACT_FILE, keepImpact, seedImpact } from "../src/checks/impact-store.ts";
2020 import { formatKept } from "../src/checks/kept.ts";
2121 import { KEPT_FILE, keepKept, seedKept } from "../src/checks/kept-store.ts";
22 import { TIMES_FILE, keepTimes, seedTimes } from "../src/checks/times-store.ts";
2223 import { addLearned, readLearned } from "../src/checks/learned.ts";
2324 import { keepGateLog, readGateLog } from "../src/checks/gate_log.ts";
2425 import { keepGate, readGate } from "../src/checks/gate_last.ts";
@@ -83,7 +84,7 @@ test("a finished run's home is removed, a live run's survives", async () => {
8384 expect(existsSync(home("no-such-run"))).toBe(false);
8485 });
8586
86 test("the sweep leaves what the checks keep between runs: the map, the kept results, what a pull learned, the gates' logs (#12178)", async () => {
87 test("the sweep leaves what the checks keep between runs: the map, the kept results, the test times, what a pull learned, the gates' logs (#12178)", async () => {
8788 // Every store sits under <data>/checks/ beside the checkouts. Swept as a stranger's checkout,
8889 // a map kept by main's verification was gone within 30 seconds and no check was ever seeded.
8990 const checkout = join(dataDir, "a-checkout");
@@ -94,6 +95,8 @@ test("the sweep leaves what the checks keep between runs: the map, the kept resu
9495 writeFileSync(join(checkout, KEPT_FILE), formatKept(new Map([[key, { test: "test/a.test.ts", commit: "b".repeat(40) }]])));
9596 expect(keepImpact(dataDir, repoId, checkout)).toBe("impact map kept: 1 tests");
9697 expect(keepKept(dataDir, repoId, checkout, true)).toBe("kept results kept: 1 tests");
98 writeFileSync(join(checkout, TIMES_FILE), JSON.stringify({ "test/a.test.ts": 2 }));
99 expect(keepTimes(dataDir, repoId, checkout, true)).toBe("test times kept: 1 files");
97100 addLearned(dataDir, "pull1", ["test/a.test.ts"]);
98101 const gate = keepGateLog(dataDir, "alice/garden", "c".repeat(40), true, ["test/a.test.ts:", "(pass) a"], Date.now());
99102 keepGate(dataDir, ["test/a.test.ts:", "(pass) a"]);
@@ -110,6 +113,7 @@ test("the sweep leaves what the checks keep between runs: the map, the kept resu
110113 expect(seedImpact(dataDir, repoId, next)).toBe(1);
111114 expect(readFileSync(join(next, IMPACT_FILE), "utf8")).toBe(map);
112115 expect(seedKept(dataDir, repoId, next)).toBe(1);
116 expect(seedTimes(dataDir, repoId, next)).toBe(1);
113117 expect(readLearned(dataDir, "pull1")).toEqual(["test/a.test.ts"]);
114118 expect(readGateLog(dataDir, "alice", "garden", gate!)).toBeDefined();
115119 expect(readGate(dataDir)).toEqual(lastGate);

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs