Public: anyone can read this; the crew can change it.
Faster pull request checks without weaker ones: measured selection, result cache, measured shards #14335
Check · Review · Land · Live — Its check is red.
torque/forge:torque/14322-faster-pull-request-checks-without-weake → main 2 commits fork 2 threads · 2 unresolved
13 files
changed +539
−86
.gitignore +2 −1
| @@ -24,9 +24,10 @@ typesafe-api-key | ||
| 24 | 24 | # what a check keeps for the next one (src/checks/cache.ts) |
| 25 | 25 | /.forge-cache/ |
| 26 | 26 | |
| 27 | # what a check hands between runs: the impact map and the kept results, never committed | |
| 27 | # what a check hands between runs: the impact map, the kept results and the test times, never committed | |
| 28 | 28 | /.forge-impact.json |
| 29 | 29 | /.forge-kept.json |
| 30 | /.forge-times.json | |
| 30 | 31 | |
| 31 | 32 | # empty stubs an agent sandbox mounts over home-directory config; never the |
| 32 | 33 | # forge's own files, and never committed (#1148) |
docs/DEVELOP.md +7 −2
| @@ -180,8 +180,13 @@ stderr as `full tier: … changed`), or the arguments or git failed. The | ||
| 180 | 180 | |
| 181 | 181 | Files go to shards by how long they take, not how big they are: longest |
| 182 | 182 | first onto the least-loaded shard, weighed from `scripts/test-times.json` |
| 183 | (file → seconds, from one recorded run). A file the table does not name | |
| 184 | weighs its bytes at the table's own seconds-per-byte. If the table is | |
| 183 | (file → seconds, from one recorded run). In a forge check, a file the table | |
| 184 | does not name weighs the seconds the default branch's last whole | |
| 185 | verification measured for it, rescaled to the table's clock (the forge | |
| 186 | keeps them in `checks/times/`, `src/checks/times-store.ts`, and hands them | |
| 187 | over as `.forge-times.json`); only a file neither names weighs its bytes at | |
| 188 | the table's own seconds-per-byte. Times only weigh shards: they never | |
| 189 | decide which test runs. If the table is | |
| 185 | 190 | malformed, the log says it was refused and the split falls back to bytes. |
| 186 | 191 | A stale table only makes the split less even; it never drops a file. |
| 187 | 192 | Refresh it with one command after adding a slow test file, or when the |
scripts/shards.ts +53 −66
| @@ -9,6 +9,10 @@ | ||
| 9 | 9 | |
| 10 | 10 | import { auditLine } from "../src/checks/kept.ts"; |
| 11 | 11 | import { isSummaryLine, nameUnnamed } from "../src/checks/bun-test.ts"; |
| 12 | import { TEST_FILE, formatTimes, normaliseTest } from "../src/checks/test_times.ts"; | |
| 13 | ||
| 14 | // The times table is the forge's too (`times-store.ts` validates what a check hands back). | |
| 15 | export { TEST_FILE, fillTimes, formatTimes, parseTimes } from "../src/checks/test_times.ts"; | |
| 12 | 16 | |
| 13 | 17 | /** |
| 14 | 18 | * No more than this many `bun test` processes at once, whatever the host: |
| @@ -189,9 +193,6 @@ export function splitCapped( | ||
| 189 | 193 | ); |
| 190 | 194 | } |
| 191 | 195 | |
| 192 | /** How a test file is named: the same rule `scripts/test-shards.ts` uses. */ | |
| 193 | export const TEST_FILE = /\.test\.[cm]?[jt]sx?$/; | |
| 194 | ||
| 195 | 196 | /** A test's short name after `--`: `crew-wrapup` is `test/crew-wrapup.test.ts`. */ |
| 196 | 197 | const TEST_NAME = /^[a-z0-9][a-z0-9-]{0,99}$/; |
| 197 | 198 | |
| @@ -220,9 +221,6 @@ export function splitArgs( | ||
| 220 | 221 | return { ok: true, flags, files }; |
| 221 | 222 | } |
| 222 | 223 | |
| 223 | /** A sane bound on one entry of the times table: a day, in seconds. */ | |
| 224 | const TIME_MAX = 86_400; | |
| 225 | ||
| 226 | 224 | /** |
| 227 | 225 | * Seconds per byte when the table says nothing about any file in the list: |
| 228 | 226 | * about what this suite showed when the table was first recorded (a 40 kB |
| @@ -233,48 +231,6 @@ const TIME_MAX = 86_400; | ||
| 233 | 231 | */ |
| 234 | 232 | export const SECONDS_PER_BYTE_DEFAULT = 1 / 2_000; |
| 235 | 233 | |
| 236 | /** | |
| 237 | * Reads `scripts/test-times.json`: one object, test file path to the | |
| 238 | * seconds it took alone, e.g. `{ "test/crew.test.ts": 50.1 }`. Anything | |
| 239 | * else — not JSON, not an object, a key that is not a test file, a value | |
| 240 | * that is not a finite number from 0 to a day — refuses the whole table, | |
| 241 | * so a half-trusted table never steers the split. | |
| 242 | */ | |
| 243 | export function parseTimes( | |
| 244 | text: string, | |
| 245 | ): { ok: true; value: Map<string, number> } | { ok: false; error: string } { | |
| 246 | let data: unknown; | |
| 247 | try { | |
| 248 | data = JSON.parse(text); | |
| 249 | } catch { | |
| 250 | return { ok: false, error: "not JSON" }; | |
| 251 | } | |
| 252 | if (typeof data !== "object" || data === null || Array.isArray(data)) | |
| 253 | return { ok: false, error: "not an object of file to seconds" }; | |
| 254 | const times = new Map<string, number>(); | |
| 255 | for (const [file, seconds] of Object.entries(data)) { | |
| 256 | if (!TEST_FILE.test(file) || file.length > 512) | |
| 257 | return { ok: false, error: `${JSON.stringify(file)} is not a test file` }; | |
| 258 | if ( | |
| 259 | typeof seconds !== "number" || | |
| 260 | !Number.isFinite(seconds) || | |
| 261 | seconds < 0 || | |
| 262 | seconds > TIME_MAX | |
| 263 | ) | |
| 264 | return { | |
| 265 | ok: false, | |
| 266 | error: `${file}: ${JSON.stringify(seconds)} is not seconds from 0 to ${TIME_MAX}`, | |
| 267 | }; | |
| 268 | times.set(normalise(file), seconds); | |
| 269 | } | |
| 270 | return { ok: true, value: times }; | |
| 271 | } | |
| 272 | ||
| 273 | /** `./test/a.test.ts` and `test/a.test.ts` are the same file. */ | |
| 274 | function normalise(file: string): string { | |
| 275 | return file.startsWith("./") ? file.slice(2) : file; | |
| 276 | } | |
| 277 | ||
| 278 | 234 | /** |
| 279 | 235 | * How long each file is expected to take, in seconds: its measured time |
| 280 | 236 | * from the table when there is one, else its size in bytes scaled to |
| @@ -292,7 +248,7 @@ export function weigher( | ||
| 292 | 248 | let seconds = 0; |
| 293 | 249 | let size = 0; |
| 294 | 250 | for (const file of files) { |
| 295 | const measured = times.get(normalise(file)); | |
| 251 | const measured = times.get(normaliseTest(file)); | |
| 296 | 252 | const b = bytes.get(file); |
| 297 | 253 | if (measured === undefined || b === undefined) continue; |
| 298 | 254 | seconds += measured; |
| @@ -301,22 +257,7 @@ export function weigher( | ||
| 301 | 257 | const perByte = |
| 302 | 258 | seconds > 0 && size > 0 ? seconds / size : SECONDS_PER_BYTE_DEFAULT; |
| 303 | 259 | return (file) => |
| 304 | times.get(normalise(file)) ?? (bytes.get(file) ?? 0) * perByte; | |
| 305 | } | |
| 306 | ||
| 307 | /** | |
| 308 | * The table as it is checked in: keys sorted, seconds to one decimal, one | |
| 309 | * file per line, so a refresh reads as a small diff. | |
| 310 | */ | |
| 311 | export function formatTimes(times: ReadonlyMap<string, number>): string { | |
| 312 | const entries = [...times] | |
| 313 | .map(([file, seconds]) => [normalise(file), seconds] as const) | |
| 314 | .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); | |
| 315 | const lines = entries.map( | |
| 316 | ([file, seconds]) => | |
| 317 | ` ${JSON.stringify(file)}: ${Math.round(seconds * 10) / 10}`, | |
| 318 | ); | |
| 319 | return lines.length === 0 ? "{}\n" : `{\n${lines.join(",\n")}\n}\n`; | |
| 260 | times.get(normaliseTest(file)) ?? (bytes.get(file) ?? 0) * perByte; | |
| 320 | 261 | } |
| 321 | 262 | |
| 322 | 263 | /** |
| @@ -355,10 +296,56 @@ export function refreshTimes( | ||
| 355 | 296 | ): Map<string, number> { |
| 356 | 297 | const fresh = new Map<string, number>(); |
| 357 | 298 | for (const [file, seconds] of old) if (exists(file)) fresh.set(file, seconds); |
| 358 | for (const { file, seconds } of ran) fresh.set(normalise(file), seconds); | |
| 299 | for (const { file, seconds } of ran) fresh.set(normaliseTest(file), seconds); | |
| 359 | 300 | return fresh; |
| 360 | 301 | } |
| 361 | 302 | |
| 303 | /** | |
| 304 | * The time of every one-file shard that ran green, for the times table: a | |
| 305 | * red file's time is not its time, and a shard of many files times none of | |
| 306 | * them alone. | |
| 307 | */ | |
| 308 | export function greenTimes( | |
| 309 | results: readonly { readonly files: readonly string[]; readonly code: number; readonly seconds?: number }[], | |
| 310 | ): { file: string; seconds: number }[] { | |
| 311 | return results.flatMap((result) => | |
| 312 | result.code === 0 && result.files.length === 1 && result.seconds !== undefined | |
| 313 | ? [{ file: normaliseTest(result.files[0]!), seconds: result.seconds }] | |
| 314 | : [], | |
| 315 | ); | |
| 316 | } | |
| 317 | ||
| 318 | /** | |
| 319 | * Whether this run writes the handed times back (#14322): the whole gate of | |
| 320 | * the full tier, unsliced and not a bisection step, measured one file per | |
| 321 | * process (`--impact`), and only when the forge handed a table over. | |
| 322 | */ | |
| 323 | export function writesHandedTimes(mode: { | |
| 324 | readonly measure: boolean; | |
| 325 | readonly gate: boolean; | |
| 326 | readonly tier: string; | |
| 327 | readonly unsliced: boolean; | |
| 328 | readonly only: boolean; | |
| 329 | readonly handed: boolean; | |
| 330 | }): boolean { | |
| 331 | return mode.measure && mode.gate && mode.tier === "full" && mode.unsliced && !mode.only && mode.handed; | |
| 332 | } | |
| 333 | ||
| 334 | /** | |
| 335 | * The handed table after a whole verification: the old one refreshed by | |
| 336 | * every file that ran green alone (`greenTimes`), or `undefined` when none | |
| 337 | * did, so a run with nothing timed leaves the table as it was. | |
| 338 | */ | |
| 339 | export function handBackTimes( | |
| 340 | results: Parameters<typeof greenTimes>[0], | |
| 341 | old: ReadonlyMap<string, number>, | |
| 342 | exists: (file: string) => boolean, | |
| 343 | ): { text: string; files: number } | undefined { | |
| 344 | const ran = greenTimes(results); | |
| 345 | if (ran.length === 0) return undefined; | |
| 346 | return { text: formatTimes(refreshTimes(old, ran, exists)), files: ran.length }; | |
| 347 | } | |
| 348 | ||
| 362 | 349 | /** One shard, finished: what it ran, how it exited, everything it printed. */ |
| 363 | 350 | export interface ShardResult { |
| 364 | 351 | readonly files: readonly string[]; |
scripts/test-shards.ts +45 −2
| @@ -61,7 +61,12 @@ | ||
| 61 | 61 | * |
| 62 | 62 | * Files are weighed by how long they took, from `scripts/test-times.json` |
| 63 | 63 | * overlaid by `.forge-cache/test-times.json` (a file neither names, or a |
| 64 | * table it refuses, falls back to bytes). | |
| 64 | * table it refuses, falls back to bytes). Packing also reads the times the | |
| 65 | * default branch's whole verification measured (`FORGE_CHECK_TIMES`, | |
| 66 | * `src/checks/times-store.ts`), rescaled to that table's clock, for every | |
| 67 | * file the table lacks (`fillTimes`); that verification writes the time of | |
| 68 | * each file it ran green back to the same file (#14322). Times only weigh | |
| 69 | * shards: which tests run is never theirs to decide. | |
| 65 | 70 | * `--record` refreshes that table: the same number of workers, but each |
| 66 | 71 | * file is its own `bun test`, timed alone from spawn to exit, and the |
| 67 | 72 | * table is rewritten from those times when every file is green. |
| @@ -72,12 +77,15 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, s | ||
| 72 | 77 | import { join, resolve } from "node:path"; |
| 73 | 78 | import { resolveLimits } from "../src/apps/contain.ts"; |
| 74 | 79 | import { |
| 80 | fillTimes, | |
| 75 | 81 | formatTimes, |
| 82 | handBackTimes, | |
| 76 | 83 | heaviestFirst, |
| 77 | 84 | MEASURE_WORKER_MEMORY_BYTES, |
| 78 | 85 | memoryLimit, |
| 79 | 86 | overlayTimes, |
| 80 | 87 | refreshTimes, |
| 88 | writesHandedTimes, | |
| 81 | 89 | parseShardSpec, |
| 82 | 90 | parseTimes, |
| 83 | 91 | SHARDS_MAX, |
| @@ -125,6 +133,10 @@ const IMPACT = handed ?? new URL("./test-impact.json", import.meta.url).pathname | ||
| 125 | 133 | const IMPACT_NAME = handed ?? "scripts/test-impact.json"; |
| 126 | 134 | // Test results kept by what each reads (`src/checks/kept.ts`): handed over like the map, by name. |
| 127 | 135 | const KEPT = handedImpact(process.env["FORGE_CHECK_KEPT"]); |
| 136 | // Each file's seconds from the default branch's last whole verification | |
| 137 | // (`src/checks/times-store.ts`), handed over like the map: they weigh the | |
| 138 | // shards, never which test runs, and that verification writes them back. | |
| 139 | const HANDED_TIMES = handedImpact(process.env["FORGE_CHECK_TIMES"]); | |
| 128 | 140 | // What the forge keeps between checks (`src/checks/cache.ts`): the full |
| 129 | 141 | // gate's own per-file times, fresher than the committed table. |
| 130 | 142 | const CACHED_TIMES = `${CHECK_CACHE}/test-times.json`; |
| @@ -171,6 +183,21 @@ function loadTimes(): Map<string, number> { | ||
| 171 | 183 | ); |
| 172 | 184 | } |
| 173 | 185 | |
| 186 | /** The verification's handed times; none when the forge handed none or they are refused. */ | |
| 187 | function handedTimes(): Map<string, number> { | |
| 188 | if (HANDED_TIMES === undefined) return new Map(); | |
| 189 | let text: string; | |
| 190 | try { | |
| 191 | text = readFileSync(HANDED_TIMES, "utf8"); | |
| 192 | } catch { | |
| 193 | return new Map(); | |
| 194 | } | |
| 195 | const parsed = parseTimes(text); | |
| 196 | if (parsed.ok) return parsed.value; | |
| 197 | console.log(`test-shards: ${HANDED_TIMES} refused (${parsed.error}); weighing those files by bytes`); | |
| 198 | return new Map(); | |
| 199 | } | |
| 200 | ||
| 174 | 201 | /** Pull `--shard N/M` or `--shard=N/M` out of argv so it never reaches bun test. */ |
| 175 | 202 | function takeShardArg(raw: string[]): { |
| 176 | 203 | rest: string[]; |
| @@ -478,7 +505,10 @@ delete process.env["FORGE_CHECK_SHARDS"]; | ||
| 478 | 505 | delete process.env["FORGE_CHECK_SHARD_INDEX"]; |
| 479 | 506 | delete process.env["FORGE_CHECK_SHARD_TOTAL"]; |
| 480 | 507 | const times = loadTimes(); |
| 481 | const weight = weigher(files, size, times); | |
| 508 | // Packing alone also reads the verification's times, rescaled to the | |
| 509 | // table's clock: a file the table never timed is weighed by seconds, not bytes. | |
| 510 | const weighed = fillTimes(times, handedTimes()); | |
| 511 | const weight = weigher(files, size, weighed); | |
| 482 | 512 | // Recording and measuring run one file per process, so every file is its |
| 483 | 513 | // own slice: Bun's coverage is per process, and a shard of many files would |
| 484 | 514 | // merge theirs into one (#1794). |
| @@ -1089,6 +1119,19 @@ if (measure) { | ||
| 1089 | 1119 | } |
| 1090 | 1120 | } |
| 1091 | 1121 | } |
| 1122 | // The whole verification times every file it runs, one process each: the | |
| 1123 | // files that ran green refresh the handed table the forge keeps (#14322). | |
| 1124 | if ( | |
| 1125 | HANDED_TIMES !== undefined && | |
| 1126 | writesHandedTimes({ measure, gate, tier, unsliced: "none" in shardSpec, only: only !== undefined, handed: true }) | |
| 1127 | ) { | |
| 1128 | const back = handBackTimes(results, handedTimes(), (file) => existsSync(file)); | |
| 1129 | if (back === undefined) console.log(`test-shards: not writing ${HANDED_TIMES}: no file ran green`); | |
| 1130 | else { | |
| 1131 | writeFileSync(HANDED_TIMES, back.text); | |
| 1132 | console.log(`test-shards: wrote ${back.files} times to ${HANDED_TIMES}`); | |
| 1133 | } | |
| 1134 | } | |
| 1092 | 1135 | // The timed full gate is the check on the cell itself: its log ends with |
| 1093 | 1136 | // what the cell did, so nobody has to measure it by hand (#2065). |
| 1094 | 1137 | if (timed) |
src/checks/handover.ts added +30
| @@ -0,0 +1,30 @@ | ||
| 1 | /** | |
| 2 | * What crosses between the forge and a check's checkout, both ways (#14322): | |
| 3 | * the files the forge seeds a check with, named in its environment, and which | |
| 4 | * run may hand the shared times table back. One place, so the local path | |
| 5 | * (`run_tier.ts`) and a runner's (`tier_runner.ts`) cannot drift apart. | |
| 6 | */ | |
| 7 | ||
| 8 | import { IMPACT_FILE } from "./impact-store.ts"; | |
| 9 | import { KEPT_FILE } from "./kept-store.ts"; | |
| 10 | import { TIMES_FILE } from "./times-store.ts"; | |
| 11 | import type { Execution } from "./local_runner.ts"; | |
| 12 | ||
| 13 | /** The seeded files' names, as every check's environment carries them. */ | |
| 14 | export function handoverEnv(): Record<"FORGE_CHECK_IMPACT" | "FORGE_CHECK_KEPT" | "FORGE_CHECK_TIMES", string> { | |
| 15 | return { | |
| 16 | FORGE_CHECK_IMPACT: IMPACT_FILE, | |
| 17 | FORGE_CHECK_KEPT: KEPT_FILE, | |
| 18 | FORGE_CHECK_TIMES: TIMES_FILE, | |
| 19 | }; | |
| 20 | } | |
| 21 | ||
| 22 | /** | |
| 23 | * Whether this run's times may replace the table every check is packed by: | |
| 24 | * the default branch's own (`keepImpact`) verification of the whole gate, | |
| 25 | * never a bisection step (`only`) and never a pull's, whose tree could write | |
| 26 | * any table. Green or red alike: a red verification's green files were timed. | |
| 27 | */ | |
| 28 | export function handsTimesBack(x: Pick<Execution, "tier" | "only" | "keepImpact">): boolean { | |
| 29 | return x.tier === "verify" && x.only === undefined && x.keepImpact === true; | |
| 30 | } |
src/checks/run_tier.ts +17 −4
| @@ -11,8 +11,10 @@ import { readLearned } from "./learned.ts"; | ||
| 11 | 11 | import { keepCache, restoreCache } from "./cache.ts"; |
| 12 | 12 | import { clearedIn } from "./base_red.ts"; |
| 13 | 13 | import { scratchEnv } from "./disk_full.ts"; |
| 14 | import { IMPACT_FILE, keepImpact, seedImpact } from "./impact-store.ts"; | |
| 15 | import { KEPT_FILE, keepKept, seedKept } from "./kept-store.ts"; | |
| 14 | import { keepImpact, seedImpact } from "./impact-store.ts"; | |
| 15 | import { keepKept, seedKept } from "./kept-store.ts"; | |
| 16 | import { keepTimes, seedTimes } from "./times-store.ts"; | |
| 17 | import { handoverEnv, handsTimesBack } from "./handover.ts"; | |
| 16 | 18 | import { redDetail } from "./report.ts"; |
| 17 | 19 | import { rerunFileCommand, rerunRed, withFlaky } from "./rerun.ts"; |
| 18 | 20 | import { failedFiles, onlyEnv, passedFiles } from "./verify.ts"; |
| @@ -142,6 +144,13 @@ export async function runTier(lane: LaneState, x: Execution): Promise<CheckOutco | ||
| 142 | 144 | } catch (error) { |
| 143 | 145 | write(`kept results not seeded: ${String(error)}`); |
| 144 | 146 | } |
| 147 | // Each test file's seconds from the last whole verification: the shards are packed by them. | |
| 148 | try { | |
| 149 | const seeded = seedTimes(app.config.dataDir, repo.id, dir); | |
| 150 | if (seeded > 0) write(`test times: ${seeded} files, from ${repo.defaultBranch}'s verification`); | |
| 151 | } catch (error) { | |
| 152 | write(`test times not seeded: ${String(error)}`); | |
| 153 | } | |
| 145 | 154 | // The check is repository code: it runs in a cell of its own |
| 146 | 155 | // (`apps/contain.ts`), its checkout and a home its only places to write, |
| 147 | 156 | // the forge, the registry and what it declared its only way out. |
| @@ -206,8 +215,7 @@ export async function runTier(lane: LaneState, x: Execution): Promise<CheckOutco | ||
| 206 | 215 | FORGE_URL: options.forgeUrl, |
| 207 | 216 | FORGE_CHECK: run.id, |
| 208 | 217 | FORGE_COMMIT: x.commitOid, |
| 209 | FORGE_CHECK_IMPACT: IMPACT_FILE, | |
| 210 | FORGE_CHECK_KEPT: KEPT_FILE, | |
| 218 | ...handoverEnv(), | |
| 211 | 219 | ...(x.only === undefined ? {} : { FORGE_CHECK_ONLY: onlyEnv(x.only) }), |
| 212 | 220 | // The cell's CPU limit, which `availableParallelism()` may not |
| 213 | 221 | // report inside it; the shard count follows this (#2257). |
| @@ -331,6 +339,9 @@ export async function runTier(lane: LaneState, x: Execution): Promise<CheckOutco | ||
| 331 | 339 | if (kept !== undefined) write(kept); |
| 332 | 340 | const results = keepKept(app.config.dataDir, repo.id, dir, true); |
| 333 | 341 | if (results !== undefined) write(results); |
| 342 | // A red verification's green files were timed all the same. | |
| 343 | const times = keepTimes(app.config.dataDir, repo.id, dir, handsTimesBack(x)); | |
| 344 | if (times !== undefined) write(times); | |
outdated
Guard differs from the green branch (wholeVerify && x.keepImpact === true). Use the same guard here so only the default branch hands the table back, and test it. |
||
| 334 | 345 | } catch (error) { |
| 335 | 346 | write(`impact map not kept: ${String(error)}`); |
| 336 | 347 | } |
| @@ -358,6 +369,8 @@ export async function runTier(lane: LaneState, x: Execution): Promise<CheckOutco | ||
| 358 | 369 | // Only the default branch's own run hands results back: a pull's tree could write any. |
| 359 | 370 | const results = keepKept(app.config.dataDir, repo.id, dir, x.keepImpact === true); |
| 360 | 371 | if (results !== undefined) write(results); |
| 372 | const times = keepTimes(app.config.dataDir, repo.id, dir, handsTimesBack(x)); | |
| 373 | if (times !== undefined) write(times); | |
| 361 | 374 | } catch (error) { |
| 362 | 375 | write(`impact map not kept: ${String(error)}`); |
| 363 | 376 | } |
src/checks/sweep.ts +6 −6
| @@ -8,6 +8,7 @@ import { gateHolding } from "./gate_record.ts"; | ||
| 8 | 8 | import { diskRecovered } from "./disk_red.ts"; |
| 9 | 9 | import { IMPACT_DIR } from "./impact-store.ts"; |
| 10 | 10 | import { KEPT_DIR } from "./kept-store.ts"; |
| 11 | import { TIMES_DIR } from "./times-store.ts"; | |
| 11 | 12 | import { createLog } from "../kernel/log.ts"; |
| 12 | 13 | import { STALE_AFTER_MS } from "../runs/lease.ts"; |
| 13 | 14 | import { CHECK_TIMEOUT_MAX } from "../apps/manifest.ts"; |
| @@ -70,13 +71,12 @@ export const checksRoot = (dataDir: string): string => join(dataDir, "checks"); | ||
| 70 | 71 | /** |
| 71 | 72 | * What is kept under the checks' root that is no checkout: every |
| 72 | 73 | * repository's impact map (`impact-store.ts`) and kept results |
| 73 | * (`kept-store.ts`), the tests a red gate taught each pull (`learned.ts`), | |
| 74 | * the gates' logs (`gate_log.ts`) and the last gate's results | |
| 75 | * (`gate_last.ts`). The sweep took each for a stranger's checkout and | |
| 76 | * removed it at every boot and every 30 seconds after: a map lived half a | |
| 77 | * minute, so no check was ever seeded with one (#12178). | |
| 74 | * (`kept-store.ts`) and test times (`times-store.ts`), the tests a red gate taught each pull | |
| 75 | * (`learned.ts`), the gates' logs (`gate_log.ts`) and the last gate's results (`gate_last.ts`). The | |
| 76 | * sweep took each for a stranger's checkout and removed it at every boot and every 30 seconds | |
| 77 | * after: a map lived half a minute, so no check was ever seeded with one (#12178). | |
| 78 | 78 | */ |
| 79 | const STORES: ReadonlySet<string> = new Set([IMPACT_DIR, KEPT_DIR, "learned", "gates", "gate-last.json"]); | |
| 79 | const STORES: ReadonlySet<string> = new Set([IMPACT_DIR, KEPT_DIR, TIMES_DIR, "learned", "gates", "gate-last.json"]); | |
| 80 | 80 | |
| 81 | 81 | /** A gate's checkout (`gateRunId`): it has no run, only its `check_gate` row. */ |
| 82 | 82 | const GATE_DIR = /^gate-([0-9a-f]{12})$/; |
src/checks/test_times.ts added +101
| @@ -0,0 +1,101 @@ | ||
| 1 | /** | |
| 2 | * The times table: test file to the seconds it took (#1797, #14322). The | |
| 3 | * committed `scripts/test-times.json`, the full gate's cache and the times a | |
| 4 | * default-branch verification measured (`times-store.ts`) all share this | |
| 5 | * form, so the forge can validate what a check hands back before keeping | |
| 6 | * it, and the gate's script can read what the forge seeds. | |
| 7 | * | |
| 8 | * Plain data in, plain data out. | |
| 9 | */ | |
| 10 | ||
| 11 | /** How a test file is named: the same rule `scripts/test-shards.ts` uses. */ | |
| 12 | export const TEST_FILE = /\.test\.[cm]?[jt]sx?$/; | |
| 13 | ||
| 14 | /** A sane bound on one entry of the times table: a day, in seconds. */ | |
| 15 | const TIME_MAX = 86_400; | |
| 16 | ||
| 17 | /** `./test/a.test.ts` and `test/a.test.ts` are the same file. */ | |
| 18 | export function normaliseTest(file: string): string { | |
| 19 | return file.startsWith("./") ? file.slice(2) : file; | |
| 20 | } | |
| 21 | ||
| 22 | /** | |
| 23 | * Reads a times table: one object, test file path to the seconds it took, | |
| 24 | * e.g. `{ "test/crew.test.ts": 50.1 }`. Anything else — not JSON, not an | |
| 25 | * object, a key that is not a test file, a value that is not a finite | |
| 26 | * number from 0 to a day — refuses the whole table, so a half-trusted | |
| 27 | * table never steers the split. | |
| 28 | */ | |
| 29 | export function parseTimes( | |
| 30 | text: string, | |
| 31 | ): { ok: true; value: Map<string, number> } | { ok: false; error: string } { | |
| 32 | let data: unknown; | |
| 33 | try { | |
| 34 | data = JSON.parse(text); | |
| 35 | } catch { | |
| 36 | return { ok: false, error: "not JSON" }; | |
| 37 | } | |
| 38 | if (typeof data !== "object" || data === null || Array.isArray(data)) | |
| 39 | return { ok: false, error: "not an object of file to seconds" }; | |
| 40 | const times = new Map<string, number>(); | |
| 41 | for (const [file, seconds] of Object.entries(data)) { | |
| 42 | if (!TEST_FILE.test(file) || file.length > 512) | |
| 43 | return { ok: false, error: `${JSON.stringify(file)} is not a test file` }; | |
| 44 | if ( | |
| 45 | typeof seconds !== "number" || | |
| 46 | !Number.isFinite(seconds) || | |
| 47 | seconds < 0 || | |
| 48 | seconds > TIME_MAX | |
| 49 | ) | |
| 50 | return { | |
| 51 | ok: false, | |
| 52 | error: `${file}: ${JSON.stringify(seconds)} is not seconds from 0 to ${TIME_MAX}`, | |
| 53 | }; | |
| 54 | times.set(normaliseTest(file), seconds); | |
| 55 | } | |
| 56 | return { ok: true, value: times }; | |
| 57 | } | |
| 58 | ||
| 59 | /** | |
| 60 | * The table as it is checked in: keys sorted, seconds to one decimal, one | |
| 61 | * file per line, so a refresh reads as a small diff. | |
| 62 | */ | |
| 63 | export function formatTimes(times: ReadonlyMap<string, number>): string { | |
| 64 | const entries = [...times] | |
| 65 | .map(([file, seconds]) => [normaliseTest(file), seconds] as const) | |
| 66 | .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); | |
| 67 | const lines = entries.map( | |
| 68 | ([file, seconds]) => | |
| 69 | ` ${JSON.stringify(file)}: ${Math.round(seconds * 10) / 10}`, | |
| 70 | ); | |
| 71 | return lines.length === 0 ? "{}\n" : `{\n${lines.join(",\n")}\n}\n`; | |
| 72 | } | |
| 73 | ||
| 74 | /** | |
| 75 | * `base` with every file only `measured` names added, rescaled to `base`'s | |
| 76 | * clock. The verification times each file under coverage beside fifteen | |
| 77 | * others, which is slower than the committed table's file timed alone; only | |
| 78 | * the ratio between files steers a split, so the measured times are scaled | |
| 79 | * by k = Σ base ÷ Σ measured over the files both name (k = 1 when they share | |
| 80 | * none, or either sum is 0). A file `base` names keeps its own time. | |
| 81 | * | |
| 82 | * Example: base {a: 10}, measured {a: 20, b: 4} → k = 10 ÷ 20 = 0.5, so | |
| 83 | * {a: 10, b: 2}. | |
| 84 | */ | |
| 85 | export function fillTimes( | |
| 86 | base: ReadonlyMap<string, number>, | |
| 87 | measured: ReadonlyMap<string, number>, | |
| 88 | ): Map<string, number> { | |
| 89 | let baseSum = 0; | |
| 90 | let measuredSum = 0; | |
| 91 | for (const [file, seconds] of measured) { | |
| 92 | const own = base.get(file); | |
| 93 | if (own === undefined) continue; | |
| 94 | baseSum += own; | |
| 95 | measuredSum += seconds; | |
| 96 | } | |
| 97 | const k = baseSum > 0 && measuredSum > 0 ? baseSum / measuredSum : 1; | |
| 98 | const filled = new Map(base); | |
| 99 | for (const [file, seconds] of measured) if (!filled.has(file)) filled.set(file, seconds * k); | |
| 100 | return filled; | |
| 101 | } |
src/checks/tier_runner.ts +2 −4
| @@ -6,8 +6,7 @@ import { recordProven } from "./proven.ts"; | ||
| 6 | 6 | import { headDepth } from "./history.ts"; |
| 7 | 7 | import { clearedIn } from "./base_red.ts"; |
| 8 | 8 | import { recordCrashFlakes } from "./flake.ts"; |
| 9 | import { IMPACT_FILE } from "./impact-store.ts"; | |
| 10 | import { KEPT_FILE } from "./kept-store.ts"; | |
| 9 | import { handoverEnv } from "./handover.ts"; | |
| 11 | 10 | import { egressSetting } from "../core/capacity.ts"; |
| 12 | 11 | import { STEP_HOSTS } from "../apps/contain.ts"; |
| 13 | 12 | import { runRunnerStep, runnerTakes, runnerVerdict } from "./runner_step.ts"; |
| @@ -66,8 +65,7 @@ export async function tryRunner(lane: LaneState, x: Execution, c: RunnerCheck): | ||
| 66 | 65 | FORGE_URL: options.forgeUrl, |
| 67 | 66 | FORGE_CHECK: run.id, |
| 68 | 67 | FORGE_COMMIT: x.commitOid, |
| 69 | FORGE_CHECK_IMPACT: IMPACT_FILE, | |
| 70 | FORGE_CHECK_KEPT: KEPT_FILE, | |
| 68 | ...handoverEnv(), | |
| 71 | 69 | ...(base === undefined ? {} : { FORGE_CHECK_BASE: base, FORGE_CHECK_LEARNED: learned }), |
| 72 | 70 | FORGE_CELLS: "off", |
| 73 | 71 | }, |
src/checks/times-store.ts added +83
| @@ -0,0 +1,83 @@ | ||
| 1 | /** | |
| 2 | * The test times a verification measured, kept beside the forge (#14322), | |
| 3 | * as the impact map is (`impact-store.ts`). The whole gate runs only as the | |
| 4 | * default branch's verification now, one file per process; its times died | |
| 5 | * with its checkout, so the committed table named 66 of ~3,760 files and a | |
| 6 | * pull's shards weighed every other file by its bytes. Every check's | |
| 7 | * checkout is seeded with the kept times; only the default branch's whole | |
| 8 | * verification hands them back, so a pull's code never fills what another | |
| 9 | * check reads. What comes back is repository code's output: size-capped and | |
| 10 | * parsed by `parseTimes` before it is kept. | |
| 11 | * | |
| 12 | * Times only weigh shards: they never decide which test runs. | |
| 13 | */ | |
| 14 | ||
| 15 | import { lstatSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs"; | |
| 16 | import { join } from "node:path"; | |
| 17 | import { formatTimes, parseTimes } from "./test_times.ts"; | |
| 18 | ||
| 19 | /** Where in the checkout the times are handed over, both ways. */ | |
| 20 | export const TIMES_FILE = ".forge-times.json"; | |
| 21 | ||
| 22 | /** ~4,000 files × ~40 bytes a line is ~160 kB; this is room and a bound. */ | |
| 23 | const MAX_BYTES = 1024 * 1024; | |
| 24 | ||
| 25 | /** The directory under `<data>/checks/` the times are kept in: beside the checkouts, not one. */ | |
| 26 | export const TIMES_DIR = "times"; | |
| 27 | ||
| 28 | const storeOf = (dataDir: string, repoId: string): string => | |
| 29 | join(dataDir, "checks", TIMES_DIR, `${repoId.replace(/[^A-Za-z0-9_-]/g, "_")}.json`); | |
| 30 | ||
| 31 | function readValid(path: string): Map<string, number> | undefined { | |
| 32 | try { | |
| 33 | if (statSync(path).size > MAX_BYTES) return undefined; | |
| 34 | const parsed = parseTimes(readFileSync(path, "utf8")); | |
| 35 | return parsed.ok && parsed.value.size > 0 ? parsed.value : undefined; | |
| 36 | } catch { | |
| 37 | return undefined; | |
| 38 | } | |
| 39 | } | |
| 40 | ||
| 41 | /** Copies the kept times into a checkout; how many files they time, or 0 for none. */ | |
| 42 | export function seedTimes(dataDir: string, repoId: string, checkout: string): number { | |
| 43 | const times = readValid(storeOf(dataDir, repoId)); | |
| 44 | if (times === undefined) return 0; | |
| 45 | // Whatever the checkout put at this name (a symlink out, say) goes. | |
| 46 | const path = join(checkout, TIMES_FILE); | |
| 47 | rmSync(path, { force: true, recursive: true }); | |
| 48 | writeFileSync(path, formatTimes(times), { flag: "wx" }); | |
| 49 | return times.size; | |
| 50 | } | |
| 51 | ||
| 52 | /** | |
| 53 | * Keeps the times a default-branch verification left in its checkout; the | |
| 54 | * log line, or `undefined` when it left none or the run is not one | |
| 55 | * (`byDefaultBranch`). | |
| 56 | */ | |
| 57 | export function keepTimes( | |
| 58 | dataDir: string, | |
| 59 | repoId: string, | |
| 60 | checkout: string, | |
| 61 | byDefaultBranch: boolean, | |
| 62 | ): string | undefined { | |
| 63 | if (!byDefaultBranch) return undefined; | |
| 64 | const path = join(checkout, TIMES_FILE); | |
| 65 | let size: number; | |
| 66 | try { | |
| 67 | const stat = lstatSync(path); | |
| 68 | if (!stat.isFile()) return "test times not kept: not a plain file"; | |
| 69 | size = stat.size; | |
| 70 | } catch { | |
| 71 | return undefined; | |
| 72 | } | |
| 73 | if (size > MAX_BYTES) return `test times not kept: ${size} bytes is over ${MAX_BYTES}`; | |
| 74 | const parsed = parseTimes(readFileSync(path, "utf8")); | |
| 75 | if (!parsed.ok) return `test times not kept: ${parsed.error}`; | |
| 76 | if (parsed.value.size === 0) return undefined; | |
| 77 | mkdirSync(join(dataDir, "checks", TIMES_DIR), { recursive: true }); | |
| 78 | // Whole or not at all: a check seeding mid-write reads the old table. | |
| 79 | const partial = `${storeOf(dataDir, repoId)}.${process.pid}.tmp`; | |
| 80 | writeFileSync(partial, formatTimes(parsed.value)); | |
| 81 | renameSync(partial, storeOf(dataDir, repoId)); | |
| 82 | return `test times kept: ${parsed.value.size} files`; | |
| 83 | } |
src/kernel/destroy_reach.ts +1
| @@ -74,6 +74,7 @@ export const ALLOWED_REMOVALS: readonly { unit: string; why: string }[] = [ | ||
| 74 | 74 | "src/checks/cache.ts:keepCache", |
| 75 | 75 | "src/checks/impact-store.ts:seedImpact", |
| 76 | 76 | "src/checks/kept-store.ts:seedKept", |
| 77 | "src/checks/times-store.ts:seedTimes", | |
| 77 | 78 | "src/checks/install.ts:prefillInstallCache", |
| 78 | 79 | "src/checks/install.ts:holdFile", |
| 79 | 80 | "src/checks/install_evict.ts:evictInstallCache", |
test/check-times.test.ts added +187
| @@ -0,0 +1,187 @@ | ||
| 1 | /** | |
| 2 | * Every file weighed by seconds (#14322): the default branch's whole | |
| 3 | * verification times each file it runs, the forge keeps that table beside | |
| 4 | * the impact map (`src/checks/times-store.ts`), and every check's shards are | |
| 5 | * packed by it, rescaled to the committed table's clock (`fillTimes`). | |
| 6 | * | |
| 7 | * @not-a-tree-scan: it lists only its own temporary directories. | |
| 8 | */ | |
| 9 | import { describe, expect, test } from "bun:test"; | |
| 10 | import { mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; | |
| 11 | import { tmpdir } from "node:os"; | |
| 12 | import { join } from "node:path"; | |
| 13 | import { fillTimes, formatTimes, parseTimes } from "../src/checks/test_times.ts"; | |
| 14 | import { TIMES_FILE, keepTimes, seedTimes } from "../src/checks/times-store.ts"; | |
| 15 | import { greenTimes, handBackTimes, writesHandedTimes } from "../scripts/shards.ts"; | |
| 16 | import { handoverEnv, handsTimesBack } from "../src/checks/handover.ts"; | |
| 17 | ||
| 18 | describe("fillTimes: the verification's times on the table's clock", () => { | |
| 19 | test("by hand: k = Σ table ÷ Σ measured over the shared files", () => { | |
| 20 | // Shared: a (10 vs 20) and b (6 vs 12): k = 16 ÷ 32 = 0.5. c is only measured: 9 × 0.5 = 4.5. | |
| 21 | const filled = fillTimes( | |
| 22 | new Map([["test/a.test.ts", 10], ["test/b.test.ts", 6]]), | |
| 23 | new Map([["test/a.test.ts", 20], ["test/b.test.ts", 12], ["test/c.test.ts", 9]]), | |
| 24 | ); | |
| 25 | expect([...filled]).toEqual([ | |
| 26 | ["test/a.test.ts", 10], | |
| 27 | ["test/b.test.ts", 6], | |
| 28 | ["test/c.test.ts", 4.5], | |
| 29 | ]); | |
| 30 | }); | |
| 31 | ||
| 32 | test("nothing shared is k = 1; a file the table names keeps its own time", () => { | |
| 33 | expect(fillTimes(new Map([["test/a.test.ts", 3]]), new Map([["test/c.test.ts", 9]])).get("test/c.test.ts")).toBe(9); | |
| 34 | expect(fillTimes(new Map(), new Map([["test/c.test.ts", 9]])).get("test/c.test.ts")).toBe(9); | |
| 35 | // A shared file at 0 measured seconds cannot divide by zero. | |
| 36 | const zero = fillTimes(new Map([["test/a.test.ts", 2]]), new Map([["test/a.test.ts", 0], ["test/c.test.ts", 4]])); | |
| 37 | expect(zero.get("test/a.test.ts")).toBe(2); | |
| 38 | expect(zero.get("test/c.test.ts")).toBe(4); | |
| 39 | }); | |
| 40 | }); | |
| 41 | ||
| 42 | describe("greenTimes: only a green file timed alone is a time", () => { | |
| 43 | test("a red file and a shard of many are left out", () => { | |
| 44 | expect( | |
| 45 | greenTimes([ | |
| 46 | { files: ["./test/a.test.ts"], code: 0, seconds: 1.5 }, | |
| 47 | { files: ["test/b.test.ts"], code: 1, seconds: 9 }, | |
| 48 | { files: ["test/c.test.ts", "test/d.test.ts"], code: 0, seconds: 4 }, | |
| 49 | { files: ["test/e.test.ts"], code: 0 }, | |
| 50 | ]), | |
| 51 | ).toEqual([{ file: "test/a.test.ts", seconds: 1.5 }]); | |
| 52 | }); | |
| 53 | }); | |
| 54 | ||
| 55 | describe("the test times beside the forge", () => { | |
| 56 | test("the file handed between checks is never committed", () => { | |
| 57 | const ignored = readFileSync(join(import.meta.dir, "..", ".gitignore"), "utf8").split("\n"); | |
| 58 | expect(ignored).toContain(`/${TIMES_FILE}`); | |
| 59 | }); | |
| 60 | ||
| 61 | test("seed what was kept; refuse what is not a table; only the default branch's verification hands back", () => { | |
| 62 | const data = mkdtempSync(join(tmpdir(), "times-data-")); | |
| 63 | const first = mkdtempSync(join(tmpdir(), "times-co-")); | |
| 64 | const second = mkdtempSync(join(tmpdir(), "times-co-")); | |
| 65 | try { | |
| 66 | expect(seedTimes(data, "r1", first)).toBe(0); | |
| 67 | expect(keepTimes(data, "r1", first, true)).toBeUndefined(); | |
| 68 | writeFileSync(join(first, TIMES_FILE), JSON.stringify({ "src/a.ts": 1 })); | |
| 69 | expect(keepTimes(data, "r1", first, true)).toContain("not kept"); | |
| 70 | writeFileSync(join(first, TIMES_FILE), formatTimes(new Map([["test/a.test.ts", 2.5], ["test/b.test.ts", 40]]))); | |
| 71 | expect(keepTimes(data, "r1", first, false)).toBeUndefined(); | |
| 72 | expect(seedTimes(data, "r1", second)).toBe(0); | |
| 73 | expect(keepTimes(data, "r1", first, true)).toBe("test times kept: 2 files"); | |
| 74 | // A planted link at the seeded name is replaced, never followed. | |
| 75 | const outside = join(data, "outside.txt"); | |
| 76 | writeFileSync(outside, "mine"); | |
| 77 | symlinkSync(outside, join(second, TIMES_FILE)); | |
| 78 | expect(seedTimes(data, "r1", second)).toBe(2); | |
| 79 | expect(readFileSync(outside, "utf8")).toBe("mine"); | |
| 80 | const seeded = parseTimes(readFileSync(join(second, TIMES_FILE), "utf8")); | |
| 81 | expect(seeded.ok && [...seeded.value]).toEqual([["test/a.test.ts", 2.5], ["test/b.test.ts", 40]]); | |
| 82 | expect(seedTimes(data, "r2", second)).toBe(0); | |
| 83 | // A link handed back is no table. | |
| 84 | rmSync(join(first, TIMES_FILE)); | |
| 85 | symlinkSync(outside, join(first, TIMES_FILE)); | |
| 86 | expect(keepTimes(data, "r1", first, true)).toBe("test times not kept: not a plain file"); | |
| 87 | } finally { | |
| 88 | for (const dir of [data, first, second]) rmSync(dir, { recursive: true, force: true }); | |
| 89 | } | |
| 90 | }); | |
| 91 | }); | |
| 92 | ||
| 93 | describe("the runner packs by the handed times", () => { | |
| 94 | test("two files the committed table never timed are weighed by the verification's seconds, not their bytes", async () => { | |
| 95 | const dir = mkdtempSync(join(tmpdir(), "shards-times-")); | |
| 96 | try { | |
| 97 | for (const name of ["a", "b"]) | |
| 98 | writeFileSync(join(dir, `${name}.test.ts`), `import { test } from "bun:test";\ntest("${name}", () => {});\n`); | |
| 99 | writeFileSync(join(dir, TIMES_FILE), formatTimes(new Map([["a.test.ts", 30], ["b.test.ts", 7]]))); | |
| 100 | const script = new URL("../scripts/test-shards.ts", import.meta.url).pathname; | |
| 101 | const run = async (env: Record<string, string>): Promise<string> => { | |
| 102 | const child = Bun.spawn(["bun", script, "a.test.ts", "b.test.ts"], { | |
| 103 | cwd: dir, | |
| 104 | env: { ...process.env, FORGE_CHECK_SHARDS: "2", ...env }, | |
| 105 | stdin: "ignore", | |
| 106 | stdout: "pipe", | |
| 107 | stderr: "pipe", | |
| 108 | }); | |
| 109 | const [out] = await Promise.all([new Response(child.stdout).text(), new Response(child.stderr).text()]); | |
| 110 | expect(await child.exited).toBe(0); | |
| 111 | return out; | |
| 112 | }; | |
| 113 | expect(await run({ FORGE_CHECK_TIMES: TIMES_FILE })).toContain("2 files in 2 shards, expected 30s 7s"); | |
| 114 | // Without them, ~60 bytes each weigh next to nothing. | |
| 115 | expect(await run({})).toContain("2 files in 2 shards, expected 0s 0s"); | |
| 116 | } finally { | |
| 117 | rmSync(dir, { recursive: true, force: true }); | |
| 118 | } | |
| 119 | }, 30_000); | |
| 120 | }); | |
| 121 | ||
| 122 | describe("the whole verification writes the handed table back", () => { | |
| 123 | const whole = { measure: true, gate: true, tier: "full", unsliced: true, only: false, handed: true }; | |
| 124 | ||
| 125 | test("only the measured, unsliced whole gate of the full tier, handed a table, writes it", () => { | |
| 126 | expect(writesHandedTimes(whole)).toBe(true); | |
| 127 | for (const off of [ | |
| 128 | { measure: false }, | |
| 129 | { gate: false }, | |
| 130 | { tier: "fast" }, | |
| 131 | { unsliced: false }, | |
| 132 | { only: true }, | |
| 133 | { handed: false }, | |
| 134 | ]) | |
| 135 | expect(writesHandedTimes({ ...whole, ...off })).toBe(false); | |
| 136 | }); | |
| 137 | ||
| 138 | test("green files alone refresh the table at their seconds; a red file keeps its old time; none green writes nothing", () => { | |
| 139 | const old = new Map([["test/a.test.ts", 5], ["test/b.test.ts", 8], ["test/gone.test.ts", 3]]); | |
| 140 | const exists = (file: string): boolean => file !== "test/gone.test.ts"; | |
| 141 | const back = handBackTimes( | |
| 142 | [ | |
| 143 | { files: ["test/a.test.ts"], code: 0, seconds: 1.2 }, | |
| 144 | { files: ["test/b.test.ts"], code: 1, seconds: 90 }, | |
| 145 | { files: ["test/c.test.ts"], code: 0, seconds: 4 }, | |
| 146 | ], | |
| 147 | old, | |
| 148 | exists, | |
| 149 | ); | |
| 150 | expect(back?.files).toBe(2); | |
| 151 | const parsed = parseTimes(back?.text ?? ""); | |
| 152 | // a: 1.2 measured; b: red, its 8 kept, never its 90; c: new at 4; gone: no longer on disk. | |
| 153 | expect(parsed.ok && [...parsed.value]).toEqual([["test/a.test.ts", 1.2], ["test/b.test.ts", 8], ["test/c.test.ts", 4]]); | |
| 154 | expect(handBackTimes([{ files: ["test/a.test.ts"], code: 1, seconds: 2 }], old, exists)).toBeUndefined(); | |
| 155 | }); | |
| 156 | ||
| 157 | test("the script writes through these: its whole mode names both", () => { | |
| 158 | // The gate itself cannot run on a temporary tree (its exclusions audit | |
| 159 | // the repository), so this holds the script to the two tested functions. | |
| 160 | const script = readFileSync(join(import.meta.dir, "..", "scripts", "test-shards.ts"), "utf8"); | |
| 161 | expect(script).toMatch(/writesHandedTimes\(\{ measure, gate, tier, unsliced: "none" in shardSpec, only: only !== undefined/); | |
| 162 | expect(script).toMatch(/handBackTimes\(results, handedTimes\(\)/); | |
| 163 | expect(script).toMatch(/writeFileSync\(HANDED_TIMES, back\.text\)/); | |
| 164 | }); | |
| 165 | }); | |
| 166 | ||
| 167 | describe("the forge's side of the handover", () => { | |
| 168 | test("every check is told where the times are", () => { | |
| 169 | expect(handoverEnv().FORGE_CHECK_TIMES).toBe(TIMES_FILE); | |
| 170 | }); | |
| 171 | ||
| 172 | test("only the default branch's whole verification hands the table back, green or red", () => { | |
| 173 | expect(handsTimesBack({ tier: "verify", keepImpact: true })).toBe(true); | |
| 174 | // A pull's own verification, a bisection step, any other tier: never. | |
| 175 | expect(handsTimesBack({ tier: "verify" })).toBe(false); | |
| 176 | expect(handsTimesBack({ tier: "verify", keepImpact: false })).toBe(false); | |
| 177 | expect(handsTimesBack({ tier: "verify", keepImpact: true, only: ["test/a.test.ts"] })).toBe(false); | |
| 178 | expect(handsTimesBack({ tier: "full", keepImpact: true })).toBe(false); | |
| 179 | }); | |
| 180 | ||
| 181 | test("both of run_tier's keeps ask the same guard", () => { | |
| 182 | const source = readFileSync(join(import.meta.dir, "..", "src", "checks", "run_tier.ts"), "utf8"); | |
| 183 | const keeps = source.match(/keepTimes\([^;]*\);/g) ?? []; | |
| 184 | expect(keeps).toHaveLength(2); | |
| 185 | for (const keep of keeps) expect(keep).toContain("handsTimesBack(x)"); | |
| 186 | }); | |
| 187 | }); |
test/checks-home-sweep.test.ts +5 −1
| @@ -19,6 +19,7 @@ import { formatImpact, packImpact } from "../src/checks/impact.ts"; | ||
| 19 | 19 | import { IMPACT_FILE, keepImpact, seedImpact } from "../src/checks/impact-store.ts"; |
| 20 | 20 | import { formatKept } from "../src/checks/kept.ts"; |
| 21 | 21 | import { KEPT_FILE, keepKept, seedKept } from "../src/checks/kept-store.ts"; |
| 22 | import { TIMES_FILE, keepTimes, seedTimes } from "../src/checks/times-store.ts"; | |
| 22 | 23 | import { addLearned, readLearned } from "../src/checks/learned.ts"; |
| 23 | 24 | import { keepGateLog, readGateLog } from "../src/checks/gate_log.ts"; |
| 24 | 25 | import { keepGate, readGate } from "../src/checks/gate_last.ts"; |
| @@ -83,7 +84,7 @@ test("a finished run's home is removed, a live run's survives", async () => { | ||
| 83 | 84 | expect(existsSync(home("no-such-run"))).toBe(false); |
| 84 | 85 | }); |
| 85 | 86 | |
| 86 | test("the sweep leaves what the checks keep between runs: the map, the kept results, what a pull learned, the gates' logs (#12178)", async () => { | |
| 87 | test("the sweep leaves what the checks keep between runs: the map, the kept results, the test times, what a pull learned, the gates' logs (#12178)", async () => { | |
| 87 | 88 | // Every store sits under <data>/checks/ beside the checkouts. Swept as a stranger's checkout, |
| 88 | 89 | // a map kept by main's verification was gone within 30 seconds and no check was ever seeded. |
| 89 | 90 | const checkout = join(dataDir, "a-checkout"); |
| @@ -94,6 +95,8 @@ test("the sweep leaves what the checks keep between runs: the map, the kept resu | ||
| 94 | 95 | writeFileSync(join(checkout, KEPT_FILE), formatKept(new Map([[key, { test: "test/a.test.ts", commit: "b".repeat(40) }]]))); |
| 95 | 96 | expect(keepImpact(dataDir, repoId, checkout)).toBe("impact map kept: 1 tests"); |
| 96 | 97 | expect(keepKept(dataDir, repoId, checkout, true)).toBe("kept results kept: 1 tests"); |
| 98 | writeFileSync(join(checkout, TIMES_FILE), JSON.stringify({ "test/a.test.ts": 2 })); | |
| 99 | expect(keepTimes(dataDir, repoId, checkout, true)).toBe("test times kept: 1 files"); | |
| 97 | 100 | addLearned(dataDir, "pull1", ["test/a.test.ts"]); |
| 98 | 101 | const gate = keepGateLog(dataDir, "alice/garden", "c".repeat(40), true, ["test/a.test.ts:", "(pass) a"], Date.now()); |
| 99 | 102 | keepGate(dataDir, ["test/a.test.ts:", "(pass) a"]); |
| @@ -110,6 +113,7 @@ test("the sweep leaves what the checks keep between runs: the map, the kept resu | ||
| 110 | 113 | expect(seedImpact(dataDir, repoId, next)).toBe(1); |
| 111 | 114 | expect(readFileSync(join(next, IMPACT_FILE), "utf8")).toBe(map); |
| 112 | 115 | expect(seedKept(dataDir, repoId, next)).toBe(1); |
| 116 | expect(seedTimes(dataDir, repoId, next)).toBe(1); | |
| 113 | 117 | expect(readLearned(dataDir, "pull1")).toEqual(["test/a.test.ts"]); |
| 114 | 118 | expect(readGateLog(dataDir, "alice", "garden", gate!)).toBeDefined(); |
| 115 | 119 | expect(readGate(dataDir)).toEqual(lastGate); |
new above
offline
The write-back has no test that fails without it. Add one that runs the recorded whole mode and asserts the handed file is written for green files and not written when none ran green.