8fc2fa1
@@ -368,6 +368,7 @@
attempted; Backblaze audits find nothing yet. No dependency was added.
bun test              # protocol-level + adversarial suite
bun x tsc --noEmit    # zero type errors is the standard
bun run check         # exactly what the forge runs against every push
bun run check:record  # the same, and rewrites scripts/test-times.json
```
`bun run check` is the gate: the same command `forge.json` declares, so what
@@ -377,8 +378,26 @@
test files named in `check.run` as parallel `bun test` shards
and `FORGE_CHECK_SHARDS=2` turns that down on a small or busy host. Each
shard's output is written whole as it finishes; the log ends with every
failing test from every shard, or on green with one line — how many tests,
across how many files, in how long. Every file is independent (its own forge,
its own database, port 0), so a test that fails only beside another is a bug
across how many files, in how long, then the slowest and fastest shard.
Files go to shards by how long they take, not how big they are: longest
first onto the least-loaded shard, weighed from `scripts/test-times.json`
(file → seconds, from one recorded run). A file the table does not name
weighs its bytes at the table's own seconds-per-byte. If the table is
malformed, the log says it was refused and the split falls back to bytes.
A stale table only makes the split less even; it never drops a file.
Refresh it with one command after adding a slow test file, or when the
log's slowest shard is well over 1.5× its fastest:
```sh
bun run check:record  # the gate's own command, with test-shards.ts --record
```
`--record` runs each file as its own `bun test`, with as many running at
once as the gate runs shards. It rewrites the table only when every file
is green. Files it did not run keep their old times.
Every file is independent (its own forge, its own database, port 0), so a test that fails only beside another is a bug
in that test. Two things it will not do:
- **It never borrows your git configuration.** A test drives a real `git`
@@ -8,7 +8,8 @@
    "start": "bun run src/main.ts",
    "test": "bun test",
    "typecheck": "bun x tsc --noEmit",
    "check": "bun run scripts/check.ts"
    "check": "bun run scripts/check.ts",
    "check:record": "bun run scripts/check.ts --record"
  },
  "devDependencies": {
    "@types/bun": "^1.2.0",
@@ -6,6 +6,11 @@
 * way that is true is if there is exactly one copy of the command: this
 * reads it from the manifest and hands it to `/bin/sh -c`, which is what
 * `apps/process.ts` does with an app's check on the deploy side.
 *
 * `--record` runs the same string with `scripts/test-shards.ts --record`
 * in place of `scripts/test-shards.ts`, which rewrites
 * `scripts/test-times.json` from this run, so the file list still lives
 * only in `forge.json`.
 */
const manifest = (await Bun.file(
@@ -18,7 +23,15 @@
if (typeof command !== "string" || command.trim() === "") {
  process.exit(2);
}
const proc = Bun.spawn(["/bin/sh", "-c", command], {
const SHARDS = "scripts/test-shards.ts ";
const record = process.argv.slice(2).includes("--record");
if (record && command.split(SHARDS).length !== 2) {
  console.error("--record: check.run must run scripts/test-shards.ts once");
  process.exit(2);
}
const run = record ? command.replace(SHARDS, `${SHARDS}--record `) : command;
const proc = Bun.spawn(["/bin/sh", "-c", run], {
  cwd: new URL("..", import.meta.url).pathname,
  stdio: ["inherit", "inherit", "inherit"],
});
@@ -36,7 +36,9 @@
export function shardCount(
/**
 * Splits `files` into at most `n` disjoint, non-empty slices that together
 * hold every file once. Heaviest file first onto the lightest slice, so a
 * hold every file once. `weight` is how long a file is expected to take
 * ({@link weigher}). Heaviest file first onto the lightest slice (LPT:
 * the longest shard is at most 4/3 of the best possible), so a
 * big file does not share a shard with other big ones; ties break on the
 * file's place in the list, so the same input always gives the same split.
 * Each slice keeps the list's own order.
@@ -74,6 +76,108 @@
export function splitFiles(
    );
}
/** How a test file is named: the same rule `scripts/test-shards.ts` uses. */
export const TEST_FILE = /\.test\.[cm]?[jt]sx?$/;
/** A sane bound on one entry of the times table: a day, in seconds. */
const TIME_MAX = 86_400;
/**
 * Seconds per byte when the table says nothing about any file in the list:
 * about what this suite showed when the table was first recorded (a 40 kB
 * file of forge-booting tests runs for tens of seconds). Only the ratio
 * between fallback files matters then, so the constant's exact value does
 * not change the split; it only has to be in the right range beside
 * measured seconds.
 */
export const SECONDS_PER_BYTE_DEFAULT = 1 / 2_000;
/**
 * Reads `scripts/test-times.json`: one object, test file path to the
 * seconds it took alone, e.g. `{ "test/crew.test.ts": 50.1 }`. Anything
 * else — not JSON, not an object, a key that is not a test file, a value
 * that is not a finite number from 0 to a day — refuses the whole table,
 * so a half-trusted table never steers the split.
 */
export function parseTimes(
  text: string,
): { ok: true; value: Map<string, number> } | { ok: false; error: string } {
  let data: unknown;
  try {
    data = JSON.parse(text);
  } catch {
    return { ok: false, error: "not JSON" };
  }
  if (typeof data !== "object" || data === null || Array.isArray(data))
    return { ok: false, error: "not an object of file to seconds" };
  const times = new Map<string, number>();
  for (const [file, seconds] of Object.entries(data)) {
    if (!TEST_FILE.test(file) || file.length > 512)
      return { ok: false, error: `${JSON.stringify(file)} is not a test file` };
    if (
      typeof seconds !== "number" ||
      !Number.isFinite(seconds) ||
      seconds < 0 ||
      seconds > TIME_MAX
    )
      return {
        ok: false,
        error: `${file}: ${JSON.stringify(seconds)} is not seconds from 0 to ${TIME_MAX}`,
      };
    times.set(normalise(file), seconds);
  }
  return { ok: true, value: times };
}
/** `./test/a.test.ts` and `test/a.test.ts` are the same file. */
function normalise(file: string): string {
  return file.startsWith("./") ? file.slice(2) : file;
}
/**
 * How long each file is expected to take, in seconds: its measured time
 * from the table when there is one, else its size in bytes scaled to
 * seconds. The scale is fitted on the files that have both (total
 * measured seconds ÷ their total bytes), so a new file is weighed like
 * its neighbours; with none, {@link SECONDS_PER_BYTE_DEFAULT}. A pure
 * function of the list, the sizes and the table: the same three always
 * give the same weights, so the same split.
 */
export function weigher(
  files: readonly string[],
  bytes: ReadonlyMap<string, number>,
  times: ReadonlyMap<string, number>,
): (file: string) => number {
  let seconds = 0;
  let size = 0;
  for (const file of files) {
    const measured = times.get(normalise(file));
    const b = bytes.get(file);
    if (measured === undefined || b === undefined) continue;
    seconds += measured;
    size += b;
  }
  const perByte =
    seconds > 0 && size > 0 ? seconds / size : SECONDS_PER_BYTE_DEFAULT;
  return (file) =>
    times.get(normalise(file)) ?? (bytes.get(file) ?? 0) * perByte;
}
/**
 * The table as it is checked in: keys sorted, seconds to one decimal, one
 * file per line, so a refresh reads as a small diff.
 */
export function formatTimes(times: ReadonlyMap<string, number>): string {
  const entries = [...times]
    .map(([file, seconds]) => [normalise(file), seconds] as const)
    .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
  const lines = entries.map(
    ([file, seconds]) =>
      `  ${JSON.stringify(file)}: ${Math.round(seconds * 10) / 10}`,
  );
  return lines.length === 0 ? "{}\n" : `{\n${lines.join(",\n")}\n}\n`;
}
/** One shard, finished: what it ran, how it exited, everything it printed. */
export interface ShardResult {
  readonly files: readonly string[];
@@ -2,6 +2,7 @@
 * `bun test`, over the named files, in parallel shards.
 *
 *   bun scripts/test-shards.ts [bun test flags…] test/a.test.ts test/b.test.ts …
 *   bun scripts/test-shards.ts --record [bun test flags…] test/a.test.ts …
 *
 * The files come from the command line, so `forge.json`'s `check.run`
 * stays the one place that names them. Each shard is its own `bun test`
@@ -9,15 +10,32 @@
 * shard's output is written through whole when it finishes, so the log
 * reads one shard at a time, and the tail names every failing test from
 * every shard. Exits non-zero if any shard does.
 *
 * Files are weighed by how long they took, from `scripts/test-times.json`
 * (a file it does not name, or a table it refuses, falls back to bytes).
 * `--record` refreshes that table: the same number of workers, but each
 * file is its own `bun test`, timed alone from spawn to exit, and the
 * table is rewritten from those times when every file is green.
 */
import { availableParallelism } from "node:os";
import { statSync } from "node:fs";
import { shardCount, splitFiles, tally, type ShardResult } from "./shards.ts";
import { readFileSync, statSync, writeFileSync } from "node:fs";
import {
  TEST_FILE,
  formatTimes,
  parseTimes,
  shardCount,
  splitFiles,
  tally,
  weigher,
  type ShardResult,
} from "./shards.ts";
const TEST_FILE = /\.test\.[cm]?[jt]sx?$/;
const TIMES = new URL("./test-times.json", import.meta.url).pathname;
const args = process.argv.slice(2);
const argv = process.argv.slice(2);
const record = argv.includes("--record");
const args = argv.filter((arg) => arg !== "--record");
const files = args.filter((arg) => TEST_FILE.test(arg));
const flags = args.filter((arg) => !TEST_FILE.test(arg));
@@ -50,11 +68,35 @@
if (!count.ok) {
  console.error(`test-shards: ${count.error}`);
  process.exit(2);
}
// Bytes stand in for how long a file takes: no timings are kept between runs.
const slices = splitFiles(files, count.value, (file) => size.get(file) ?? 0);
// Measured seconds where the table has them, bytes scaled to seconds
// where it does not. A missing or refused table is said out loud and the
// gate still runs, on bytes alone.
let times = new Map<string, number>();
let table = "";
try {
  table = readFileSync(TIMES, "utf8");
} catch {
  console.log("test-shards: no scripts/test-times.json; weighing by bytes");
}
if (table !== "") {
  const parsed = parseTimes(table);
  if (parsed.ok) times = parsed.value;
  else
    console.log(
      `test-shards: scripts/test-times.json refused (${parsed.error}); weighing by bytes`,
    );
}
const weight = weigher(files, size, times);
// Recording runs one file per process, so every file is its own slice.
const slices = record
  ? files.map((file) => [file])
  : splitFiles(files, count.value, weight);
console.log(
  `test-shards: ${files.length} files in ${slices.length} shard${slices.length === 1 ? "" : "s"}`,
  record
    ? `test-shards: recording ${files.length} files, ${count.value} at a time`
    : `test-shards: ${files.length} files in ${slices.length} shard${slices.length === 1 ? "" : "s"}, ` +
        `expected ${slices.map((slice) => slice.reduce((sum, file) => sum + weight(file), 0).toFixed(0) + "s").join(" ")}`,
);
const children = new Set<Bun.Subprocess>();
@@ -74,7 +116,9 @@
async function drain(
  into.push(decoder.decode());
}
async function runShard(slice: string[], index: number): Promise<ShardResult> {
type Timed = ShardResult & { seconds: number };
async function runShard(slice: string[], index: number): Promise<Timed> {
  const started = performance.now();
  // An exact path, never a filter: `test/web.test.ts` alone would also
  // match any file whose path contains it, in some other shard.
@@ -98,11 +142,51 @@
async function runShard(slice: string[], index: number): Promise<ShardResult> {
  process.stdout.write(
    `\n── shard ${index + 1}/${slices.length} (exit ${code}, ${seconds}s): ${slice.join(" ")}\n${text}${text.endsWith("\n") ? "" : "\n"}`,
  );
  return { files: slice, code, output: text };
  return {
    files: slice,
    code,
    output: text,
    seconds: (performance.now() - started) / 1000,
  };
}
/** Runs every slice, at most `width` at once, in the slices' order. */
async function pool(width: number): Promise<Timed[]> {
  const results: Timed[] = [];
  let next = 0;
  async function worker(): Promise<void> {
    while (next < slices.length) {
      const index = next++;
      results[index] = await runShard(slices[index]!, index);
    }
  }
  await Promise.all(Array.from({ length: width }, worker));
  return results;
}
const started = performance.now();
const results = await Promise.all(slices.map(runShard));
const results = await pool(record ? count.value : slices.length);
const ending = tally(results, (performance.now() - started) / 1000);
const spread = results.map((result) => result.seconds);
console.log(`\n${ending.lines.join("\n")}`);
if (!record && results.length > 1)
  console.log(
    `Slowest shard ${Math.max(...spread).toFixed(1)}s, fastest ${Math.min(...spread).toFixed(1)}s.`,
  );
if (record) {
  if (ending.code !== 0) {
    console.log(
      "test-shards: not recording: a file is red, so its time is not its time",
    );
  } else {
    // Files this run did not name keep their old times: a partial run
    // refreshes what it ran and forgets nothing else.
    const fresh = new Map(times);
    for (const result of results) fresh.set(result.files[0]!, result.seconds);
    writeFileSync(TIMES, formatTimes(fresh));
    console.log(
      `test-shards: wrote ${results.length} times to scripts/test-times.json`,
    );
  }
}
process.exit(ending.code);
@@ -0,0 +1,43 @@
{
  "test/apps-adversarial.test.ts": 145.9,
  "test/apps-busy.test.ts": 26.4,
  "test/apps-token.test.ts": 19.8,
  "test/apps.test.ts": 148.5,
  "test/asks.test.ts": 7.3,
  "test/autonomy.test.ts": 18,
  "test/board-windows.test.ts": 6,
  "test/checks.test.ts": 26.3,
  "test/cloud.test.ts": 7.1,
  "test/crew-limits.test.ts": 0.5,
  "test/crew-sandbox.test.ts": 5.3,
  "test/crew-wrapup.test.ts": 23.6,
  "test/crew.test.ts": 70.1,
  "test/cross-repo-pull.test.ts": 13,
  "test/decide.test.ts": 3.1,
  "test/dependencies.test.ts": 10.8,
  "test/feed-page.test.ts": 7.1,
  "test/feed.test.ts": 8.3,
  "test/forge-cloudflare.test.ts": 0.1,
  "test/fork-flow.test.ts": 48.4,
  "test/forks.test.ts": 28.9,
  "test/git-trust.test.ts": 0.3,
  "test/helpers.test.ts": 0.7,
  "test/landing.test.ts": 9.4,
  "test/ledger.test.ts": 12.8,
  "test/migrate.test.ts": 41.9,
  "test/offline.test.ts": 3.5,
  "test/pulls.test.ts": 14.6,
  "test/run-secrets.test.ts": 2.8,
  "test/shards.test.ts": 0.3,
  "test/share.test.ts": 16.2,
  "test/spawn-plan.test.ts": 0.2,
  "test/spawn.test.ts": 11.1,
  "test/standing.test.ts": 2.8,
  "test/steer.test.ts": 16.2,
  "test/studio-words.test.ts": 17,
  "test/studio.test.ts": 196.9,
  "test/usage.test.ts": 17.5,
  "test/vitals.test.ts": 7.1,
  "test/web-apps.test.ts": 18.7,
  "test/web.test.ts": 16.1
}
@@ -4,11 +4,16 @@
 * red run whose tail names every failing test from every shard.
 */
import { describe, expect, test } from "bun:test";
import { readFileSync } from "node:fs";
import {
  SECONDS_PER_BYTE_DEFAULT,
  SHARDS_MAX,
  formatTimes,
  parseTimes,
  shardCount,
  splitFiles,
  tally,
  weigher,
  type ShardResult,
} from "../scripts/shards.ts";
@@ -46,6 +51,155 @@
describe("splitFiles", () => {
  });
});
describe("weighing by measured time", () => {
  // Four files of one size: bytes cannot tell them apart, the table can.
  const list = [
    "test/slow.test.ts",
    "test/a.test.ts",
    "test/b.test.ts",
    "test/c.test.ts",
  ];
  const bytes = new Map(list.map((file) => [file, 10_000]));
  test("a known slow file is not stacked with others when a fast one of its size exists", () => {
    const times = new Map([
      ["test/slow.test.ts", 50],
      ["test/a.test.ts", 20],
      ["test/b.test.ts", 15],
      ["test/c.test.ts", 14],
    ]);
    const slices = splitFiles(list, 2, weigher(list, bytes, times));
    // Hand count: 50 alone; 20 + 15 + 14 = 49 beside it.
    expect(slices).toEqual([
      ["test/slow.test.ts"],
      ["test/a.test.ts", "test/b.test.ts", "test/c.test.ts"],
    ]);
    // By bytes alone the same four split two and two: slow carries a neighbour.
    const byBytes = splitFiles(list, 2, (file) => bytes.get(file)!);
    expect(byBytes.find((s) => s.includes("test/slow.test.ts"))!.length).toBe(
      2,
    );
  });
  test("a file missing from the table weighs its bytes at the table's own rate", () => {
    const sizes = new Map([
      ["test/slow.test.ts", 10_000],
      ["test/a.test.ts", 30_000],
      ["test/new.test.ts", 5_000],
    ]);
    const times = new Map([
      ["test/slow.test.ts", 50],
      ["test/a.test.ts", 10],
    ]);
    const files = [...sizes.keys()];
    const weight = weigher(files, sizes, times);
    // 60 s over 40 000 bytes = 0.0015 s/byte; 5 000 bytes → 7.5 s.
    expect(weight("test/new.test.ts")).toBeCloseTo(7.5, 9);
    expect(weight("test/slow.test.ts")).toBe(50);
    expect(weight("./test/slow.test.ts")).toBe(50);
    const slices = splitFiles(files, 2, weight);
    expect(slices.flat().sort()).toEqual([...files].sort());
    expect(slices).toEqual([
      ["test/slow.test.ts"],
      ["test/a.test.ts", "test/new.test.ts"],
    ]);
  });
  test("with no table at all, bytes at the default rate", () => {
    const weight = weigher(list, bytes, new Map());
    expect(weight("test/a.test.ts")).toBeCloseTo(
      10_000 * SECONDS_PER_BYTE_DEFAULT,
      9,
    );
    expect(weight("test/unknown.test.ts")).toBe(0);
  });
  test("the same list and the same table give the same split", () => {
    const times = new Map([["test/slow.test.ts", 50]]);
    const first = splitFiles(list, 3, weigher(list, bytes, times));
    for (let i = 0; i < 5; i++)
      expect(splitFiles(list, 3, weigher(list, bytes, new Map(times)))).toEqual(
        first,
      );
  });
});
describe("parseTimes", () => {
  test("reads file to seconds", () => {
    const parsed = parseTimes(
      '{ "test/crew.test.ts": 50.1, "./test/a.test.ts": 0 }',
    );
    expect(parsed).toEqual({
      ok: true,
      value: new Map([
        ["test/crew.test.ts", 50.1],
        ["test/a.test.ts", 0],
      ]),
    });
  });
  test("a malformed table is refused whole", () => {
    for (const bad of [
      "",
      "{",
      "[]",
      "null",
      "50",
      '{ "test/a.test.ts": "50" }',
      '{ "test/a.test.ts": -1 }',
      '{ "test/a.test.ts": 1e9 }',
      '{ "test/a.test.ts": null }',
      '{ "src/app.ts": 3 }',
      '{ "test/a.test.ts": 3, "README.md": 1 }',
    ])
      expect(parseTimes(bad).ok).toBe(false);
  });
  test("a refused table still splits, on bytes", () => {
    const parsed = parseTimes("not json");
    const times = parsed.ok ? parsed.value : new Map<string, number>();
    const files = ["test/a.test.ts", "test/b.test.ts", "test/c.test.ts"];
    const sizes = new Map([
      ["test/a.test.ts", 300],
      ["test/b.test.ts", 200],
      ["test/c.test.ts", 100],
    ]);
    expect(splitFiles(files, 2, weigher(files, sizes, times))).toEqual([
      ["test/a.test.ts"],
      ["test/b.test.ts", "test/c.test.ts"],
    ]);
  });
  test("formatTimes writes what parseTimes reads, sorted, to a tenth", () => {
    const text = formatTimes(
      new Map([
        ["./test/b.test.ts", 1.26],
        ["test/a.test.ts", 50.04],
      ]),
    );
    expect(text).toBe(
      '{\n  "test/a.test.ts": 50,\n  "test/b.test.ts": 1.3\n}\n',
    );
    expect(parseTimes(text).ok).toBe(true);
    expect(formatTimes(new Map())).toBe("{}\n");
  });
  test("the checked-in table is accepted and names only files that exist", () => {
    const parsed = parseTimes(
      readFileSync(
        new URL("../scripts/test-times.json", import.meta.url),
        "utf8",
      ),
    );
    expect(parsed.ok).toBe(true);
    if (!parsed.ok) return;
    for (const file of parsed.value.keys())
      expect(
        Bun.file(new URL(`../${file}`, import.meta.url)).size,
      ).toBeGreaterThan(0);
  });
});
describe("shardCount", () => {
  test("one per CPU, capped, at least one, never more than files", () => {
    expect(shardCount(80, 22)).toEqual({ ok: true, value: SHARDS_MAX });
		@@ -368,6 +368,7 @@ attempted; Backblaze audits find nothing yet. No dependency was added.
368	368	bun test # protocol-level + adversarial suite
369	369	bun x tsc --noEmit # zero type errors is the standard
370	370	bun run check # exactly what the forge runs against every push
	371	bun run check:record # the same, and rewrites scripts/test-times.json
371	372	```
372	373
373	374	`bun run check` is the gate: the same command `forge.json` declares, so what
		@@ -377,8 +378,26 @@ test files named in `check.run` as parallel `bun test` shards
377	378	and `FORGE_CHECK_SHARDS=2` turns that down on a small or busy host. Each
378	379	shard's output is written whole as it finishes; the log ends with every
379	380	failing test from every shard, or on green with one line — how many tests,
380		across how many files, in how long. Every file is independent (its own forge,
381		its own database, port 0), so a test that fails only beside another is a bug
	381	across how many files, in how long, then the slowest and fastest shard.
	382
	383	Files go to shards by how long they take, not how big they are: longest
	384	first onto the least-loaded shard, weighed from `scripts/test-times.json`
	385	(file → seconds, from one recorded run). A file the table does not name
	386	weighs its bytes at the table's own seconds-per-byte. If the table is
	387	malformed, the log says it was refused and the split falls back to bytes.
	388	A stale table only makes the split less even; it never drops a file.
	389	Refresh it with one command after adding a slow test file, or when the
	390	log's slowest shard is well over 1.5× its fastest:
	391
	392	```sh
	393	bun run check:record # the gate's own command, with test-shards.ts --record
	394	```
	395
	396	`--record` runs each file as its own `bun test`, with as many running at
	397	once as the gate runs shards. It rewrites the table only when every file
	398	is green. Files it did not run keep their old times.
	399
	400	Every file is independent (its own forge, its own database, port 0), so a test that fails only beside another is a bug
382	401	in that test. Two things it will not do:
383	402
384	403	- It never borrows your git configuration. A test drives a real `git`
		@@ -8,7 +8,8 @@
8	8	"start": "bun run src/main.ts",
9	9	"test": "bun test",
10	10	"typecheck": "bun x tsc --noEmit",
11		"check": "bun run scripts/check.ts"
	11	"check": "bun run scripts/check.ts",
	12	"check:record": "bun run scripts/check.ts --record"
12	13	},
13	14	"devDependencies": {
14	15	"@types/bun": "^1.2.0",
		@@ -6,6 +6,11 @@
6	6	* way that is true is if there is exactly one copy of the command: this
7	7	* reads it from the manifest and hands it to `/bin/sh -c`, which is what
8	8	* `apps/process.ts` does with an app's check on the deploy side.
	9	*
	10	* `--record` runs the same string with `scripts/test-shards.ts --record`
	11	* in place of `scripts/test-shards.ts`, which rewrites
	12	* `scripts/test-times.json` from this run, so the file list still lives
	13	* only in `forge.json`.
9	14	*/
10	15
11	16	const manifest = (await Bun.file(
		@@ -18,7 +23,15 @@ if (typeof command !== "string" \|\| command.trim() === "") {
18	23	process.exit(2);
19	24	}
20	25
21		const proc = Bun.spawn(["/bin/sh", "-c", command], {
	26	const SHARDS = "scripts/test-shards.ts ";
	27	const record = process.argv.slice(2).includes("--record");
	28	if (record && command.split(SHARDS).length !== 2) {
	29	console.error("--record: check.run must run scripts/test-shards.ts once");
	30	process.exit(2);
	31	}
	32	const run = record ? command.replace(SHARDS, `${SHARDS}--record `) : command;
	33
	34	const proc = Bun.spawn(["/bin/sh", "-c", run], {
22	35	cwd: new URL("..", import.meta.url).pathname,
23	36	stdio: ["inherit", "inherit", "inherit"],
24	37	});
		@@ -36,7 +36,9 @@ export function shardCount(
36	36
37	37	/**
38	38	* Splits `files` into at most `n` disjoint, non-empty slices that together
39		* hold every file once. Heaviest file first onto the lightest slice, so a
	39	* hold every file once. `weight` is how long a file is expected to take
	40	* ({@link weigher}). Heaviest file first onto the lightest slice (LPT:
	41	* the longest shard is at most 4/3 of the best possible), so a
40	42	* big file does not share a shard with other big ones; ties break on the
41	43	* file's place in the list, so the same input always gives the same split.
42	44	* Each slice keeps the list's own order.
		@@ -74,6 +76,108 @@ export function splitFiles(
74	76	);
75	77	}
76	78
	79	/** How a test file is named: the same rule `scripts/test-shards.ts` uses. */
	80	export const TEST_FILE = /\.test\.[cm]?[jt]sx?$/;
	81
	82	/** A sane bound on one entry of the times table: a day, in seconds. */
	83	const TIME_MAX = 86_400;
	84
	85	/**
	86	* Seconds per byte when the table says nothing about any file in the list:
	87	* about what this suite showed when the table was first recorded (a 40 kB
	88	* file of forge-booting tests runs for tens of seconds). Only the ratio
	89	* between fallback files matters then, so the constant's exact value does
	90	* not change the split; it only has to be in the right range beside
	91	* measured seconds.
	92	*/
	93	export const SECONDS_PER_BYTE_DEFAULT = 1 / 2_000;
	94
	95	/**
	96	* Reads `scripts/test-times.json`: one object, test file path to the
	97	* seconds it took alone, e.g. `{ "test/crew.test.ts": 50.1 }`. Anything
	98	* else — not JSON, not an object, a key that is not a test file, a value
	99	* that is not a finite number from 0 to a day — refuses the whole table,
	100	* so a half-trusted table never steers the split.
	101	*/
	102	export function parseTimes(
	103	text: string,
	104	): { ok: true; value: Map<string, number> } \| { ok: false; error: string } {
	105	let data: unknown;
	106	try {
	107	data = JSON.parse(text);
	108	} catch {
	109	return { ok: false, error: "not JSON" };
	110	}
	111	if (typeof data !== "object" \|\| data === null \|\| Array.isArray(data))
	112	return { ok: false, error: "not an object of file to seconds" };
	113	const times = new Map<string, number>();
	114	for (const [file, seconds] of Object.entries(data)) {
	115	if (!TEST_FILE.test(file) \|\| file.length > 512)
	116	return { ok: false, error: `${JSON.stringify(file)} is not a test file` };
	117	if (
	118	typeof seconds !== "number" \|\|
	119	!Number.isFinite(seconds) \|\|
	120	seconds < 0 \|\|
	121	seconds > TIME_MAX
	122	)
	123	return {
	124	ok: false,
	125	error: `${file}: ${JSON.stringify(seconds)} is not seconds from 0 to ${TIME_MAX}`,
	126	};
	127	times.set(normalise(file), seconds);
	128	}
	129	return { ok: true, value: times };
	130	}
	131
	132	/** `./test/a.test.ts` and `test/a.test.ts` are the same file. */
	133	function normalise(file: string): string {
	134	return file.startsWith("./") ? file.slice(2) : file;
	135	}
	136
	137	/**
	138	* How long each file is expected to take, in seconds: its measured time
	139	* from the table when there is one, else its size in bytes scaled to
	140	* seconds. The scale is fitted on the files that have both (total
	141	* measured seconds ÷ their total bytes), so a new file is weighed like
	142	* its neighbours; with none, {@link SECONDS_PER_BYTE_DEFAULT}. A pure
	143	* function of the list, the sizes and the table: the same three always
	144	* give the same weights, so the same split.
	145	*/
	146	export function weigher(
	147	files: readonly string[],
	148	bytes: ReadonlyMap<string, number>,
	149	times: ReadonlyMap<string, number>,
	150	): (file: string) => number {
	151	let seconds = 0;
	152	let size = 0;
	153	for (const file of files) {
	154	const measured = times.get(normalise(file));
	155	const b = bytes.get(file);
	156	if (measured === undefined \|\| b === undefined) continue;
	157	seconds += measured;
	158	size += b;
	159	}
	160	const perByte =
	161	seconds > 0 && size > 0 ? seconds / size : SECONDS_PER_BYTE_DEFAULT;
	162	return (file) =>
	163	times.get(normalise(file)) ?? (bytes.get(file) ?? 0) * perByte;
	164	}
	165
	166	/**
	167	* The table as it is checked in: keys sorted, seconds to one decimal, one
	168	* file per line, so a refresh reads as a small diff.
	169	*/
	170	export function formatTimes(times: ReadonlyMap<string, number>): string {
	171	const entries = [...times]
	172	.map(([file, seconds]) => [normalise(file), seconds] as const)
	173	.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
	174	const lines = entries.map(
	175	([file, seconds]) =>
	176	` ${JSON.stringify(file)}: ${Math.round(seconds * 10) / 10}`,
	177	);
	178	return lines.length === 0 ? "{}\n" : `{\n${lines.join(",\n")}\n}\n`;
	179	}
	180
77	181	/** One shard, finished: what it ran, how it exited, everything it printed. */
78	182	export interface ShardResult {
79	183	readonly files: readonly string[];
		@@ -2,6 +2,7 @@
2	2	* `bun test`, over the named files, in parallel shards.
3	3	*
4	4	* bun scripts/test-shards.ts [bun test flags…] test/a.test.ts test/b.test.ts …
	5	* bun scripts/test-shards.ts --record [bun test flags…] test/a.test.ts …
5	6	*
6	7	* The files come from the command line, so `forge.json`'s `check.run`
7	8	* stays the one place that names them. Each shard is its own `bun test`
		@@ -9,15 +10,32 @@
9	10	* shard's output is written through whole when it finishes, so the log
10	11	* reads one shard at a time, and the tail names every failing test from
11	12	* every shard. Exits non-zero if any shard does.
	13	*
	14	* Files are weighed by how long they took, from `scripts/test-times.json`
	15	* (a file it does not name, or a table it refuses, falls back to bytes).
	16	* `--record` refreshes that table: the same number of workers, but each
	17	* file is its own `bun test`, timed alone from spawn to exit, and the
	18	* table is rewritten from those times when every file is green.
12	19	*/
13	20
14	21	import { availableParallelism } from "node:os";
15		import { statSync } from "node:fs";
16		import { shardCount, splitFiles, tally, type ShardResult } from "./shards.ts";
	22	import { readFileSync, statSync, writeFileSync } from "node:fs";
	23	import {
	24	TEST_FILE,
	25	formatTimes,
	26	parseTimes,
	27	shardCount,
	28	splitFiles,
	29	tally,
	30	weigher,
	31	type ShardResult,
	32	} from "./shards.ts";
17	33
18		const TEST_FILE = /\.test\.[cm]?[jt]sx?$/;
	34	const TIMES = new URL("./test-times.json", import.meta.url).pathname;
19	35
20		const args = process.argv.slice(2);
	36	const argv = process.argv.slice(2);
	37	const record = argv.includes("--record");
	38	const args = argv.filter((arg) => arg !== "--record");
21	39	const files = args.filter((arg) => TEST_FILE.test(arg));
22	40	const flags = args.filter((arg) => !TEST_FILE.test(arg));
23	41
		@@ -50,11 +68,35 @@ if (!count.ok) {
50	68	console.error(`test-shards: ${count.error}`);
51	69	process.exit(2);
52	70	}
53		// Bytes stand in for how long a file takes: no timings are kept between runs.
54		const slices = splitFiles(files, count.value, (file) => size.get(file) ?? 0);
	71	// Measured seconds where the table has them, bytes scaled to seconds
	72	// where it does not. A missing or refused table is said out loud and the
	73	// gate still runs, on bytes alone.
	74	let times = new Map<string, number>();
	75	let table = "";
	76	try {
	77	table = readFileSync(TIMES, "utf8");
	78	} catch {
	79	console.log("test-shards: no scripts/test-times.json; weighing by bytes");
	80	}
	81	if (table !== "") {
	82	const parsed = parseTimes(table);
	83	if (parsed.ok) times = parsed.value;
	84	else
	85	console.log(
	86	`test-shards: scripts/test-times.json refused (${parsed.error}); weighing by bytes`,
	87	);
	88	}
	89	const weight = weigher(files, size, times);
	90	// Recording runs one file per process, so every file is its own slice.
	91	const slices = record
	92	? files.map((file) => [file])
	93	: splitFiles(files, count.value, weight);
55	94
56	95	console.log(
57		`test-shards: ${files.length} files in ${slices.length} shard${slices.length === 1 ? "" : "s"}`,
	96	record
	97	? `test-shards: recording ${files.length} files, ${count.value} at a time`
	98	: `test-shards: ${files.length} files in ${slices.length} shard${slices.length === 1 ? "" : "s"}, ` +
	99	`expected ${slices.map((slice) => slice.reduce((sum, file) => sum + weight(file), 0).toFixed(0) + "s").join(" ")}`,
58	100	);
59	101
60	102	const children = new Set<Bun.Subprocess>();
		@@ -74,7 +116,9 @@ async function drain(
74	116	into.push(decoder.decode());
75	117	}
76	118
77		async function runShard(slice: string[], index: number): Promise<ShardResult> {
	119	type Timed = ShardResult & { seconds: number };
	120
	121	async function runShard(slice: string[], index: number): Promise<Timed> {
78	122	const started = performance.now();
79	123	// An exact path, never a filter: `test/web.test.ts` alone would also
80	124	// match any file whose path contains it, in some other shard.
		@@ -98,11 +142,51 @@ async function runShard(slice: string[], index: number): Promise<ShardResult> {
98	142	process.stdout.write(
99	143	`\n── shard ${index + 1}/${slices.length} (exit ${code}, ${seconds}s): ${slice.join(" ")}\n${text}${text.endsWith("\n") ? "" : "\n"}`,
100	144	);
101		return { files: slice, code, output: text };
	145	return {
	146	files: slice,
	147	code,
	148	output: text,
	149	seconds: (performance.now() - started) / 1000,
	150	};
	151	}
	152
	153	/** Runs every slice, at most `width` at once, in the slices' order. */
	154	async function pool(width: number): Promise<Timed[]> {
	155	const results: Timed[] = [];
	156	let next = 0;
	157	async function worker(): Promise<void> {
	158	while (next < slices.length) {
	159	const index = next++;
	160	results[index] = await runShard(slices[index]!, index);
	161	}
	162	}
	163	await Promise.all(Array.from({ length: width }, worker));
	164	return results;
102	165	}
103	166
104	167	const started = performance.now();
105		const results = await Promise.all(slices.map(runShard));
	168	const results = await pool(record ? count.value : slices.length);
106	169	const ending = tally(results, (performance.now() - started) / 1000);
	170	const spread = results.map((result) => result.seconds);
107	171	console.log(`\n${ending.lines.join("\n")}`);
	172	if (!record && results.length > 1)
	173	console.log(
	174	`Slowest shard ${Math.max(...spread).toFixed(1)}s, fastest ${Math.min(...spread).toFixed(1)}s.`,
	175	);
	176	if (record) {
	177	if (ending.code !== 0) {
	178	console.log(
	179	"test-shards: not recording: a file is red, so its time is not its time",
	180	);
	181	} else {
	182	// Files this run did not name keep their old times: a partial run
	183	// refreshes what it ran and forgets nothing else.
	184	const fresh = new Map(times);
	185	for (const result of results) fresh.set(result.files[0]!, result.seconds);
	186	writeFileSync(TIMES, formatTimes(fresh));
	187	console.log(
	188	`test-shards: wrote ${results.length} times to scripts/test-times.json`,
	189	);
	190	}
	191	}
108	192	process.exit(ending.code);
		@@ -0,0 +1,43 @@
	1	{
	2	"test/apps-adversarial.test.ts": 145.9,
	3	"test/apps-busy.test.ts": 26.4,
	4	"test/apps-token.test.ts": 19.8,
	5	"test/apps.test.ts": 148.5,
	6	"test/asks.test.ts": 7.3,
	7	"test/autonomy.test.ts": 18,
	8	"test/board-windows.test.ts": 6,
	9	"test/checks.test.ts": 26.3,
	10	"test/cloud.test.ts": 7.1,
	11	"test/crew-limits.test.ts": 0.5,
	12	"test/crew-sandbox.test.ts": 5.3,
	13	"test/crew-wrapup.test.ts": 23.6,
	14	"test/crew.test.ts": 70.1,
	15	"test/cross-repo-pull.test.ts": 13,
	16	"test/decide.test.ts": 3.1,
	17	"test/dependencies.test.ts": 10.8,
	18	"test/feed-page.test.ts": 7.1,
	19	"test/feed.test.ts": 8.3,
	20	"test/forge-cloudflare.test.ts": 0.1,
	21	"test/fork-flow.test.ts": 48.4,
	22	"test/forks.test.ts": 28.9,
	23	"test/git-trust.test.ts": 0.3,
	24	"test/helpers.test.ts": 0.7,
	25	"test/landing.test.ts": 9.4,
	26	"test/ledger.test.ts": 12.8,
	27	"test/migrate.test.ts": 41.9,
	28	"test/offline.test.ts": 3.5,
	29	"test/pulls.test.ts": 14.6,
	30	"test/run-secrets.test.ts": 2.8,
	31	"test/shards.test.ts": 0.3,
	32	"test/share.test.ts": 16.2,
	33	"test/spawn-plan.test.ts": 0.2,
	34	"test/spawn.test.ts": 11.1,
	35	"test/standing.test.ts": 2.8,
	36	"test/steer.test.ts": 16.2,
	37	"test/studio-words.test.ts": 17,
	38	"test/studio.test.ts": 196.9,
	39	"test/usage.test.ts": 17.5,
	40	"test/vitals.test.ts": 7.1,
	41	"test/web-apps.test.ts": 18.7,
	42	"test/web.test.ts": 16.1
	43	}
		@@ -4,11 +4,16 @@
4	4	* red run whose tail names every failing test from every shard.
5	5	*/
6	6	import { describe, expect, test } from "bun:test";
	7	import { readFileSync } from "node:fs";
7	8	import {
	9	SECONDS_PER_BYTE_DEFAULT,
8	10	SHARDS_MAX,
	11	formatTimes,
	12	parseTimes,
9	13	shardCount,
10	14	splitFiles,
11	15	tally,
	16	weigher,
12	17	type ShardResult,
13	18	} from "../scripts/shards.ts";
14	19
		@@ -46,6 +51,155 @@ describe("splitFiles", () => {
46	51	});
47	52	});
48	53
	54	describe("weighing by measured time", () => {
	55	// Four files of one size: bytes cannot tell them apart, the table can.
	56	const list = [
	57	"test/slow.test.ts",
	58	"test/a.test.ts",
	59	"test/b.test.ts",
	60	"test/c.test.ts",
	61	];
	62	const bytes = new Map(list.map((file) => [file, 10_000]));
	63
	64	test("a known slow file is not stacked with others when a fast one of its size exists", () => {
	65	const times = new Map([
	66	["test/slow.test.ts", 50],
	67	["test/a.test.ts", 20],
	68	["test/b.test.ts", 15],
	69	["test/c.test.ts", 14],
	70	]);
	71	const slices = splitFiles(list, 2, weigher(list, bytes, times));
	72	// Hand count: 50 alone; 20 + 15 + 14 = 49 beside it.
	73	expect(slices).toEqual([
	74	["test/slow.test.ts"],
	75	["test/a.test.ts", "test/b.test.ts", "test/c.test.ts"],
	76	]);
	77	// By bytes alone the same four split two and two: slow carries a neighbour.
	78	const byBytes = splitFiles(list, 2, (file) => bytes.get(file)!);
	79	expect(byBytes.find((s) => s.includes("test/slow.test.ts"))!.length).toBe(
	80	2,
	81	);
	82	});
	83
	84	test("a file missing from the table weighs its bytes at the table's own rate", () => {
	85	const sizes = new Map([
	86	["test/slow.test.ts", 10_000],
	87	["test/a.test.ts", 30_000],
	88	["test/new.test.ts", 5_000],
	89	]);
	90	const times = new Map([
	91	["test/slow.test.ts", 50],
	92	["test/a.test.ts", 10],
	93	]);
	94	const files = [...sizes.keys()];
	95	const weight = weigher(files, sizes, times);
	96	// 60 s over 40 000 bytes = 0.0015 s/byte; 5 000 bytes → 7.5 s.
	97	expect(weight("test/new.test.ts")).toBeCloseTo(7.5, 9);
	98	expect(weight("test/slow.test.ts")).toBe(50);
	99	expect(weight("./test/slow.test.ts")).toBe(50);
	100	const slices = splitFiles(files, 2, weight);
	101	expect(slices.flat().sort()).toEqual([...files].sort());
	102	expect(slices).toEqual([
	103	["test/slow.test.ts"],
	104	["test/a.test.ts", "test/new.test.ts"],
	105	]);
	106	});
	107
	108	test("with no table at all, bytes at the default rate", () => {
	109	const weight = weigher(list, bytes, new Map());
	110	expect(weight("test/a.test.ts")).toBeCloseTo(
	111	10_000 * SECONDS_PER_BYTE_DEFAULT,
	112	9,
	113	);
	114	expect(weight("test/unknown.test.ts")).toBe(0);
	115	});
	116
	117	test("the same list and the same table give the same split", () => {
	118	const times = new Map([["test/slow.test.ts", 50]]);
	119	const first = splitFiles(list, 3, weigher(list, bytes, times));
	120	for (let i = 0; i < 5; i++)
	121	expect(splitFiles(list, 3, weigher(list, bytes, new Map(times)))).toEqual(
	122	first,
	123	);
	124	});
	125	});
	126
	127	describe("parseTimes", () => {
	128	test("reads file to seconds", () => {
	129	const parsed = parseTimes(
	130	'{ "test/crew.test.ts": 50.1, "./test/a.test.ts": 0 }',
	131	);
	132	expect(parsed).toEqual({
	133	ok: true,
	134	value: new Map([
	135	["test/crew.test.ts", 50.1],
	136	["test/a.test.ts", 0],
	137	]),
	138	});
	139	});
	140
	141	test("a malformed table is refused whole", () => {
	142	for (const bad of [
	143	"",
	144	"{",
	145	"[]",
	146	"null",
	147	"50",
	148	'{ "test/a.test.ts": "50" }',
	149	'{ "test/a.test.ts": -1 }',
	150	'{ "test/a.test.ts": 1e9 }',
	151	'{ "test/a.test.ts": null }',
	152	'{ "src/app.ts": 3 }',
	153	'{ "test/a.test.ts": 3, "README.md": 1 }',
	154	])
	155	expect(parseTimes(bad).ok).toBe(false);
	156	});
	157
	158	test("a refused table still splits, on bytes", () => {
	159	const parsed = parseTimes("not json");
	160	const times = parsed.ok ? parsed.value : new Map<string, number>();
	161	const files = ["test/a.test.ts", "test/b.test.ts", "test/c.test.ts"];
	162	const sizes = new Map([
	163	["test/a.test.ts", 300],
	164	["test/b.test.ts", 200],
	165	["test/c.test.ts", 100],
	166	]);
	167	expect(splitFiles(files, 2, weigher(files, sizes, times))).toEqual([
	168	["test/a.test.ts"],
	169	["test/b.test.ts", "test/c.test.ts"],
	170	]);
	171	});
	172
	173	test("formatTimes writes what parseTimes reads, sorted, to a tenth", () => {
	174	const text = formatTimes(
	175	new Map([
	176	["./test/b.test.ts", 1.26],
	177	["test/a.test.ts", 50.04],
	178	]),
	179	);
	180	expect(text).toBe(
	181	'{\n "test/a.test.ts": 50,\n "test/b.test.ts": 1.3\n}\n',
	182	);
	183	expect(parseTimes(text).ok).toBe(true);
	184	expect(formatTimes(new Map())).toBe("{}\n");
	185	});
	186
	187	test("the checked-in table is accepted and names only files that exist", () => {
	188	const parsed = parseTimes(
	189	readFileSync(
	190	new URL("../scripts/test-times.json", import.meta.url),
	191	"utf8",
	192	),
	193	);
	194	expect(parsed.ok).toBe(true);
	195	if (!parsed.ok) return;
	196	for (const file of parsed.value.keys())
	197	expect(
	198	Bun.file(new URL(`../${file}`, import.meta.url)).size,
	199	).toBeGreaterThan(0);
	200	});
	201	});
	202
49	203	describe("shardCount", () => {
50	204	test("one per CPU, capped, at least one, never more than files", () => {
51	205	expect(shardCount(80, 22)).toEqual({ ok: true, value: SHARDS_MAX });