e032238 · 1d6d801 · 6ed2e67 · 2c3dd7e
@@ -209,3 +209,13 @@
carries today, each entry under the issue that owns it; an entry that is no
longer hollow turns the check red, so the list only shrinks.
`bun scripts/hollow.ts` prints the same lines on their own.
## Budget tests run last
A test file that calls `expectBudget(` (a call in code, not a mention in a
comment or string) bounds a speed and only passes with the machine quiet. The
sharded gate (`scripts/test-shards.ts`) sets those files aside and runs them
one at a time after every other shard is green; a red shard skips them. The
rule is `callsBudget` in `scripts/shards.ts`. Each shard also gets its own
`TMPDIR` and `FORGE_APP_DATA` (nested under the cell's, when it set one),
removed when the shard ends.
@@ -189,6 +189,32 @@
export function splitCapped(
  );
}
/** Whether a test file's code (comments dropped) calls `expectBudget(`. */
export function callsBudget(text: string | undefined): boolean {
  if (text === undefined) return false;
  const code = text.replace(/\/\*[\s\S]*?\*\/|\/\/[^\n]*/g, "");
  return /\bexpectBudget\s*\(/.test(code);
}
/**
 * Splits `files` into those that bound a speed (a call to `expectBudget(`, which only
 * passes with the cell quiet enough) and the rest. The runner runs the
 * budget files last, one at a time, after every other shard is done, so the
 * load of the gate's own shards cannot redden them (#7617). `read` returns
 * a file's text; one it cannot read counts as no budget. Each list keeps
 * the input's order.
 */
export function budgetLast(
  files: readonly string[],
  read: (file: string) => string | undefined,
): { rest: string[]; budget: string[] } {
  const rest: string[] = [];
  const budget: string[] = [];
  for (const file of files)
    (callsBudget(read(file)) ? budget : rest).push(file);
  return { rest, budget };
}
/** How a test file is named: the same rule `scripts/test-shards.ts` uses. */
export const TEST_FILE = /\.test\.[cm]?[jt]sx?$/;
@@ -71,6 +71,7 @@
import {
  tally,
  weigher,
  besideOthers,
  budgetLast,
  failingFiles,
  onlyTestFailures,
  stopsTheRest,
@@ -408,10 +409,23 @@
const serialFiles = !record && count.value === 1;
// refreshes the time of every file it ran (#1797).
const timed = !record && !serialFiles && gate && tier === "full" && "none" in shardSpec;
const perFile = record || measure || serialFiles || timed;
const slices =
  perFile
    ? heaviestFirst(files, weight).map((file) => [file])
    : splitCapped(files, count.value, weight);
// Files that bound a speed run last and alone, after every other shard,
// so the gate's own load cannot redden them (#7617).
const ordered = budgetLast(files, (file) => {
  try {
    return readFileSync(file, "utf8");
  } catch {
    return undefined;
  }
});
const slicesOf = (list: string[]): string[][] =>
  list.length === 0
    ? []
    : perFile
      ? heaviestFirst(list, weight).map((file) => [file])
      : splitCapped(list, count.value, weight);
const mainSlices = slicesOf(ordered.rest);
const slices = [...mainSlices, ...ordered.budget.map((file) => [file])];
const work = files.reduce((sum, file) => sum + weight(file), 0);
const banner = record
@@ -428,6 +442,8 @@
const banner = record
        `expected ${slices.map((slice) => slice.reduce((sum, file) => sum + weight(file), 0).toFixed(0) + "s").join(" ")}`;
console.log(banner);
process.stderr.write(banner + "\n");
if (ordered.budget.length > 0)
  console.log(`test-shards: ${ordered.budget.length} budget files last, one at a time`);
const children = new Set<Bun.Subprocess>();
for (const signal of ["SIGINT", "SIGTERM"] as const)
@@ -469,10 +485,24 @@
type Timed = ShardResult & { seconds: number };
 * output is held and written through whole when the shard ends, so shards
 * running side by side never interleave.
 */
async function runShard(
async function runShard(slice: string[], index: number, live: boolean): Promise<Timed> {
  // Its own temp and data directories: shards side by side never share a
  // path. They sit under the TMPDIR the cell set (tmpdir() reads it), and a
  // FORGE_APP_DATA the cell set is nested under, never replaced. Removed
  // however the shard ends.
  const home = mkdtempSync(join(tmpdir(), "shard-"));
  try {
    return await runShardIn(slice, index, live, home);
  } finally {
    rmSync(home, { recursive: true, force: true });
  }
}
async function runShardIn(
  slice: string[],
  index: number,
  live: boolean,
  home: string,
): Promise<Timed> {
  const started = performance.now();
  const where = `${index + 1}/${slices.length}`;
@@ -486,6 +516,10 @@
async function runShard(
    file.startsWith("/") || file.startsWith("./") ? file : `./${file}`,
  );
  const childEnv = { ...process.env };
  childEnv["TMPDIR"] = join(home, "tmp");
  childEnv["FORGE_APP_DATA"] = join(process.env["FORGE_APP_DATA"] ?? home, `data-${index}`);
  mkdirSync(childEnv["TMPDIR"], { recursive: true });
  mkdirSync(childEnv["FORGE_APP_DATA"], { recursive: true });
  delete childEnv["FORGE_CHECK_SHARDS"];
  delete childEnv["FORGE_CHECK_SHARD_INDEX"];
  delete childEnv["FORGE_CHECK_SHARD_TOTAL"];
@@ -568,6 +602,8 @@
async function runShard(
      if (verdict.red.length === 0) code = 0;
    }
  }
  if (process.env["FORGE_APP_DATA"])
    rmSync(childEnv["FORGE_APP_DATA"], { recursive: true, force: true });
  const text = output.join("");
  const seconds = ((performance.now() - started) / 1000).toFixed(1);
  process.stdout.write(
@@ -594,23 +630,20 @@
async function runShard(
  };
}
/** Runs every slice, at most `width` at once, in the slices' order.
/** Runs slices `from` up to `to`, at most `width` at once, in the slices' order.
 * Stops starting new slices after the first non-zero exit and signals
 * in-flight children so a red shard does not burn the rest of the wall. */
async function pool(width: number): Promise<Timed[]> {
async function pool(width: number, from: number, to: number): Promise<Timed[]> {
  const results: Timed[] = [];
  let next = 0;
  let stop = false;
  let next = from;
  async function worker(): Promise<void> {
    while (next < slices.length) {
      if (stop) break;
    while (next < to) {
      if (halted) break;
      const index = next++;
      if (stop) break;
      results[index] = await runShard(slices[index]!, index, width === 1);
      // A shard red only on budget tests lets the rest run (#7354);
      // a verification after landing runs everything (#7786).
      if (!keepGoing && stopsTheRest(results[index]!)) {
        stop = true;
        halted = true;
        for (const child of children) child.kill("SIGTERM");
      }
@@ -624,10 +657,16 @@
async function pool(width: number): Promise<Timed[]> {
const started = performance.now();
// Serial one-file processes when SHARDS=1; record keeps its worker width;
// multi-shard still runs every slice side by side.
const results = await pool(
  // Never more side by side than the count: capped slices outnumber it.
  serialFiles ? 1 : Math.min(count.value, slices.length),
);
const results = [
  ...(await pool(
    // Never more side by side than the count: capped slices outnumber it.
    serialFiles ? 1 : Math.max(1, Math.min(count.value, mainSlices.length)),
    0,
    mainSlices.length,
  )),
  // Budget files, one at a time, once the rest are done and green.
  ...(await pool(1, mainSlices.length, slices.length)),
];
if (results.length < slices.length) {
  const skipped = slices.length - results.length;
  console.log(
@@ -4,7 +4,7 @@
 * red run whose tail names every failing test from every shard.
 */
import { afterAll, describe, expect, test } from "bun:test";
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { resolveLimits } from "../src/apps/contain.ts";
@@ -12,6 +12,8 @@
import {
  SECONDS_PER_BYTE_DEFAULT,
  SHARDS_MAX,
  besideOthers,
  budgetLast,
  callsBudget,
  failingFiles,
  onlyTestFailures,
  WORKER_MEMORY_BYTES,
@@ -948,3 +950,149 @@
describe("a failing file is run once more alone", () => {
    expect(got.out).not.toContain("fails only beside others");
  }, 30_000);
});
describe("budget files last, and a fixture suite in six shards (#7617)", () => {
  const script = new URL("../scripts/test-shards.ts", import.meta.url).pathname;
  const fixture = (files: Record<string, string>): string => {
    const dir = mkdtempSync(join(tmpdir(), "shards-fixture-"));
    for (const [name, text] of Object.entries(files)) writeFileSync(join(dir, name), text);
    return dir;
  };
  const run = async (
    dir: string,
    shards: string,
    names: string[],
    env: Record<string, string> = {},
  ): Promise<{ code: number; out: string; stdout: string }> => {
    const child = Bun.spawn(["bun", script, ...names], {
      cwd: dir,
      env: { ...process.env, FORGE_CHECK_SHARDS: shards, ...env },
      stdin: "ignore",
      stdout: "pipe",
      stderr: "pipe",
    });
    const [stdout, err] = await Promise.all([new Response(child.stdout).text(), new Response(child.stderr).text()]);
    const code = await child.exited;
    return { code, out: stdout + err, stdout };
  };
  const passing = (n: number): string =>
    `import { test } from "bun:test";\ntest("passes ${n}", () => {});\n`;
  const budgeted = `import { test } from "bun:test";\ntest("budgeted", () => { expectBudget(() => {}, 1); });\nfunction expectBudget(f: () => void, _s: number) { f(); }\n`;
  /** The files named on the `── shard N/M…: files` lines, in the order shards ran. */
  const finished = (stdout: string): string[] =>
    [...stdout.matchAll(/^── shard \d+\/\d+(?: \(exit \d+, [\d.]+s\))?: (.*)$/gm)].map((m) => m[1]!);
  test("callsBudget sees a call, not a mention in a comment or a string", () => {
    expect(callsBudget("expectBudget(1, 2)")).toBe(true);
    expect(callsBudget("await expectBudget (x)")).toBe(true);
    expect(callsBudget("// expectBudget(1)\nlet a = 1")).toBe(false);
    expect(callsBudget("/* expectBudget(1) */")).toBe(false);
    expect(callsBudget('const note = "expectBudget is a helper"')).toBe(false);
    expect(callsBudget(undefined)).toBe(false);
  });
  test("budgetLast sets aside the files that call expectBudget, in order", () => {
    const text: Record<string, string> = { "a": "x", "b": "expectBudget(1)", "c": "y", "d": "expectBudget(2)", "e": "// expectBudget(3)" };
    const got = budgetLast(["a", "b", "c", "d", "e", "gone"], (file) => text[file]);
    expect(got.rest).toEqual(["a", "c", "e", "gone"]);
    expect(got.budget).toEqual(["b", "d"]);
  });
  test("twelve files in six shards: the plan has six slices and each file runs exactly once", async () => {
    const names = Array.from({ length: 12 }, (_, i) => `s${i}.test.ts`);
    const dir = fixture(Object.fromEntries(names.map((name, i) => [name, passing(i)])));
    const six = await run(dir, "6", names);
    rmSync(dir, { recursive: true, force: true });
    expect(six.code).toBe(0);
    expect(six.out).toContain("12 files in 6 shards");
    expect(six.out.match(/test-shards: start \d+\/6 /g)?.length).toBe(6);
    const ran = finished(six.stdout).flatMap((line) => line.split(" "));
    for (const name of names) expect(ran.filter((file) => file === name).length).toBe(1);
    expect(six.out).toMatch(/Ran 12 tests across 12 files/);
  }, 60_000);
  test("a failure in any shard reds the run and names the test", async () => {
    const files: Record<string, string> = {};
    for (let i = 0; i < 5; i++) files[`ok${i}.test.ts`] = passing(i);
    files["bad.test.ts"] = `import { test, expect } from "bun:test";\ntest("the broken one", () => { expect(1).toBe(2); });\n`;
    const dir = fixture(files);
    const got = await run(dir, "3", Object.keys(files));
    rmSync(dir, { recursive: true, force: true });
    expect(got.code).not.toBe(0);
    expect(got.out).toContain("(fail) the broken one");
  }, 60_000);
  test("budget files run last, alone, after the others", async () => {
    const dir = fixture({
      "a.test.ts": passing(1),
      "b.test.ts": passing(2),
      "c.test.ts": passing(3),
      "slow.test.ts": budgeted,
    });
    const got = await run(dir, "4", ["slow.test.ts", "a.test.ts", "b.test.ts", "c.test.ts"]);
    rmSync(dir, { recursive: true, force: true });
    expect(got.code).toBe(0);
    expect(got.out).toContain("1 budget files last");
    const order = finished(got.stdout);
    expect(order.at(-1)).toBe("slow.test.ts");
    expect(order.slice(0, -1).join(" ").split(" ").sort()).toEqual(["a.test.ts", "b.test.ts", "c.test.ts"]);
  }, 60_000);
  test("a red shard skips the budget files: the run is red and nothing crashes", async () => {
    const dir = fixture({
      "bad.test.ts": `import { test, expect } from "bun:test";\ntest("the broken one", () => { expect(1).toBe(2); });\n`,
      "a.test.ts": passing(1),
      "slow.test.ts": budgeted,
    });
    const got = await run(dir, "2", ["bad.test.ts", "a.test.ts", "slow.test.ts"]);
    rmSync(dir, { recursive: true, force: true });
    expect(got.code).not.toBe(0);
    expect(got.out).toContain("(fail) the broken one");
    expect(got.out).toMatch(/fail-fast — skipped \d+ remaining shards? after a red exit/);
    expect(finished(got.stdout).flat().join(" ")).not.toContain("slow.test.ts");
    expect(got.out).not.toContain("undefined");
    expect(got.out).not.toContain("TypeError");
  }, 60_000);
  test("each shard gets its own empty FORGE_APP_DATA and TMPDIR, removed after", async () => {
    const probe = (n: number): string =>
      `import { test, expect } from "bun:test";
import { readdirSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
test("probe ${n}", () => {
  const data = process.env.FORGE_APP_DATA!;
  expect(readdirSync(data)).toEqual([]);
  expect(readdirSync(tmpdir())).toEqual([]);
  writeFileSync(data + "/mark", "${n}");
  console.log("PROBE-DATA " + data + " PROBE-TMP " + tmpdir());
});
`;
    const names = ["p0.test.ts", "p1.test.ts", "p2.test.ts"];
    const dir = fixture(Object.fromEntries(names.map((name, i) => [name, probe(i)])));
    const root = mkdtempSync(join(tmpdir(), "shards-root-"));
    const got = await run(dir, "3", names, { TMPDIR: root, FORGE_APP_DATA: root });
    rmSync(dir, { recursive: true, force: true });
    const seen = [...got.out.matchAll(/PROBE-DATA (\S+) PROBE-TMP (\S+)/g)];
    const leaked = readdirSync(root);
    rmSync(root, { recursive: true, force: true });
    expect(got.code).toBe(0);
    expect(new Set(seen.map((m) => m[1])).size).toBe(3);
    expect(new Set(seen.map((m) => m[2])).size).toBe(3);
    for (const m of seen) expect(m[1]!.startsWith(root)).toBe(true);
    expect(leaked).toEqual([]);
  }, 60_000);
  test("a FORGE_APP_DATA the cell set is nested under, not replaced", async () => {
    const dir = fixture({
      "d.test.ts": `import { test } from "bun:test";\ntest("d", () => { console.log("PROBE-DATA " + process.env.FORGE_APP_DATA); });\n`,
    });
    const data = mkdtempSync(join(tmpdir(), "shards-data-"));
    const got = await run(dir, "1", ["d.test.ts"], { FORGE_APP_DATA: data });
    rmSync(dir, { recursive: true, force: true });
    const leaked = readdirSync(data);
    rmSync(data, { recursive: true, force: true });
    expect(got.code).toBe(0);
    expect(got.out).toContain(`PROBE-DATA ${data}/data-0`);
    expect(leaked).toEqual([]);
  }, 60_000);
});
		@@ -209,3 +209,13 @@ carries today, each entry under the issue that owns it; an entry that is no
209	209	longer hollow turns the check red, so the list only shrinks.
210	210	`bun scripts/hollow.ts` prints the same lines on their own.
211	211
	212
	213	## Budget tests run last
	214
	215	A test file that calls `expectBudget(` (a call in code, not a mention in a
	216	comment or string) bounds a speed and only passes with the machine quiet. The
	217	sharded gate (`scripts/test-shards.ts`) sets those files aside and runs them
	218	one at a time after every other shard is green; a red shard skips them. The
	219	rule is `callsBudget` in `scripts/shards.ts`. Each shard also gets its own
	220	`TMPDIR` and `FORGE_APP_DATA` (nested under the cell's, when it set one),
	221	removed when the shard ends.
		@@ -189,6 +189,32 @@ export function splitCapped(
189	189	);
190	190	}
191	191
	192	/** Whether a test file's code (comments dropped) calls `expectBudget(`. */
	193	export function callsBudget(text: string \| undefined): boolean {
	194	if (text === undefined) return false;
	195	const code = text.replace(/\/\[\s\S]?\\/\|\/\/[^\n]/g, "");
	196	return /\bexpectBudget\s*\(/.test(code);
	197	}
	198
	199	/**
	200	* Splits `files` into those that bound a speed (a call to `expectBudget(`, which only
	201	* passes with the cell quiet enough) and the rest. The runner runs the
	202	* budget files last, one at a time, after every other shard is done, so the
	203	* load of the gate's own shards cannot redden them (#7617). `read` returns
	204	* a file's text; one it cannot read counts as no budget. Each list keeps
	205	* the input's order.
	206	*/
	207	export function budgetLast(
	208	files: readonly string[],
	209	read: (file: string) => string \| undefined,
	210	): { rest: string[]; budget: string[] } {
	211	const rest: string[] = [];
	212	const budget: string[] = [];
	213	for (const file of files)
	214	(callsBudget(read(file)) ? budget : rest).push(file);
	215	return { rest, budget };
	216	}
	217
192	218	/** How a test file is named: the same rule `scripts/test-shards.ts` uses. */
193	219	export const TEST_FILE = /\.test\.[cm]?[jt]sx?$/;
194	220
		@@ -71,6 +71,7 @@ import {
71	71	tally,
72	72	weigher,
73	73	besideOthers,
	74	budgetLast,
74	75	failingFiles,
75	76	onlyTestFailures,
76	77	stopsTheRest,
		@@ -408,10 +409,23 @@ const serialFiles = !record && count.value === 1;
408	409	// refreshes the time of every file it ran (#1797).
409	410	const timed = !record && !serialFiles && gate && tier === "full" && "none" in shardSpec;
410	411	const perFile = record \|\| measure \|\| serialFiles \|\| timed;
411		const slices =
412		perFile
413		? heaviestFirst(files, weight).map((file) => [file])
414		: splitCapped(files, count.value, weight);
	412	// Files that bound a speed run last and alone, after every other shard,
	413	// so the gate's own load cannot redden them (#7617).
	414	const ordered = budgetLast(files, (file) => {
	415	try {
	416	return readFileSync(file, "utf8");
	417	} catch {
	418	return undefined;
	419	}
	420	});
	421	const slicesOf = (list: string[]): string[][] =>
	422	list.length === 0
	423	? []
	424	: perFile
	425	? heaviestFirst(list, weight).map((file) => [file])
	426	: splitCapped(list, count.value, weight);
	427	const mainSlices = slicesOf(ordered.rest);
	428	const slices = [...mainSlices, ...ordered.budget.map((file) => [file])];
415	429	const work = files.reduce((sum, file) => sum + weight(file), 0);
416	430
417	431	const banner = record
		@@ -428,6 +442,8 @@ const banner = record
428	442	`expected ${slices.map((slice) => slice.reduce((sum, file) => sum + weight(file), 0).toFixed(0) + "s").join(" ")}`;
429	443	console.log(banner);
430	444	process.stderr.write(banner + "\n");
	445	if (ordered.budget.length > 0)
	446	console.log(`test-shards: ${ordered.budget.length} budget files last, one at a time`);
431	447
432	448	const children = new Set<Bun.Subprocess>();
433	449	for (const signal of ["SIGINT", "SIGTERM"] as const)
		@@ -469,10 +485,24 @@ type Timed = ShardResult & { seconds: number };
469	485	* output is held and written through whole when the shard ends, so shards
470	486	* running side by side never interleave.
471	487	*/
472		async function runShard(
	488	async function runShard(slice: string[], index: number, live: boolean): Promise<Timed> {
	489	// Its own temp and data directories: shards side by side never share a
	490	// path. They sit under the TMPDIR the cell set (tmpdir() reads it), and a
	491	// FORGE_APP_DATA the cell set is nested under, never replaced. Removed
	492	// however the shard ends.
	493	const home = mkdtempSync(join(tmpdir(), "shard-"));
	494	try {
	495	return await runShardIn(slice, index, live, home);
	496	} finally {
	497	rmSync(home, { recursive: true, force: true });
	498	}
	499	}
	500
	501	async function runShardIn(
473	502	slice: string[],
474	503	index: number,
475	504	live: boolean,
	505	home: string,
476	506	): Promise<Timed> {
477	507	const started = performance.now();
478	508	const where = `${index + 1}/${slices.length}`;
		@@ -486,6 +516,10 @@ async function runShard(
486	516	file.startsWith("/") \|\| file.startsWith("./") ? file : `./${file}`,
487	517	);
488	518	const childEnv = { ...process.env };
	519	childEnv["TMPDIR"] = join(home, "tmp");
	520	childEnv["FORGE_APP_DATA"] = join(process.env["FORGE_APP_DATA"] ?? home, `data-${index}`);
	521	mkdirSync(childEnv["TMPDIR"], { recursive: true });
	522	mkdirSync(childEnv["FORGE_APP_DATA"], { recursive: true });
489	523	delete childEnv["FORGE_CHECK_SHARDS"];
490	524	delete childEnv["FORGE_CHECK_SHARD_INDEX"];
491	525	delete childEnv["FORGE_CHECK_SHARD_TOTAL"];
		@@ -568,6 +602,8 @@ async function runShard(
568	602	if (verdict.red.length === 0) code = 0;
569	603	}
570	604	}
	605	if (process.env["FORGE_APP_DATA"])
	606	rmSync(childEnv["FORGE_APP_DATA"], { recursive: true, force: true });
571	607	const text = output.join("");
572	608	const seconds = ((performance.now() - started) / 1000).toFixed(1);
573	609	process.stdout.write(
		@@ -594,23 +630,20 @@ async function runShard(
594	630	};
595	631	}
596	632
597		/** Runs every slice, at most `width` at once, in the slices' order.
	633	/** Runs slices `from` up to `to`, at most `width` at once, in the slices' order.
598	634	* Stops starting new slices after the first non-zero exit and signals
599	635	* in-flight children so a red shard does not burn the rest of the wall. */
600		async function pool(width: number): Promise<Timed[]> {
	636	async function pool(width: number, from: number, to: number): Promise<Timed[]> {
601	637	const results: Timed[] = [];
602		let next = 0;
603		let stop = false;
	638	let next = from;
604	639	async function worker(): Promise<void> {
605		while (next < slices.length) {
606		if (stop) break;
	640	while (next < to) {
	641	if (halted) break;
607	642	const index = next++;
608		if (stop) break;
609	643	results[index] = await runShard(slices[index]!, index, width === 1);
610	644	// A shard red only on budget tests lets the rest run (#7354);
611	645	// a verification after landing runs everything (#7786).
612	646	if (!keepGoing && stopsTheRest(results[index]!)) {
613		stop = true;
614	647	halted = true;
615	648	for (const child of children) child.kill("SIGTERM");
616	649	}
		@@ -624,10 +657,16 @@ async function pool(width: number): Promise<Timed[]> {
624	657	const started = performance.now();
625	658	// Serial one-file processes when SHARDS=1; record keeps its worker width;
626	659	// multi-shard still runs every slice side by side.
627		const results = await pool(
628		// Never more side by side than the count: capped slices outnumber it.
629		serialFiles ? 1 : Math.min(count.value, slices.length),
630		);
	660	const results = [
	661	...(await pool(
	662	// Never more side by side than the count: capped slices outnumber it.
	663	serialFiles ? 1 : Math.max(1, Math.min(count.value, mainSlices.length)),
	664	0,
	665	mainSlices.length,
	666	)),
	667	// Budget files, one at a time, once the rest are done and green.
	668	...(await pool(1, mainSlices.length, slices.length)),
	669	];
631	670	if (results.length < slices.length) {
632	671	const skipped = slices.length - results.length;
633	672	console.log(
		@@ -4,7 +4,7 @@
4	4	* red run whose tail names every failing test from every shard.
5	5	*/
6	6	import { afterAll, describe, expect, test } from "bun:test";
7		import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
	7	import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
8	8	import { tmpdir } from "node:os";
9	9	import { join } from "node:path";
10	10	import { resolveLimits } from "../src/apps/contain.ts";
		@@ -12,6 +12,8 @@ import {
12	12	SECONDS_PER_BYTE_DEFAULT,
13	13	SHARDS_MAX,
14	14	besideOthers,
	15	budgetLast,
	16	callsBudget,
15	17	failingFiles,
16	18	onlyTestFailures,
17	19	WORKER_MEMORY_BYTES,
		@@ -948,3 +950,149 @@ describe("a failing file is run once more alone", () => {
948	950	expect(got.out).not.toContain("fails only beside others");
949	951	}, 30_000);
950	952	});
	953
	954	describe("budget files last, and a fixture suite in six shards (#7617)", () => {
	955	const script = new URL("../scripts/test-shards.ts", import.meta.url).pathname;
	956	const fixture = (files: Record<string, string>): string => {
	957	const dir = mkdtempSync(join(tmpdir(), "shards-fixture-"));
	958	for (const [name, text] of Object.entries(files)) writeFileSync(join(dir, name), text);
	959	return dir;
	960	};
	961	const run = async (
	962	dir: string,
	963	shards: string,
	964	names: string[],
	965	env: Record<string, string> = {},
	966	): Promise<{ code: number; out: string; stdout: string }> => {
	967	const child = Bun.spawn(["bun", script, ...names], {
	968	cwd: dir,
	969	env: { ...process.env, FORGE_CHECK_SHARDS: shards, ...env },
	970	stdin: "ignore",
	971	stdout: "pipe",
	972	stderr: "pipe",
	973	});
	974	const [stdout, err] = await Promise.all([new Response(child.stdout).text(), new Response(child.stderr).text()]);
	975	const code = await child.exited;
	976	return { code, out: stdout + err, stdout };
	977	};
	978	const passing = (n: number): string =>
	979	`import { test } from "bun:test";\ntest("passes ${n}", () => {});\n`;
	980	const budgeted = `import { test } from "bun:test";\ntest("budgeted", () => { expectBudget(() => {}, 1); });\nfunction expectBudget(f: () => void, _s: number) { f(); }\n`;
	981	/** The files named on the `── shard N/M…: files` lines, in the order shards ran. */
	982	const finished = (stdout: string): string[] =>
	983	[...stdout.matchAll(/^── shard \d+\/\d+(?: \(exit \d+, [\d.]+s\))?: (.*)$/gm)].map((m) => m[1]!);
	984
	985	test("callsBudget sees a call, not a mention in a comment or a string", () => {
	986	expect(callsBudget("expectBudget(1, 2)")).toBe(true);
	987	expect(callsBudget("await expectBudget (x)")).toBe(true);
	988	expect(callsBudget("// expectBudget(1)\nlet a = 1")).toBe(false);
	989	expect(callsBudget("/* expectBudget(1) */")).toBe(false);
	990	expect(callsBudget('const note = "expectBudget is a helper"')).toBe(false);
	991	expect(callsBudget(undefined)).toBe(false);
	992	});
	993
	994	test("budgetLast sets aside the files that call expectBudget, in order", () => {
	995	const text: Record<string, string> = { "a": "x", "b": "expectBudget(1)", "c": "y", "d": "expectBudget(2)", "e": "// expectBudget(3)" };
	996	const got = budgetLast(["a", "b", "c", "d", "e", "gone"], (file) => text[file]);
	997	expect(got.rest).toEqual(["a", "c", "e", "gone"]);
	998	expect(got.budget).toEqual(["b", "d"]);
	999	});
	1000
	1001	test("twelve files in six shards: the plan has six slices and each file runs exactly once", async () => {
	1002	const names = Array.from({ length: 12 }, (_, i) => `s${i}.test.ts`);
	1003	const dir = fixture(Object.fromEntries(names.map((name, i) => [name, passing(i)])));
	1004	const six = await run(dir, "6", names);
	1005	rmSync(dir, { recursive: true, force: true });
	1006	expect(six.code).toBe(0);
	1007	expect(six.out).toContain("12 files in 6 shards");
	1008	expect(six.out.match(/test-shards: start \d+\/6 /g)?.length).toBe(6);
	1009	const ran = finished(six.stdout).flatMap((line) => line.split(" "));
	1010	for (const name of names) expect(ran.filter((file) => file === name).length).toBe(1);
	1011	expect(six.out).toMatch(/Ran 12 tests across 12 files/);
	1012	}, 60_000);
	1013
	1014	test("a failure in any shard reds the run and names the test", async () => {
	1015	const files: Record<string, string> = {};
	1016	for (let i = 0; i < 5; i++) files[`ok${i}.test.ts`] = passing(i);
	1017	files["bad.test.ts"] = `import { test, expect } from "bun:test";\ntest("the broken one", () => { expect(1).toBe(2); });\n`;
	1018	const dir = fixture(files);
	1019	const got = await run(dir, "3", Object.keys(files));
	1020	rmSync(dir, { recursive: true, force: true });
	1021	expect(got.code).not.toBe(0);
	1022	expect(got.out).toContain("(fail) the broken one");
	1023	}, 60_000);
	1024
	1025	test("budget files run last, alone, after the others", async () => {
	1026	const dir = fixture({
	1027	"a.test.ts": passing(1),
	1028	"b.test.ts": passing(2),
	1029	"c.test.ts": passing(3),
	1030	"slow.test.ts": budgeted,
	1031	});
	1032	const got = await run(dir, "4", ["slow.test.ts", "a.test.ts", "b.test.ts", "c.test.ts"]);
	1033	rmSync(dir, { recursive: true, force: true });
	1034	expect(got.code).toBe(0);
	1035	expect(got.out).toContain("1 budget files last");
	1036	const order = finished(got.stdout);
	1037	expect(order.at(-1)).toBe("slow.test.ts");
	1038	expect(order.slice(0, -1).join(" ").split(" ").sort()).toEqual(["a.test.ts", "b.test.ts", "c.test.ts"]);
	1039	}, 60_000);
	1040
	1041	test("a red shard skips the budget files: the run is red and nothing crashes", async () => {
	1042	const dir = fixture({
	1043	"bad.test.ts": `import { test, expect } from "bun:test";\ntest("the broken one", () => { expect(1).toBe(2); });\n`,
	1044	"a.test.ts": passing(1),
	1045	"slow.test.ts": budgeted,
	1046	});
	1047	const got = await run(dir, "2", ["bad.test.ts", "a.test.ts", "slow.test.ts"]);
	1048	rmSync(dir, { recursive: true, force: true });
	1049	expect(got.code).not.toBe(0);
	1050	expect(got.out).toContain("(fail) the broken one");
	1051	expect(got.out).toMatch(/fail-fast — skipped \d+ remaining shards? after a red exit/);
	1052	expect(finished(got.stdout).flat().join(" ")).not.toContain("slow.test.ts");
	1053	expect(got.out).not.toContain("undefined");
	1054	expect(got.out).not.toContain("TypeError");
	1055	}, 60_000);
	1056
	1057	test("each shard gets its own empty FORGE_APP_DATA and TMPDIR, removed after", async () => {
	1058	const probe = (n: number): string =>
	1059	`import { test, expect } from "bun:test";
	1060	import { readdirSync, writeFileSync } from "node:fs";
	1061	import { tmpdir } from "node:os";
	1062	test("probe ${n}", () => {
	1063	const data = process.env.FORGE_APP_DATA!;
	1064	expect(readdirSync(data)).toEqual([]);
	1065	expect(readdirSync(tmpdir())).toEqual([]);
	1066	writeFileSync(data + "/mark", "${n}");
	1067	console.log("PROBE-DATA " + data + " PROBE-TMP " + tmpdir());
	1068	});
	1069	`;
	1070	const names = ["p0.test.ts", "p1.test.ts", "p2.test.ts"];
	1071	const dir = fixture(Object.fromEntries(names.map((name, i) => [name, probe(i)])));
	1072	const root = mkdtempSync(join(tmpdir(), "shards-root-"));
	1073	const got = await run(dir, "3", names, { TMPDIR: root, FORGE_APP_DATA: root });
	1074	rmSync(dir, { recursive: true, force: true });
	1075	const seen = [...got.out.matchAll(/PROBE-DATA (\S+) PROBE-TMP (\S+)/g)];
	1076	const leaked = readdirSync(root);
	1077	rmSync(root, { recursive: true, force: true });
	1078	expect(got.code).toBe(0);
	1079	expect(new Set(seen.map((m) => m[1])).size).toBe(3);
	1080	expect(new Set(seen.map((m) => m[2])).size).toBe(3);
	1081	for (const m of seen) expect(m[1]!.startsWith(root)).toBe(true);
	1082	expect(leaked).toEqual([]);
	1083	}, 60_000);
	1084
	1085	test("a FORGE_APP_DATA the cell set is nested under, not replaced", async () => {
	1086	const dir = fixture({
	1087	"d.test.ts": `import { test } from "bun:test";\ntest("d", () => { console.log("PROBE-DATA " + process.env.FORGE_APP_DATA); });\n`,
	1088	});
	1089	const data = mkdtempSync(join(tmpdir(), "shards-data-"));
	1090	const got = await run(dir, "1", ["d.test.ts"], { FORGE_APP_DATA: data });
	1091	rmSync(dir, { recursive: true, force: true });
	1092	const leaked = readdirSync(data);
	1093	rmSync(data, { recursive: true, force: true });
	1094	expect(got.code).toBe(0);
	1095	expect(got.out).toContain(`PROBE-DATA ${data}/data-0`);
	1096	expect(leaked).toEqual([]);
	1097	}, 60_000);
	1098	});