7f8f857 · 5856136
@@ -0,0 +1,38 @@
name: outside-gate
# Nightly: every file in EXCLUDED (scripts/gate.ts), one at a time with the
# gate's 60 s timeout, so a suite outside the gate that breaks is seen the
# day it breaks. A newly red file opens one `Red: test/….test.ts` issue
# naming its first failing test; a file green again closes it
# (scripts/flake-hunt.ts outside). The list is EXCLUDED itself: a file moved
# into the gate stops being run here.
#
# Needs secret FORGE_TOKEN and variable FORGE_URL, as flake-hunt does.
on:
  schedule:
    - cron: "47 3 * * *"
  workflow_dispatch: {}
jobs:
  outside:
    runs-on: isolated
    timeout-minutes: 30
    env:
      REGISTRY: forgejo-http.forgejo.svc.cluster.local:3000
    steps:
      - name: clone main
        env:
          TOKEN: ${{ github.token }}
        run: |
          set -eu
          git clone -q "http://x-access-token:$TOKEN@$REGISTRY/$GITHUB_REPOSITORY.git" src
          cd src
          bun install --frozen-lockfile
      - name: run the excluded files
        working-directory: src
        env:
          FORGE_API: ${{ vars.FORGE_URL }}/api/v1
          FORGE_REPO: ${{ github.repository }}
          FORGE_TOKEN: ${{ secrets.FORGE_TOKEN }}
          FORGE_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_number }}
        run: bun scripts/flake-hunt.ts outside --file-issues
@@ -1282,6 +1282,16 @@
takes tens of ms): assert relative to a baseline from the same run, or bound
only the failure the test names. The workflow needs a
`FORGE_TOKEN` secret and a `FORGE_URL` variable on the mirror.
**No unseen red outside the gate.** Nightly, `.forgejo/workflows/outside-gate.yml`
runs `bun scripts/flake-hunt.ts outside --file-issues`: every file in
`EXCLUDED` (the complement of the gate, from the same list, so a file moved
into the gate is no longer run there), once, one at a time, with the gate's
60 s timeout. A file that newly fails opens one `Red: test/x.test.ts` issue
naming its first failing test; still red on another test, it updates that
issue; still red on the same test, it says nothing, so the owner's asks show
it once; passing again, it closes the issue. Same `FORGE_TOKEN` and
`FORGE_URL` as the hunt.
Bun's own `--randomize --seed=N` drives every hunt and admission run. Not
used: `--rerun-each` (one seed for every rerun, so ten reruns explore one
order) and `--parallel` (it changes the load the gate is judged under).
@@ -9,6 +9,12 @@
 *       `--file-issues` each gets an issue carrying the failure text, the
 *       seed and the run (FORGE_API, FORGE_REPO, FORGE_TOKEN; FORGE_RUN_URL
 *       names the run). Exits 1 when it found one.
 *   bun scripts/flake-hunt.ts outside [--file-issues]
 *       Every file in EXCLUDED once, one at a time, with the gate's timeout:
 *       the schedule for what the gate does not run (#812). With
 *       `--file-issues` a newly red file opens one `Red: test/….test.ts`
 *       issue naming its first failing test, a file red on another test
 *       updates it, and a file green again closes it.
 *   bun scripts/flake-hunt.ts quarantine
 *       Puts every file an open `Flaky: test/….test.ts` issue the hunt filed
 *       (author is the token's user, not untrusted) names into
@@ -30,12 +36,16 @@
import {
  huntedFile,
  HUNT_RUNS,
  issueFor,
  redActions,
  redFile,
  redIssueFor,
  type OpenRed,
  quarantineFlaky,
  readRun,
  shuffled,
  type FileRun,
} from "./flake.ts";
import { gateFiles, presentTests } from "./gate.ts";
import { gateFiles, outsideFiles, presentTests } from "./gate.ts";
import { parseShardSpec, sliceFiles } from "./shards.ts";
const GATE_FILE = new URL("./gate.ts", import.meta.url).pathname;
@@ -66,7 +76,7 @@
function runsOf(fallback: number): number {
const freshSeed = (): number => crypto.getRandomValues(new Uint32Array(1))[0]!;
/** One file, one seed, one `bun test`: the gate's own unit. */
/** One file, one `bun test`: the gate's own unit (random order when seeded). */
async function runFile(file: string, seed: number): Promise<FileRun> {
  const env = { ...process.env };
  // The gate hands its children none of these (scripts/test-shards.ts).
@@ -84,8 +94,7 @@
async function runFile(file: string, seed: number): Promise<FileRun> {
      "test",
      "--timeout",
      "60000",
      "--randomize",
      `--seed=${seed}`,
      ...(seed === 0 ? [] : ["--randomize", `--seed=${seed}`]),
      `./${file}`,
    ],
    { stdin: "ignore", stdout: "pipe", stderr: "pipe", env },
@@ -171,6 +180,102 @@
async function openFlakyIssues(): Promise<Map<string, number>> {
  return found;
}
async function forgeJson(
  method: string,
  path: string,
  body: unknown,
): Promise<Response> {
  const { api, repo, token } = forgeApi();
  return fetch(`${api}/repos/${repo}${path}`, {
    method,
    headers: {
      authorization: `Bearer ${token}`,
      "content-type": "application/json",
    },
    body: JSON.stringify(body),
  });
}
/** Open `Red:` issues the scheduled run filed: file → issue. */
async function openRedIssues(): Promise<Map<string, OpenRed>> {
  const { api, repo, token } = forgeApi();
  const self = await huntUserId(api, token);
  const found = new Map<string, OpenRed>();
  let cursor: string | number | null = "";
  for (let page = 0; page < 100 && cursor !== null; page++) {
    const query = cursor === "" ? "" : `&cursor=${encodeURIComponent(cursor)}`;
    const res = await fetch(
      `${api}/repos/${repo}/issues?state=open&limit=100${query}`,
      { headers: { authorization: `Bearer ${token}` } },
    );
    if (!res.ok) fail(`listing issues: ${res.status}`);
    const body = (await res.json()) as {
      items?: {
        number?: unknown;
        title?: unknown;
        body?: unknown;
        authorId?: unknown;
        untrusted?: unknown;
      }[];
      next_cursor?: string | number | null;
    };
    for (const issue of body.items ?? []) {
      if (issue.authorId !== self || (issue.untrusted ?? false) !== false)
        continue;
      const file =
        typeof issue.title === "string" ? redFile(issue.title) : undefined;
      if (file !== undefined && Number.isInteger(issue.number))
        found.set(file, {
          number: issue.number as number,
          ...(typeof issue.body === "string" ? { body: issue.body } : {}),
        });
    }
    cursor = body.next_cursor ?? null;
  }
  return found;
}
/** The files outside the gate, one at a time; red ones reported to the forge. */
async function outside(): Promise<number> {
  const files = outsideFiles(presentTests());
  const fileIssues = rest.includes("--file-issues");
  const open = fileIssues ? await openRedIssues() : new Map<string, OpenRed>();
  const runs: FileRun[] = [];
  for (const file of files) {
    const run = await runFile(file, 0);
    console.log(`${run.green ? "green" : "red  "}: ${file}`);
    runs.push(run);
  }
  const where = process.env["FORGE_RUN_URL"] ?? "";
  for (const action of redActions(runs, open)) {
    if (!fileIssues) continue;
    if (action.kind === "open") {
      const res = await forgeJson("POST", "/issues", redIssueFor(action.run, where));
      const made = (await res.json().catch(() => ({}))) as { number?: unknown };
      if (!res.ok) fail(`filing the issue for ${action.run.file}: ${res.status}`);
      console.log(`  filed #${made.number} for ${action.run.file}`);
    } else if (action.kind === "update") {
      const { body } = redIssueFor(action.run, where);
      const res = await forgeJson("PATCH", `/issues/${action.number}`, { body });
      if (!res.ok) {
        const note = await forgeJson("POST", `/issues/${action.number}/comments`, { body });
        if (!note.ok) fail(`updating #${action.number}: ${note.status}`);
      }
      console.log(`  updated #${action.number} for ${action.run.file}`);
    } else {
      await forgeJson("POST", `/issues/${action.number}/comments`, {
        body: `\`${action.file}\` passed in the scheduled run${where ? ` (${where})` : ""}.`,
      });
      const res = await forgeJson("PATCH", `/issues/${action.number}`, { state: "closed" });
      if (!res.ok) fail(`closing #${action.number}: ${res.status}`);
      console.log(`  closed #${action.number} for ${action.file}`);
    }
  }
  const red = runs.filter((run) => !run.green).length;
  console.log(`${files.length} files outside the gate, ${red} red`);
  return 0;
}
async function hunt(): Promise<number> {
  const runs = runsOf(HUNT_RUNS);
  const spec = parseShardSpec(valueOf("--shard"));
@@ -263,10 +368,11 @@
async function admit(): Promise<number> {
  return admitted(results) ? 0 : 1;
}
if (mode === "hunt") process.exit(await hunt());
if (mode === "outside") process.exit(await outside());
else if (mode === "hunt") process.exit(await hunt());
else if (mode === "quarantine") process.exit(await quarantineAll());
else if (mode === "admit") process.exit(await admit());
else
  fail(
    "usage: flake-hunt.ts hunt [--runs N] [--shard N/M] [--file-issues] | quarantine | admit test/x.test.ts [--runs N]",
    "usage: flake-hunt.ts hunt [--runs N] [--shard N/M] [--file-issues] | outside [--file-issues] | quarantine | admit test/x.test.ts [--runs N]",
  );
@@ -21,6 +21,9 @@
export const HUNT_RUNS = 5;
/** The title prefix of the issue the hunt files; quarantine reads it back. */
export const FLAKY_TITLE = "Flaky: ";
/** The title prefix of the issue the scheduled run files for a red file. */
export const RED_TITLE = "Red: ";
/**
 * The least a test may bound a request or a spawn by, in wall-clock ms. A
 * loopback fetch on a loaded shared runner takes tens of milliseconds; a
@@ -321,3 +324,72 @@
export function shuffled(files: readonly string[], seed: number): string[] {
  }
  return out;
}
/** The first failing test's name as the issue words it. */
export function firstFailing(run: FileRun): string {
  return run.failing[0] !== undefined
    ? `\`${run.failing[0]}\``
    : "none named; the file failed outside a test";
}
/** The issue for a file outside the gate that is red. */
export function redIssueFor(
  run: FileRun,
  where: string,
): { title: string; body: string } {
  const fence = "```";
  return {
    title: `${RED_TITLE}${run.file}`,
    body: [
      `\`${run.file}\` is outside the gate and was red in the scheduled run. This issue closes itself when the file passes again.`,
      "",
      `- first failing test: ${firstFailing(run)}`,
      `- run: ${where || "(a local run)"}`,
      `- reproduce: \`bun test --timeout 60000 ./${run.file}\``,
      "",
      fence,
      run.text || "(no failure text captured)",
      fence,
    ].join("\n"),
  };
}
/** The file an issue titled `Red: …` names, or undefined. */
export function redFile(title: string): string | undefined {
  if (!title.startsWith(RED_TITLE)) return undefined;
  return flakyFile(`${FLAKY_TITLE}${title.slice(RED_TITLE.length)}`);
}
/** An open `Red:` issue as the scheduled run sees it. */
export type OpenRed = { number: number; body?: string };
export type RedAction =
  | { kind: "open"; run: FileRun }
  | { kind: "update"; run: FileRun; number: number }
  | { kind: "close"; file: string; number: number };
/**
 * What the forge must be told after a scheduled run: a newly red file opens
 * one issue; a file still red whose first failing test is not in its issue
 * updates that issue; a file green again closes it. A file that stays red
 * on the same test says nothing, so the asks show it once.
 */
export function redActions(
  runs: readonly FileRun[],
  open: ReadonlyMap<string, OpenRed>,
): RedAction[] {
  const actions: RedAction[] = [];
  for (const run of runs) {
    const issue = open.get(run.file);
    if (run.green) {
      if (issue)
        actions.push({ kind: "close", file: run.file, number: issue.number });
    } else if (!issue) actions.push({ kind: "open", run });
    else if (
      typeof issue.body === "string" &&
      !issue.body.includes(firstFailing(run))
    )
      actions.push({ kind: "update", run, number: issue.number });
  }
  return actions;
}
@@ -15,6 +15,9 @@
 * order under load (`bun scripts/flake-hunt.ts admit <file>`). A flaky file
 * leaves it only through `bun scripts/flake-hunt.ts quarantine`, under the
 * issue the nightly hunt filed (`.forgejo/workflows/flake-hunt.yml`).
 *
 * The excluded files are not unseen: `.forgejo/workflows/outside-gate.yml`
 * runs `outsideFiles` nightly and files a `Red:` issue for each that fails (#812).
 */
import { readdirSync } from "node:fs";
@@ -135,6 +138,20 @@
export function gateFiles(
  return present.filter((file) => !out.has(file));
}
/**
 * What the scheduled run runs (`bun scripts/flake-hunt.ts outside`): the
 * present files `EXCLUDED` names, in name order. Derived from the same list
 * as `gateFiles`, so a file is in exactly one of the two: moving it into
 * the gate stops the schedule running it.
 */
export function outsideFiles(
  present: readonly string[] = presentTests(),
  excluded: readonly Exclusion[] = EXCLUDED,
): string[] {
  const gated = new Set(gateFiles(present, excluded));
  return present.filter((file) => !gated.has(file));
}
/**
 * The gate a checked-out tree's own `scripts/gate.ts` names, whichever shape
 * it has: a tree from before #2071 lists it in `GATE`; a later one derives
@@ -9,7 +9,7 @@
import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
import { readdirSync, readFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { auditGate, gateFiles, type Exclusion } from "../scripts/gate.ts";
import { auditGate, EXCLUDED, gateFiles, outsideFiles, presentTests, type Exclusion } from "../scripts/gate.ts";
import {
  admissionRecord,
  admitted,
@@ -22,6 +22,9 @@
import {
  issueFor,
  quarantine,
  quarantineFlaky,
  redActions,
  redFile,
  redIssueFor,
  readRun,
  retries,
  ROOT_CAUSES,
@@ -346,3 +349,60 @@
describe("no tight wall-clock bound in a test", () => {
    expect(files.flatMap((path) => tightBounds(path, read(path)))).toEqual([]);
  });
});
describe("the scheduled run of the files outside the gate", () => {
  const present = ["test/a.test.ts", "test/b.test.ts", "test/c.test.ts"];
  const excluded: Exclusion[] = [
    { file: "test/b.test.ts", reason: "r", issue: 1 },
  ];
  const run = (file: string, failing: string[] = []) =>
    readRun(file, 0, failing.length ? 1 : 0, failing.map((f) => `(fail) ${f}`).join("\n"));
  test("it runs exactly the files the gate does not", () => {
    expect(outsideFiles(present, excluded)).toEqual(["test/b.test.ts"]);
    const both = [...gateFiles(present, excluded), ...outsideFiles(present, excluded)];
    expect(both.sort()).toEqual(present);
    // A file moved into the gate is no longer run by the schedule.
    expect(outsideFiles(present, [])).toEqual([]);
  });
  test("the real lists partition the test files", () => {
    const all = presentTests();
    expect(outsideFiles(all)).toEqual(EXCLUDED.map((e) => e.file).sort());
    expect(new Set([...gateFiles(all), ...outsideFiles(all)]).size).toBe(all.length);
  });
  test("a newly red file opens one issue naming it and its first failing test", () => {
    const red = run("test/b.test.ts", ["first one", "second one"]);
    const actions = redActions([red], new Map());
    expect(actions).toEqual([{ kind: "open", run: red }]);
    const issue = redIssueFor(red, "http://run/1");
    expect(issue.title).toBe("Red: test/b.test.ts");
    expect(redFile(issue.title)).toBe("test/b.test.ts");
    expect(issue.body).toContain("`first one`");
    expect(issue.body).not.toContain("second one` ");
  });
  test("a file still red on the same test says nothing; on another it updates", () => {
    const red = run("test/b.test.ts", ["first one"]);
    const body = redIssueFor(red, "").body;
    const open = new Map([["test/b.test.ts", { number: 7, body }]]);
    expect(redActions([red], open)).toEqual([]);
    const other = run("test/b.test.ts", ["another"]);
    expect(redActions([other], open)).toEqual([{ kind: "update", run: other, number: 7 }]);
  });
  test("a file green again closes its issue; green with no issue does nothing", () => {
    const green = run("test/b.test.ts");
    expect(redActions([green], new Map([["test/b.test.ts", { number: 7 }]]))).toEqual([
      { kind: "close", file: "test/b.test.ts", number: 7 },
    ]);
    expect(redActions([green], new Map())).toEqual([]);
  });
  test("only Red: titles naming a test file count", () => {
    expect(redFile("Red: ../x.test.ts")).toBeUndefined();
    expect(redFile("Flaky: test/a.test.ts")).toBeUndefined();
    expect(redFile("Red: test/a.test.ts")).toBe("test/a.test.ts");
  });
});
		@@ -0,0 +1,38 @@
	1	name: outside-gate
	2	# Nightly: every file in EXCLUDED (scripts/gate.ts), one at a time with the
	3	# gate's 60 s timeout, so a suite outside the gate that breaks is seen the
	4	# day it breaks. A newly red file opens one `Red: test/….test.ts` issue
	5	# naming its first failing test; a file green again closes it
	6	# (scripts/flake-hunt.ts outside). The list is EXCLUDED itself: a file moved
	7	# into the gate stops being run here.
	8	#
	9	# Needs secret FORGE_TOKEN and variable FORGE_URL, as flake-hunt does.
	10	on:
	11	schedule:
	12	- cron: "47 3 * * *"
	13	workflow_dispatch: {}
	14
	15	jobs:
	16	outside:
	17	runs-on: isolated
	18	timeout-minutes: 30
	19	env:
	20	REGISTRY: forgejo-http.forgejo.svc.cluster.local:3000
	21	steps:
	22	- name: clone main
	23	env:
	24	TOKEN: ${{ github.token }}
	25	run: \|
	26	set -eu
	27	git clone -q "http://x-access-token:$TOKEN@$REGISTRY/$GITHUB_REPOSITORY.git" src
	28	cd src
	29	bun install --frozen-lockfile
	30
	31	- name: run the excluded files
	32	working-directory: src
	33	env:
	34	FORGE_API: ${{ vars.FORGE_URL }}/api/v1
	35	FORGE_REPO: ${{ github.repository }}
	36	FORGE_TOKEN: ${{ secrets.FORGE_TOKEN }}
	37	FORGE_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_number }}
	38	run: bun scripts/flake-hunt.ts outside --file-issues
		@@ -1282,6 +1282,16 @@ takes tens of ms): assert relative to a baseline from the same run, or bound
1282	1282	only the failure the test names. The workflow needs a
1283	1283	`FORGE_TOKEN` secret and a `FORGE_URL` variable on the mirror.
1284	1284
	1285	No unseen red outside the gate. Nightly, `.forgejo/workflows/outside-gate.yml`
	1286	runs `bun scripts/flake-hunt.ts outside --file-issues`: every file in
	1287	`EXCLUDED` (the complement of the gate, from the same list, so a file moved
	1288	into the gate is no longer run there), once, one at a time, with the gate's
	1289	60 s timeout. A file that newly fails opens one `Red: test/x.test.ts` issue
	1290	naming its first failing test; still red on another test, it updates that
	1291	issue; still red on the same test, it says nothing, so the owner's asks show
	1292	it once; passing again, it closes the issue. Same `FORGE_TOKEN` and
	1293	`FORGE_URL` as the hunt.
	1294
1285	1295	Bun's own `--randomize --seed=N` drives every hunt and admission run. Not
1286	1296	used: `--rerun-each` (one seed for every rerun, so ten reruns explore one
1287	1297	order) and `--parallel` (it changes the load the gate is judged under).
		@@ -9,6 +9,12 @@
9	9	* `--file-issues` each gets an issue carrying the failure text, the
10	10	* seed and the run (FORGE_API, FORGE_REPO, FORGE_TOKEN; FORGE_RUN_URL
11	11	* names the run). Exits 1 when it found one.
	12	* bun scripts/flake-hunt.ts outside [--file-issues]
	13	* Every file in EXCLUDED once, one at a time, with the gate's timeout:
	14	* the schedule for what the gate does not run (#812). With
	15	* `--file-issues` a newly red file opens one `Red: test/….test.ts`
	16	* issue naming its first failing test, a file red on another test
	17	* updates it, and a file green again closes it.
12	18	* bun scripts/flake-hunt.ts quarantine
13	19	* Puts every file an open `Flaky: test/….test.ts` issue the hunt filed
14	20	* (author is the token's user, not untrusted) names into
		@@ -30,12 +36,16 @@ import {
30	36	huntedFile,
31	37	HUNT_RUNS,
32	38	issueFor,
	39	redActions,
	40	redFile,
	41	redIssueFor,
	42	type OpenRed,
33	43	quarantineFlaky,
34	44	readRun,
35	45	shuffled,
36	46	type FileRun,
37	47	} from "./flake.ts";
38		import { gateFiles, presentTests } from "./gate.ts";
	48	import { gateFiles, outsideFiles, presentTests } from "./gate.ts";
39	49	import { parseShardSpec, sliceFiles } from "./shards.ts";
40	50
41	51	const GATE_FILE = new URL("./gate.ts", import.meta.url).pathname;
		@@ -66,7 +76,7 @@ function runsOf(fallback: number): number {
66	76
67	77	const freshSeed = (): number => crypto.getRandomValues(new Uint32Array(1))[0]!;
68	78
69		/** One file, one seed, one `bun test`: the gate's own unit. */
	79	/** One file, one `bun test`: the gate's own unit (random order when seeded). */
70	80	async function runFile(file: string, seed: number): Promise<FileRun> {
71	81	const env = { ...process.env };
72	82	// The gate hands its children none of these (scripts/test-shards.ts).
		@@ -84,8 +94,7 @@ async function runFile(file: string, seed: number): Promise<FileRun> {
84	94	"test",
85	95	"--timeout",
86	96	"60000",
87		"--randomize",
88		`--seed=${seed}`,
	97	...(seed === 0 ? [] : ["--randomize", `--seed=${seed}`]),
89	98	`./${file}`,
90	99	],
91	100	{ stdin: "ignore", stdout: "pipe", stderr: "pipe", env },
		@@ -171,6 +180,102 @@ async function openFlakyIssues(): Promise<Map<string, number>> {
171	180	return found;
172	181	}
173	182
	183	async function forgeJson(
	184	method: string,
	185	path: string,
	186	body: unknown,
	187	): Promise<Response> {
	188	const { api, repo, token } = forgeApi();
	189	return fetch(`${api}/repos/${repo}${path}`, {
	190	method,
	191	headers: {
	192	authorization: `Bearer ${token}`,
	193	"content-type": "application/json",
	194	},
	195	body: JSON.stringify(body),
	196	});
	197	}
	198
	199	/** Open `Red:` issues the scheduled run filed: file → issue. */
	200	async function openRedIssues(): Promise<Map<string, OpenRed>> {
	201	const { api, repo, token } = forgeApi();
	202	const self = await huntUserId(api, token);
	203	const found = new Map<string, OpenRed>();
	204	let cursor: string \| number \| null = "";
	205	for (let page = 0; page < 100 && cursor !== null; page++) {
	206	const query = cursor === "" ? "" : `&cursor=${encodeURIComponent(cursor)}`;
	207	const res = await fetch(
	208	`${api}/repos/${repo}/issues?state=open&limit=100${query}`,
	209	{ headers: { authorization: `Bearer ${token}` } },
	210	);
	211	if (!res.ok) fail(`listing issues: ${res.status}`);
	212	const body = (await res.json()) as {
	213	items?: {
	214	number?: unknown;
	215	title?: unknown;
	216	body?: unknown;
	217	authorId?: unknown;
	218	untrusted?: unknown;
	219	}[];
	220	next_cursor?: string \| number \| null;
	221	};
	222	for (const issue of body.items ?? []) {
	223	if (issue.authorId !== self \|\| (issue.untrusted ?? false) !== false)
	224	continue;
	225	const file =
	226	typeof issue.title === "string" ? redFile(issue.title) : undefined;
	227	if (file !== undefined && Number.isInteger(issue.number))
	228	found.set(file, {
	229	number: issue.number as number,
	230	...(typeof issue.body === "string" ? { body: issue.body } : {}),
	231	});
	232	}
	233	cursor = body.next_cursor ?? null;
	234	}
	235	return found;
	236	}
	237
	238	/** The files outside the gate, one at a time; red ones reported to the forge. */
	239	async function outside(): Promise<number> {
	240	const files = outsideFiles(presentTests());
	241	const fileIssues = rest.includes("--file-issues");
	242	const open = fileIssues ? await openRedIssues() : new Map<string, OpenRed>();
	243	const runs: FileRun[] = [];
	244	for (const file of files) {
	245	const run = await runFile(file, 0);
	246	console.log(`${run.green ? "green" : "red "}: ${file}`);
	247	runs.push(run);
	248	}
	249	const where = process.env["FORGE_RUN_URL"] ?? "";
	250	for (const action of redActions(runs, open)) {
	251	if (!fileIssues) continue;
	252	if (action.kind === "open") {
	253	const res = await forgeJson("POST", "/issues", redIssueFor(action.run, where));
	254	const made = (await res.json().catch(() => ({}))) as { number?: unknown };
	255	if (!res.ok) fail(`filing the issue for ${action.run.file}: ${res.status}`);
	256	console.log(` filed #${made.number} for ${action.run.file}`);
	257	} else if (action.kind === "update") {
	258	const { body } = redIssueFor(action.run, where);
	259	const res = await forgeJson("PATCH", `/issues/${action.number}`, { body });
	260	if (!res.ok) {
	261	const note = await forgeJson("POST", `/issues/${action.number}/comments`, { body });
	262	if (!note.ok) fail(`updating #${action.number}: ${note.status}`);
	263	}
	264	console.log(` updated #${action.number} for ${action.run.file}`);
	265	} else {
	266	await forgeJson("POST", `/issues/${action.number}/comments`, {
	267	body: `\`${action.file}\` passed in the scheduled run${where ? ` (${where})` : ""}.`,
	268	});
	269	const res = await forgeJson("PATCH", `/issues/${action.number}`, { state: "closed" });
	270	if (!res.ok) fail(`closing #${action.number}: ${res.status}`);
	271	console.log(` closed #${action.number} for ${action.file}`);
	272	}
	273	}
	274	const red = runs.filter((run) => !run.green).length;
	275	console.log(`${files.length} files outside the gate, ${red} red`);
	276	return 0;
	277	}
	278
174	279	async function hunt(): Promise<number> {
175	280	const runs = runsOf(HUNT_RUNS);
176	281	const spec = parseShardSpec(valueOf("--shard"));
		@@ -263,10 +368,11 @@ async function admit(): Promise<number> {
263	368	return admitted(results) ? 0 : 1;
264	369	}
265	370
266		if (mode === "hunt") process.exit(await hunt());
	371	if (mode === "outside") process.exit(await outside());
	372	else if (mode === "hunt") process.exit(await hunt());
267	373	else if (mode === "quarantine") process.exit(await quarantineAll());
268	374	else if (mode === "admit") process.exit(await admit());
269	375	else
270	376	fail(
271		"usage: flake-hunt.ts hunt [--runs N] [--shard N/M] [--file-issues] \| quarantine \| admit test/x.test.ts [--runs N]",
	377	"usage: flake-hunt.ts hunt [--runs N] [--shard N/M] [--file-issues] \| outside [--file-issues] \| quarantine \| admit test/x.test.ts [--runs N]",
272	378	);
		@@ -21,6 +21,9 @@ export const HUNT_RUNS = 5;
21	21	/** The title prefix of the issue the hunt files; quarantine reads it back. */
22	22	export const FLAKY_TITLE = "Flaky: ";
23	23
	24	/** The title prefix of the issue the scheduled run files for a red file. */
	25	export const RED_TITLE = "Red: ";
	26
24	27	/**
25	28	* The least a test may bound a request or a spawn by, in wall-clock ms. A
26	29	* loopback fetch on a loaded shared runner takes tens of milliseconds; a
		@@ -321,3 +324,72 @@ export function shuffled(files: readonly string[], seed: number): string[] {
321	324	}
322	325	return out;
323	326	}
	327
	328	/** The first failing test's name as the issue words it. */
	329	export function firstFailing(run: FileRun): string {
	330	return run.failing[0] !== undefined
	331	? `\`${run.failing[0]}\``
	332	: "none named; the file failed outside a test";
	333	}
	334
	335	/** The issue for a file outside the gate that is red. */
	336	export function redIssueFor(
	337	run: FileRun,
	338	where: string,
	339	): { title: string; body: string } {
	340	const fence = "```";
	341	return {
	342	title: `${RED_TITLE}${run.file}`,
	343	body: [
	344	`\`${run.file}\` is outside the gate and was red in the scheduled run. This issue closes itself when the file passes again.`,
	345	"",
	346	`- first failing test: ${firstFailing(run)}`,
	347	`- run: ${where \|\| "(a local run)"}`,
	348	`- reproduce: \`bun test --timeout 60000 ./${run.file}\``,
	349	"",
	350	fence,
	351	run.text \|\| "(no failure text captured)",
	352	fence,
	353	].join("\n"),
	354	};
	355	}
	356
	357	/** The file an issue titled `Red: …` names, or undefined. */
	358	export function redFile(title: string): string \| undefined {
	359	if (!title.startsWith(RED_TITLE)) return undefined;
	360	return flakyFile(`${FLAKY_TITLE}${title.slice(RED_TITLE.length)}`);
	361	}
	362
	363	/** An open `Red:` issue as the scheduled run sees it. */
	364	export type OpenRed = { number: number; body?: string };
	365
	366	export type RedAction =
	367	\| { kind: "open"; run: FileRun }
	368	\| { kind: "update"; run: FileRun; number: number }
	369	\| { kind: "close"; file: string; number: number };
	370
	371	/**
	372	* What the forge must be told after a scheduled run: a newly red file opens
	373	* one issue; a file still red whose first failing test is not in its issue
	374	* updates that issue; a file green again closes it. A file that stays red
	375	* on the same test says nothing, so the asks show it once.
	376	*/
	377	export function redActions(
	378	runs: readonly FileRun[],
	379	open: ReadonlyMap<string, OpenRed>,
	380	): RedAction[] {
	381	const actions: RedAction[] = [];
	382	for (const run of runs) {
	383	const issue = open.get(run.file);
	384	if (run.green) {
	385	if (issue)
	386	actions.push({ kind: "close", file: run.file, number: issue.number });
	387	} else if (!issue) actions.push({ kind: "open", run });
	388	else if (
	389	typeof issue.body === "string" &&
	390	!issue.body.includes(firstFailing(run))
	391	)
	392	actions.push({ kind: "update", run, number: issue.number });
	393	}
	394	return actions;
	395	}
		@@ -15,6 +15,9 @@
15	15	* order under load (`bun scripts/flake-hunt.ts admit <file>`). A flaky file
16	16	* leaves it only through `bun scripts/flake-hunt.ts quarantine`, under the
17	17	* issue the nightly hunt filed (`.forgejo/workflows/flake-hunt.yml`).
	18	*
	19	* The excluded files are not unseen: `.forgejo/workflows/outside-gate.yml`
	20	* runs `outsideFiles` nightly and files a `Red:` issue for each that fails (#812).
18	21	*/
19	22
20	23	import { readdirSync } from "node:fs";
		@@ -135,6 +138,20 @@ export function gateFiles(
135	138	return present.filter((file) => !out.has(file));
136	139	}
137	140
	141	/**
	142	* What the scheduled run runs (`bun scripts/flake-hunt.ts outside`): the
	143	* present files `EXCLUDED` names, in name order. Derived from the same list
	144	* as `gateFiles`, so a file is in exactly one of the two: moving it into
	145	* the gate stops the schedule running it.
	146	*/
	147	export function outsideFiles(
	148	present: readonly string[] = presentTests(),
	149	excluded: readonly Exclusion[] = EXCLUDED,
	150	): string[] {
	151	const gated = new Set(gateFiles(present, excluded));
	152	return present.filter((file) => !gated.has(file));
	153	}
	154
138	155	/**
139	156	* The gate a checked-out tree's own `scripts/gate.ts` names, whichever shape
140	157	* it has: a tree from before #2071 lists it in `GATE`; a later one derives
		@@ -9,7 +9,7 @@ import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
9	9	import { readdirSync, readFileSync } from "node:fs";
10	10	import { tmpdir } from "node:os";
11	11	import { join } from "node:path";
12		import { auditGate, gateFiles, type Exclusion } from "../scripts/gate.ts";
	12	import { auditGate, EXCLUDED, gateFiles, outsideFiles, presentTests, type Exclusion } from "../scripts/gate.ts";
13	13	import {
14	14	admissionRecord,
15	15	admitted,
		@@ -22,6 +22,9 @@ import {
22	22	issueFor,
23	23	quarantine,
24	24	quarantineFlaky,
	25	redActions,
	26	redFile,
	27	redIssueFor,
25	28	readRun,
26	29	retries,
27	30	ROOT_CAUSES,
		@@ -346,3 +349,60 @@ describe("no tight wall-clock bound in a test", () => {
346	349	expect(files.flatMap((path) => tightBounds(path, read(path)))).toEqual([]);
347	350	});
348	351	});
	352
	353	describe("the scheduled run of the files outside the gate", () => {
	354	const present = ["test/a.test.ts", "test/b.test.ts", "test/c.test.ts"];
	355	const excluded: Exclusion[] = [
	356	{ file: "test/b.test.ts", reason: "r", issue: 1 },
	357	];
	358	const run = (file: string, failing: string[] = []) =>
	359	readRun(file, 0, failing.length ? 1 : 0, failing.map((f) => `(fail) ${f}`).join("\n"));
	360
	361	test("it runs exactly the files the gate does not", () => {
	362	expect(outsideFiles(present, excluded)).toEqual(["test/b.test.ts"]);
	363	const both = [...gateFiles(present, excluded), ...outsideFiles(present, excluded)];
	364	expect(both.sort()).toEqual(present);
	365	// A file moved into the gate is no longer run by the schedule.
	366	expect(outsideFiles(present, [])).toEqual([]);
	367	});
	368
	369	test("the real lists partition the test files", () => {
	370	const all = presentTests();
	371	expect(outsideFiles(all)).toEqual(EXCLUDED.map((e) => e.file).sort());
	372	expect(new Set([...gateFiles(all), ...outsideFiles(all)]).size).toBe(all.length);
	373	});
	374
	375	test("a newly red file opens one issue naming it and its first failing test", () => {
	376	const red = run("test/b.test.ts", ["first one", "second one"]);
	377	const actions = redActions([red], new Map());
	378	expect(actions).toEqual([{ kind: "open", run: red }]);
	379	const issue = redIssueFor(red, "http://run/1");
	380	expect(issue.title).toBe("Red: test/b.test.ts");
	381	expect(redFile(issue.title)).toBe("test/b.test.ts");
	382	expect(issue.body).toContain("`first one`");
	383	expect(issue.body).not.toContain("second one` ");
	384	});
	385
	386	test("a file still red on the same test says nothing; on another it updates", () => {
	387	const red = run("test/b.test.ts", ["first one"]);
	388	const body = redIssueFor(red, "").body;
	389	const open = new Map([["test/b.test.ts", { number: 7, body }]]);
	390	expect(redActions([red], open)).toEqual([]);
	391	const other = run("test/b.test.ts", ["another"]);
	392	expect(redActions([other], open)).toEqual([{ kind: "update", run: other, number: 7 }]);
	393	});
	394
	395	test("a file green again closes its issue; green with no issue does nothing", () => {
	396	const green = run("test/b.test.ts");
	397	expect(redActions([green], new Map([["test/b.test.ts", { number: 7 }]]))).toEqual([
	398	{ kind: "close", file: "test/b.test.ts", number: 7 },
	399	]);
	400	expect(redActions([green], new Map())).toEqual([]);
	401	});
	402
	403	test("only Red: titles naming a test file count", () => {
	404	expect(redFile("Red: ../x.test.ts")).toBeUndefined();
	405	expect(redFile("Flaky: test/a.test.ts")).toBeUndefined();
	406	expect(redFile("Red: test/a.test.ts")).toBe("test/a.test.ts");
	407	});
	408	});