fa0b994
@@ -205,6 +205,13 @@
A `check` that is present and wrong fails with the reason, so a commit is
never silently unchecked. A check under way when the forge restarts is
queued again, and the forge that carries on runs it from the start.
Checks share `FORGE_CHECKS_CONCURRENCY` slots (default `1`), and when one
frees, it goes to the most recently requested check waiting, not the one
queued longest — a fresh push is never stuck behind pulls that queued hours
earlier. The one exception is starvation: a check that has waited ten
minutes to start outranks a newer one regardless, so nothing waits forever.
A pull's status names how long its queued check has been waiting.
### Environment
What an app needs that the repository must not carry — an admin password,
@@ -62,6 +62,44 @@
export interface ChecksHandle {
/** How often queued checks another forge left are looked for. */
const TICK_MS = 5_000;
/**
 * How long a queued check may wait before it outranks a newer one however
 * fresh: the bound that keeps `pickNext` from starving a pull nobody has
 * pushed to in a while.
 */
export const STARVATION_MS = 10 * 60 * 1000;
/**
 * Which queued check takes a slot when one frees: the one requested most
 * recently, so a push's own check is never stuck behind pulls that queued
 * hours earlier — unless something has already waited `STARVATION_MS` or
 * more, in which case the longest-waiting check goes first regardless of
 * how fresh the rest are. Ties (down to the same instant) fall back to id
 * order, which is itself creation order (`kernel/id.ts`).
 */
export function pickNext(
  queued: readonly Run[],
  now: number = Date.now(),
): Run | undefined {
  if (queued.length === 0) return undefined;
  let oldest = queued[0]!;
  let newest = queued[0]!;
  for (const run of queued) {
    if (
      run.createdAt < oldest.createdAt ||
      (run.createdAt === oldest.createdAt && run.id < oldest.id)
    )
      oldest = run;
    if (
      run.createdAt > newest.createdAt ||
      (run.createdAt === newest.createdAt && run.id > newest.id)
    )
      newest = run;
  }
  if (now - Date.parse(oldest.createdAt) >= STARVATION_MS) return oldest;
  return newest;
}
/** Where a check's checkout lives while it runs. */
const checksRoot = (dataDir: string): string => join(dataDir, "checks");
@@ -72,7 +110,8 @@
export const checksFor = (app: App): ChecksHandle | undefined =>
  handles.get(app);
export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
  const gate = new Gate(options.concurrency ?? 1);
  const capacity = options.concurrency ?? 1;
  const gate = new Gate(capacity);
  const inFlight = new Map<
    string,
    { promise: Promise<void>; abort: AbortController }
@@ -116,7 +155,7 @@
export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
      ...(actorId === undefined ? {} : { actorId }),
    });
    announceRun(app, run, actorId);
    start(run, repo);
    schedule();
    return run;
  }
@@ -131,20 +170,33 @@
export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
          error: String(error),
        }),
      )
      .finally(() => inFlight.delete(run.id));
      .finally(() => {
        inFlight.delete(run.id);
        // The slot this run held is free again: give it to whichever queued
        // check now deserves it most, not whoever asked first.
        schedule();
      });
    inFlight.set(run.id, { promise, abort });
  }
  /**
   * Queued checks are whichever forge's picks them up: one the forge before
   * this one never reached, or one a forge stopping beside this one handed
   * back. Claiming is what decides, so two forges never run the same one.
   * Fill every free slot with the queued check `pickNext` says deserves it —
   * one the forge before this one never reached, one a forge stopping
   * beside this one handed back, or simply the newest push once a slot
   * opens up. Claiming is what decides between two forges, so calling this
   * from both never runs the same check twice.
   */
  function pickUp(): void {
    for (const run of listActive(app.db, "check")) {
      if (run.state !== "queued" || inFlight.has(run.id)) continue;
      const repo = getRepoById(app.db, run.repoId);
      if (repo) start(run, repo);
  function schedule(): void {
    if (stopped) return;
    while (inFlight.size < capacity) {
      const queued = listActive(app.db, "check").filter(
        (run) => run.state === "queued" && !inFlight.has(run.id),
      );
      const next = pickNext(queued);
      if (!next) return;
      const repo = getRepoById(app.db, next.repoId);
      if (!repo) return;
      start(next, repo);
    }
  }
@@ -406,9 +458,9 @@
export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
  }
  const timer = setInterval(() => {
    if (!stopped) pickUp();
    if (!stopped) schedule();
  }, TICK_MS);
  pickUp();
  schedule();
  void catchUp();
  const handle: ChecksHandle = {
@@ -1039,7 +1039,9 @@
export function checksText(
    case "running":
      return "checks: deploying this head";
    case "queued":
      return "checks: a deploy of this head is queued";
      return `checks: a deploy of this head is queued${
        run === null ? "" : ` (queued ${relTime(run.createdAt)})`
      }`;
    case "cancelled":
      return `checks: the deploy was cancelled${summary}`;
  }
@@ -72,6 +72,7 @@
import {
  labelCss,
  listCard,
  originTag,
  queuedPhrase,
  sha,
  stateBadge,
  time,
@@ -226,12 +227,17 @@
function checksLine(
      ? html`<span class="ok">${icon("check")}Checks passed</span>${link}`
      : check.state === "failed"
        ? html`<span class="bad">${icon("x")}Checks failed</span>${link}`
        : check.state === "running" || check.state === "queued"
          ? html`<span class="now">${icon("loader")}Checks ${check.state}</span
              >${link}`
          : check.state === "cancelled"
            ? html`<span>${icon("ban")}Checks cancelled</span>${link}`
            : html`<span>${icon("clock")}No checks on this head yet</span>`;
        : check.state === "queued"
          ? html`<span class="now"
              >${icon("loader")}Checks
              ${run ? queuedPhrase(run.createdAt) : "queued"}</span
            >${link}`
          : check.state === "running"
            ? html`<span class="now">${icon("loader")}Checks running</span
                >${link}`
            : check.state === "cancelled"
              ? html`<span>${icon("ban")}Checks cancelled</span>${link}`
              : html`<span>${icon("clock")}No checks on this head yet</span>`;
  return html`<p class="checks-line">
    ${body}${when(
      required,
@@ -65,6 +65,22 @@
export function servedPhrase(env: EnvStatus): string | undefined {
  return `${(ms / 1000).toFixed(1)}s after the push`;
}
/**
 * How long a queued run has waited, worded like the run list's own line
 * (`queued 12m ago`) but compact enough to sit beside a state word: `queued
 * 12m`. `just now` under a minute, so a check that only just queued never
 * reads as `queued 0m`.
 */
export function queuedPhrase(createdAt: string, now = Date.now()): string {
  const seconds = Math.max(0, (now - Date.parse(createdAt)) / 1000);
  if (seconds < 60) return "queued just now";
  const minutes = Math.floor(seconds / 60);
  if (minutes < 60) return `queued ${minutes}m`;
  const hours = Math.floor(minutes / 60);
  if (hours < 24) return `queued ${hours}h`;
  return `queued ${Math.floor(hours / 24)}d`;
}
/** Bytes, three significant figures, never more precision than is true. */
export function fileSize(bytes: number): string {
  if (bytes < 1024) return `${bytes} B`;
@@ -11,8 +11,20 @@
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { openPull } from "../src/actions/pulls.ts";
import { commitToBranch, createRepo, forkRepo } from "../src/actions/repos.ts";
import { startChecks, type ChecksHandle } from "../src/checks/service.ts";
import { checksForHead, listRuns, type Run } from "../src/core/runs.ts";
import {
  pickNext,
  startChecks,
  STARVATION_MS,
  type ChecksHandle,
} from "../src/checks/service.ts";
import {
  cancelRun,
  checksForHead,
  createRun,
  getRun,
  listRuns,
  type Run,
} from "../src/core/runs.ts";
import { createUser, type User } from "../src/core/users.ts";
import type { Repo } from "../src/core/repos.ts";
import { serve, type Servers } from "../src/http/server.ts";
@@ -225,6 +237,106 @@
describe("checks", () => {
    expect(middle?.summary).toContain("superseded by");
  }, 60_000);
  test("pickNext takes the newest queued check, unless one has starved", () => {
    const a = createRun(app.db, { repoId: repo.id, kind: "check", env: "a" });
    const b = createRun(app.db, { repoId: repo.id, kind: "check", env: "b" });
    const c = createRun(app.db, { repoId: repo.id, kind: "check", env: "c" });
    // ids are creation-ordered, so this is oldest-to-newest already.
    expect(pickNext([a, b, c])?.id).toBe(c.id);
    expect(pickNext([c, a, b])?.id).toBe(c.id);
    expect(pickNext([a])?.id).toBe(a.id);
    expect(pickNext([])).toBeUndefined();
    // `a` has waited past the starvation bound: it outranks `c`, however new.
    app.db
      .query("UPDATE run SET created_at = ?2 WHERE id = ?1")
      .run(a.id, new Date(Date.now() - STARVATION_MS - 1_000).toISOString());
    const starved = getRun(app.db, a.id)!;
    expect(pickNext([starved, b, c])?.id).toBe(starved.id);
    // Just short of the bound, the newest still wins.
    app.db
      .query("UPDATE run SET created_at = ?2 WHERE id = ?1")
      .run(a.id, new Date(Date.now() - STARVATION_MS + 1_000).toISOString());
    expect(pickNext([getRun(app.db, a.id)!, b, c])?.id).toBe(c.id);
    // These never named a real commit: cancel them rather than leave them
    // for the live checks service running beside this test to trip over.
    for (const run of [a, b, c]) cancelRun(app.db, run.id);
  });
  test("when a slot frees, the newest of several queued checks runs first", async () => {
    // A slow check on its own environment holds the only slot.
    const held = await commitToBranch(app, alice, repo, {
      branch: "order-hold",
      message: "Hold the slot",
      files: [
        { path: "forge.json", content: checkJson("sleep 2 && test -f OK.md") },
      ],
    });
    if (!held.ok) throw new Error(JSON.stringify(held.error));
    const holder = await checks.request(
      repo,
      "order-hold",
      held.value.commitOid,
      "push",
    );
    expect(holder).toBeDefined();
    // Three more, on three environments so none supersedes another, queue
    // behind it — oldest requested first.
    const oldest = await commitToBranch(app, alice, repo, {
      branch: "order-a",
      message: "A",
      files: [{ path: "forge.json", content: checkJson("test -f OK.md") }],
    });
    if (!oldest.ok) throw new Error(JSON.stringify(oldest.error));
    const middle = await commitToBranch(app, alice, repo, {
      branch: "order-b",
      message: "B",
      files: [{ path: "forge.json", content: checkJson("test -f OK.md") }],
    });
    if (!middle.ok) throw new Error(JSON.stringify(middle.error));
    const newest = await commitToBranch(app, alice, repo, {
      branch: "order-c",
      message: "C",
      files: [{ path: "forge.json", content: checkJson("test -f OK.md") }],
    });
    if (!newest.ok) throw new Error(JSON.stringify(newest.error));
    const oldestRun = await checks.request(
      repo,
      "order-a",
      oldest.value.commitOid,
      "push",
    );
    const middleRun = await checks.request(
      repo,
      "order-b",
      middle.value.commitOid,
      "push",
    );
    const newestRun = await checks.request(
      repo,
      "order-c",
      newest.value.commitOid,
      "push",
    );
    expect(oldestRun && middleRun && newestRun).toBeTruthy();
    expect(oldestRun?.state).toBe("queued");
    expect(middleRun?.state).toBe("queued");
    expect(newestRun?.state).toBe("queued");
    await settled((r) => r.id === newestRun!.id);
    await settled((r) => r.id === middleRun!.id);
    await settled((r) => r.id === oldestRun!.id);
    const runs = listRuns(app.db, repo.id, { kind: "check", limit: 50 });
    const startedAt = (id: string): number =>
      Date.parse(runs.find((r) => r.id === id)!.startedAt!);
    expect(startedAt(newestRun!.id)).toBeLessThan(startedAt(middleRun!.id));
    expect(startedAt(middleRun!.id)).toBeLessThan(startedAt(oldestRun!.id));
  }, 30_000);
  test("a check in flight when the forge stops is queued again, and the next forge runs it", async () => {
    const slow = await commitToBranch(app, alice, repo, {
      message: "A slower check",
		@@ -205,6 +205,13 @@ A `check` that is present and wrong fails with the reason, so a commit is
205	205	never silently unchecked. A check under way when the forge restarts is
206	206	queued again, and the forge that carries on runs it from the start.
207	207
	208	Checks share `FORGE_CHECKS_CONCURRENCY` slots (default `1`), and when one
	209	frees, it goes to the most recently requested check waiting, not the one
	210	queued longest — a fresh push is never stuck behind pulls that queued hours
	211	earlier. The one exception is starvation: a check that has waited ten
	212	minutes to start outranks a newer one regardless, so nothing waits forever.
	213	A pull's status names how long its queued check has been waiting.
	214
208	215	### Environment
209	216
210	217	What an app needs that the repository must not carry — an admin password,
		@@ -62,6 +62,44 @@ export interface ChecksHandle {
62	62	/** How often queued checks another forge left are looked for. */
63	63	const TICK_MS = 5_000;
64	64
	65	/**
	66	* How long a queued check may wait before it outranks a newer one however
	67	* fresh: the bound that keeps `pickNext` from starving a pull nobody has
	68	* pushed to in a while.
	69	*/
	70	export const STARVATION_MS = 10 * 60 * 1000;
	71
	72	/**
	73	* Which queued check takes a slot when one frees: the one requested most
	74	* recently, so a push's own check is never stuck behind pulls that queued
	75	* hours earlier — unless something has already waited `STARVATION_MS` or
	76	* more, in which case the longest-waiting check goes first regardless of
	77	* how fresh the rest are. Ties (down to the same instant) fall back to id
	78	* order, which is itself creation order (`kernel/id.ts`).
	79	*/
	80	export function pickNext(
	81	queued: readonly Run[],
	82	now: number = Date.now(),
	83	): Run \| undefined {
	84	if (queued.length === 0) return undefined;
	85	let oldest = queued[0]!;
	86	let newest = queued[0]!;
	87	for (const run of queued) {
	88	if (
	89	run.createdAt < oldest.createdAt \|\|
	90	(run.createdAt === oldest.createdAt && run.id < oldest.id)
	91	)
	92	oldest = run;
	93	if (
	94	run.createdAt > newest.createdAt \|\|
	95	(run.createdAt === newest.createdAt && run.id > newest.id)
	96	)
	97	newest = run;
	98	}
	99	if (now - Date.parse(oldest.createdAt) >= STARVATION_MS) return oldest;
	100	return newest;
	101	}
	102
65	103	/** Where a check's checkout lives while it runs. */
66	104	const checksRoot = (dataDir: string): string => join(dataDir, "checks");
67	105
		@@ -72,7 +110,8 @@ export const checksFor = (app: App): ChecksHandle \| undefined =>
72	110	handles.get(app);
73	111
74	112	export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
75		const gate = new Gate(options.concurrency ?? 1);
	113	const capacity = options.concurrency ?? 1;
	114	const gate = new Gate(capacity);
76	115	const inFlight = new Map<
77	116	string,
78	117	{ promise: Promise<void>; abort: AbortController }
		@@ -116,7 +155,7 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
116	155	...(actorId === undefined ? {} : { actorId }),
117	156	});
118	157	announceRun(app, run, actorId);
119		start(run, repo);
	158	schedule();
120	159	return run;
121	160	}
122	161
		@@ -131,20 +170,33 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
131	170	error: String(error),
132	171	}),
133	172	)
134		.finally(() => inFlight.delete(run.id));
	173	.finally(() => {
	174	inFlight.delete(run.id);
	175	// The slot this run held is free again: give it to whichever queued
	176	// check now deserves it most, not whoever asked first.
	177	schedule();
	178	});
135	179	inFlight.set(run.id, { promise, abort });
136	180	}
137	181
138	182	/**
139		* Queued checks are whichever forge's picks them up: one the forge before
140		* this one never reached, or one a forge stopping beside this one handed
141		* back. Claiming is what decides, so two forges never run the same one.
	183	* Fill every free slot with the queued check `pickNext` says deserves it —
	184	* one the forge before this one never reached, one a forge stopping
	185	* beside this one handed back, or simply the newest push once a slot
	186	* opens up. Claiming is what decides between two forges, so calling this
	187	* from both never runs the same check twice.
142	188	*/
143		function pickUp(): void {
144		for (const run of listActive(app.db, "check")) {
145		if (run.state !== "queued" \|\| inFlight.has(run.id)) continue;
146		const repo = getRepoById(app.db, run.repoId);
147		if (repo) start(run, repo);
	189	function schedule(): void {
	190	if (stopped) return;
	191	while (inFlight.size < capacity) {
	192	const queued = listActive(app.db, "check").filter(
	193	(run) => run.state === "queued" && !inFlight.has(run.id),
	194	);
	195	const next = pickNext(queued);
	196	if (!next) return;
	197	const repo = getRepoById(app.db, next.repoId);
	198	if (!repo) return;
	199	start(next, repo);
148	200	}
149	201	}
150	202
		@@ -406,9 +458,9 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
406	458	}
407	459
408	460	const timer = setInterval(() => {
409		if (!stopped) pickUp();
	461	if (!stopped) schedule();
410	462	}, TICK_MS);
411		pickUp();
	463	schedule();
412	464	void catchUp();
413	465
414	466	const handle: ChecksHandle = {
		@@ -1039,7 +1039,9 @@ export function checksText(
1039	1039	case "running":
1040	1040	return "checks: deploying this head";
1041	1041	case "queued":
1042		return "checks: a deploy of this head is queued";
	1042	return `checks: a deploy of this head is queued${
	1043	run === null ? "" : ` (queued ${relTime(run.createdAt)})`
	1044	}`;
1043	1045	case "cancelled":
1044	1046	return `checks: the deploy was cancelled${summary}`;
1045	1047	}
		@@ -72,6 +72,7 @@ import {
72	72	labelCss,
73	73	listCard,
74	74	originTag,
	75	queuedPhrase,
75	76	sha,
76	77	stateBadge,
77	78	time,
		@@ -226,12 +227,17 @@ function checksLine(
226	227	? html`<span class="ok">${icon("check")}Checks passed</span>${link}`
227	228	: check.state === "failed"
228	229	? html`<span class="bad">${icon("x")}Checks failed</span>${link}`
229		: check.state === "running" \|\| check.state === "queued"
230		? html`<span class="now">${icon("loader")}Checks ${check.state}</span
231		>${link}`
232		: check.state === "cancelled"
233		? html`<span>${icon("ban")}Checks cancelled</span>${link}`
234		: html`<span>${icon("clock")}No checks on this head yet</span>`;
	230	: check.state === "queued"
	231	? html`<span class="now"
	232	>${icon("loader")}Checks
	233	${run ? queuedPhrase(run.createdAt) : "queued"}</span
	234	>${link}`
	235	: check.state === "running"
	236	? html`<span class="now">${icon("loader")}Checks running</span
	237	>${link}`
	238	: check.state === "cancelled"
	239	? html`<span>${icon("ban")}Checks cancelled</span>${link}`
	240	: html`<span>${icon("clock")}No checks on this head yet</span>`;
235	241	return html`<p class="checks-line">
236	242	${body}${when(
237	243	required,
		@@ -65,6 +65,22 @@ export function servedPhrase(env: EnvStatus): string \| undefined {
65	65	return `${(ms / 1000).toFixed(1)}s after the push`;
66	66	}
67	67
	68	/**
	69	* How long a queued run has waited, worded like the run list's own line
	70	* (`queued 12m ago`) but compact enough to sit beside a state word: `queued
	71	* 12m`. `just now` under a minute, so a check that only just queued never
	72	* reads as `queued 0m`.
	73	*/
	74	export function queuedPhrase(createdAt: string, now = Date.now()): string {
	75	const seconds = Math.max(0, (now - Date.parse(createdAt)) / 1000);
	76	if (seconds < 60) return "queued just now";
	77	const minutes = Math.floor(seconds / 60);
	78	if (minutes < 60) return `queued ${minutes}m`;
	79	const hours = Math.floor(minutes / 60);
	80	if (hours < 24) return `queued ${hours}h`;
	81	return `queued ${Math.floor(hours / 24)}d`;
	82	}
	83
68	84	/** Bytes, three significant figures, never more precision than is true. */
69	85	export function fileSize(bytes: number): string {
70	86	if (bytes < 1024) return `${bytes} B`;
		@@ -11,8 +11,20 @@ import { join } from "node:path";
11	11	import { createApp, type App } from "../src/app.ts";
12	12	import { openPull } from "../src/actions/pulls.ts";
13	13	import { commitToBranch, createRepo, forkRepo } from "../src/actions/repos.ts";
14		import { startChecks, type ChecksHandle } from "../src/checks/service.ts";
15		import { checksForHead, listRuns, type Run } from "../src/core/runs.ts";
	14	import {
	15	pickNext,
	16	startChecks,
	17	STARVATION_MS,
	18	type ChecksHandle,
	19	} from "../src/checks/service.ts";
	20	import {
	21	cancelRun,
	22	checksForHead,
	23	createRun,
	24	getRun,
	25	listRuns,
	26	type Run,
	27	} from "../src/core/runs.ts";
16	28	import { createUser, type User } from "../src/core/users.ts";
17	29	import type { Repo } from "../src/core/repos.ts";
18	30	import { serve, type Servers } from "../src/http/server.ts";
		@@ -225,6 +237,106 @@ describe("checks", () => {
225	237	expect(middle?.summary).toContain("superseded by");
226	238	}, 60_000);
227	239
	240	test("pickNext takes the newest queued check, unless one has starved", () => {
	241	const a = createRun(app.db, { repoId: repo.id, kind: "check", env: "a" });
	242	const b = createRun(app.db, { repoId: repo.id, kind: "check", env: "b" });
	243	const c = createRun(app.db, { repoId: repo.id, kind: "check", env: "c" });
	244	// ids are creation-ordered, so this is oldest-to-newest already.
	245	expect(pickNext([a, b, c])?.id).toBe(c.id);
	246	expect(pickNext([c, a, b])?.id).toBe(c.id);
	247	expect(pickNext([a])?.id).toBe(a.id);
	248	expect(pickNext([])).toBeUndefined();
	249
	250	// `a` has waited past the starvation bound: it outranks `c`, however new.
	251	app.db
	252	.query("UPDATE run SET created_at = ?2 WHERE id = ?1")
	253	.run(a.id, new Date(Date.now() - STARVATION_MS - 1_000).toISOString());
	254	const starved = getRun(app.db, a.id)!;
	255	expect(pickNext([starved, b, c])?.id).toBe(starved.id);
	256	// Just short of the bound, the newest still wins.
	257	app.db
	258	.query("UPDATE run SET created_at = ?2 WHERE id = ?1")
	259	.run(a.id, new Date(Date.now() - STARVATION_MS + 1_000).toISOString());
	260	expect(pickNext([getRun(app.db, a.id)!, b, c])?.id).toBe(c.id);
	261
	262	// These never named a real commit: cancel them rather than leave them
	263	// for the live checks service running beside this test to trip over.
	264	for (const run of [a, b, c]) cancelRun(app.db, run.id);
	265	});
	266
	267	test("when a slot frees, the newest of several queued checks runs first", async () => {
	268	// A slow check on its own environment holds the only slot.
	269	const held = await commitToBranch(app, alice, repo, {
	270	branch: "order-hold",
	271	message: "Hold the slot",
	272	files: [
	273	{ path: "forge.json", content: checkJson("sleep 2 && test -f OK.md") },
	274	],
	275	});
	276	if (!held.ok) throw new Error(JSON.stringify(held.error));
	277	const holder = await checks.request(
	278	repo,
	279	"order-hold",
	280	held.value.commitOid,
	281	"push",
	282	);
	283	expect(holder).toBeDefined();
	284
	285	// Three more, on three environments so none supersedes another, queue
	286	// behind it — oldest requested first.
	287	const oldest = await commitToBranch(app, alice, repo, {
	288	branch: "order-a",
	289	message: "A",
	290	files: [{ path: "forge.json", content: checkJson("test -f OK.md") }],
	291	});
	292	if (!oldest.ok) throw new Error(JSON.stringify(oldest.error));
	293	const middle = await commitToBranch(app, alice, repo, {
	294	branch: "order-b",
	295	message: "B",
	296	files: [{ path: "forge.json", content: checkJson("test -f OK.md") }],
	297	});
	298	if (!middle.ok) throw new Error(JSON.stringify(middle.error));
	299	const newest = await commitToBranch(app, alice, repo, {
	300	branch: "order-c",
	301	message: "C",
	302	files: [{ path: "forge.json", content: checkJson("test -f OK.md") }],
	303	});
	304	if (!newest.ok) throw new Error(JSON.stringify(newest.error));
	305
	306	const oldestRun = await checks.request(
	307	repo,
	308	"order-a",
	309	oldest.value.commitOid,
	310	"push",
	311	);
	312	const middleRun = await checks.request(
	313	repo,
	314	"order-b",
	315	middle.value.commitOid,
	316	"push",
	317	);
	318	const newestRun = await checks.request(
	319	repo,
	320	"order-c",
	321	newest.value.commitOid,
	322	"push",
	323	);
	324	expect(oldestRun && middleRun && newestRun).toBeTruthy();
	325	expect(oldestRun?.state).toBe("queued");
	326	expect(middleRun?.state).toBe("queued");
	327	expect(newestRun?.state).toBe("queued");
	328
	329	await settled((r) => r.id === newestRun!.id);
	330	await settled((r) => r.id === middleRun!.id);
	331	await settled((r) => r.id === oldestRun!.id);
	332
	333	const runs = listRuns(app.db, repo.id, { kind: "check", limit: 50 });
	334	const startedAt = (id: string): number =>
	335	Date.parse(runs.find((r) => r.id === id)!.startedAt!);
	336	expect(startedAt(newestRun!.id)).toBeLessThan(startedAt(middleRun!.id));
	337	expect(startedAt(middleRun!.id)).toBeLessThan(startedAt(oldestRun!.id));
	338	}, 30_000);
	339
228	340	test("a check in flight when the forge stops is queued again, and the next forge runs it", async () => {
229	341	const slow = await commitToBranch(app, alice, repo, {
230	342	message: "A slower check",