2b94d3e · 5d847af
@@ -162,6 +162,27 @@
export function heldChecks(db: Database): { red: RedMain; held: number }[] {
  return out;
}
/**
 * The queued checks no red main holds (startable now): how many, and the
 * oldest one's queue time. A held check's age says nothing about the lane.
 */
export function freeChecks(
  db: Database,
  redRepoIds: readonly string[],
  now: number,
): { n: number; oldest: string | null } {
  const row = db
    .query<{ n: number; oldest: string | null }, [string, string]>(
      `SELECT COUNT(*) AS n, MIN(created_at) AS oldest FROM run
        WHERE state = 'queued' AND kind = 'check'
          AND (not_before IS NULL OR not_before <= ?1)
          AND NOT (issue_id IS NOT NULL AND reason <> '${ASKED}'
                   AND repo_id IN (SELECT value FROM json_each(?2)))`,
    )
    .get(new Date(now).toISOString(), JSON.stringify(redRepoIds));
  return row ?? { n: 0, oldest: null };
      
        junipercrew
        25d ago
        
      
      Add a created_at > now - RED_HOLD_MS condition to the NOT(...) clause so checks past the hold count as free, matching heldByRed.
    
}
/** The card's sentence: what is red, by which test, and what waits. */
export function redSentence(red: RedMain, held: number): string {
  const first = red.failing[0];
@@ -32,7 +32,7 @@
import { behindStanding } from "../release/behind.ts";
import { newestDrills } from "../core/domain_drills.ts";
import { DOMAINS } from "./domains.ts";
import { domainDrillNeeds } from "./domain_drill.ts";
import { heldChecks, redSentence } from "../checks/red_main.ts";
import { freeChecks, heldChecks, redSentence } from "../checks/red_main.ts";
import { flakeHref, sharedCause, sharedSentence, sharedTests } from "../checks/shared_red.ts";
/** Free disk under this share of the volume is something to say. */
@@ -249,16 +249,18 @@
export function laneSentence(
  oldest: string,
  lane: Lane,
  now: number,
  held = 0,
): string {
  const since = new Date(oldest).toISOString().slice(11, 16);
  const head = `${queued} check${queued === 1 ? "" : "s"} ${queued === 1 ? "waits" : "wait"} behind one lane; the oldest since ${since}`;
  if (lane.running === null) return `${head}; the lane runs nothing.`;
  const more = held > 0 ? `; ${held} more held by a red main` : "";
  if (lane.running === null) return `${head}; the lane runs nothing${more}.`;
  const age = Math.max(
    0,
    Math.round((now - new Date(lane.running.since).getTime()) / 60_000),
  );
  const what = lane.running.pull === null ? "a check" : `#${lane.running.pull}`;
  return `${head}; the lane runs ${what} for ${age} min (limit ${BUDGETS.gate / 60_000}).`;
  return `${head}; the lane runs ${what} for ${age} min (limit ${BUDGETS.gate / 60_000})${more}.`;
}
export function vitals(input: VitalsInput): Vitals {
@@ -307,17 +309,25 @@
export function vitals(input: VitalsInput): Vitals {
  const running = listRunning(input.db);
  const lane = readLane(input.db, running, now);
  const queue: Record<string, number> = {};
  const reds = heldChecks(input.db);
  const held = reds.reduce((n, r) => n + r.held, 0);
  const free = freeChecks(input.db, reds.map((r) => r.red.repoId), now);
  for (const row of queueDepth(input.db, now)) {
    queue[row.kind] = row.queued;
    if (row.kind === "check") {
      // Checks a red main holds are not waiting for the lane: the red: card names them.
      // Age and "since" come from the free checks alone.
      
        junipercrew
        25d ago
        outdated
      
      free is a count only. waited and row.oldest still come from the whole queue, so held checks decide when this fires and what the sentence says. Take the oldest from non-held checks.
    
      if (free.oldest !== null && now - new Date(free.oldest).getTime() > BUDGETS.laneWait)
        need(
          "queue:check",
          laneSentence(free.n, free.oldest, lane, now, Math.min(held, row.queued)),
          new Date(free.oldest).toISOString(),
          "/settings/capacity",
        );
      continue;
    }
    const waited = now - new Date(row.oldest).getTime();
    if (row.kind === "check" && waited > BUDGETS.laneWait)
      need(
        "queue:check",
        laneSentence(row.queued, row.oldest, lane, now),
        new Date(row.oldest).toISOString(),
        "/settings/capacity",
      );
    else if (row.kind !== "check" && waited > QUEUE_WAIT_MAX_MS)
    if (waited > QUEUE_WAIT_MAX_MS)
      need(
        `queue:${row.kind}`,
        `The ${row.kind} queue has waited over ${QUEUE_WAIT_MAX_MS / 60_000} minutes.`,
@@ -364,7 +374,7 @@
export function vitals(input: VitalsInput): Vitals {
    );
  // A red main holding its pulls' checks is one card, naming the test.
  for (const { red, held } of heldChecks(input.db))
  for (const { red, held } of reds)
    need(
      `red:${red.repoId}`,
      redSentence(red, held),
@@ -179,3 +179,44 @@
describe("one fix issue for a red main (#1971)", () => {
    expect(coveredByRedFix(db, "r1", dup)).toBe(false);
  });
});
describe("the lane sentence and a red main", () => {
  const old = (ids: string[]) =>
    ids.forEach((id) =>
      db.run("UPDATE run SET created_at = ?2 WHERE id = ?1", [id, new Date(Date.now() - 20 * 60_000).toISOString()]),
    );
  const read = () =>
    vitals({
      db,
      signals: { errors: new ErrorLedger(), refused: new RefusedLedger() } as never,
      dataDir: tmpdir(),
      disk: () => ({ freeBytes: 95, totalBytes: 100 }),
    }).degraded;
  test("checks all held by a red main, with an idle lane, raise red: and not queue:check", () => {
    main("b", "failed", ["x"]);
    old(Array.from({ length: 10 }, (_, i) => pullCheck(String(i)).id));
    const causes = read().map((d) => d.cause);
    expect(causes).toContain("red:r1");
    expect(causes).not.toContain("queue:check");
  });
  test("some held, some free: the sentence counts the free ones and names the held", () => {
    main("b", "failed", ["x"]);
    const held = [pullCheck("1"), pullCheck("2"), pullCheck("3")];
    const asked = pullCheck("4", "asked");
    old([...held, asked].map((r) => r.id));
    const lane = read().find((d) => d.cause === "queue:check");
      
        junipercrew
        25d ago
        
      
      Here the free check is aged as well, so the test passes with the oldest-overall bug. Add a case where only the held checks are old and the free one is fresh.
    
    expect(lane?.sentence).toStartWith("1 check waits behind one lane;");
    expect(lane?.sentence).toEndWith("; 3 more held by a red main.");
  });
  test("old held checks and a fresh free one: the lane is not blamed", () => {
    main("b", "failed", ["x"]);
    old([pullCheck("1"), pullCheck("2"), pullCheck("3")].map((r) => r.id));
    createRun(db, { repoId: "r1", kind: "check", env: "pull-7", issueId: pull, commitOid: OID("4"), reason: "asked" });
    const causes = read().map((d) => d.cause);
    expect(causes).toContain("red:r1");
    expect(causes).not.toContain("queue:check");
  });
});
@@ -167,8 +167,8 @@
test("the number of queries does not grow with runs", () => {
    if (i % 2) app.db.query("UPDATE run SET state='running' WHERE id=?1").run(r.id);
  }
  expect(count()).toBe(before);
  // The queue, the running runs, the runs killed twice for their budget, the lane's depth and finished checks, the red mains holding pulls' checks, the tests red on many pulls, and the newest drill of every domain.
  expect(before).toBe(8);
  // The queue, the running runs, the runs killed twice for their budget, the lane's depth and finished checks, the red mains holding pulls' checks, the tests red on many pulls, and the newest drill of every domain, and the checks no red main holds.
  expect(before).toBe(9);
});
test("errors are counted, bounded, redacted, and a burst is said", async () => {
		@@ -162,6 +162,27 @@ export function heldChecks(db: Database): { red: RedMain; held: number }[] {
162	162	return out;
163	163	}
164	164
	165	/**
	166	* The queued checks no red main holds (startable now): how many, and the
	167	* oldest one's queue time. A held check's age says nothing about the lane.
	168	*/
	169	export function freeChecks(
	170	db: Database,
	171	redRepoIds: readonly string[],
	172	now: number,
	173	): { n: number; oldest: string \| null } {
	174	const row = db
	175	.query<{ n: number; oldest: string \| null }, [string, string]>(
	176	`SELECT COUNT(*) AS n, MIN(created_at) AS oldest FROM run
	177	WHERE state = 'queued' AND kind = 'check'
	178	AND (not_before IS NULL OR not_before <= ?1)
	179	AND NOT (issue_id IS NOT NULL AND reason <> '${ASKED}'
	180	AND repo_id IN (SELECT value FROM json_each(?2)))`,
	181	)
	182	.get(new Date(now).toISOString(), JSON.stringify(redRepoIds));
	183	return row ?? { n: 0, oldest: null };
		junipercrew 25d ago Add a created_at > now - RED_HOLD_MS condition to the NOT(...) clause so checks past the hold count as free, matching heldByRed.
	184	}
	185
165	186	/** The card's sentence: what is red, by which test, and what waits. */
166	187	export function redSentence(red: RedMain, held: number): string {
167	188	const first = red.failing[0];
		@@ -32,7 +32,7 @@ import { behindStanding } from "../release/behind.ts";
32	32	import { newestDrills } from "../core/domain_drills.ts";
33	33	import { DOMAINS } from "./domains.ts";
34	34	import { domainDrillNeeds } from "./domain_drill.ts";
35		import { heldChecks, redSentence } from "../checks/red_main.ts";
	35	import { freeChecks, heldChecks, redSentence } from "../checks/red_main.ts";
36	36	import { flakeHref, sharedCause, sharedSentence, sharedTests } from "../checks/shared_red.ts";
37	37
38	38	/** Free disk under this share of the volume is something to say. */
		@@ -249,16 +249,18 @@ export function laneSentence(
249	249	oldest: string,
250	250	lane: Lane,
251	251	now: number,
	252	held = 0,
252	253	): string {
253	254	const since = new Date(oldest).toISOString().slice(11, 16);
254	255	const head = `${queued} check${queued === 1 ? "" : "s"} ${queued === 1 ? "waits" : "wait"} behind one lane; the oldest since ${since}`;
255		if (lane.running === null) return `${head}; the lane runs nothing.`;
	256	const more = held > 0 ? `; ${held} more held by a red main` : "";
	257	if (lane.running === null) return `${head}; the lane runs nothing${more}.`;
256	258	const age = Math.max(
257	259	0,
258	260	Math.round((now - new Date(lane.running.since).getTime()) / 60_000),
259	261	);
260	262	const what = lane.running.pull === null ? "a check" : `#${lane.running.pull}`;
261		return `${head}; the lane runs ${what} for ${age} min (limit ${BUDGETS.gate / 60_000}).`;
	263	return `${head}; the lane runs ${what} for ${age} min (limit ${BUDGETS.gate / 60_000})${more}.`;
262	264	}
263	265
264	266	export function vitals(input: VitalsInput): Vitals {
		@@ -307,17 +309,25 @@ export function vitals(input: VitalsInput): Vitals {
307	309	const running = listRunning(input.db);
308	310	const lane = readLane(input.db, running, now);
309	311	const queue: Record<string, number> = {};
	312	const reds = heldChecks(input.db);
	313	const held = reds.reduce((n, r) => n + r.held, 0);
	314	const free = freeChecks(input.db, reds.map((r) => r.red.repoId), now);
310	315	for (const row of queueDepth(input.db, now)) {
311	316	queue[row.kind] = row.queued;
	317	if (row.kind === "check") {
	318	// Checks a red main holds are not waiting for the lane: the red: card names them.
	319	// Age and "since" come from the free checks alone.
		junipercrew 25d ago outdated `free` is a count only. `waited` and `row.oldest` still come from the whole queue, so held checks decide when this fires and what the sentence says. Take the oldest from non-held checks.
	320	if (free.oldest !== null && now - new Date(free.oldest).getTime() > BUDGETS.laneWait)
	321	need(
	322	"queue:check",
	323	laneSentence(free.n, free.oldest, lane, now, Math.min(held, row.queued)),
	324	new Date(free.oldest).toISOString(),
	325	"/settings/capacity",
	326	);
	327	continue;
	328	}
312	329	const waited = now - new Date(row.oldest).getTime();
313		if (row.kind === "check" && waited > BUDGETS.laneWait)
314		need(
315		"queue:check",
316		laneSentence(row.queued, row.oldest, lane, now),
317		new Date(row.oldest).toISOString(),
318		"/settings/capacity",
319		);
320		else if (row.kind !== "check" && waited > QUEUE_WAIT_MAX_MS)
	330	if (waited > QUEUE_WAIT_MAX_MS)
321	331	need(
322	332	`queue:${row.kind}`,
323	333	`The ${row.kind} queue has waited over ${QUEUE_WAIT_MAX_MS / 60_000} minutes.`,
		@@ -364,7 +374,7 @@ export function vitals(input: VitalsInput): Vitals {
364	374	);
365	375
366	376	// A red main holding its pulls' checks is one card, naming the test.
367		for (const { red, held } of heldChecks(input.db))
	377	for (const { red, held } of reds)
368	378	need(
369	379	`red:${red.repoId}`,
370	380	redSentence(red, held),
		@@ -179,3 +179,44 @@ describe("one fix issue for a red main (#1971)", () => {
179	179	expect(coveredByRedFix(db, "r1", dup)).toBe(false);
180	180	});
181	181	});
	182
	183	describe("the lane sentence and a red main", () => {
	184	const old = (ids: string[]) =>
	185	ids.forEach((id) =>
	186	db.run("UPDATE run SET created_at = ?2 WHERE id = ?1", [id, new Date(Date.now() - 20 * 60_000).toISOString()]),
	187	);
	188	const read = () =>
	189	vitals({
	190	db,
	191	signals: { errors: new ErrorLedger(), refused: new RefusedLedger() } as never,
	192	dataDir: tmpdir(),
	193	disk: () => ({ freeBytes: 95, totalBytes: 100 }),
	194	}).degraded;
	195
	196	test("checks all held by a red main, with an idle lane, raise red: and not queue:check", () => {
	197	main("b", "failed", ["x"]);
	198	old(Array.from({ length: 10 }, (_, i) => pullCheck(String(i)).id));
	199	const causes = read().map((d) => d.cause);
	200	expect(causes).toContain("red:r1");
	201	expect(causes).not.toContain("queue:check");
	202	});
	203
	204	test("some held, some free: the sentence counts the free ones and names the held", () => {
	205	main("b", "failed", ["x"]);
	206	const held = [pullCheck("1"), pullCheck("2"), pullCheck("3")];
	207	const asked = pullCheck("4", "asked");
	208	old([...held, asked].map((r) => r.id));
	209	const lane = read().find((d) => d.cause === "queue:check");
		junipercrew 25d ago Here the free check is aged as well, so the test passes with the oldest-overall bug. Add a case where only the held checks are old and the free one is fresh.
	210	expect(lane?.sentence).toStartWith("1 check waits behind one lane;");
	211	expect(lane?.sentence).toEndWith("; 3 more held by a red main.");
	212	});
	213
	214	test("old held checks and a fresh free one: the lane is not blamed", () => {
	215	main("b", "failed", ["x"]);
	216	old([pullCheck("1"), pullCheck("2"), pullCheck("3")].map((r) => r.id));
	217	createRun(db, { repoId: "r1", kind: "check", env: "pull-7", issueId: pull, commitOid: OID("4"), reason: "asked" });
	218	const causes = read().map((d) => d.cause);
	219	expect(causes).toContain("red:r1");
	220	expect(causes).not.toContain("queue:check");
	221	});
	222	});
		@@ -167,8 +167,8 @@ test("the number of queries does not grow with runs", () => {
167	167	if (i % 2) app.db.query("UPDATE run SET state='running' WHERE id=?1").run(r.id);
168	168	}
169	169	expect(count()).toBe(before);
170		// The queue, the running runs, the runs killed twice for their budget, the lane's depth and finished checks, the red mains holding pulls' checks, the tests red on many pulls, and the newest drill of every domain.
171		expect(before).toBe(8);
	170	// The queue, the running runs, the runs killed twice for their budget, the lane's depth and finished checks, the red mains holding pulls' checks, the tests red on many pulls, and the newest drill of every domain, and the checks no red main holds.
	171	expect(before).toBe(9);
172	172	});
173	173
174	174	test("errors are counted, bounded, redacted, and a burst is said", async () => {