d664e96
@@ -111,18 +111,24 @@
It starts the old forge on a copy of `--from` (never the live data
directory: restore a snapshot first, `docs/RESTORE.md`), sends requests to
`/`, `/healthz` and `/explore`, starts the new forge beside it on the same
copy, and moves rotation the way the Service does: a forge joins when its
`/readyz` says 200 and leaves when it does not, probed every 5 s as the
chart probes. Once the new forge is in, the old one is sent SIGTERM. A minute
later it prints its verdict and exits:
`/readyz` says 200 and leaves when it says 503 or nothing is listening,
probed every 5 s as the chart probes. A probe that only times out is a busy
machine, not a forge going away: like the chart's (Kubernetes' default
failure threshold), it takes three in a row to leave. Once the new forge is
in, the old one is sent SIGTERM. A minute later it prints its verdict and
exits:
| exit | means                                                          |
| ---- | -------------------------------------------------------------- |
| 0    | zero refused: roll                                             |
| 1    | a request got a 5xx or no answer, or nothing rolled: do not roll |
| 1    | a request got a 5xx or a refused connection, or nothing rolled: do not roll |
| 2    | an argument is wrong, or a forge would not start               |
Exit 1 is a bug in the release, not a reason to ask for a dark roll: the
logs it names say which forge refused and when. Release checklist: step 1
Exit 1 is a bug in the release, not a reason to ask for a dark roll: each
`miss` line says when, which forge was asked, what came back (the client's
error code with it), and whether each process was serving, draining or gone;
the logs it names say the rest. A request the client gave up on is counted
`slow` and listed, but is not a refusal: the forge held its connection. Release checklist: step 1
exits 0, then this exits 0, then roll. Quote the `zero refused` line in the
release notes.
@@ -98,20 +98,40 @@
export function parseArgs(argv: readonly string[], here: string): RehearsalArgs
  };
}
/**
 * What one readiness probe heard: a 200, a not-ready answer (a 503 or a
 * refused connection — nothing there to take work), or silence past the
 * probe's timeout, which says only that the machine is busy.
 */
export type ProbeAnswer = "ready" | "not-ready" | "silent";
/**
 * Silent probes in a row before a member leaves: the chart's readiness
 * probe keeps Kubernetes' default failureThreshold, 3. One slow probe on
 * a loaded machine is not a forge going away.
 */
export const SILENT_PROBES_TO_LEAVE = 3;
/**
 * Who takes requests, as a Service's endpoints do: a forge joins when a
 * probe sees it ready and leaves when a probe sees it not, never between
 * probe sees it ready and leaves when a probe hears it is not, or hears
 * nothing {@link SILENT_PROBES_TO_LEAVE} times in a row — never between
 * probes. Requests go round the members in turn.
 */
export class Rotation<T> {
  private members: T[] = [];
  private silent = new Map<T, number>();
  private turn = 0;
  /** What one probe of `member` saw. */
  saw(member: T, ready: boolean): void {
  /** What one probe of `member` heard; `true` and `false` are ready and not. */
  saw(member: T, answer: ProbeAnswer | boolean): void {
    const heard = answer === true ? "ready" : answer === false ? "not-ready" : answer;
    const at = this.members.indexOf(member);
    if (ready && at < 0) this.members.push(member);
    if (!ready && at >= 0) this.members.splice(at, 1);
    const quiet = heard === "silent" ? (this.silent.get(member) ?? 0) + 1 : 0;
    this.silent.set(member, quiet);
    const leaves = heard === "not-ready" || quiet >= SILENT_PROBES_TO_LEAVE;
    if (heard === "ready" && at < 0) this.members.push(member);
    if (leaves && at >= 0) this.members.splice(at, 1);
  }
  has(member: T): boolean {
@@ -126,8 +146,26 @@
export class Rotation<T> {
  }
}
/** One request's outcome: a status, or "refused" when nothing answered. */
export type Outcome = number | "refused" | "nobody";
/**
 * One request's outcome: a status; "refused" when the connection was
 * refused or cut (nothing was there to answer); "slow" when the client gave
 * up waiting on a forge that held the connection open; "nobody" when no
 * forge was in rotation to ask.
 */
export type Outcome = number | "refused" | "slow" | "nobody";
/** A request that did not come back answered, and the moment around it. */
export interface Miss {
  /** Milliseconds from the first request. */
  readonly atMs: number;
  /** The forge asked, or null when nobody was in rotation. */
  readonly forge: string | null;
  readonly outcome: Outcome;
  /** The client's error code ("ConnectionRefused", "TimeoutError"…), or null. */
  readonly code: string | null;
  /** Each forge's process at that moment: serving, draining or exited. */
  readonly forges: Readonly<Record<string, string>>;
}
export interface Tally {
  requests: number;
@@ -135,22 +173,35 @@
export interface Tally {
  byOutcome: Record<string, number>;
  /** Requests each forge answered, by its name. */
  byForge: Record<string, number>;
  /** Every request that was not answered below 500, in order. */
  misses: Miss[];
}
export function emptyTally(): Tally {
  return { requests: 0, byOutcome: {}, byForge: {} };
  return { requests: 0, byOutcome: {}, byForge: {}, misses: [] };
}
export function count(tally: Tally, forge: string | null, outcome: Outcome): void {
export function count(
  tally: Tally,
  forge: string | null,
  outcome: Outcome,
  miss?: Omit<Miss, "forge" | "outcome">,
): void {
  tally.requests += 1;
  const key = String(outcome);
  tally.byOutcome[key] = (tally.byOutcome[key] ?? 0) + 1;
  if (forge !== null) tally.byForge[forge] = (tally.byForge[forge] ?? 0) + 1;
  if (miss !== undefined && (outcome === "slow" || isRefusal(outcome)))
    tally.misses.push({ ...miss, forge, outcome });
}
/** Whether a person would have seen the forge down. */
/**
 * Whether a person would have seen the forge down. A slow answer is not
 * one: the forge held the connection, and a busy machine is not a roll
 * that went dark (#2453).
 */
export function isRefusal(outcome: Outcome): boolean {
  return typeof outcome === "number" ? outcome >= 500 : true;
  return typeof outcome === "number" ? outcome >= 500 : outcome !== "slow";
}
export function refusals(tally: Tally): number {
@@ -205,6 +256,16 @@
export function judge(tally: Tally, timeline: Timeline): Verdict {
const secs = (ms: number | null): string =>
  ms === null ? "never" : `${(ms / 1000).toFixed(1)}s`;
const MISSES_SHOWN = 20;
/** One miss as a line: when, whom, what came back, and where each process was. */
export function missLine(m: Miss): string {
  const forges = Object.entries(m.forges)
    .map(([k, state]) => `${k} ${state}`)
    .join(", ");
  return `miss       ${m.atMs} ms ${m.forge ?? "nobody in rotation"}: ${m.outcome}${m.code === null ? "" : ` (${m.code})`}; ${forges}`;
}
/** The verdict as the release checklist reads it. */
export function report(v: Verdict): string {
  const outcomes = Object.entries(v.tally.byOutcome)
@@ -225,5 +286,9 @@
export function report(v: Verdict): string {
    `new ready  ${secs(v.timeline.newInRotationMs)} after the first request`,
    `old out    ${secs(v.timeline.oldOutOfRotationMs)} after its SIGTERM`,
    `old exit   ${secs(v.timeline.oldExitMs)} after its SIGTERM`,
    ...v.tally.misses.slice(0, MISSES_SHOWN).map(missLine),
    ...(v.tally.misses.length > MISSES_SHOWN
      ? [`           and ${v.tally.misses.length - MISSES_SHOWN} more`]
      : []),
  ].join("\n");
}
@@ -30,6 +30,7 @@
import {
  report,
  Rotation,
  type Outcome,
  type ProbeAnswer,
  type RehearsalArgs,
  type Timeline,
  type Verdict,
@@ -83,23 +84,41 @@
function start(
  return { name, url: `http://127.0.0.1:${port}`, proc, log };
}
async function ready(forge: Forge, timeoutMs: number): Promise<boolean> {
/** One readiness probe: ready, not ready (a 503, or nothing listening), or silent past its timeout. */
async function probe(forge: Forge, timeoutMs: number): Promise<ProbeAnswer> {
  try {
    const res = await fetch(`${forge.url}/readyz`, { signal: AbortSignal.timeout(timeoutMs) });
    await res.arrayBuffer();
    return res.status === 200;
  } catch {
    return false;
    return res.status === 200 ? "ready" : "not-ready";
  } catch (error) {
    return errorCode(error) === "TimeoutError" ? "silent" : "not-ready";
  }
}
async function ask(url: string): Promise<Outcome> {
async function ready(forge: Forge, timeoutMs: number): Promise<boolean> {
  return (await probe(forge, timeoutMs)) === "ready";
}
/** The client's name for what went wrong: "ConnectionRefused", "TimeoutError"… */
function errorCode(error: unknown): string {
  const e = error as { code?: unknown; name?: unknown };
  if (e?.name === "TimeoutError" || e?.name === "AbortError") return "TimeoutError";
  return typeof e?.code === "string" ? e.code : String(e?.name ?? "Error");
}
/**
 * One request. A connection refused or cut is "refused"; one the forge
 * held open past the client's patience is "slow" — the machine, not the
 * roll (#2453).
 */
async function ask(url: string): Promise<{ outcome: Outcome; code: string | null }> {
  try {
    const res = await fetch(url, { redirect: "manual", signal: AbortSignal.timeout(10_000) });
    await res.arrayBuffer();
    return res.status;
  } catch {
    return "refused";
    return { outcome: res.status, code: null };
  } catch (error) {
    const code = errorCode(error);
    return { outcome: code === "TimeoutError" ? "slow" : "refused", code };
  }
}
@@ -135,17 +154,26 @@
export async function rehearse(
    const began = Date.now();
    const until = began + args.seconds * 1000;
    let termAt: number | null = null;
    // Each process as a miss saw it: whether it still ran, and was told to go.
    const states = (): Record<string, string> =>
      Object.fromEntries(
        forges.map((f) => [
          f.name,
          f.proc.exitCode !== null ? "exited" : f === old && termAt !== null ? "draining" : "serving",
        ]),
      );
    const load = async (lane: number): Promise<void> => {
      for (let i = lane; Date.now() < until; i += args.concurrency) {
        const forge = rotation.next();
        if (forge === null) {
          count(tally, null, "nobody");
          count(tally, null, "nobody", { atMs: Date.now() - began, code: null, forges: states() });
          await Bun.sleep(10);
          continue;
        }
        const path = args.paths[i % args.paths.length] ?? "/";
        count(tally, forge.name, await ask(forge.url + path));
        const { outcome, code } = await ask(forge.url + path);
        count(tally, forge.name, outcome, { atMs: Date.now() - began, code, forges: states() });
      }
    };
@@ -161,10 +189,14 @@
export async function rehearse(
      });
      while (Date.now() < until) {
        for (const forge of forges) {
          const up = await ready(forge, Math.min(1000, args.probeMs));
          const answer = await probe(forge, Math.min(1000, args.probeMs));
          const was = rotation.has(forge);
          rotation.saw(forge, up);
          if (forge === next && up && !was) {
          rotation.saw(forge, answer);
          const up = rotation.has(forge);
          if (up !== was)
            say(`${forge.name} forge ${up ? "joins" : "leaves"} rotation at ${Date.now() - began} ms: probe ${answer}`);
          // Signalled once: a new forge that leaves and rejoins is not a second roll.
          if (forge === next && up && !was && termAt === null) {
            timeline.newInRotationMs = Date.now() - began;
            say("new forge in rotation; the old one is sent SIGTERM");
            termAt = Date.now();
@@ -13,10 +13,12 @@
import {
  emptyTally,
  isRefusal,
  judge,
  missLine,
  parseArgs,
  refusals,
  report,
  Rotation,
  SILENT_PROBES_TO_LEAVE,
  type RehearsalArgs,
} from "../scripts/rehearsal.ts";
@@ -70,6 +72,23 @@
describe("rotation", () => {
    r.saw("old", false);
    expect([r.next(), r.next(), r.has("old")]).toEqual(["new", "new", false]);
  });
  test("a silent probe is a busy machine: a member leaves only after three in a row (#2453)", () => {
    const r = new Rotation<string>();
    r.saw("old", "ready");
    for (let i = 1; i < SILENT_PROBES_TO_LEAVE; i++) r.saw("old", "silent");
    r.saw("old", "ready");
    for (let i = 1; i < SILENT_PROBES_TO_LEAVE; i++) r.saw("old", "silent");
    expect(r.has("old")).toBe(true);
    r.saw("old", "silent");
    expect(r.has("old")).toBe(false);
    // Silence never lets anyone in; a not-ready answer sends a member out at once.
    r.saw("new", "silent");
    expect(r.has("new")).toBe(false);
    r.saw("new", "ready");
    r.saw("new", "not-ready");
    expect(r.has("new")).toBe(false);
  });
});
describe("the verdict", () => {
@@ -80,6 +99,24 @@
describe("the verdict", () => {
    expect([200, 302, 404].some((o) => isRefusal(o))).toBe(false);
  });
  test("a slow answer is not a refusal, but every miss is kept with its moment", () => {
    expect(isRefusal("slow")).toBe(false);
    const t = emptyTally();
    const at = { atMs: 1234, code: "TimeoutError", forges: { old: "draining", new: "serving" } };
    count(t, "old", 200, at);
    count(t, "new", "slow", at);
    count(t, null, "nobody", { ...at, code: null });
    expect(refusals(t)).toBe(1);
    expect(t.misses.map((m) => [m.forge, m.outcome])).toEqual([
      ["new", "slow"],
      [null, "nobody"],
    ]);
    expect(missLine(t.misses[0]!)).toBe(
      "miss       1234 ms new: slow (TimeoutError); old draining, new serving",
    );
    expect(report(judge(t, rolled))).toContain("nobody in rotation: nobody; old draining");
  });
  test("zero refused over a roll that happened is a pass", () => {
    const t = emptyTally();
    count(t, "old", 200);
@@ -131,6 +168,8 @@
describe("a real roll of two forge processes on one data directory", () => {
  test("an old forge that lingers past a probe period refuses nobody", async () => {
    const v = await rehearse(short(5));
    expect(v.problem).toBeNull();
    // Each refusal says when, whom, what came back and where both processes were.
    expect(v.tally.misses.filter((m) => isRefusal(m.outcome)).map(missLine)).toEqual([]);
    expect(v.refused).toBe(0);
    expect(v.tally.byForge["old"]).toBeGreaterThan(0);
    expect(v.tally.byForge["new"]).toBeGreaterThan(0);
		@@ -111,18 +111,24 @@ It starts the old forge on a copy of `--from` (never the live data
111	111	directory: restore a snapshot first, `docs/RESTORE.md`), sends requests to
112	112	`/`, `/healthz` and `/explore`, starts the new forge beside it on the same
113	113	copy, and moves rotation the way the Service does: a forge joins when its
114		`/readyz` says 200 and leaves when it does not, probed every 5 s as the
115		chart probes. Once the new forge is in, the old one is sent SIGTERM. A minute
116		later it prints its verdict and exits:
	114	`/readyz` says 200 and leaves when it says 503 or nothing is listening,
	115	probed every 5 s as the chart probes. A probe that only times out is a busy
	116	machine, not a forge going away: like the chart's (Kubernetes' default
	117	failure threshold), it takes three in a row to leave. Once the new forge is
	118	in, the old one is sent SIGTERM. A minute later it prints its verdict and
	119	exits:
117	120
118	121	\| exit \| means \|
119	122	\| ---- \| -------------------------------------------------------------- \|
120	123	\| 0 \| zero refused: roll \|
121		\| 1 \| a request got a 5xx or no answer, or nothing rolled: do not roll \|
	124	\| 1 \| a request got a 5xx or a refused connection, or nothing rolled: do not roll \|
122	125	\| 2 \| an argument is wrong, or a forge would not start \|
123	126
124		Exit 1 is a bug in the release, not a reason to ask for a dark roll: the
125		logs it names say which forge refused and when. Release checklist: step 1
	127	Exit 1 is a bug in the release, not a reason to ask for a dark roll: each
	128	`miss` line says when, which forge was asked, what came back (the client's
	129	error code with it), and whether each process was serving, draining or gone;
	130	the logs it names say the rest. A request the client gave up on is counted
	131	`slow` and listed, but is not a refusal: the forge held its connection. Release checklist: step 1
126	132	exits 0, then this exits 0, then roll. Quote the `zero refused` line in the
127	133	release notes.
128	134
		@@ -98,20 +98,40 @@ export function parseArgs(argv: readonly string[], here: string): RehearsalArgs
98	98	};
99	99	}
100	100
	101	/**
	102	* What one readiness probe heard: a 200, a not-ready answer (a 503 or a
	103	* refused connection — nothing there to take work), or silence past the
	104	* probe's timeout, which says only that the machine is busy.
	105	*/
	106	export type ProbeAnswer = "ready" \| "not-ready" \| "silent";
	107
	108	/**
	109	* Silent probes in a row before a member leaves: the chart's readiness
	110	* probe keeps Kubernetes' default failureThreshold, 3. One slow probe on
	111	* a loaded machine is not a forge going away.
	112	*/
	113	export const SILENT_PROBES_TO_LEAVE = 3;
	114
101	115	/**
102	116	* Who takes requests, as a Service's endpoints do: a forge joins when a
103		* probe sees it ready and leaves when a probe sees it not, never between
	117	* probe sees it ready and leaves when a probe hears it is not, or hears
	118	* nothing {@link SILENT_PROBES_TO_LEAVE} times in a row — never between
104	119	* probes. Requests go round the members in turn.
105	120	*/
106	121	export class Rotation<T> {
107	122	private members: T[] = [];
	123	private silent = new Map<T, number>();
108	124	private turn = 0;
109	125
110		/** What one probe of `member` saw. */
111		saw(member: T, ready: boolean): void {
	126	/** What one probe of `member` heard; `true` and `false` are ready and not. */
	127	saw(member: T, answer: ProbeAnswer \| boolean): void {
	128	const heard = answer === true ? "ready" : answer === false ? "not-ready" : answer;
112	129	const at = this.members.indexOf(member);
113		if (ready && at < 0) this.members.push(member);
114		if (!ready && at >= 0) this.members.splice(at, 1);
	130	const quiet = heard === "silent" ? (this.silent.get(member) ?? 0) + 1 : 0;
	131	this.silent.set(member, quiet);
	132	const leaves = heard === "not-ready" \|\| quiet >= SILENT_PROBES_TO_LEAVE;
	133	if (heard === "ready" && at < 0) this.members.push(member);
	134	if (leaves && at >= 0) this.members.splice(at, 1);
115	135	}
116	136
117	137	has(member: T): boolean {
		@@ -126,8 +146,26 @@ export class Rotation<T> {
126	146	}
127	147	}
128	148
129		/** One request's outcome: a status, or "refused" when nothing answered. */
130		export type Outcome = number \| "refused" \| "nobody";
	149	/**
	150	* One request's outcome: a status; "refused" when the connection was
	151	* refused or cut (nothing was there to answer); "slow" when the client gave
	152	* up waiting on a forge that held the connection open; "nobody" when no
	153	* forge was in rotation to ask.
	154	*/
	155	export type Outcome = number \| "refused" \| "slow" \| "nobody";
	156
	157	/** A request that did not come back answered, and the moment around it. */
	158	export interface Miss {
	159	/** Milliseconds from the first request. */
	160	readonly atMs: number;
	161	/** The forge asked, or null when nobody was in rotation. */
	162	readonly forge: string \| null;
	163	readonly outcome: Outcome;
	164	/** The client's error code ("ConnectionRefused", "TimeoutError"…), or null. */
	165	readonly code: string \| null;
	166	/** Each forge's process at that moment: serving, draining or exited. */
	167	readonly forges: Readonly<Record<string, string>>;
	168	}
131	169
132	170	export interface Tally {
133	171	requests: number;
		@@ -135,22 +173,35 @@ export interface Tally {
135	173	byOutcome: Record<string, number>;
136	174	/** Requests each forge answered, by its name. */
137	175	byForge: Record<string, number>;
	176	/** Every request that was not answered below 500, in order. */
	177	misses: Miss[];
138	178	}
139	179
140	180	export function emptyTally(): Tally {
141		return { requests: 0, byOutcome: {}, byForge: {} };
	181	return { requests: 0, byOutcome: {}, byForge: {}, misses: [] };
142	182	}
143	183
144		export function count(tally: Tally, forge: string \| null, outcome: Outcome): void {
	184	export function count(
	185	tally: Tally,
	186	forge: string \| null,
	187	outcome: Outcome,
	188	miss?: Omit<Miss, "forge" \| "outcome">,
	189	): void {
145	190	tally.requests += 1;
146	191	const key = String(outcome);
147	192	tally.byOutcome[key] = (tally.byOutcome[key] ?? 0) + 1;
148	193	if (forge !== null) tally.byForge[forge] = (tally.byForge[forge] ?? 0) + 1;
	194	if (miss !== undefined && (outcome === "slow" \|\| isRefusal(outcome)))
	195	tally.misses.push({ ...miss, forge, outcome });
149	196	}
150	197
151		/** Whether a person would have seen the forge down. */
	198	/**
	199	* Whether a person would have seen the forge down. A slow answer is not
	200	* one: the forge held the connection, and a busy machine is not a roll
	201	* that went dark (#2453).
	202	*/
152	203	export function isRefusal(outcome: Outcome): boolean {
153		return typeof outcome === "number" ? outcome >= 500 : true;
	204	return typeof outcome === "number" ? outcome >= 500 : outcome !== "slow";
154	205	}
155	206
156	207	export function refusals(tally: Tally): number {
		@@ -205,6 +256,16 @@ export function judge(tally: Tally, timeline: Timeline): Verdict {
205	256	const secs = (ms: number \| null): string =>
206	257	ms === null ? "never" : `${(ms / 1000).toFixed(1)}s`;
207	258
	259	const MISSES_SHOWN = 20;
	260
	261	/** One miss as a line: when, whom, what came back, and where each process was. */
	262	export function missLine(m: Miss): string {
	263	const forges = Object.entries(m.forges)
	264	.map(([k, state]) => `${k} ${state}`)
	265	.join(", ");
	266	return `miss ${m.atMs} ms ${m.forge ?? "nobody in rotation"}: ${m.outcome}${m.code === null ? "" : ` (${m.code})`}; ${forges}`;
	267	}
	268
208	269	/** The verdict as the release checklist reads it. */
209	270	export function report(v: Verdict): string {
210	271	const outcomes = Object.entries(v.tally.byOutcome)
		@@ -225,5 +286,9 @@ export function report(v: Verdict): string {
225	286	`new ready ${secs(v.timeline.newInRotationMs)} after the first request`,
226	287	`old out ${secs(v.timeline.oldOutOfRotationMs)} after its SIGTERM`,
227	288	`old exit ${secs(v.timeline.oldExitMs)} after its SIGTERM`,
	289	...v.tally.misses.slice(0, MISSES_SHOWN).map(missLine),
	290	...(v.tally.misses.length > MISSES_SHOWN
	291	? [` and ${v.tally.misses.length - MISSES_SHOWN} more`]
	292	: []),
228	293	].join("\n");
229	294	}
		@@ -30,6 +30,7 @@ import {
30	30	report,
31	31	Rotation,
32	32	type Outcome,
	33	type ProbeAnswer,
33	34	type RehearsalArgs,
34	35	type Timeline,
35	36	type Verdict,
		@@ -83,23 +84,41 @@ function start(
83	84	return { name, url: `http://127.0.0.1:${port}`, proc, log };
84	85	}
85	86
86		async function ready(forge: Forge, timeoutMs: number): Promise<boolean> {
	87	/** One readiness probe: ready, not ready (a 503, or nothing listening), or silent past its timeout. */
	88	async function probe(forge: Forge, timeoutMs: number): Promise<ProbeAnswer> {
87	89	try {
88	90	const res = await fetch(`${forge.url}/readyz`, { signal: AbortSignal.timeout(timeoutMs) });
89	91	await res.arrayBuffer();
90		return res.status === 200;
91		} catch {
92		return false;
	92	return res.status === 200 ? "ready" : "not-ready";
	93	} catch (error) {
	94	return errorCode(error) === "TimeoutError" ? "silent" : "not-ready";
93	95	}
94	96	}
95	97
96		async function ask(url: string): Promise<Outcome> {
	98	async function ready(forge: Forge, timeoutMs: number): Promise<boolean> {
	99	return (await probe(forge, timeoutMs)) === "ready";
	100	}
	101
	102	/** The client's name for what went wrong: "ConnectionRefused", "TimeoutError"… */
	103	function errorCode(error: unknown): string {
	104	const e = error as { code?: unknown; name?: unknown };
	105	if (e?.name === "TimeoutError" \|\| e?.name === "AbortError") return "TimeoutError";
	106	return typeof e?.code === "string" ? e.code : String(e?.name ?? "Error");
	107	}
	108
	109	/**
	110	* One request. A connection refused or cut is "refused"; one the forge
	111	* held open past the client's patience is "slow" — the machine, not the
	112	* roll (#2453).
	113	*/
	114	async function ask(url: string): Promise<{ outcome: Outcome; code: string \| null }> {
97	115	try {
98	116	const res = await fetch(url, { redirect: "manual", signal: AbortSignal.timeout(10_000) });
99	117	await res.arrayBuffer();
100		return res.status;
101		} catch {
102		return "refused";
	118	return { outcome: res.status, code: null };
	119	} catch (error) {
	120	const code = errorCode(error);
	121	return { outcome: code === "TimeoutError" ? "slow" : "refused", code };
103	122	}
104	123	}
105	124
		@@ -135,17 +154,26 @@ export async function rehearse(
135	154	const began = Date.now();
136	155	const until = began + args.seconds * 1000;
137	156	let termAt: number \| null = null;
	157	// Each process as a miss saw it: whether it still ran, and was told to go.
	158	const states = (): Record<string, string> =>
	159	Object.fromEntries(
	160	forges.map((f) => [
	161	f.name,
	162	f.proc.exitCode !== null ? "exited" : f === old && termAt !== null ? "draining" : "serving",
	163	]),
	164	);
138	165
139	166	const load = async (lane: number): Promise<void> => {
140	167	for (let i = lane; Date.now() < until; i += args.concurrency) {
141	168	const forge = rotation.next();
142	169	if (forge === null) {
143		count(tally, null, "nobody");
	170	count(tally, null, "nobody", { atMs: Date.now() - began, code: null, forges: states() });
144	171	await Bun.sleep(10);
145	172	continue;
146	173	}
147	174	const path = args.paths[i % args.paths.length] ?? "/";
148		count(tally, forge.name, await ask(forge.url + path));
	175	const { outcome, code } = await ask(forge.url + path);
	176	count(tally, forge.name, outcome, { atMs: Date.now() - began, code, forges: states() });
149	177	}
150	178	};
151	179
		@@ -161,10 +189,14 @@ export async function rehearse(
161	189	});
162	190	while (Date.now() < until) {
163	191	for (const forge of forges) {
164		const up = await ready(forge, Math.min(1000, args.probeMs));
	192	const answer = await probe(forge, Math.min(1000, args.probeMs));
165	193	const was = rotation.has(forge);
166		rotation.saw(forge, up);
167		if (forge === next && up && !was) {
	194	rotation.saw(forge, answer);
	195	const up = rotation.has(forge);
	196	if (up !== was)
	197	say(`${forge.name} forge ${up ? "joins" : "leaves"} rotation at ${Date.now() - began} ms: probe ${answer}`);
	198	// Signalled once: a new forge that leaves and rejoins is not a second roll.
	199	if (forge === next && up && !was && termAt === null) {
168	200	timeline.newInRotationMs = Date.now() - began;
169	201	say("new forge in rotation; the old one is sent SIGTERM");
170	202	termAt = Date.now();
		@@ -13,10 +13,12 @@ import {
13	13	emptyTally,
14	14	isRefusal,
15	15	judge,
	16	missLine,
16	17	parseArgs,
17	18	refusals,
18	19	report,
19	20	Rotation,
	21	SILENT_PROBES_TO_LEAVE,
20	22	type RehearsalArgs,
21	23	} from "../scripts/rehearsal.ts";
22	24
		@@ -70,6 +72,23 @@ describe("rotation", () => {
70	72	r.saw("old", false);
71	73	expect([r.next(), r.next(), r.has("old")]).toEqual(["new", "new", false]);
72	74	});
	75
	76	test("a silent probe is a busy machine: a member leaves only after three in a row (#2453)", () => {
	77	const r = new Rotation<string>();
	78	r.saw("old", "ready");
	79	for (let i = 1; i < SILENT_PROBES_TO_LEAVE; i++) r.saw("old", "silent");
	80	r.saw("old", "ready");
	81	for (let i = 1; i < SILENT_PROBES_TO_LEAVE; i++) r.saw("old", "silent");
	82	expect(r.has("old")).toBe(true);
	83	r.saw("old", "silent");
	84	expect(r.has("old")).toBe(false);
	85	// Silence never lets anyone in; a not-ready answer sends a member out at once.
	86	r.saw("new", "silent");
	87	expect(r.has("new")).toBe(false);
	88	r.saw("new", "ready");
	89	r.saw("new", "not-ready");
	90	expect(r.has("new")).toBe(false);
	91	});
73	92	});
74	93
75	94	describe("the verdict", () => {
		@@ -80,6 +99,24 @@ describe("the verdict", () => {
80	99	expect([200, 302, 404].some((o) => isRefusal(o))).toBe(false);
81	100	});
82	101
	102	test("a slow answer is not a refusal, but every miss is kept with its moment", () => {
	103	expect(isRefusal("slow")).toBe(false);
	104	const t = emptyTally();
	105	const at = { atMs: 1234, code: "TimeoutError", forges: { old: "draining", new: "serving" } };
	106	count(t, "old", 200, at);
	107	count(t, "new", "slow", at);
	108	count(t, null, "nobody", { ...at, code: null });
	109	expect(refusals(t)).toBe(1);
	110	expect(t.misses.map((m) => [m.forge, m.outcome])).toEqual([
	111	["new", "slow"],
	112	[null, "nobody"],
	113	]);
	114	expect(missLine(t.misses[0]!)).toBe(
	115	"miss 1234 ms new: slow (TimeoutError); old draining, new serving",
	116	);
	117	expect(report(judge(t, rolled))).toContain("nobody in rotation: nobody; old draining");
	118	});
	119
83	120	test("zero refused over a roll that happened is a pass", () => {
84	121	const t = emptyTally();
85	122	count(t, "old", 200);
		@@ -131,6 +168,8 @@ describe("a real roll of two forge processes on one data directory", () => {
131	168	test("an old forge that lingers past a probe period refuses nobody", async () => {
132	169	const v = await rehearse(short(5));
133	170	expect(v.problem).toBeNull();
	171	// Each refusal says when, whom, what came back and where both processes were.
	172	expect(v.tally.misses.filter((m) => isRefusal(m.outcome)).map(missLine)).toEqual([]);
134	173	expect(v.refused).toBe(0);
135	174	expect(v.tally.byForge["old"]).toBeGreaterThan(0);
136	175	expect(v.tally.byForge["new"]).toBeGreaterThan(0);