cf76e2b · 948b416
@@ -104,6 +104,9 @@
answer `/healthz` as that version, then runs the smoke checks (`src/release/smok
keeps the verdict in `$FORGE_APP_DATA/release/proof.json`. A failure is a
`Release X is unproven: …` card on an admin's board and on the terminal's first
screen (the same cards), and is paged once; a later passing version clears it.
A failed proof is tried again after 5 minutes, doubling to an hour; one that
failed only because an answer was slow (`slow: true` in the proof) is tried
again after a minute per failure in a row (1, 2, 3, 4 minutes), then the doubling.
**The outsider suite** (#663) is the proof a person reads before flipping signup
to open. It adds the proof only: it removes no gate and reopens nothing.
@@ -41,11 +41,20 @@
export interface Proof {
  readonly since?: string;
  /** How many proofs of this version have failed in a row; 1 when unset. */
  readonly failures?: number;
  /** The first failing check was only a slow answer (`Failure.slow`); retried sooner. */
  readonly slow?: boolean;
}
/** The wait after a version's `failures`th failed proof in a row: `RETRY_MS` doubled per failure, capped at `MAX_RETRY_MS`. */
export function retryAfter(failures: number): number {
  return Math.min(MAX_RETRY_MS, RETRY_MS * 2 ** Math.max(0, Math.min(failures, 20) - 1));
/**
 * The wait after a version's `failures`th failed proof in a row: `RETRY_MS`
 * doubled per failure, capped at `MAX_RETRY_MS`. A proof that failed only
 * because an answer was slow (`slow`) is a moment's load, not a broken
 * release, so it waits `min(that, TICK_MS * failures)`: 1, 2, 3, 4 minutes,
 * then the doubling takes over (#9238).
 */
export function retryAfter(failures: number, slow = false): number {
  const backoff = Math.min(MAX_RETRY_MS, RETRY_MS * 2 ** Math.max(0, Math.min(failures, 20) - 1));
  return slow ? Math.min(backoff, TICK_MS * Math.max(1, failures)) : backoff;
}
const proofFile = (dataDir: string): string => join(dataDir, "release", "proof.json");
@@ -73,7 +82,7 @@
function keep(dataDir: string, proof: Proof): Proof {
export function proofNeed(dataDir: string): { sentence: string; since: string } | undefined {
  const p = lastProof(dataDir);
  if (p === undefined || p.ok) return undefined;
  const next = Date.parse(p.at) + retryAfter(p.failures ?? 1);
  const next = Date.parse(p.at) + retryAfter(p.failures ?? 1, p.slow === true);
  // When it is tried again, so a person reading the card knows it is not stuck.
  const again = Number.isFinite(next) ? ` It is proven again at ${new Date(next).toISOString().slice(11, 16)} UTC.` : "";
  return { sentence: `Release ${p.version} is unproven: ${p.failure ?? "a smoke check failed"}.${again}`, since: p.since ?? p.at };
@@ -107,7 +116,7 @@
export function proofDue(last: Proof | undefined, version: string, now: number):
  if (last?.version !== version) return true;
  if (last.ok) return false;
  const at = Date.parse(last.at);
  return !Number.isFinite(at) || now - at >= retryAfter(last.failures ?? 1);
  return !Number.isFinite(at) || now - at >= retryAfter(last.failures ?? 1, last.slow === true);
}
/**
@@ -121,7 +130,7 @@
export async function proveRelease(o: ProveOptions): Promise<Proof | undefined>
  const sleep = o.sleep ?? Bun.sleep;
  const doFetch = o.fetch ?? fetch;
  const failedBefore = last?.version === o.version && !last.ok ? last : undefined;
  const finish = (failure?: string): Proof => {
  const finish = (failure?: string, slow = false): Proof => {
    const at = new Date(now()).toISOString();
    return keep(o.dataDir, {
      version: o.version,
@@ -129,7 +138,7 @@
export async function proveRelease(o: ProveOptions): Promise<Proof | undefined>
      ok: failure === undefined,
      ...(failure === undefined
        ? {}
        : { failure, since: failedBefore ? (failedBefore.since ?? failedBefore.at) : at, failures: (failedBefore?.failures ?? (failedBefore ? 1 : 0)) + 1 }),
        : { failure, since: failedBefore ? (failedBefore.since ?? failedBefore.at) : at, failures: (failedBefore?.failures ?? (failedBefore ? 1 : 0)) + 1, ...(slow ? { slow } : {}) }),
    });
  };
  try {
@@ -151,7 +160,7 @@
export async function proveRelease(o: ProveOptions): Promise<Proof | undefined>
      anonymousBoard: true,
      heavy: { version: o.version, probe: ownProbeHeaders(o.dataDir), ...o.heavy }, ...(o.appHost ? { appHost: o.appHost } : {}),
    });
    return finish(failures[0] === undefined ? undefined : checkLine(failures[0]));
    return finish(failures[0] === undefined ? undefined : checkLine(failures[0]), failures[0]?.slow === true);
  } catch (error) {
    return finish(`the smoke checks could not run: ${String(error)}`);
  }
@@ -65,6 +65,8 @@
export interface Failure {
  status: number | string;
  contentType: string;
  why: string;
  /** The check failed only because the answer was slower than allowed. */
  slow?: boolean;
}
/** The first stylesheet href in a page, or null. */
@@ -168,7 +170,7 @@
async function watch(root: URL, token: string, heavy: Heavy, fetch: typeof globa
      if (took > slowMs) {
        // The forge's own time says whether the release was slow or the way to it was (#9022).
        const own = forgeMs(res);
        if (own === undefined || own > slowMs) return { failure: failureFor(`${path} took ${took} ms, over ${slowMs} ms`, res.status), slow: true };
        if (own === undefined || own > slowMs) return { failure: { ...failureFor(`${path} took ${took} ms, over ${slowMs} ms`, res.status), slow: true }, slow: true };
        console.warn(`smoke: ${path} took ${took} ms, the forge ${Math.round(own)} ms of it; the rest was the network`);
      }
      if (path === REPO_HOME && !heavy.repo && res.status === 404) return { failure: null, slow: false };
@@ -6,7 +6,7 @@
 */
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
import { mkdtempSync, rmSync } from "node:fs";
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { lastProof, proofDue, proofNeed, proveRelease, proveRunning, retryAfter, RETRY_MS, startProving, type Proof } from "../src/release/prove.ts";
@@ -147,7 +147,7 @@
describe("proving a release", () => {
  });
  test("each failure in a row doubles the wait, up to an hour", async () => {
    expect([1, 2, 3, 4, 5, 99].map(retryAfter)).toEqual([5, 10, 20, 40, 60, 60].map((m) => m * 60_000));
    expect([1, 2, 3, 4, 5, 99].map((n) => retryAfter(n))).toEqual([5, 10, 20, 40, 60, 60].map((m) => m * 60_000));
    let t = Date.parse("2026-10-28T12:00:00Z");
    const clock = { now: () => t };
    const broken = () => site("4.3.0", { "/": () => new Response("x", { status: 500 }) }).fake;
@@ -237,3 +237,29 @@
describe("proving a release", () => {
    expect(s.hits).toEqual([]);
  });
});
describe("a slow-only proof is asked again sooner (#9238)", () => {
  const slowProof: Proof = { version: "1.0.0", at: "2026-11-03T03:00:00.000Z", ok: false, failure: "https://x/healthz as desktop /healthz took 2032 ms, over 1000 ms (status 200)", failures: 1, slow: true };
  const at = Date.parse(slowProof.at);
  test("retryAfter", () => {
    expect(retryAfter(1, true)).toBe(60_000);
    expect(retryAfter(3, true)).toBe(180_000);
    expect(retryAfter(5, true)).toBe(300_000);
    expect(retryAfter(1)).toBe(RETRY_MS);
  });
  test("proofDue is due after a minute for slow, not for the same text without the flag", () => {
    expect(proofDue(slowProof, "1.0.0", at + 61_000)).toBe(true);
    const plain: Proof = { ...slowProof, slow: false };
    expect(proofDue(plain, "1.0.0", at + 61_000)).toBe(false);
  });
  test("proofNeed says the shortened time", () => {
    const dir = mkdtempSync(join(tmpdir(), "prove-"));
    try {
      mkdirSync(join(dir, "release"), { recursive: true });
      writeFileSync(join(dir, "release", "proof.json"), JSON.stringify(slowProof));
      expect(proofNeed(dir)!.sentence).toEndWith("It is proven again at 03:01 UTC.");
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });
});
@@ -93,7 +93,12 @@
test("a /healthz that is slow once in the watch is asked again and passes (#8917
test("a /healthz that is slow twice in a row in the watch fails", async () => {
  // hit 1 is the pre-watch check of /healthz by smoke; the watch's polls follow
  const failures = await smoke(forge(0, [], () => undefined, (p, hit) => (p === "/healthz" && (hit === 4 || hit === 5) ? 300 : 0)), "t", { heavy });
  expect(failures.some((f) => f.url.endsWith("/healthz") && f.why.includes("took"))).toBe(true);
  expect(failures.some((f) => f.url.endsWith("/healthz") && f.why.includes("took") && f.slow === true)).toBe(true);
});
test("a failure that is not slowness is not marked slow (#9238)", async () => {
  const failures = await smoke(forge(0, [], () => undefined, () => 0), "t", { heavy });
  expect(failures.every((f) => f.slow !== true)).toBe(true);
});
test("a polled heavy page that is slow once in the watch is asked again and passes", async () => {
		@@ -104,6 +104,9 @@ answer `/healthz` as that version, then runs the smoke checks (`src/release/smok
104	104	keeps the verdict in `$FORGE_APP_DATA/release/proof.json`. A failure is a
105	105	`Release X is unproven: …` card on an admin's board and on the terminal's first
106	106	screen (the same cards), and is paged once; a later passing version clears it.
	107	A failed proof is tried again after 5 minutes, doubling to an hour; one that
	108	failed only because an answer was slow (`slow: true` in the proof) is tried
	109	again after a minute per failure in a row (1, 2, 3, 4 minutes), then the doubling.
107	110
108	111	The outsider suite (#663) is the proof a person reads before flipping signup
109	112	to open. It adds the proof only: it removes no gate and reopens nothing.
		@@ -41,11 +41,20 @@ export interface Proof {
41	41	readonly since?: string;
42	42	/** How many proofs of this version have failed in a row; 1 when unset. */
43	43	readonly failures?: number;
	44	/** The first failing check was only a slow answer (`Failure.slow`); retried sooner. */
	45	readonly slow?: boolean;
44	46	}
45	47
46		/** The wait after a version's `failures`th failed proof in a row: `RETRY_MS` doubled per failure, capped at `MAX_RETRY_MS`. */
47		export function retryAfter(failures: number): number {
48		return Math.min(MAX_RETRY_MS, RETRY_MS * 2 ** Math.max(0, Math.min(failures, 20) - 1));
	48	/**
	49	* The wait after a version's `failures`th failed proof in a row: `RETRY_MS`
	50	* doubled per failure, capped at `MAX_RETRY_MS`. A proof that failed only
	51	* because an answer was slow (`slow`) is a moment's load, not a broken
	52	* release, so it waits `min(that, TICK_MS * failures)`: 1, 2, 3, 4 minutes,
	53	* then the doubling takes over (#9238).
	54	*/
	55	export function retryAfter(failures: number, slow = false): number {
	56	const backoff = Math.min(MAX_RETRY_MS, RETRY_MS * 2 ** Math.max(0, Math.min(failures, 20) - 1));
	57	return slow ? Math.min(backoff, TICK_MS * Math.max(1, failures)) : backoff;
49	58	}
50	59
51	60	const proofFile = (dataDir: string): string => join(dataDir, "release", "proof.json");
		@@ -73,7 +82,7 @@ function keep(dataDir: string, proof: Proof): Proof {
73	82	export function proofNeed(dataDir: string): { sentence: string; since: string } \| undefined {
74	83	const p = lastProof(dataDir);
75	84	if (p === undefined \|\| p.ok) return undefined;
76		const next = Date.parse(p.at) + retryAfter(p.failures ?? 1);
	85	const next = Date.parse(p.at) + retryAfter(p.failures ?? 1, p.slow === true);
77	86	// When it is tried again, so a person reading the card knows it is not stuck.
78	87	const again = Number.isFinite(next) ? ` It is proven again at ${new Date(next).toISOString().slice(11, 16)} UTC.` : "";
79	88	return { sentence: `Release ${p.version} is unproven: ${p.failure ?? "a smoke check failed"}.${again}`, since: p.since ?? p.at };
		@@ -107,7 +116,7 @@ export function proofDue(last: Proof \| undefined, version: string, now: number):
107	116	if (last?.version !== version) return true;
108	117	if (last.ok) return false;
109	118	const at = Date.parse(last.at);
110		return !Number.isFinite(at) \|\| now - at >= retryAfter(last.failures ?? 1);
	119	return !Number.isFinite(at) \|\| now - at >= retryAfter(last.failures ?? 1, last.slow === true);
111	120	}
112	121
113	122	/**
		@@ -121,7 +130,7 @@ export async function proveRelease(o: ProveOptions): Promise<Proof \| undefined>
121	130	const sleep = o.sleep ?? Bun.sleep;
122	131	const doFetch = o.fetch ?? fetch;
123	132	const failedBefore = last?.version === o.version && !last.ok ? last : undefined;
124		const finish = (failure?: string): Proof => {
	133	const finish = (failure?: string, slow = false): Proof => {
125	134	const at = new Date(now()).toISOString();
126	135	return keep(o.dataDir, {
127	136	version: o.version,
		@@ -129,7 +138,7 @@ export async function proveRelease(o: ProveOptions): Promise<Proof \| undefined>
129	138	ok: failure === undefined,
130	139	...(failure === undefined
131	140	? {}
132		: { failure, since: failedBefore ? (failedBefore.since ?? failedBefore.at) : at, failures: (failedBefore?.failures ?? (failedBefore ? 1 : 0)) + 1 }),
	141	: { failure, since: failedBefore ? (failedBefore.since ?? failedBefore.at) : at, failures: (failedBefore?.failures ?? (failedBefore ? 1 : 0)) + 1, ...(slow ? { slow } : {}) }),
133	142	});
134	143	};
135	144	try {
		@@ -151,7 +160,7 @@ export async function proveRelease(o: ProveOptions): Promise<Proof \| undefined>
151	160	anonymousBoard: true,
152	161	heavy: { version: o.version, probe: ownProbeHeaders(o.dataDir), ...o.heavy }, ...(o.appHost ? { appHost: o.appHost } : {}),
153	162	});
154		return finish(failures[0] === undefined ? undefined : checkLine(failures[0]));
	163	return finish(failures[0] === undefined ? undefined : checkLine(failures[0]), failures[0]?.slow === true);
155	164	} catch (error) {
156	165	return finish(`the smoke checks could not run: ${String(error)}`);
157	166	}
		@@ -65,6 +65,8 @@ export interface Failure {
65	65	status: number \| string;
66	66	contentType: string;
67	67	why: string;
	68	/** The check failed only because the answer was slower than allowed. */
	69	slow?: boolean;
68	70	}
69	71
70	72	/** The first stylesheet href in a page, or null. */
		@@ -168,7 +170,7 @@ async function watch(root: URL, token: string, heavy: Heavy, fetch: typeof globa
168	170	if (took > slowMs) {
169	171	// The forge's own time says whether the release was slow or the way to it was (#9022).
170	172	const own = forgeMs(res);
171		if (own === undefined \|\| own > slowMs) return { failure: failureFor(`${path} took ${took} ms, over ${slowMs} ms`, res.status), slow: true };
	173	if (own === undefined \|\| own > slowMs) return { failure: { ...failureFor(`${path} took ${took} ms, over ${slowMs} ms`, res.status), slow: true }, slow: true };
172	174	console.warn(`smoke: ${path} took ${took} ms, the forge ${Math.round(own)} ms of it; the rest was the network`);
173	175	}
174	176	if (path === REPO_HOME && !heavy.repo && res.status === 404) return { failure: null, slow: false };
		@@ -6,7 +6,7 @@
6	6	*/
7	7
8	8	import { afterEach, beforeEach, describe, expect, test } from "bun:test";
9		import { mkdtempSync, rmSync } from "node:fs";
	9	import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
10	10	import { tmpdir } from "node:os";
11	11	import { join } from "node:path";
12	12	import { lastProof, proofDue, proofNeed, proveRelease, proveRunning, retryAfter, RETRY_MS, startProving, type Proof } from "../src/release/prove.ts";
		@@ -147,7 +147,7 @@ describe("proving a release", () => {
147	147	});
148	148
149	149	test("each failure in a row doubles the wait, up to an hour", async () => {
150		expect([1, 2, 3, 4, 5, 99].map(retryAfter)).toEqual([5, 10, 20, 40, 60, 60].map((m) => m * 60_000));
	150	expect([1, 2, 3, 4, 5, 99].map((n) => retryAfter(n))).toEqual([5, 10, 20, 40, 60, 60].map((m) => m * 60_000));
151	151	let t = Date.parse("2026-10-28T12:00:00Z");
152	152	const clock = { now: () => t };
153	153	const broken = () => site("4.3.0", { "/": () => new Response("x", { status: 500 }) }).fake;
		@@ -237,3 +237,29 @@ describe("proving a release", () => {
237	237	expect(s.hits).toEqual([]);
238	238	});
239	239	});
	240
	241	describe("a slow-only proof is asked again sooner (#9238)", () => {
	242	const slowProof: Proof = { version: "1.0.0", at: "2026-11-03T03:00:00.000Z", ok: false, failure: "https://x/healthz as desktop /healthz took 2032 ms, over 1000 ms (status 200)", failures: 1, slow: true };
	243	const at = Date.parse(slowProof.at);
	244	test("retryAfter", () => {
	245	expect(retryAfter(1, true)).toBe(60_000);
	246	expect(retryAfter(3, true)).toBe(180_000);
	247	expect(retryAfter(5, true)).toBe(300_000);
	248	expect(retryAfter(1)).toBe(RETRY_MS);
	249	});
	250	test("proofDue is due after a minute for slow, not for the same text without the flag", () => {
	251	expect(proofDue(slowProof, "1.0.0", at + 61_000)).toBe(true);
	252	const plain: Proof = { ...slowProof, slow: false };
	253	expect(proofDue(plain, "1.0.0", at + 61_000)).toBe(false);
	254	});
	255	test("proofNeed says the shortened time", () => {
	256	const dir = mkdtempSync(join(tmpdir(), "prove-"));
	257	try {
	258	mkdirSync(join(dir, "release"), { recursive: true });
	259	writeFileSync(join(dir, "release", "proof.json"), JSON.stringify(slowProof));
	260	expect(proofNeed(dir)!.sentence).toEndWith("It is proven again at 03:01 UTC.");
	261	} finally {
	262	rmSync(dir, { recursive: true, force: true });
	263	}
	264	});
	265	});
		@@ -93,7 +93,12 @@ test("a /healthz that is slow once in the watch is asked again and passes (#8917
93	93	test("a /healthz that is slow twice in a row in the watch fails", async () => {
94	94	// hit 1 is the pre-watch check of /healthz by smoke; the watch's polls follow
95	95	const failures = await smoke(forge(0, [], () => undefined, (p, hit) => (p === "/healthz" && (hit === 4 \|\| hit === 5) ? 300 : 0)), "t", { heavy });
96		expect(failures.some((f) => f.url.endsWith("/healthz") && f.why.includes("took"))).toBe(true);
	96	expect(failures.some((f) => f.url.endsWith("/healthz") && f.why.includes("took") && f.slow === true)).toBe(true);
	97	});
	98
	99	test("a failure that is not slowness is not marked slow (#9238)", async () => {
	100	const failures = await smoke(forge(0, [], () => undefined, () => 0), "t", { heavy });
	101	expect(failures.every((f) => f.slow !== true)).toBe(true);
97	102	});
98	103
99	104	test("a polled heavy page that is slow once in the watch is asked again and passes", async () => {