Public: anyone can read this; the crew can change it.
Sentinel: release was degraded 40 minutes today: Release 0.7.471 is unproven: the public URL never answered as 0.7.471. It is proven again at 10:19 UTC. #10148
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/10138-sentinel-release-was-degraded-40-minutes → main 2 commits fork
2 files
changed +61
−5
src/release/prove.ts +30 −4
| @@ -11,7 +11,10 @@ | ||
| 11 | 11 | * failure up to an hour), so one slow answer in the watch no longer stands |
| 12 | 12 | * for the version's whole life (#7622) and a version that stays broken |
| 13 | 13 | * costs its public URL a few loads an hour, not one every 5 minutes. It |
| 14 | * gives up after ten minutes; a give-up is retried as a slow answer is, since the swap may only be late (#9942). | |
| 14 | * gives up after ten minutes. A give-up is retried as a slow answer is, | |
| 15 | * since the swap may only be late (#9942). When a newer release answers | |
| 16 | * instead, this process is being replaced: it keeps nothing and never polls | |
| 17 | * again (#10138). | |
| 15 | 18 | */ |
| 16 | 19 | |
| 17 | 20 | import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; |
| @@ -57,6 +60,20 @@ export function retryAfter(failures: number, slow = false): number { | ||
| 57 | 60 | return slow ? Math.min(backoff, TICK_MS * Math.max(1, failures)) : backoff; |
| 58 | 61 | } |
| 59 | 62 | |
| 63 | /** Is dotted version `a` later than `b`; false when either is not numeric dotted. */ | |
| 64 | export function isNewer(a: string, b: string): boolean { | |
| 65 | const x = a.split("."), y = b.split("."); | |
| 66 | if (![...x, ...y].every((n) => /^\d+$/.test(n))) return false; | |
| 67 | for (let i = 0; i < Math.max(x.length, y.length); i++) { | |
| 68 | const d = Number(x[i] ?? 0) - Number(y[i] ?? 0); | |
| 69 | if (d !== 0) return d > 0; | |
| 70 | } | |
| 71 | return false; | |
| 72 | } | |
| 73 | ||
| 74 | /** Versions this process saw replaced by a newer one; never polled again. */ | |
| 75 | const superseded = new Set<string>(); | |
| 76 | ||
| 60 | 77 | const proofFile = (dataDir: string): string => join(dataDir, "release", "proof.json"); |
| 61 | 78 | |
| 62 | 79 | /** The last verdict kept, or undefined when none is or the file is not one. */ |
| @@ -126,7 +143,7 @@ export function proofDue(last: Proof | undefined, version: string, now: number): | ||
| 126 | 143 | export async function proveRelease(o: ProveOptions): Promise<Proof | undefined> { |
| 127 | 144 | const now = o.now ?? Date.now; |
| 128 | 145 | const last = lastProof(o.dataDir); |
| 129 | if (!proofDue(last, o.version, now())) return undefined; | |
| 146 | if (superseded.has(o.version) || !proofDue(last, o.version, now())) return undefined; | |
| 130 | 147 | const sleep = o.sleep ?? Bun.sleep; |
| 131 | 148 | const doFetch = o.fetch ?? fetch; |
| 132 | 149 | const failedBefore = last?.version === o.version && !last.ok ? last : undefined; |
| @@ -144,15 +161,24 @@ export async function proveRelease(o: ProveOptions): Promise<Proof | undefined> | ||
| 144 | 161 | try { |
| 145 | 162 | const health = new URL("/healthz", o.url).href; |
| 146 | 163 | const deadline = now() + (o.giveUpMs ?? GIVE_UP_MS); |
| 164 | let seen = ""; | |
| 147 | 165 | for (;;) { |
| 148 | 166 | try { |
| 149 | 167 | const res = await doFetch(health, { redirect: "manual", headers: ownProbeHeaders(o.dataDir), signal: AbortSignal.timeout(5_000) }); |
| 150 | 168 | await res.body?.cancel(); |
| 151 | if (res.headers.get("x-forge-version") === o.version) break; | |
| 169 | seen = res.headers.get("x-forge-version") ?? ""; | |
| 170 | if (seen === o.version) break; | |
| 152 | 171 | } catch { |
| 153 | 172 | // not up yet at its public address |
| 154 | 173 | } |
| 155 | if (now() + POLL_MS > deadline) return finish(`the public URL never answered as ${o.version}`, true); | |
| 174 | if (now() + POLL_MS > deadline) { | |
| 175 | // A newer release answers: this process is being replaced, and a failure kept now would stand over the newer version's own proof. | |
| 176 | if (isNewer(seen, o.version)) { | |
| 177 | superseded.add(o.version); | |
| 178 | return undefined; | |
| 179 | } | |
| 180 | return finish(`the public URL never answered as ${o.version}`, true); | |
| 181 | } | |
| 156 | 182 | await sleep(POLL_MS); |
| 157 | 183 | } |
| 158 | 184 | const failures = await smoke(o.url, o.token ?? "", { |
test/release-prove.test.ts +31 −1
| @@ -9,7 +9,7 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; | ||
| 9 | 9 | import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; |
| 10 | 10 | import { tmpdir } from "node:os"; |
| 11 | 11 | import { join } from "node:path"; |
| 12 | import { lastProof, proofDue, proofNeed, proveRelease, proveRunning, retryAfter, RETRY_MS, startProving, type Proof } from "../src/release/prove.ts"; | |
| 12 | import { isNewer, lastProof, proofDue, proofNeed, proveRelease, proveRunning, retryAfter, RETRY_MS, startProving, type Proof } from "../src/release/prove.ts"; | |
| 13 | 13 | import { owned } from "../src/crew/incidents.ts"; |
| 14 | 14 | |
| 15 | 15 | const CSS = '<html><link rel="stylesheet" href="/a.css"></html>'; |
| @@ -115,6 +115,36 @@ describe("proving a release", () => { | ||
| 115 | 115 | expect(old.hits.every((h) => h === "/healthz")).toBe(true); |
| 116 | 116 | }); |
| 117 | 117 | |
| 118 | test("it keeps no failure when a newer release answers instead (#10138)", async () => { | |
| 119 | let t = 0; | |
| 120 | const p = await run("1.0.0", site("1.0.1").fake, { now: () => t, sleep: async (ms: number) => void (t += ms) }); | |
| 121 | expect(p).toBeUndefined(); | |
| 122 | expect(proofNeed(dir)).toBeUndefined(); | |
| 123 | const again = site("1.0.1"); | |
| 124 | expect(await run("1.0.0", again.fake, { now: () => t, sleep: async (ms: number) => void (t += ms) })).toBeUndefined(); | |
| 125 | expect(again.hits).toEqual([]); // a replaced process does not poll again | |
| 126 | }); | |
| 127 | ||
| 128 | test("an older or non-numeric answer is still a kept failure (#10138)", async () => { | |
| 129 | for (const [mine, answered] of [["1.0.1", "1.0.0"], ["1.0.2", "dev"]] as const) { | |
| 130 | let t = 0; | |
| 131 | const p = await run(mine, site(answered).fake, { now: () => t, sleep: async (ms: number) => void (t += ms) }); | |
| 132 | expect(p?.ok).toBe(false); | |
| 133 | expect(proofNeed(dir)?.sentence).toContain(`never answered as ${mine}`); | |
| 134 | } | |
| 135 | }); | |
| 136 | ||
| 137 | test("isNewer compares dotted numbers", () => { | |
| 138 | expect(isNewer("1.0.1", "1.0.0")).toBe(true); | |
| 139 | expect(isNewer("1.0.0", "1.0.0")).toBe(false); | |
| 140 | expect(isNewer("1.0.0", "1.0.1")).toBe(false); | |
| 141 | expect(isNewer("1.0.1", "1.0")).toBe(true); | |
| 142 | expect(isNewer("1.0", "1.0.0")).toBe(false); | |
| 143 | expect(isNewer("0.7.472", "0.7.471")).toBe(true); | |
| 144 | expect(isNewer("dev", "1.0.0")).toBe(false); | |
| 145 | expect(isNewer("", "1.0.0")).toBe(false); | |
| 146 | }); | |
| 147 | ||
| 118 | 148 | test("a failed version is proven again after RETRY_MS, and a pass clears the card (#7622)", async () => { |
| 119 | 149 | let t = Date.parse("2026-10-28T12:00:00Z"); |
| 120 | 150 | const clock = { now: () => t }; |
new above
offline