Public: anyone can read this; the crew can change it.
Sentinel: release was degraded 36 minutes today: Release 0.7.473 is unproven: the public URL never answered as 0.7.473. It is proven again at 10:57 UTC. #10174
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/10168-sentinel-release-was-degraded-36-minutes → main 2 commits fork
2 files
changed +39
−9
src/release/prove.ts +17 −9
| @@ -11,8 +11,10 @@ | ||
| 11 | 11 | * failure up to an hour), so one slow answer in the watch no longer stands |
| 12 | 12 | * for the version's whole life (#7622) and a version that stays broken |
| 13 | 13 | * costs its public URL a few loads an hour, not one every 5 minutes. It |
| 14 | * gives up after ten minutes. A give-up is retried as a slow answer is, | |
| 15 | * since the swap may only be late (#9942). When a newer release answers | |
| 14 | * gives up after ten minutes. A first give-up is retried after one tick, | |
| 15 | * since the swap may only be late (#9942, #10168); a repeated one backs off | |
| 16 | * by the plain doubling, so a version that never answers is not polled | |
| 17 | * without pause. When a newer release answers | |
| 16 | 18 | * instead, this process is being replaced: it keeps nothing and never polls |
| 17 | 19 | * again (#10138). |
| 18 | 20 | */ |
| @@ -46,6 +48,8 @@ export interface Proof { | ||
| 46 | 48 | readonly failures?: number; |
| 47 | 49 | /** The first failing check was only a slow, empty or hung answer (`Failure.slow`, `.empty`, `.hung`); retried sooner (#9358, #9482). */ |
| 48 | 50 | readonly slow?: boolean; |
| 51 | /** The failure was the give-up: ten minutes already spent polling; only its first retry skips the backoff (#10168). */ | |
| 52 | readonly gaveUp?: boolean; | |
| 49 | 53 | } |
| 50 | 54 | |
| 51 | 55 | /** |
| @@ -53,9 +57,13 @@ export interface Proof { | ||
| 53 | 57 | * doubled per failure, capped at `MAX_RETRY_MS`. A proof that failed only |
| 54 | 58 | * because an answer was slow (`slow`) is a moment's load, not a broken |
| 55 | 59 | * release, so it waits `min(that, TICK_MS * failures)`: 1, 2, 3, 4 minutes, |
| 56 | * then the doubling takes over (#9238). | |
| 60 | * then the doubling takes over (#9238). A first give-up (`gaveUp`) | |
| 61 | * already polled ten minutes, so it waits one tick; a repeated give-up is | |
| 62 | * not a slow answer and backs off by the plain doubling (#10168). | |
| 57 | 63 | */ |
| 58 | export function retryAfter(failures: number, slow = false): number { | |
| 64 | export function retryAfter(failures: number, slow = false, gaveUp = false): number { | |
| 65 | // The first give-up already polled ten minutes, so it is asked again after one tick; a repeat backs off by the plain doubling. | |
| 66 | if (gaveUp) return failures <= 1 ? TICK_MS : Math.min(MAX_RETRY_MS, RETRY_MS * 2 ** (Math.min(failures, 20) - 1)); | |
| 59 | 67 | const backoff = Math.min(MAX_RETRY_MS, RETRY_MS * 2 ** Math.max(0, Math.min(failures, 20) - 1)); |
| 60 | 68 | return slow ? Math.min(backoff, TICK_MS * Math.max(1, failures)) : backoff; |
| 61 | 69 | } |
| @@ -99,7 +107,7 @@ function keep(dataDir: string, proof: Proof): Proof { | ||
| 99 | 107 | export function proofNeed(dataDir: string): { sentence: string; since: string } | undefined { |
| 100 | 108 | const p = lastProof(dataDir); |
| 101 | 109 | if (p === undefined || p.ok) return undefined; |
| 102 | const next = Date.parse(p.at) + retryAfter(p.failures ?? 1, p.slow === true); | |
| 110 | const next = Date.parse(p.at) + retryAfter(p.failures ?? 1, p.slow === true, p.gaveUp === true); | |
| 103 | 111 | // When it is tried again, so a person reading the card knows it is not stuck. |
| 104 | 112 | const again = Number.isFinite(next) ? ` It is proven again at ${new Date(next).toISOString().slice(11, 16)} UTC.` : ""; |
| 105 | 113 | return { sentence: `Release ${p.version} is unproven: ${p.failure ?? "a smoke check failed"}.${again}`, since: p.since ?? p.at }; |
| @@ -133,7 +141,7 @@ export function proofDue(last: Proof | undefined, version: string, now: number): | ||
| 133 | 141 | if (last?.version !== version) return true; |
| 134 | 142 | if (last.ok) return false; |
| 135 | 143 | const at = Date.parse(last.at); |
| 136 | return !Number.isFinite(at) || now - at >= retryAfter(last.failures ?? 1, last.slow === true); | |
| 144 | return !Number.isFinite(at) || now - at >= retryAfter(last.failures ?? 1, last.slow === true, last.gaveUp === true); | |
| 137 | 145 | } |
| 138 | 146 | |
| 139 | 147 | /** |
| @@ -147,7 +155,7 @@ export async function proveRelease(o: ProveOptions): Promise<Proof | undefined> | ||
| 147 | 155 | const sleep = o.sleep ?? Bun.sleep; |
| 148 | 156 | const doFetch = o.fetch ?? fetch; |
| 149 | 157 | const failedBefore = last?.version === o.version && !last.ok ? last : undefined; |
| 150 | const finish = (failure?: string, slow = false): Proof => { | |
| 158 | const finish = (failure?: string, slow = false, gaveUp = false): Proof => { | |
| 151 | 159 | const at = new Date(now()).toISOString(); |
| 152 | 160 | return keep(o.dataDir, { |
| 153 | 161 | version: o.version, |
| @@ -155,7 +163,7 @@ export async function proveRelease(o: ProveOptions): Promise<Proof | undefined> | ||
| 155 | 163 | ok: failure === undefined, |
| 156 | 164 | ...(failure === undefined |
| 157 | 165 | ? {} |
| 158 | : { failure, since: failedBefore ? (failedBefore.since ?? failedBefore.at) : at, failures: (failedBefore?.failures ?? (failedBefore ? 1 : 0)) + 1, ...(slow ? { slow } : {}) }), | |
| 166 | : { failure, since: failedBefore ? (failedBefore.since ?? failedBefore.at) : at, failures: (failedBefore?.failures ?? (failedBefore ? 1 : 0)) + 1, ...(slow ? { slow } : {}), ...(gaveUp ? { gaveUp } : {}) }), | |
| 159 | 167 | }); |
| 160 | 168 | }; |
| 161 | 169 | try { |
| @@ -177,7 +185,7 @@ export async function proveRelease(o: ProveOptions): Promise<Proof | undefined> | ||
| 177 | 185 | superseded.add(o.version); |
| 178 | 186 | return undefined; |
| 179 | 187 | } |
| 180 | return finish(`the public URL never answered as ${o.version}`, true); | |
| 188 | return finish(`the public URL never answered as ${o.version}`, true, true); | |
| 181 | 189 | } |
| 182 | 190 | await sleep(POLL_MS); |
| 183 | 191 | } |
test/release-prove.test.ts +22
| @@ -115,6 +115,28 @@ describe("proving a release", () => { | ||
| 115 | 115 | expect(old.hits.every((h) => h === "/healthz")).toBe(true); |
| 116 | 116 | }); |
| 117 | 117 | |
| 118 | test("only a first give-up is retried after one tick; a repeat backs off (#10168)", () => { | |
| 119 | expect(retryAfter(1, true, true)).toBe(60_000); | |
| 120 | expect(retryAfter(2, true, true)).toBe(RETRY_MS * 2); | |
| 121 | expect(retryAfter(9, true, true)).toBe(60 * 60_000); | |
| 122 | expect(retryAfter(4, true)).toBe(4 * 60_000); | |
| 123 | }); | |
| 124 | ||
| 125 | test("a silent public URL keeps gaveUp, and due and the card both wait one tick (#10168)", async () => { | |
| 126 | let t = 0; | |
| 127 | const p = await run("1.0.0", site("0.9.0").fake, { now: () => t, sleep: async (ms: number) => void (t += ms) }); | |
| 128 | expect(p?.gaveUp).toBe(true); | |
| 129 | expect(lastProof(dir)?.gaveUp).toBe(true); | |
| 130 | const at = Date.parse(p!.at); | |
| 131 | expect(proofDue(p, "1.0.0", at + 59_000)).toBe(false); | |
| 132 | expect(proofDue(p, "1.0.0", at + 60_000)).toBe(true); | |
| 133 | const when = new Date(at + 60_000).toISOString().slice(11, 16); | |
| 134 | expect(proofNeed(dir)?.sentence).toEndWith(`It is proven again at ${when} UTC.`); | |
| 135 | // a smoke failure is not a give-up | |
| 136 | const smoke = await run("1.0.1", site("1.0.1", { "/": () => new Response("x", { status: 500 }) }).fake); | |
| 137 | expect(smoke?.gaveUp).toBeUndefined(); | |
| 138 | }); | |
| 139 | ||
| 118 | 140 | test("it keeps no failure when a newer release answers instead (#10138)", async () => { |
| 119 | 141 | let t = 0; |
| 120 | 142 | const p = await run("1.0.0", site("1.0.1").fake, { now: () => t, sleep: async (ms: number) => void (t += ms) }); |
new above
offline