Public: anyone can read this; the crew can change it.
Sentinel: release was degraded 178 minutes today: Release 0.7.479 is unproven: the public URL never answered as 0.7.479. It is proven again at 15:55 UTC. #10267
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/10248-sentinel-release-was-degraded-178-minute → main 10 commits fork 1 thread · 1 unresolved
7 files
changed +80
−8
scripts/unreached-allow.json +1
| @@ -15,6 +15,7 @@ | ||
| 15 | 15 | "src/flow/review_rounds.ts", |
| 16 | 16 | "src/flow/shared_red.ts", |
| 17 | 17 | "src/flow/stale_fold.ts", |
| 18 | "src/spawn/pod.ts", | |
| 18 | 19 | "src/tui/gallery.ts", |
| 19 | 20 | "src/tui/ui/qr.ts", |
| 20 | 21 | "src/web/highlight-client.ts", |
src/kernel/vitals.ts +2 −1
| @@ -32,6 +32,7 @@ import { lastMirror, mirrorNeed } from "../snapshot/mirror.ts"; | ||
| 32 | 32 | import { lastSnapshotOutcome, snapshotNeed } from "../snapshot/outcome.ts"; |
| 33 | 33 | import { drillNeed, firstSeen as drillFirstSeen, lastVerdict } from "../snapshot/verdict.ts"; |
| 34 | 34 | import { proofNeed } from "../release/prove.ts"; |
| 35 | import { running as runningVersion } from "./version.ts"; | |
| 35 | 36 | import { rollbackNeed } from "../release/launch.ts"; |
| 36 | 37 | import { behindStanding } from "../release/behind.ts"; |
| 37 | 38 | import { newestDrills } from "../core/domain_drills.ts"; |
| @@ -515,7 +516,7 @@ export function vitals(input: VitalsInput): Vitals { | ||
| 515 | 516 | if (behind !== undefined) need("mirror", behind.sentence, behind.since, "/settings/backups"); |
| 516 | 517 | |
| 517 | 518 | // A release the forge could not prove against its own public URL is a card until a release does. |
| 518 | const unproven = proofNeed(input.dataDir); | |
| 519 | const unproven = proofNeed(input.dataDir, runningVersion().version); | |
| 519 | 520 | if (unproven !== undefined) need("release", unproven.sentence, unproven.since, "/board"); |
| 520 | 521 | |
| 521 | 522 | // A roll the image's launcher undid is a card until a later version reads ready. |
src/release/prove.ts +16 −6
| @@ -16,7 +16,8 @@ | ||
| 16 | 16 | * #10168, #10222). After that the plain doubling takes over, so a version |
| 17 | 17 | * that never answers costs its public URL a few loads an hour. When a newer |
| 18 | 18 | * release answers instead, this process is being replaced: it keeps nothing |
| 19 | * and never polls again (#10138). | |
| 19 | * and does not poll again for `GIVE_UP_MS` (#10138); still running after | |
| 20 | * that, it was not replaced and proves its version again (#10248). | |
| 20 | 21 | */ |
| 21 | 22 | |
| 22 | 23 | import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; |
| @@ -82,8 +83,14 @@ export function isNewer(a: string, b: string): boolean { | ||
| 82 | 83 | return false; |
| 83 | 84 | } |
| 84 | 85 | |
| 85 | /** Versions this process saw replaced by a newer one; never polled again. */ | |
| 86 | const superseded = new Set<string>(); | |
| 86 | /** | |
| 87 | * Versions this process saw a newer release answer for, and until when it | |
| 88 | * does not poll for them again. A replaced process is gone well within | |
| 89 | * `GIVE_UP_MS`; one still running after it was not replaced (a roll the | |
| 90 | * launcher undid), so it proves its version again rather than leave the last | |
| 91 | * kept failure, and its past retry time, standing for hours (#10248). | |
| 92 | */ | |
| 93 | const superseded = new Map<string, number>(); | |
| 87 | 94 | |
| 88 | 95 | const proofFile = (dataDir: string): string => join(dataDir, "release", "proof.json"); |
| 89 | 96 | |
| @@ -107,9 +114,12 @@ function keep(dataDir: string, proof: Proof): Proof { | ||
| 107 | 114 | } |
| 108 | 115 | |
| 109 | 116 | /** The card an unproven release is, saying when it is tried again; undefined while the last verdict is a pass. */ |
| 110 | export function proofNeed(dataDir: string): { sentence: string; since: string } | undefined { | |
| 117 | export function proofNeed(dataDir: string, running?: string): { sentence: string; since: string } | undefined { | |
| 111 | 118 | const p = lastProof(dataDir); |
| 112 | 119 | if (p === undefined || p.ok) return undefined; |
| 120 | // Only the process running a version proves it: a failure kept for another version is not | |
| 121 | // this process's to report, and stood for hours after a roll or a newer boot (#10248). | |
| 122 | if (running !== undefined && running !== UNSTAMPED && p.version !== running) return undefined; | |
| 113 | 123 | const next = Date.parse(p.at) + retryAfter(p.failures ?? 1, p.slow === true, p.gaveUp === true); |
| 114 | 124 | // When it is tried again, so a person reading the card knows it is not stuck. |
| 115 | 125 | const again = Number.isFinite(next) ? ` It is proven again at ${new Date(next).toISOString().slice(11, 16)} UTC.` : ""; |
| @@ -154,7 +164,7 @@ export function proofDue(last: Proof | undefined, version: string, now: number): | ||
| 154 | 164 | export async function proveRelease(o: ProveOptions): Promise<Proof | undefined> { |
| 155 | 165 | const now = o.now ?? Date.now; |
| 156 | 166 | const last = lastProof(o.dataDir); |
| 157 | if (superseded.has(o.version) || !proofDue(last, o.version, now())) return undefined; | |
| 167 | if (now() < (superseded.get(o.version) ?? -Infinity) || !proofDue(last, o.version, now())) return undefined; | |
| 158 | 168 | const sleep = o.sleep ?? Bun.sleep; |
| 159 | 169 | const doFetch = o.fetch ?? fetch; |
| 160 | 170 | const failedBefore = last?.version === o.version && !last.ok ? last : undefined; |
| @@ -186,7 +196,7 @@ export async function proveRelease(o: ProveOptions): Promise<Proof | undefined> | ||
| 186 | 196 | if (now() + POLL_MS > deadline) { |
| 187 | 197 | // A newer release answers: this process is being replaced, and a failure kept now would stand over the newer version's own proof. |
| 188 | 198 | if (isNewer(seen, o.version)) { |
| 189 | superseded.add(o.version); | |
| 199 | superseded.set(o.version, now() + GIVE_UP_MS); | |
| 190 | 200 | return undefined; |
| 191 | 201 | } |
| 192 | 202 | return finish(`the public URL never answered as ${o.version}`, true, true); |
test/crew-loop.test.ts +4
| @@ -256,6 +256,10 @@ describe("a crew reviewer's changes-requested wakes the builder", () => { | ||
| 256 | 256 | concurrency: 0, |
| 257 | 257 | budgetUsd: 1, |
| 258 | 258 | runTimeoutMs: 60_000, |
| 259 | // No tick inside the suite: the first one runs the autonomy pass, whose | |
| 260 | // `moveRequested` queues its own follow-up on a standing verdict, and on | |
| 261 | // a loaded shard it landed mid-test and moved the counts these assert. | |
| 262 | tickMs: 3_600_000, | |
| 259 | 263 | }); |
| 260 | 264 | }); |
| 261 | 265 |
test/release-boot.test.ts +2
| @@ -96,6 +96,8 @@ test("this tree, shipped as the image ships it (src/ alone), boots", async () => | ||
| 96 | 96 | const repo = join(import.meta.dir, ".."); |
| 97 | 97 | cpSync(join(repo, "src"), join(image, "src"), { recursive: true }); |
| 98 | 98 | for (const file of ["package.json", "tsconfig.json"]) cpSync(join(repo, file), join(image, file)); |
| 99 | // The Dockerfile copies the OpenCode plugin too: src/http/plugin_routes.ts reads it at boot (#10274). | |
| 100 | cpSync(join(repo, "plugins", "opencode"), join(image, "plugins", "opencode"), { recursive: true }); | |
| 99 | 101 | expect(existsSync(join(image, "scripts"))).toBe(false); |
| 100 | 102 | const result = await proveBoot({ tree: image, version: "9.9.9", budgetMs: 60_000 }); |
| 101 | 103 | expect(result).toMatchObject({ ok: true }); |
test/release-prove.test.ts +29 −1
| @@ -9,7 +9,7 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; | ||
| 9 | 9 | import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; |
| 10 | 10 | import { tmpdir } from "node:os"; |
| 11 | 11 | import { join } from "node:path"; |
| 12 | import { isNewer, lastProof, proofDue, proofNeed, proveRelease, proveRunning, QUICK_RETRIES, retryAfter, RETRY_MS, startProving, type Proof } from "../src/release/prove.ts"; | |
| 12 | import { GIVE_UP_MS, isNewer, lastProof, proofDue, proofNeed, proveRelease, proveRunning, QUICK_RETRIES, retryAfter, RETRY_MS, startProving, type Proof } from "../src/release/prove.ts"; | |
| 13 | 13 | import { owned } from "../src/crew/incidents.ts"; |
| 14 | 14 | |
| 15 | 15 | const CSS = '<html><link rel="stylesheet" href="/a.css"></html>'; |
| @@ -179,6 +179,34 @@ describe("proving a release", () => { | ||
| 179 | 179 | expect(again.hits).toEqual([]); // a replaced process does not poll again |
| 180 | 180 | }); |
| 181 | 181 | |
| 182 | test("a process still running after a newer release answered proves its version again (#10248)", async () => { | |
| 183 | let t = 0; | |
| 184 | const clock = { now: () => t, sleep: async (ms: number) => void (t += ms) }; | |
| 185 | // 1.4.0 failed once, then a newer release answered for a moment and was undone. | |
| 186 | expect((await run("1.4.0", site("1.3.0").fake, clock))?.ok).toBe(false); | |
| 187 | t += 60_000; | |
| 188 | expect(await run("1.4.0", site("1.4.1").fake, clock)).toBeUndefined(); | |
| 189 | expect(proofNeed(dir, "1.4.0")).toBeDefined(); | |
| 190 | const soon = site("1.4.0"); | |
| 191 | t += GIVE_UP_MS - 60_000; | |
| 192 | expect(await run("1.4.0", soon.fake, clock)).toBeUndefined(); | |
| 193 | expect(soon.hits).toEqual([]); | |
| 194 | t += 60_000; | |
| 195 | const later = site("1.4.0"); | |
| 196 | expect((await run("1.4.0", later.fake, clock))?.ok).toBe(true); | |
| 197 | expect(later.hits.length).toBeGreaterThan(0); | |
| 198 | expect(proofNeed(dir, "1.4.0")).toBeUndefined(); | |
| 199 | }); | |
| 200 | ||
| 201 | test("a failure kept for another version than the running one is no card (#10248)", async () => { | |
| 202 | let t = 0; | |
| 203 | await run("1.0.2", site("1.0.0").fake, { now: () => t, sleep: async (ms: number) => void (t += ms) }); | |
| 204 | expect(proofNeed(dir)).toBeDefined(); | |
| 205 | expect(proofNeed(dir, "1.0.2")).toBeDefined(); | |
| 206 | expect(proofNeed(dir, "1.0.3")).toBeUndefined(); | |
| 207 | expect(proofNeed(dir, "dev")).toBeDefined(); | |
| 208 | }); | |
| 209 | ||
| 182 | 210 | test("an older or non-numeric answer is still a kept failure (#10138)", async () => { |
| 183 | 211 | for (const [mine, answered] of [["1.0.1", "1.0.0"], ["1.0.2", "dev"]] as const) { |
| 184 | 212 | let t = 0; |
test/vitals-release.test.ts +26
| @@ -48,3 +48,29 @@ test("a passing proof clears it", () => { | ||
| 48 | 48 | keep({ version: "9.9.10", at: "2026-10-18T11:00:00.000Z", ok: true }); |
| 49 | 49 | expect(card()).toBeUndefined(); |
| 50 | 50 | }); |
| 51 | ||
| 52 | // The process running a version reports only that version's failure: one kept | |
| 53 | // for another version (a roll the launcher undid, a newer boot) is no card (#10248). | |
| 54 | const runningAs = (version: string, body: () => void) => { | |
| 55 | const was = process.env["FORGE_VERSION"]; | |
| 56 | process.env["FORGE_VERSION"] = version; | |
| 57 | try { | |
| 58 | body(); | |
| 59 | } finally { | |
| 60 | if (was === undefined) delete process.env["FORGE_VERSION"]; | |
| 61 | else process.env["FORGE_VERSION"] = was; | |
| 62 | } | |
| 63 | }; | |
| 64 | ||
| 65 | test("a failed proof kept for another version is no card for the running one", () => { | |
| 66 | keep({ version: "0.7.479", at: "2026-11-09T12:57:00.000Z", ok: false, failure: "the public URL never answered as 0.7.479" }); | |
| 67 | runningAs("0.7.478", () => expect(card()).toBeUndefined()); | |
| 68 | }); | |
| 69 | ||
| 70 | test("a failed proof for the running version is still a card", () => { | |
| 71 | keep({ version: "0.7.479", at: "2026-11-09T12:57:00.000Z", ok: false, failure: "the public URL never answered as 0.7.479" }); | |
| 72 | runningAs("0.7.479", () => | |
| 73 | expect(card()).toMatchObject({ href: "/board", since: "2026-11-09T12:57:00.000Z" }), | |
| 74 | ); | |
| 75 | runningAs("v0.7.479", () => expect(card()?.sentence).toStartWith("Release 0.7.479 is unproven")); | |
| 76 | }); |
This argument is the fix, and nothing tests it. Add a vitals() test (test/vitals-release.test.ts) that fails when this line goes back to proofNeed(input.dataDir): a failed proof for another version gives no release card, and one for the running version still does.