Public: anyone can read this; the crew can change it.
Turn the release keeper on in production and prove it with one canary release #7948
torque/forge:torque/7947-turn-the-release-keeper-on-in-production → main 2 commits fork 4 threads · 4 unresolved
3 files
changed +110
−2
docs/features/release-keeper.md +15
| @@ -100,3 +100,18 @@ Assumption: the keeper promotes by setting the image directly, as the | ||
| 100 | 100 | steward's rollback did. If Flux reconciles the image back, the keeper sets |
| 101 | 101 | it again on its next tick while the release is rolling. If it never takes, |
| 102 | 102 | the 3 h deadline rolls it back and holds. |
| 103 | ||
| 104 | ## Staying on the verified image | |
| 105 | ||
| 106 | Production's reconcile can render an older chart tag after a verify. On every | |
| 107 | idle tick the keeper compares the Deployment's tag with the last verified | |
| 108 | release's (or, after a rollback, the known-good one). If the Deployment names | |
| 109 | an older version it sets the settled image again and writes `release.restored` | |
| 110 | (only when the set was accepted; a refused set is retried next tick). A newer | |
| 111 | tag, or an image with no tag (a digest), is left alone. It is a repair: a | |
| 112 | reconcile that owns the image tag still rolls the pod each time, so the | |
| 113 | Deployment's manifests should leave the forge container's image to the keeper. | |
| 114 | ||
| 115 | `FORGE_REPO` also drives `releaseOnLand` (`checks/landing.ts`), which checks | |
| 116 | out and preflights a landed tree of that repository only, and swaps the | |
| 117 | process in only with `FORGE_SELF_SWAP=1`, which a clustered forge leaves unset. |
src/keeper/keeper.ts +28 −2
| @@ -29,7 +29,8 @@ import type { Database } from "bun:sqlite"; | ||
| 29 | 29 | import type { EventBus } from "../core/events.ts"; |
| 30 | 30 | import { createIssue } from "../core/issues.ts"; |
| 31 | 31 | import { decideCut } from "./cadence.ts"; |
| 32 | import { imageAt, type Deployer } from "./deploy.ts"; | |
| 32 | import { compareVersions } from "../release/shipped.ts"; | |
| 33 | import { imageAt, tagOf, type Deployer } from "./deploy.ts"; | |
| 33 | 34 | import { blocking, liftFixed, names, setCause, setHold, standingHolds, type Hold } from "./hold.ts"; |
| 34 | 35 | import { countRollProbe, dropCut, finish, getKeeperRelease, inFlight, keepTally, listKeeperReleases, markLive, recordCut, type KeeperRelease } from "./record.ts"; |
| 35 | 36 | import { rollBackToNamed } from "./rollback.ts"; |
| @@ -164,9 +165,11 @@ export async function tick(p: KeeperPorts): Promise<string> { | ||
| 164 | 165 | if (flying.stage === "rolling" && (await p.deployer.image()) !== want) await promote(p, flying.version, want); |
| 165 | 166 | return `${flying.version} is ${flying.stage}`; |
| 166 | 167 | } |
| 168 | const last = listKeeperReleases(p.db, 1)[0]; | |
| 169 | const kept = await keepVerified(p, last); | |
A new repair loop on every idle tick, off the pipeline-first direction and not what #7948 asked. Move it to its own issue and pull. |
||
| 170 | if (kept !== undefined) return kept; | |
| 167 | 171 | const main = await p.main(); |
| 168 | 172 | const newest = await p.newest(); |
| 169 | const last = listKeeperReleases(p.db, 1)[0]; | |
| 170 | 173 | const decision = decideCut({ |
| 171 | 174 | main: main ?? "", |
| 172 | 175 | newest, |
| @@ -191,6 +194,29 @@ export async function tick(p: KeeperPorts): Promise<string> { | ||
| 191 | 194 | return `cut ${r.version} at ${main.slice(0, 12)}; ${sent}; known good: ${knownGood.version}`; |
| 192 | 195 | } |
| 193 | 196 | |
| 197 | /** | |
| 198 | * The image the keeper last settled on stays: production's reconcile renders the chart tag from gitops and | |
| 199 | * can set an older image back after the verify. When the Deployment names an older version than the one | |
| 200 | * verified (or the known-good one after a rollback), the keeper names the settled image again. A newer tag, | |
| 201 | * which a person set in gitops, is never touched. | |
| 202 | */ | |
| 203 | async function keepVerified(p: KeeperPorts, last: KeeperRelease | undefined): Promise<string | undefined> { | |
| 204 | if (last === undefined || (last.stage !== "verified" && last.stage !== "rolled_back")) return undefined; | |
| 205 | const want = last.stage === "verified" ? imageAt(last.knownGood.image, last.version) : last.knownGood.image; | |
| 206 | const now = await p.deployer.image(); | |
| 207 | const have = tagOf(now); | |
| 208 | const settled = tagOf(want); | |
| 209 | // A digest-pinned or untagged image names no version to compare: leave it. | |
| 210 | if (have === "" || settled === "" || have === settled || compareVersions(have, settled) <= 0) return undefined; | |
| 211 | try { | |
| 212 | await p.deployer.setImage(want); | |
outdated
Append |
||
| 213 | } catch (error) { | |
| 214 | return `the Deployment named ${now}; keeping ${settled} refused (${error instanceof Error ? error.message : String(error)}); sent again next tick`; | |
| 215 | } | |
| 216 | p.bus.append({ kind: "release.restored", repoId: p.repo.id, payload: { version: settled, was: now, image: want } }); | |
| 217 | return `the Deployment named ${now}; set ${want} to keep ${settled}`; | |
| 218 | } | |
|
||
| 219 | ||
| 194 | 220 | /** Make the release's tag; when it is refused the cut is dropped and the line says why. */ |
| 195 | 221 | async function tagged(p: KeeperPorts, r: KeeperRelease): Promise<string | undefined> { |
| 196 | 222 | const refused = await p.tag(r.version, r.sha); |
test/release-keeper-rollback-named.test.ts +67
| @@ -95,3 +95,70 @@ describe("the named rollback", () => { | ||
| 95 | 95 | expect(tagOf("reg.test:5000/o/forge")).toBe(""); |
| 96 | 96 | }); |
| 97 | 97 | }); |
| 98 | ||
| 99 | describe("the verified image is kept", () => { | |
| 100 | test("a reconcile that sets an older tag back is answered by naming the verified image again; a newer tag is left alone", async () => { | |
| 101 | const f = await keeperForge(); | |
| 102 | await tick(f.ports); | |
| 103 | f.world.live = "0.7.284"; | |
| 104 | expect(await tick(f.ports)).toContain("0.7.284 verified live"); | |
| 105 | // The standby reconcile renders the older chart tag. | |
| 106 | f.state.image = "reg.test/o/forge:0.7.283"; | |
| 107 | f.world.live = "0.7.283"; | |
| 108 | f.world.now += 60_000; | |
| 109 | expect(await tick(f.ports)).toContain("to keep 0.7.284"); | |
| 110 | expect(f.state.image).toBe("reg.test/o/forge:0.7.284"); | |
| 111 | expect(f.events("release.restored")).toHaveLength(1); | |
| 112 | // A person's newer gitops tag is not fought. | |
| 113 | f.state.image = "reg.test/o/forge:0.7.290"; | |
| 114 | const sets = f.state.set.length; | |
| 115 | await tick(f.ports); | |
| 116 | expect(f.state.set).toHaveLength(sets); | |
| 117 | }); | |
| 118 | ||
| 119 | test("after a rollback the known-good image is the one kept", async () => { | |
| 120 | const f = await keeperForge(); | |
| 121 | await tick(f.ports); | |
| 122 | f.world.live = "0.7.284"; | |
| 123 | f.world.healthz = (n) => (n >= 20 ? { status: 503, ms: 4 } : { status: 200, ms: 4 }); | |
| 124 | await tick(f.ports); | |
| 125 | expect(f.state.image).toBe("reg.test/o/forge:0.7.283"); | |
| 126 | f.state.image = "reg.test/o/forge:0.7.282"; | |
| 127 | f.world.live = "0.7.283"; | |
| 128 | await tick(f.ports); | |
| 129 | expect(f.state.image).toBe("reg.test/o/forge:0.7.283"); | |
| 130 | }); | |
| 131 | ||
| 132 | test("a refused set claims no restore and is sent again next tick", async () => { | |
| 133 | const f = await keeperForge(); | |
| 134 | await tick(f.ports); | |
| 135 | f.world.live = "0.7.284"; | |
| 136 | await tick(f.ports); | |
| 137 | f.state.image = "reg.test/o/forge:0.7.283"; | |
| 138 | const real = f.ports.deployer.setImage; | |
| 139 | f.ports.deployer.setImage = async () => { | |
| 140 | throw new Error("HTTP 403"); | |
| 141 | }; | |
| 142 | expect(await tick(f.ports)).toContain("refused"); | |
| 143 | expect(f.events("release.restored")).toHaveLength(0); | |
| 144 | f.ports.deployer.setImage = real; | |
| 145 | await tick(f.ports); | |
| 146 | expect(f.state.image).toBe("reg.test/o/forge:0.7.284"); | |
| 147 | expect(f.events("release.restored")).toHaveLength(1); | |
| 148 | expect(f.events("release.released")).toHaveLength(1); | |
| 149 | }); | |
| 150 | ||
| 151 | test("a digest-pinned image names no version and is not set every tick", async () => { | |
| 152 | const f = await keeperForge(); | |
| 153 | await tick(f.ports); | |
| 154 | f.world.live = "0.7.284"; | |
| 155 | await tick(f.ports); | |
| 156 | const pinned = `reg.test/o/forge@sha256:${"a".repeat(64)}`; | |
| 157 | f.state.image = pinned; | |
| 158 | const sets = f.state.set.length; | |
| 159 | await tick(f.ports); | |
| 160 | await tick(f.ports); | |
| 161 | expect(f.state.set).toHaveLength(sets); | |
| 162 | expect(f.state.image).toBe(pinned); | |
| 163 | }); | |
| 164 | }); |
new above
offline
This
FORGE_REPO/releaseOnLandparagraph is about the switch-on, which this pull does not do. Move it to that pull.