Public: anyone can read this; the crew can change it.
Self-deploy rollback, part 1: `bun src/main.ts --rollback <sha>` swaps back to a kept release, and refuses one whose schema is behind the live database #1846
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/1823-self-deploy-rollback-part-1-bun-src-main → main 3 commits fork 6 threads · 6 unresolved
README.md +2 −1
Diff not shown — a line 3,133 characters wide (limit 2,000). +2 −1
View filescripts/gate.ts +1
| @@ -74,6 +74,7 @@ export const GATE: readonly string[] = [ | ||
| 74 | 74 | "test/recovery-doc.test.ts", |
| 75 | 75 | "test/release-checkout.test.ts", |
| 76 | 76 | "test/release-preflight.test.ts", |
| 77 | "test/release-rollback.test.ts", | |
| 77 | 78 | "test/release-swap.test.ts", |
| 78 | 79 | "test/threats-doc.test.ts", |
| 79 | 80 | "test/returns.test.ts", |
src/main.ts +21
| @@ -71,6 +71,27 @@ if (process.argv.includes("--preflight")) { | ||
| 71 | 71 | process.exit(result.ok ? 0 : 1); |
| 72 | 72 | } |
| 73 | 73 | |
| 74 | // `--rollback <sha>` starts a kept release beside the running forge and | |
| 75 | // exits once it reads healthy as that sha (release/rollback.ts). | |
| 76 | if (process.argv[2] === "--rollback") { | |
| 77 | const { rollback } = await import("./release/rollback.ts"); | |
| 78 | const port = Number(process.env.PORT ?? 3000); | |
| 79 | if (!Number.isInteger(port) || port < 1 || port > 65535) { | |
| 80 | console.error(`PORT must be a port number, not ${JSON.stringify(process.env.PORT ?? "").slice(0, 40)}`); | |
outdated
A non-numeric PORT becomes NaN here and reaches the swap. Validate it and exit 1 with one line. |
||
| 81 | process.exit(1); | |
| 82 | } | |
| 83 | const result = await rollback({ | |
| 84 | dataDir: dataDirOf(process.env), | |
| 85 | sha: process.argv[3] ?? "", | |
| 86 | port, | |
| 87 | env: process.env, | |
| 88 | }); | |
| 89 | // CLI answer on the stream a person or a script reads. | |
| 90 | if (result.ok) console.log(`rolled back: pid ${result.pid} is healthy; the previous forge is still running on the port, stop it yourself`); | |
| 91 | else console.error(result.line); | |
| 92 | process.exit(result.ok ? 0 : 1); | |
| 93 | } | |
The success branch (exit 0, this line) has no test, and nothing proves the release survives this process exiting. Add a spawn test with a kept stand-in release that asserts exit 0 and that health still answers after the CLI exits. |
||
| 94 | ||
| 74 | 95 | // A forge can be an app of a forge (docs/APPS.md, "The forge as an app of |
| 75 | 96 | // itself"). The host's `FORGE_APP_*` say where this one lives and what it |
| 76 | 97 | // answers to; `FORGE_URL` there names the host, not this forge. Hosted, the |
src/release/rollback.ts added +62
| @@ -0,0 +1,62 @@ | ||
| 1 | /** | |
| 2 | * `bun src/main.ts --rollback <sha>`: swap back to a kept release. The same | |
| 3 | * health-read swap as a landing (`startRelease`), after two refusals that | |
| 4 | * each take one line: the release is not kept, or its schema is behind the | |
| 5 | * live database's (a down-migration is not something this forge has). | |
| 6 | * A release whose schema is ahead of the live database is fine: it migrates | |
| 7 | * forward on boot, and the database's own refusal of a future schema is | |
| 8 | * untouched. | |
| 9 | */ | |
| 10 | ||
| 11 | import { Database } from "bun:sqlite"; | |
| 12 | import { existsSync } from "node:fs"; | |
| 13 | import { join } from "node:path"; | |
| 14 | import { releaseDir } from "./checkout.ts"; | |
| 15 | import { startRelease, type SwapInput } from "./swap.ts"; | |
| 16 | ||
| 17 | export type Rollback = { ok: true; pid: number } | { ok: false; line: string }; | |
| 18 | ||
| 19 | /** How many migrations the live database has applied; 0 when there is none yet. */ | |
| 20 | export function liveSchema(dataDir: string): number { | |
| 21 | const file = join(dataDir, "forge.db"); | |
| 22 | if (!existsSync(file)) return 0; | |
| 23 | const db = new Database(file, { readonly: true }); | |
| 24 | try { | |
| 25 | const table = db | |
| 26 | .query<{ n: number }, []>("SELECT COUNT(*) AS n FROM sqlite_master WHERE type='table' AND name='schema_migration'") | |
| 27 | .get(); | |
| 28 | if (table?.n === 1) | |
The |
||
| 29 | return db.query<{ n: number }, []>("SELECT COUNT(*) AS n FROM schema_migration").get()?.n ?? 0; | |
| 30 | return db.query<{ user_version: number }, []>("PRAGMA user_version").get()?.user_version ?? 0; | |
| 31 | } finally { | |
| 32 | db.close(); | |
| 33 | } | |
| 34 | } | |
| 35 | ||
| 36 | /** How many migrations the kept release knows, read from its own schema module. */ | |
| 37 | export async function releaseSchema(dir: string): Promise<number | undefined> { | |
| 38 | const file = join(dir, "src", "core", "schema.ts"); | |
| 39 | if (!existsSync(file)) return undefined; | |
| 40 | try { | |
| 41 | const mod = (await import(file)) as { migrations?: readonly unknown[] }; | |
| 42 | return Array.isArray(mod.migrations) ? mod.migrations.length : undefined; | |
| 43 | } catch { | |
| 44 | return undefined; | |
| 45 | } | |
| 46 | } | |
| 47 | ||
| 48 | export async function rollback( | |
| 49 | input: Omit<SwapInput, "dir" | "sha"> & { dataDir: string; sha: string }, | |
| 50 | ): Promise<Rollback> { | |
| 51 | const { dataDir, sha, ...swap } = input; | |
| 52 | const shown = /^[0-9a-f]{1,40}$/.test(sha) ? sha : "(not a commit id)"; | |
| 53 | const dir = releaseDir(dataDir, sha); | |
| 54 | if (!/^[0-9a-f]{40}$/.test(sha) || !existsSync(dir)) | |
| 55 | return { ok: false, line: `no kept release ${shown}; it was never checked out or has been pruned` }; | |
| 56 | const known = await releaseSchema(dir); | |
| 57 | if (known === undefined) return { ok: false, line: `${sha} has no readable schema; not rolling back to it` }; | |
| 58 | const live = liveSchema(dataDir); | |
| 59 | if (known < live) | |
Add a test for the |
||
| 60 | return { ok: false, line: `${sha} predates schema ${live}; rolling back would need a down-migration` }; | |
| 61 | return startRelease({ ...swap, dir, sha }); | |
This starts the kept release but never stops the forge it replaces, so the 'swap' leaves two forges on one port, with the newer code still serving. Drain the old pid here, or have the CLI say it is left running and cover that in the README. |
||
| 62 | } |
test/release-rollback.test.ts added +181
| @@ -0,0 +1,181 @@ | ||
| 1 | /** | |
| 2 | * `--rollback <sha>` (#1823): a kept release is swapped to through the same | |
| 3 | * health-read swap as a landing; an unknown sha and one whose schema is | |
| 4 | * behind the live database's are refused in one line. | |
| 5 | */ | |
| 6 | ||
| 7 | import { afterAll, describe, expect, test } from "bun:test"; | |
| 8 | import { Database } from "bun:sqlite"; | |
| 9 | import { mkdirSync, writeFileSync } from "node:fs"; | |
| 10 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 11 | import { tmpdir } from "node:os"; | |
| 12 | import { join } from "node:path"; | |
| 13 | import { releaseDir } from "../src/release/checkout.ts"; | |
| 14 | import { liveSchema, rollback } from "../src/release/rollback.ts"; | |
| 15 | ||
| 16 | const KEPT = "a".repeat(40); | |
| 17 | const OLDER = "b".repeat(40); | |
| 18 | const GONE = "c".repeat(40); | |
| 19 | ||
| 20 | const STAND_IN = ` | |
| 21 | Bun.serve({ | |
| 22 | port: Number(process.env.PORT), | |
| 23 | reusePort: true, | |
| 24 | fetch: () => Response.json({ ok: true, draining: false, commit: process.env.FORGE_COMMIT }), | |
| 25 | }); | |
| 26 | `; | |
| 27 | ||
| 28 | const dirs: string[] = []; | |
| 29 | const pids: number[] = []; | |
| 30 | afterAll(async () => { | |
| 31 | for (const pid of pids) { | |
| 32 | try { | |
| 33 | process.kill(pid, "SIGKILL"); | |
| 34 | } catch {} | |
| 35 | } | |
| 36 | for (const d of dirs) await rm(d, { recursive: true, force: true }); | |
| 37 | }); | |
| 38 | ||
| 39 | async function emptyDir(): Promise<string> { | |
| 40 | const dir = await mkdtemp(join(tmpdir(), "forge-rollback-")); | |
| 41 | dirs.push(dir); | |
| 42 | return dir; | |
| 43 | } | |
| 44 | ||
| 45 | async function dataDir(live: number): Promise<string> { | |
| 46 | const dir = await emptyDir(); | |
| 47 | const db = new Database(join(dir, "forge.db"), { create: true }); | |
| 48 | db.exec("CREATE TABLE schema_migration (id TEXT PRIMARY KEY, position INTEGER, applied_at TEXT)"); | |
| 49 | for (let i = 0; i < live; i++) db.exec(`INSERT INTO schema_migration VALUES ('m${i}', ${i + 1}, 'now')`); | |
| 50 | db.close(); | |
| 51 | return dir; | |
| 52 | } | |
| 53 | ||
| 54 | function keep(data: string, sha: string, migrations: number): void { | |
| 55 | const dir = releaseDir(data, sha); | |
| 56 | mkdirSync(join(dir, "src", "core"), { recursive: true }); | |
| 57 | writeFileSync(join(dir, "src", "main.ts"), STAND_IN); | |
| 58 | writeFileSync( | |
| 59 | join(dir, "src", "core", "schema.ts"), | |
| 60 | `export const migrations = ${JSON.stringify(Array.from({ length: migrations }, (_, i) => `SELECT ${i}`))};\n`, | |
| 61 | ); | |
| 62 | } | |
| 63 | ||
| 64 | async function freePort(): Promise<number> { | |
| 65 | const s = Bun.serve({ port: 0, fetch: () => new Response("") }); | |
| 66 | const port = s.port ?? 0; | |
| 67 | s.stop(true); | |
| 68 | return port; | |
| 69 | } | |
| 70 | ||
| 71 | describe("rollback", () => { | |
| 72 | test("a kept sha swaps and health names it", async () => { | |
| 73 | const data = await dataDir(3); | |
| 74 | keep(data, KEPT, 3); | |
| 75 | const port = await freePort(); | |
| 76 | const res = await rollback({ dataDir: data, sha: KEPT, port, env: process.env, budgetMs: 15_000 }); | |
| 77 | expect(res.ok).toBe(true); | |
| 78 | if (res.ok) pids.push(res.pid); | |
| 79 | const body = (await (await fetch(`http://127.0.0.1:${port}/api/v1/health`)).json()) as { commit: string }; | |
| 80 | expect(body.commit).toBe(KEPT); | |
| 81 | }); | |
| 82 | ||
| 83 | test("an unknown or pruned sha is refused naming it", async () => { | |
| 84 | const data = await dataDir(3); | |
| 85 | const res = await rollback({ dataDir: data, sha: GONE, port: await freePort(), env: process.env }); | |
| 86 | expect(res.ok).toBe(false); | |
| 87 | if (!res.ok) { | |
| 88 | expect(res.line).toContain(GONE); | |
| 89 | expect(res.line).not.toContain("\n"); | |
| 90 | } | |
| 91 | }); | |
| 92 | ||
| 93 | test("a sha that is not hex cannot reach the filesystem", async () => { | |
| 94 | const data = await dataDir(1); | |
| 95 | const res = await rollback({ dataDir: data, sha: "../../etc", port: await freePort(), env: process.env }); | |
| 96 | expect(res.ok).toBe(false); | |
| 97 | if (!res.ok) expect(res.line).not.toContain("etc"); | |
| 98 | }); | |
| 99 | ||
| 100 | test("a release with an older schema than the live database is refused", async () => { | |
| 101 | const data = await dataDir(5); | |
| 102 | keep(data, OLDER, 3); | |
| 103 | const res = await rollback({ dataDir: data, sha: OLDER, port: await freePort(), env: process.env }); | |
| 104 | expect(res).toEqual({ | |
| 105 | ok: false, | |
| 106 | line: `${OLDER} predates schema 5; rolling back would need a down-migration`, | |
| 107 | }); | |
| 108 | }); | |
| 109 | ||
| 110 | test("a release with a newer schema than the live database is allowed", async () => { | |
| 111 | const data = await dataDir(3); | |
| 112 | keep(data, KEPT, 5); | |
| 113 | const port = await freePort(); | |
| 114 | const res = await rollback({ dataDir: data, sha: KEPT, port, env: process.env, budgetMs: 15_000 }); | |
| 115 | expect(res.ok).toBe(true); | |
| 116 | if (res.ok) pids.push(res.pid); | |
| 117 | }); | |
| 118 | ||
| 119 | test("a release with no readable schema is refused", async () => { | |
| 120 | const data = await dataDir(1); | |
| 121 | const none = releaseDir(data, KEPT); | |
| 122 | mkdirSync(join(none, "src"), { recursive: true }); | |
| 123 | writeFileSync(join(none, "src", "main.ts"), STAND_IN); | |
| 124 | const missing = await rollback({ dataDir: data, sha: KEPT, port: await freePort(), env: process.env }); | |
| 125 | expect(missing.ok).toBe(false); | |
| 126 | if (!missing.ok) expect(missing.line).toContain("has no readable schema"); | |
| 127 | ||
| 128 | keep(data, OLDER, 1); | |
| 129 | writeFileSync(join(releaseDir(data, OLDER), "src", "core", "schema.ts"), "throw new Error('broken');\n"); | |
| 130 | const broken = await rollback({ dataDir: data, sha: OLDER, port: await freePort(), env: process.env }); | |
| 131 | expect(broken.ok).toBe(false); | |
| 132 | if (!broken.ok) expect(broken.line).toContain("has no readable schema"); | |
| 133 | }); | |
| 134 | }); | |
| 135 | ||
| 136 | describe("liveSchema", () => { | |
| 137 | test("no database is schema 0", async () => { | |
| 138 | expect(liveSchema(await emptyDir())).toBe(0); | |
| 139 | }); | |
| 140 | ||
| 141 | test("a legacy database counts its user_version", async () => { | |
| 142 | const data = await emptyDir(); | |
| 143 | const db = new Database(join(data, "forge.db"), { create: true }); | |
| 144 | db.exec("PRAGMA user_version = 4"); | |
| 145 | db.close(); | |
| 146 | expect(liveSchema(data)).toBe(4); | |
| 147 | keep(data, OLDER, 2); | |
| 148 | const res = await rollback({ dataDir: data, sha: OLDER, port: await freePort(), env: process.env }); | |
| 149 | expect(res).toEqual({ | |
| 150 | ok: false, | |
| 151 | line: `${OLDER} predates schema 4; rolling back would need a down-migration`, | |
| 152 | }); | |
| 153 | }); | |
| 154 | }); | |
| 155 | ||
| 156 | describe("--rollback command line", () => { | |
| 157 | const run = async (args: string[], env: Record<string, string>) => { | |
| 158 | const proc = Bun.spawn([process.execPath, join(import.meta.dir, "..", "src", "main.ts"), ...args], { | |
| 159 | env: { ...process.env, FORGE_SSH_PORT: "0", ...env }, | |
| 160 | stdout: "pipe", | |
| 161 | stderr: "pipe", | |
| 162 | }); | |
| 163 | const [err, code] = await Promise.all([new Response(proc.stderr).text(), proc.exited]); | |
| 164 | return { err: err.trim(), code }; | |
| 165 | }; | |
| 166 | ||
| 167 | test("a missing sha exits 1 with one line", async () => { | |
| 168 | const data = await emptyDir(); | |
| 169 | const { err, code } = await run(["--rollback"], { FORGE_DATA: data }); | |
| 170 | expect(code).toBe(1); | |
| 171 | expect(err.split("\n")).toHaveLength(1); | |
| 172 | }); | |
| 173 | ||
| 174 | test("a bad PORT exits 1 with one line", async () => { | |
| 175 | const data = await emptyDir(); | |
| 176 | const { err, code } = await run(["--rollback", KEPT], { FORGE_DATA: data, PORT: "abc" }); | |
| 177 | expect(code).toBe(1); | |
| 178 | expect(err.split("\n")).toHaveLength(1); | |
| 179 | expect(err).toContain("PORT"); | |
| 180 | }); | |
| 181 | }); |
New CLI flag with no README entry. Document it beside
--preflight(README.md:38).