Public: anyone can read this; the crew can change it.
Kept code pages cross a release swap, so the new process starts warm #9632
mason/forge:mason/9631-kept-code-pages-cross-a-release-swap-so → main 6 commits fork 4 threads · 4 unresolved
10 files
changed +583
−14
bench/code-pages-at-rate.ts +32 −13
| @@ -12,10 +12,20 @@ | ||
| 12 | 12 | * bun bench/code-pages-at-rate.ts --rate 500 --seconds 20 --addresses 4000 --crawl 3000 \ |
| 13 | 13 | * --out bench/results/9514-in-process-bypass.json |
| 14 | 14 | * |
| 15 | * A release swap (#9631): `--data DIR` keeps the copy in DIR between runs, | |
| 16 | * `--save` writes the kept pages to DIR's hand-over file when the run ends | |
| 17 | * (the old process at shutdown) and `--load` reads them before serving (the | |
| 18 | * new one at start). Each run is its own process, so everything else starts | |
| 19 | * cold, as it does after a real swap: | |
| 20 | * | |
| 21 | * bun bench/code-pages-at-rate.ts --data $TMPDIR/swap --seconds 40 --save # the old release, serving | |
| 22 | * bun bench/code-pages-at-rate.ts --data $TMPDIR/swap --warm 0 --load # the new one, right after | |
| 23 | * | |
| 15 | 24 | * What it is not: the edge run. That one needs the real deployment and is run |
| 16 | 25 | * with bench/load.ts against it (docs/features/load-test.md). |
| 17 | 26 | */ |
| 18 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 27 | import { existsSync } from "node:fs"; | |
| 28 | import { mkdir, mkdtemp, rm } from "node:fs/promises"; | |
| 19 | 29 | import { tmpdir } from "node:os"; |
| 20 | 30 | import { join } from "node:path"; |
| 21 | 31 | import { createApp } from "../src/app.ts"; |
| @@ -23,6 +33,7 @@ import { createRepo } from "../src/actions/repos.ts"; | ||
| 23 | 33 | import { createUser } from "../src/core/users.ts"; |
| 24 | 34 | import { repoDir } from "../src/git/repos.ts"; |
| 25 | 35 | import { serve } from "../src/http/server.ts"; |
| 36 | import { keptPagesPath, loadKeptPages, saveKeptPages } from "../src/web/pages/kept_pages_file.ts"; | |
| 26 | 37 | |
| 27 | 38 | const arg = (name: string, fallback: string): string => { |
| 28 | 39 | const at = process.argv.indexOf(`--${name}`); |
| @@ -33,20 +44,15 @@ const arg = (name: string, fallback: string): string => { | ||
| 33 | 44 | const out = process.argv.includes("--out") ? process.argv[process.argv.indexOf("--out") + 1] : undefined; |
| 34 | 45 | const root = join(import.meta.dir, ".."); |
| 35 | 46 | |
| 36 | const dataDir = await mkdtemp(join(tmpdir(), "forge-code-rate-")); | |
| 47 | const kept = process.argv.includes("--data") ? process.argv[process.argv.indexOf("--data") + 1]! : undefined; | |
| 48 | if (kept !== undefined) await mkdir(kept, { recursive: true }); | |
| 49 | const seeded = kept !== undefined && existsSync(join(kept, "forge.db")); | |
| 50 | const dataDir = kept ?? (await mkdtemp(join(tmpdir(), "forge-code-rate-"))); | |
| 37 | 51 | const app = createApp({ dataDir, port: 0 }); |
| 52 | if (process.argv.includes("--load")) console.error(`handover: ${loadKeptPages(keptPagesPath(dataDir))} kept pages loaded`); | |
| 38 | 53 | const servers = await serve(app); |
| 39 | 54 | try { |
| 40 | const u = await createUser(app.db, { origin: "seed", trusted: true, username: "trevato", email: "[email protected]", password: "a-good-passphrase", isAdmin: true }); | |
| 41 | if (!u.ok) throw new Error(u.error); | |
| 42 | const r = await createRepo(app, u.value, { name: "forge", visibility: "public" }); | |
| 43 | if (!r.ok) throw new Error(String(r.error)); | |
| 44 | // The clone's own history, fetched whole (a fetch runs no receive hook): every folder, file and commit as the forge holds them. | |
| 45 | const pushed = Bun.spawnSync(["git", "--git-dir", repoDir(app.reposRoot, "trevato", "forge"), "fetch", "--quiet", root, "HEAD:refs/heads/main"], { cwd: root, stderr: "pipe" }); | |
| 46 | if (pushed.exitCode !== 0) throw new Error(`copying the clone failed: ${pushed.stderr.toString()}`); | |
| 47 | const files = Bun.spawnSync(["git", "ls-files"], { cwd: root }).stdout.toString().split("\n").length - 1; | |
| 48 | console.error(`copy: trevato/forge with ${files} files at ${Bun.spawnSync(["git", "rev-parse", "HEAD"], { cwd: root }).stdout.toString().trim()}`); | |
| 49 | ||
| 55 | if (!seeded) await seed(); | |
| 50 | 56 | const args = ["--url", servers.url, "--repo", "trevato/forge", "--bypass", "--version", "in-process"]; |
| 51 | 57 | for (const name of ["rate", "seconds", "addresses", "crawl", "warm"]) args.push(`--${name}`, arg(name, { rate: "500", seconds: "20", addresses: "4000", crawl: "3000", warm: "0" }[name]!)); |
| 52 | 58 | if (out !== undefined) args.push("--out", out); |
| @@ -57,8 +63,21 @@ try { | ||
| 57 | 63 | stderr: "inherit", |
| 58 | 64 | }); |
| 59 | 65 | process.exitCode = await load.exited; |
| 66 | if (process.argv.includes("--save")) console.error(`handover: ${saveKeptPages(keptPagesPath(dataDir))} kept pages saved`); | |
| 60 | 67 | } finally { |
| 61 | 68 | servers.stop(); |
| 62 | 69 | app.close(); |
| 63 | await rm(dataDir, { recursive: true, force: true }); | |
| 70 | if (kept === undefined) await rm(dataDir, { recursive: true, force: true }); | |
| 71 | } | |
| 72 | ||
| 73 | async function seed(): Promise<void> { | |
| 74 | const u = await createUser(app.db, { origin: "seed", trusted: true, username: "trevato", email: "[email protected]", password: "a-good-passphrase", isAdmin: true }); | |
| 75 | if (!u.ok) throw new Error(u.error); | |
| 76 | const r = await createRepo(app, u.value, { name: "forge", visibility: "public" }); | |
| 77 | if (!r.ok) throw new Error(String(r.error)); | |
| 78 | // The clone's own history, fetched whole (a fetch runs no receive hook): every folder, file and commit as the forge holds them. | |
| 79 | const pushed = Bun.spawnSync(["git", "--git-dir", repoDir(app.reposRoot, "trevato", "forge"), "fetch", "--quiet", root, "HEAD:refs/heads/main"], { cwd: root, stderr: "pipe" }); | |
| 80 | if (pushed.exitCode !== 0) throw new Error(`copying the clone failed: ${pushed.stderr.toString()}`); | |
| 81 | const files = Bun.spawnSync(["git", "ls-files"], { cwd: root }).stdout.toString().split("\n").length - 1; | |
| 82 | console.error(`copy: trevato/forge with ${files} files at ${Bun.spawnSync(["git", "rev-parse", "HEAD"], { cwd: root }).stdout.toString().trim()}`); | |
| 64 | 83 | } |
bench/results/9631-swap-handover.md added +30
| @@ -0,0 +1,30 @@ | ||
| 1 | # Kept pages across a swap (#9631), measured against a cold start | |
| 2 | ||
| 3 | In-process copy of this repository, Shield at its default, 500/s across the | |
| 4 | `--crawl 3000` pinned links, 4000 made-up strangers, edge bypassed. Each run | |
| 5 | is its own process on the same data (`--data`), so everything but the | |
| 6 | hand-over file starts cold, as after a real swap. "Old release" ran 40 s and | |
| 7 | saved at the end (`--save`); the new one started right after with `--warm 0`, | |
| 8 | with (`--load`) or without the file. | |
| 9 | ||
| 10 | bun bench/code-pages-at-rate.ts --data $TMPDIR/swap --seconds 40 --save | |
| 11 | bun bench/code-pages-at-rate.ts --data $TMPDIR/swap --warm 0 --load # or without --load | |
| 12 | ||
| 13 | | run | pages handed over | refused of 9999 | p95 | | |
| 14 | | --- | ---: | ---: | ---: | | |
| 15 | | hand-over, 4 MB file (as shipped) | 295 | 4091 | 600 ms | | |
| 16 | | cold | 0 | 4487 | 659 ms | | |
| 17 | | hand-over, 4 MB file, again | 279 | 4564 | 589 ms | | |
| 18 | | cold, again | 0 | 4678 | 705 ms | | |
| 19 | | hand-over, 64 MB file (experiment, not shipped) | 1609 (23 MB) | 5566 | 717 ms | | |
| 20 | ||
| 21 | Per second, first hand-over run: `203/296 99/401 184/316 108/392 202/298 197/303 222/278 221/279 289/211 271/229 234/266 266/233 314/187 401/99 342/158 389/111 500/0 500/0 500/0 466/34`. | |
| 22 | Per second, first cold run: `232/267 162/338 66/434 212/288 188/312 195/305 277/223 247/253 286/214 218/282 249/251 314/186 263/237 138/362 292/208 356/144 480/20 336/163 501/0 500/0`. | |
| 23 | ||
| 24 | Reading: the hand-over refuses 2-9% fewer strangers at 4 MB, inside the | |
| 25 | spread between two cold runs (4487 vs 4678); handing over five times as many | |
| 26 | pages did not help at all. The cold window of #9566 is not made by the | |
| 27 | missing kept pages: the pages that still miss render cold on the one main | |
| 28 | thread, the loop falls behind and the stranger budgets divide all the same. | |
| 29 | The issue's done condition ("refuses fewer strangers than the baseline") is | |
| 30 | not met in any way that is distinguishable from noise. |
src/main.ts +9
| @@ -4,6 +4,7 @@ | ||
| 4 | 4 | * environment-only — a handful of variables, all with sane defaults. |
| 5 | 5 | */ |
| 6 | 6 | |
| 7 | import { keepPagesAcrossSwaps } from "./web/pages/kept_pages_file.ts"; | |
| 7 | 8 | import { scanNightly } from "./checks/scan.ts"; |
| 8 | 9 | import { nightlyDue } from "./checks/scan_store.ts"; |
| 9 | 10 | import { runModelReview } from "./actions/model_review.ts"; |
| @@ -189,6 +190,12 @@ if (problem !== null) { | ||
| 189 | 190 | process.exit(1); |
| 190 | 191 | } |
| 191 | 192 | |
| 193 | // A release swap hands the old process's kept code pages to this one before | |
| 194 | // it takes traffic, so strangers' pages do not all render cold (#9631). | |
| 195 | // (after the data directory is validated above) | |
| 196 | const keptAcrossSwap = keepPagesAcrossSwaps(dataDir); | |
| 197 | setInterval(keptAcrossSwap.save, 5_000).unref(); | |
| 198 | ||
| 192 | 199 | // FORGE_DRAIN_LINGER_S: seconds a SIGTERM'd forge keeps answering while |
| 193 | 200 | // routers learn it is going (`http/health.ts`). |
| 194 | 201 | const lingerMs = drainLingerMs(process.env.FORGE_DRAIN_LINGER_S); |
| @@ -317,6 +324,7 @@ const app = createApp({ | ||
| 317 | 324 | }), |
| 318 | 325 | }), |
| 319 | 326 | }); |
| 327 | app.log.info("kept pages restored", { pages: keptAcrossSwap.loaded }); | |
| 320 | 328 | |
| 321 | 329 | // A statement that holds the thread past 200 ms names its caller (#6957). |
| 322 | 330 | timeStatements(app.db, createLog("sql")); |
| @@ -908,6 +916,7 @@ async function close(): Promise<void> { | ||
| 908 | 916 | ssh?.stop(); |
| 909 | 917 | servers.stop(); |
| 910 | 918 | clearInterval(flagTimer); |
| 919 | keptAcrossSwap.saveNow(); | |
| 911 | 920 | flushFlagUses(app.db); |
| 912 | 921 | recordCleanExit(app.config.dataDir); |
| 913 | 922 | app.close(); |
src/view/cache.ts
Binary file not shown.
src/web/asset.ts +14
| @@ -243,6 +243,20 @@ export function speculationAsset(): FileAsset { | ||
| 243 | 243 | return speculation; |
| 244 | 244 | } |
| 245 | 245 | |
| 246 | /** | |
| 247 | * The build a rendered page belongs to: the content-hashed shell assets every | |
| 248 | * page names (stylesheet, shell, beacon, sprite, speculation rules). A process | |
| 249 | * serves only its own hash of each, so a page body is servable by another | |
| 250 | * process exactly when this matches; page scripts are kept on disk and served | |
| 251 | * across releases (below), so they need not be in it. A page's policy names | |
| 252 | * its inline styles by the hash of the text in that same body, so header and | |
| 253 | * body replayed together stay consistent whatever the build. | |
| 254 | */ | |
| 255 | export function shellBuild(): string { | |
| 256 | const paths = [stylesheetAsset(), shellAsset(), beaconAsset(), spriteAsset(), speculationAsset()].map((a) => a.path); | |
| 257 | return sha256(paths.join("\n")).slice(0, 32); | |
| 258 | } | |
| 259 | ||
| 246 | 260 | // The set of page scripts is static (the code is trusted, data rides in |
| 247 | 261 | // `data-*`), so these maps are never evicted: a cached page or an edge copy |
| 248 | 262 | // may name a script for as long as it lives, and it must still be served. |
src/web/pages/kept_page.ts +12
| @@ -96,6 +96,18 @@ export async function keepPage(key: string, response: Response): Promise<Respons | ||
| 96 | 96 | return new Response(body, { status: 200, headers: new Headers(headers) }); |
| 97 | 97 | } |
| 98 | 98 | |
| 99 | /** Entries of the kept pages that a successor process may start from (`kept_pages_file.ts`). */ | |
| 100 | export const liveKeptPages = (): { key: string; body: Uint8Array; headers: readonly [string, string][]; ttlMs: number }[] => | |
| 101 | store.live().map((e) => ({ key: e.key, ttlMs: e.ttlMs, ...(e.value as KeptPage) })); | |
| 102 | ||
| 103 | /** Put a page read from a predecessor back, with the time it had left. */ | |
| 104 | export function restoreKeptPage(key: string, body: Uint8Array, headers: readonly [string, string][], ttlMs: number): void { | |
| 105 | store.set(key, { body, headers }, Math.min(ttlMs, TTL_MS), body.length + 1024); | |
| 106 | } | |
| 107 | ||
| 108 | /** Changes whenever a page is kept: equal twice means nothing new to save. */ | |
| 109 | export const keptPagesVersion = (): number => store.version; | |
| 110 | ||
| 99 | 111 | export const keptPageStats = () => store.stats(); |
| 100 | 112 | |
| 101 | 113 | /** Test seam: start from nothing. */ |
src/web/pages/kept_pages_file.ts added +213
| @@ -0,0 +1,213 @@ | ||
| 1 | /** | |
| 2 | * The kept code pages, handed across a release swap (#9631). | |
| 3 | * | |
| 4 | * The old and new process overlap while a release swaps, and the new one used | |
| 5 | * to start with nothing kept, so every stranger's page rendered cold. The | |
| 6 | * running process saves its kept pages to one file under FORGE_APP_DATA (the | |
| 7 | * freshest first, up to `FILE_BYTES`), and a starting process reads it before | |
| 8 | * it takes traffic. The file records when it was written; a page returns with | |
| 9 | * the time it had left minus the file's age, and a page with none left is | |
| 10 | * dropped, so a hand-over adds no staleness and a file left from yesterday | |
| 11 | * restores nothing. The file is a cache, not truth: a missing, torn, oversized | |
| 12 | * or foreign file is ignored whole. It holds page bodies, so it is mode 0600. | |
| 13 | * | |
| 14 | * A page names the build it was rendered by (`asset.ts` `shellBuild`: the | |
| 15 | * hashed stylesheet and scripts its markup links), and a process serves only | |
| 16 | * its own. So the file records that build and a process of another one loads | |
| 17 | * nothing: a swap carries pages across exactly when the new release ships the | |
| 18 | * same stylesheet and scripts (a server-only change, a restart, a rollback to | |
| 19 | * the same assets), and starts cold when they changed. Markup a new release | |
| 20 | * would render differently is otherwise served for what is left of a page's | |
| 21 | * minute, the staleness the kept page already allows. | |
| 22 | * | |
| 23 | * Who may read a page is not in the file: a kept page is only consulted after | |
| 24 | * the route loaded the repository from the database and found it public | |
| 25 | * (`repo.ts`), so a repository turned private before, during or after a swap | |
| 26 | * is refused at once, restored copy or not. | |
| 27 | * | |
| 28 | * Layout: 8 bytes savedAt (ms, float64), u32 build length, build (utf-8), then | |
| 29 | * repeated: u32 header length, header JSON {key, headers, ttlMs}, u32 body | |
| 30 | * length, body bytes. Written to a temp name and renamed, so a reader never | |
| 31 | * sees half a file. | |
| 32 | */ | |
| 33 | ||
| 34 | import { chmodSync, closeSync, fstatSync, mkdirSync, openSync, readSync, renameSync, unlinkSync, writeFileSync } from "node:fs"; | |
| 35 | import { mkdir, rename, unlink, writeFile } from "node:fs/promises"; | |
| 36 | import { dirname, join } from "node:path"; | |
| 37 | import { shellBuild } from "../asset.ts"; | |
| 38 | import { keptPagesVersion, liveKeptPages, restoreKeptPage } from "./kept_page.ts"; | |
| 39 | ||
| 40 | /** Most bytes the file holds: ~250 pages, small enough to write in the background without a stall. */ | |
| 41 | export const FILE_BYTES = 4 * 1024 * 1024; | |
| 42 | /** Longest header block read back; anything bigger is not ours. */ | |
| 43 | const MAX_HEADER = 64 * 1024; | |
| 44 | /** Longest build id read back. */ | |
| 45 | const MAX_BUILD = 256; | |
| 46 | ||
| 47 | export const keptPagesPath = (dataDir: string): string => join(dataDir, "cache", "kept-pages.bin"); | |
| 48 | ||
outdated
Blocking multi-MB write on the event loop every 5 s under traffic, and a failed write leaks the tmp file. Async write, smaller budget, unlink on error, mode 0600. |
||
| 49 | /** The file's bytes for the live kept pages, and how many pages are in it. */ | |
| 50 | export function encodeKeptPages(now: number = Date.now(), build: string = shellBuild()): { bytes: Buffer; count: number } { | |
| 51 | const id = new TextEncoder().encode(build); | |
| 52 | const stamp = new DataView(new ArrayBuffer(12)); | |
| 53 | stamp.setFloat64(0, now); | |
| 54 | stamp.setUint32(8, id.length); | |
| 55 | const chunks: Uint8Array[] = [new Uint8Array(stamp.buffer), id]; | |
| 56 | let size = 12 + id.length; | |
| 57 | let count = 0; | |
| 58 | for (const page of liveKeptPages()) { | |
| 59 | const header = new TextEncoder().encode(JSON.stringify({ key: page.key, headers: page.headers, ttlMs: Math.round(page.ttlMs) })); | |
| 60 | const len = 8 + header.length + page.body.length; | |
| 61 | if (size + len > FILE_BYTES) break; | |
| 62 | const hlen = new DataView(new ArrayBuffer(4)); | |
| 63 | hlen.setUint32(0, header.length); | |
| 64 | const blen = new DataView(new ArrayBuffer(4)); | |
| 65 | blen.setUint32(0, page.body.length); | |
| 66 | chunks.push(new Uint8Array(hlen.buffer), header, new Uint8Array(blen.buffer), page.body); | |
| 67 | size += len; | |
| 68 | count++; | |
| 69 | } | |
| 70 | return { bytes: Buffer.concat(chunks), count }; | |
| 71 | } | |
| 72 | ||
| 73 | const tmpOf = (path: string): string => `${path}.${process.pid}.tmp`; | |
| 74 | ||
| 75 | /** Write the live kept pages, blocking; for shutdown and tests. Returns how many. */ | |
| 76 | export function saveKeptPages(path: string, now: number = Date.now(), build: string = shellBuild()): number { | |
| 77 | const { bytes, count } = encodeKeptPages(now, build); | |
| 78 | mkdirSync(dirname(path), { recursive: true }); | |
| 79 | const tmp = tmpOf(path); | |
| 80 | try { | |
| 81 | writeFileSync(tmp, bytes, { mode: 0o600 }); | |
| 82 | chmodSync(tmp, 0o600); | |
| 83 | renameSync(tmp, path); | |
| 84 | } catch (e) { | |
| 85 | try { | |
| 86 | unlinkSync(tmp); | |
| 87 | } catch { | |
| 88 | /* nothing was written */ | |
| 89 | } | |
| 90 | throw e; | |
| 91 | } | |
| 92 | return count; | |
| 93 | } | |
| 94 | ||
| 95 | /** Write the live kept pages without blocking the loop on the disk. Returns how many. */ | |
| 96 | export async function saveKeptPagesAsync(path: string, now: number = Date.now(), build: string = shellBuild()): Promise<number> { | |
| 97 | const { bytes, count } = encodeKeptPages(now, build); | |
outdated
Refuse the file when it is larger than FILE_BYTES before reading it (statSync), and record the release id in the header and compare it here. |
||
| 98 | await mkdir(dirname(path), { recursive: true }); | |
| 99 | const tmp = tmpOf(path); | |
| 100 | try { | |
| 101 | await writeFile(tmp, bytes, { mode: 0o600 }); | |
| 102 | await rename(tmp, path); | |
| 103 | } catch (e) { | |
| 104 | await unlink(tmp).catch(() => {}); | |
| 105 | throw e; | |
| 106 | } | |
| 107 | return count; | |
| 108 | } | |
| 109 | ||
| 110 | /** The file's bytes, or undefined when it is missing, unreadable or bigger than any file this writes. */ | |
| 111 | function readBounded(path: string): Uint8Array | undefined { | |
| 112 | let fd: number; | |
| 113 | try { | |
| 114 | fd = openSync(path, "r"); | |
| 115 | } catch { | |
| 116 | return undefined; | |
| 117 | } | |
| 118 | try { | |
| 119 | const size = fstatSync(fd).size; | |
| 120 | if (size > FILE_BYTES) return undefined; | |
| 121 | const bytes = new Uint8Array(size); | |
| 122 | let got = 0; | |
| 123 | while (got < size) { | |
| 124 | const n = readSync(fd, bytes, got, size - got, got); | |
| 125 | if (n <= 0) return undefined; | |
| 126 | got += n; | |
| 127 | } | |
| 128 | return bytes; | |
| 129 | } catch { | |
| 130 | return undefined; | |
| 131 | } finally { | |
| 132 | closeSync(fd); | |
| 133 | } | |
| 134 | } | |
| 135 | ||
| 136 | /** Read a predecessor's pages back into the kept pages; returns how many were restored. Never throws. */ | |
| 137 | export function loadKeptPages(path: string, now: number = Date.now(), build: string = shellBuild()): number { | |
| 138 | const bytes = readBounded(path); | |
| 139 | if (bytes === undefined || bytes.length < 12) return 0; | |
| 140 | const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); | |
| 141 | const savedAt = view.getFloat64(0); | |
| 142 | if (!Number.isFinite(savedAt)) return 0; | |
| 143 | const idLen = view.getUint32(8); | |
| 144 | if (idLen > MAX_BUILD || 12 + idLen > bytes.length) return 0; | |
| 145 | // Another build's pages link stylesheets and scripts this process does not serve. | |
| 146 | if (new TextDecoder().decode(bytes.subarray(12, 12 + idLen)) !== build) return 0; | |
| 147 | const age = Math.max(0, now - savedAt); | |
| 148 | const staged: { key: string; headers: [string, string][]; ttlMs: number; body: Uint8Array }[] = []; | |
| 149 | try { | |
| 150 | let at = 12 + idLen; | |
| 151 | while (at < bytes.length) { | |
| 152 | if (at + 4 > bytes.length) return 0; | |
| 153 | const hlen = view.getUint32(at); | |
| 154 | if (hlen > MAX_HEADER || at + 8 + hlen > bytes.length) return 0; | |
| 155 | const h = JSON.parse(new TextDecoder().decode(bytes.subarray(at + 4, at + 4 + hlen))) as Record<string, unknown>; | |
| 156 | const blen = view.getUint32(at + 4 + hlen); | |
| 157 | const start = at + 8 + hlen; | |
| 158 | if (start + blen > bytes.length) return 0; | |
| 159 | const { key, headers, ttlMs } = h; | |
| 160 | if (typeof key !== "string" || typeof ttlMs !== "number" || !Array.isArray(headers)) return 0; | |
| 161 | if (!headers.every((p) => Array.isArray(p) && p.length === 2 && typeof p[0] === "string" && typeof p[1] === "string")) return 0; | |
| 162 | staged.push({ key, headers: headers as [string, string][], ttlMs: ttlMs - age, body: bytes.slice(start, start + blen) }); | |
| 163 | at = start + blen; | |
| 164 | } | |
| 165 | } catch { | |
| 166 | return 0; | |
| 167 | } | |
| 168 | // Oldest first, so the freshest ends up most recently used. | |
| 169 | let restored = 0; | |
| 170 | for (const p of staged.reverse()) { | |
| 171 | if (!(p.ttlMs > 0)) continue; | |
| 172 | restoreKeptPage(p.key, p.body, p.headers, p.ttlMs); | |
| 173 | restored++; | |
| 174 | } | |
| 175 | return restored; | |
| 176 | } | |
| 177 | ||
| 178 | /** | |
| 179 | * Load at start. The caller ticks `save` (main.ts, every few seconds): it | |
| 180 | * writes in the background, only when something new was kept and no write is | |
| 181 | * still running. `saveNow` is the blocking write at shutdown, unconditional. | |
| 182 | */ | |
| 183 | export function keepPagesAcrossSwaps(dataDir: string): { loaded: number; save: () => void; saveNow: () => void } { | |
| 184 | const path = keptPagesPath(dataDir); | |
| 185 | const loaded = loadKeptPages(path); | |
| 186 | let saved = keptPagesVersion(); | |
| 187 | let writing = false; | |
| 188 | return { | |
| 189 | loaded, | |
| 190 | save: () => { | |
| 191 | const v = keptPagesVersion(); | |
| 192 | if (v === saved || writing) return; | |
| 193 | saved = v; | |
| 194 | writing = true; | |
| 195 | saveKeptPagesAsync(path) | |
| 196 | .catch(() => { | |
| 197 | /* a cache: the next tick tries again */ | |
| 198 | }) | |
| 199 | .finally(() => { | |
| 200 | writing = false; | |
| 201 | }); | |
| 202 | }, | |
| 203 | // Always, even with nothing new: the successor must find this process's | |
| 204 | // pages, not an older file a background write left behind. | |
| 205 | saveNow: () => { | |
| 206 | try { | |
| 207 | saveKeptPages(path); | |
| 208 | } catch { | |
| 209 | /* a cache */ | |
| 210 | } | |
| 211 | }, | |
| 212 | }; | |
| 213 | } |
test/anonymous-deep.test.ts +39 −1
| @@ -14,7 +14,8 @@ import { serve, type Servers } from "../src/http/server.ts"; | ||
| 14 | 14 | import { crawlerShare, createPerfRecord } from "../src/perf/record.ts"; |
| 15 | 15 | import { crawlReport } from "../src/perf/crawlers.ts"; |
| 16 | 16 | import { clearTreeCache, treeCacheStats } from "../src/web/pages/tree_cache.ts"; |
| 17 | import { keptPageStats } from "../src/web/pages/kept_page.ts"; | |
| 17 | import { clearKeptPages, keptPageStats } from "../src/web/pages/kept_page.ts"; | |
| 18 | import { loadKeptPages, saveKeptPages } from "../src/web/pages/kept_pages_file.ts"; | |
| 18 | 19 | |
| 19 | 20 | let dataDir: string; |
| 20 | 21 | let app: App; |
| @@ -394,3 +395,40 @@ test("a stranger's pinned page is kept whole and encoded; a session, a theme coo | ||
| 394 | 395 | await rename(join(gitDir, "objects.away"), join(gitDir, "objects")); |
| 395 | 396 | } |
| 396 | 397 | }); |
| 398 | ||
| 399 | test("a page restored from a predecessor's file is refused once the repository turned private, and served while it is public (#9631)", async () => { | |
| 400 | setCrawlClock(app, () => 18_000_000); | |
| 401 | clearTreeCache(); | |
| 402 | clearKeptPages(); | |
| 403 | const gitDir = join(dataDir, "repos", "alice", "open.git"); | |
| 404 | const page = `/alice/open/blob/${sha}/README.md`; | |
| 405 | const br = { "accept-encoding": "br" }; | |
| 406 | const first = await get(page, from("10.7.3.1", br)); | |
| 407 | expect(first.status).toBe(200); | |
| 408 | const firstBody = await first.text(); | |
| 409 | // the old process saves, the new one starts with nothing and loads the file | |
| 410 | const file = join(dataDir, "kept-handover.bin"); | |
| 411 | expect(saveKeptPages(file)).toBeGreaterThanOrEqual(1); | |
| 412 | clearKeptPages(); | |
| 413 | clearTreeCache(); | |
| 414 | expect(loadKeptPages(file)).toBeGreaterThanOrEqual(1); | |
| 415 | await rename(join(gitDir, "objects"), join(gitDir, "objects.away")); | |
| 416 | try { | |
| 417 | const hits = keptPageStats().hits; | |
| 418 | // turned private during the swap: the restored copy is never handed out | |
| 419 | app.db.run(`UPDATE repo SET visibility = 'private' WHERE name = 'open'`); | |
| 420 | const refused = await get(page, from("10.7.3.2", br)); | |
| 421 | expect(refused.status).not.toBe(200); | |
| 422 | expect(await refused.text()).not.toBe(firstBody); | |
| 423 | expect(keptPageStats().hits).toBe(hits); | |
| 424 | // public again: the restored page is served with git gone | |
| 425 | app.db.run(`UPDATE repo SET visibility = 'public' WHERE name = 'open'`); | |
| 426 | const warm = await get(page, from("10.7.3.3", br)); | |
| 427 | expect(warm.status).toBe(200); | |
| 428 | expect(await warm.text()).toBe(firstBody); | |
| 429 | expect(keptPageStats().hits).toBe(hits + 1); | |
| 430 | } finally { | |
| 431 | app.db.run(`UPDATE repo SET visibility = 'public' WHERE name = 'open'`); | |
| 432 | await rename(join(gitDir, "objects.away"), join(gitDir, "objects")); | |
| 433 | } | |
| 434 | }); |
test/kept-pages-boot.test.ts added +92
| @@ -0,0 +1,92 @@ | ||
| 1 | /** | |
| 2 | * The kept-pages hand-over through the real entry (#9631): a forge stopped | |
| 3 | * with Ctrl-C writes its strangers' kept pages at shutdown (`close` in | |
| 4 | * main.ts), and the next forge started on the same data reads them before | |
| 5 | * it takes traffic, so the same page is answered with no git at all. | |
| 6 | */ | |
| 7 | ||
| 8 | import { expect, test } from "bun:test"; | |
| 9 | import { mkdtemp, rename, rm } from "node:fs/promises"; | |
| 10 | import { tmpdir } from "node:os"; | |
| 11 | import { join } from "node:path"; | |
| 12 | import { commitToBranch, createRepo } from "../src/actions/repos.ts"; | |
| 13 | import { createApp } from "../src/app.ts"; | |
| 14 | import { getRepo } from "../src/core/repos.ts"; | |
| 15 | import { createUser } from "../src/core/users.ts"; | |
| 16 | import { keptPagesPath } from "../src/web/pages/kept_pages_file.ts"; | |
| 17 | ||
| 18 | const BROWSER = "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 Chrome/130.0 Safari/537.36"; | |
| 19 | ||
| 20 | async function boot(dataDir: string, port: number) { | |
| 21 | const env: Record<string, string> = { | |
| 22 | ...(process.env as Record<string, string>), | |
| 23 | FORGE_DATA: dataDir, | |
| 24 | PORT: String(port), | |
| 25 | FORGE_SSH_PORT: "0", | |
| 26 | FORGE_APPS: "0", | |
| 27 | }; | |
| 28 | delete env.FORGE_URL; | |
| 29 | const proc = Bun.spawn(["bun", "run", "src/main.ts"], { env, stdout: "pipe", stderr: "pipe" }); | |
| 30 | const log = { text: "" }; | |
| 31 | const drain = async (stream: ReadableStream<Uint8Array>) => { | |
| 32 | for await (const chunk of stream) log.text += new TextDecoder().decode(chunk); | |
| 33 | }; | |
| 34 | void drain(proc.stdout); | |
| 35 | void drain(proc.stderr); | |
| 36 | for (let i = 0; i < 300; i++) { | |
| 37 | try { | |
| 38 | if ((await fetch(`http://127.0.0.1:${port}/healthz`, { signal: AbortSignal.timeout(1_000) })).ok) return { proc, log }; | |
| 39 | } catch { | |
| 40 | await Bun.sleep(100); | |
| 41 | } | |
| 42 | } | |
| 43 | proc.kill(); | |
| 44 | throw new Error(`forge did not come up:\n${log.text}`); | |
| 45 | } | |
| 46 | ||
| 47 | test("shutdown saves the kept pages and the next boot serves them with git gone", async () => { | |
| 48 | const dataDir = await mkdtemp(join(tmpdir(), "forge-kept-boot-")); | |
| 49 | const seed = createApp({ dataDir, port: 0 }); | |
| 50 | const u = await createUser(seed.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", isAdmin: true }); | |
| 51 | if (!u.ok) throw new Error(u.error); | |
| 52 | const r = await createRepo(seed, u.value, { name: "open", visibility: "public" }); | |
| 53 | if (!r.ok) throw new Error(String(r.error)); | |
| 54 | const c = await commitToBranch(seed, u.value, getRepo(seed.db, "alice", "open")!, { message: "Start", files: [{ path: "a.txt", content: "hi\n" }] }); | |
| 55 | if (!c.ok) throw new Error(String(c.error)); | |
| 56 | seed.close(); | |
| 57 | const gitDir = join(dataDir, "repos", "alice", "open.git"); | |
| 58 | const sha = Bun.spawnSync(["git", "--git-dir", gitDir, "rev-parse", "HEAD"]).stdout.toString().trim(); | |
| 59 | const port = 20000 + Math.floor(Math.random() * 20000); | |
| 60 | const page = `http://127.0.0.1:${port}/alice/open/blob/${sha}/a.txt`; | |
| 61 | const headers = { "user-agent": BROWSER, "accept-encoding": "br", "x-forwarded-for": "10.9.0.1" }; | |
| 62 | let running: Awaited<ReturnType<typeof boot>> | undefined; | |
| 63 | try { | |
| 64 | running = await boot(dataDir, port); | |
| 65 | const first = await fetch(page, { headers }); | |
| 66 | expect(first.status).toBe(200); | |
| 67 | const body = await first.text(); | |
| 68 | // The background tick writes it within 5 s; take that file away, so | |
| 69 | // only the shutdown write can put one back. | |
| 70 | const file = keptPagesPath(dataDir); | |
| 71 | for (let i = 0; i < 100 && !(await Bun.file(file).exists()); i++) await Bun.sleep(100); | |
| 72 | expect(await Bun.file(file).exists()).toBe(true); | |
| 73 | await Bun.sleep(200); | |
| 74 | await rm(file); | |
| 75 | // Ctrl-C: the drain's last step writes the file before exit | |
| 76 | running.proc.kill("SIGINT"); | |
| 77 | await running.proc.exited; | |
| 78 | expect(await Bun.file(keptPagesPath(dataDir)).exists()).toBe(true); | |
| 79 | ||
| 80 | // the successor: no git objects, so only a restored page can answer | |
| 81 | await rename(join(gitDir, "objects"), join(gitDir, "objects.away")); | |
| 82 | running = await boot(dataDir, port); | |
| 83 | expect(running.log.text).toMatch(/kept pages restored[^\n]*"?pages"?[":= ]+1/); | |
| 84 | const warm = await fetch(page, { headers: { ...headers, "x-forwarded-for": "10.9.0.2" } }); | |
| 85 | expect(warm.status).toBe(200); | |
| 86 | expect(await warm.text()).toBe(body); | |
| 87 | } finally { | |
| 88 | running?.proc.kill(); | |
| 89 | await running?.proc.exited; | |
| 90 | await rm(dataDir, { recursive: true, force: true }); | |
| 91 | } | |
| 92 | }, 60_000); |
test/kept-pages-file.test.ts added +142
| @@ -0,0 +1,142 @@ | ||
| 1 | import { afterEach, expect, test } from "bun:test"; | |
| 2 | import { chmodSync, mkdirSync, mkdtempSync, readdirSync, statSync, writeFileSync } from "node:fs"; | |
| 3 | import { join } from "node:path"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { clearKeptPages, keepPage, keptPage } from "../src/web/pages/kept_page.ts"; | |
| 6 | import { shellBuild } from "../src/web/asset.ts"; | |
| 7 | import { FILE_BYTES, keepPagesAcrossSwaps, keptPagesPath, loadKeptPages, saveKeptPages, saveKeptPagesAsync } from "../src/web/pages/kept_pages_file.ts"; | |
| 8 | ||
| 9 | afterEach(clearKeptPages); | |
| 10 | const keyOf = (n: string): string => ["gzip", "000", "r1", "http://x", `/a/${n}`, ""].join("\x00"); | |
| 11 | const KEY = keyOf("b"); | |
| 12 | const html = (body: string) => new Response(body, { headers: { "content-type": "text/html", "x-request-id": "gone" } }); | |
| 13 | const dir = () => mkdtempSync(join(tmpdir(), "kept-")); | |
| 14 | ||
| 15 | test("a kept page crosses a swap: same bytes and headers, served without rendering", async () => { | |
| 16 | const first = await keepPage(KEY, html("<p>hello</p>")); | |
| 17 | const bytes = new Uint8Array(await first.arrayBuffer()); | |
| 18 | const path = join(dir(), "cache", "kept-pages.bin"); | |
| 19 | expect(saveKeptPages(path)).toBe(1); | |
| 20 | expect((statSync(path).mode & 0o777).toString(8)).toBe("600"); | |
| 21 | clearKeptPages(); | |
| 22 | expect(keptPage(KEY)).toBeUndefined(); | |
| 23 | expect(loadKeptPages(path)).toBe(1); | |
| 24 | const warm = keptPage(KEY)!; | |
| 25 | expect(warm.headers.get("content-type")).toBe("text/html"); | |
| 26 | expect(warm.headers.get("x-request-id")).toBeNull(); | |
| 27 | expect(warm.headers.get("content-encoding")).toBe("gzip"); | |
| 28 | expect(new Uint8Array(await warm.arrayBuffer())).toEqual(bytes); | |
| 29 | }); | |
| 30 | ||
| 31 | test("the file's age is taken off a page's time; a file older than a minute restores nothing", async () => { | |
| 32 | await keepPage(KEY, html("<p>x</p>")); | |
| 33 | const path = join(dir(), "k.bin"); | |
| 34 | const t0 = 1_000_000; | |
| 35 | saveKeptPages(path, t0); | |
| 36 | clearKeptPages(); | |
| 37 | expect(loadKeptPages(path, t0 + 3_600_000)).toBe(0); | |
| 38 | expect(keptPage(KEY)).toBeUndefined(); | |
| 39 | expect(loadKeptPages(path, t0 + 120_000)).toBe(0); | |
| 40 | // a clock that went backwards counts as no age, and the ttl is clamped to a minute | |
| 41 | expect(loadKeptPages(path, t0 - 10_000)).toBe(1); | |
| 42 | expect(keptPage(KEY)).toBeDefined(); | |
| 43 | }); | |
| 44 | ||
| 45 | test("a page with its time used up is dropped and not counted", async () => { | |
| 46 | await keepPage(KEY, html("<p>x</p>")); | |
| 47 | await keepPage(keyOf("c"), html("<p>y</p>")); | |
| 48 | const path = join(dir(), "k.bin"); | |
| 49 | saveKeptPages(path, Date.now()); | |
| 50 | clearKeptPages(); | |
| 51 | expect(loadKeptPages(path, Date.now() + 61_000)).toBe(0); | |
| 52 | }); | |
| 53 | ||
| 54 | test("the file holds at most FILE_BYTES; the freshest pages go first", async () => { | |
| 55 | const filler = (n: number) => Array.from({ length: 1500 }, (_, i) => `${n}-${i}-${Math.random()}`).join(" "); | |
| 56 | for (let i = 0; i < 40; i++) await keepPage(keyOf(String(i)), html(`<p>${filler(i)}</p>`)); | |
| 57 | const path = join(dir(), "k.bin"); | |
| 58 | const n = saveKeptPages(path); | |
| 59 | expect(statSync(path).size).toBeLessThanOrEqual(FILE_BYTES); | |
| 60 | expect(n).toBeGreaterThan(0); | |
| 61 | expect(n).toBeLessThanOrEqual(40); | |
| 62 | }); | |
| 63 | ||
| 64 | test("a missing, torn or foreign file loads nothing and does not throw", async () => { | |
| 65 | const d = dir(); | |
| 66 | expect(loadKeptPages(join(d, "none"))).toBe(0); | |
| 67 | await keepPage(KEY, html("<p>x</p>")); | |
| 68 | const path = join(d, "a.bin"); | |
| 69 | saveKeptPages(path); | |
| 70 | const full = await Bun.file(path).bytes(); | |
| 71 | clearKeptPages(); | |
| 72 | writeFileSync(path, full.subarray(0, full.length - 3)); | |
| 73 | expect(loadKeptPages(path)).toBe(0); | |
| 74 | writeFileSync(path, "not a kept pages file at all"); | |
| 75 | expect(loadKeptPages(path)).toBe(0); | |
| 76 | expect(keptPage(KEY)).toBeUndefined(); | |
| 77 | }); | |
| 78 | ||
| 79 | test("a failed write leaves no temp file behind", async () => { | |
| 80 | await keepPage(KEY, html("<p>x</p>")); | |
| 81 | const d = dir(); | |
| 82 | const path = join(d, "k.bin"); | |
| 83 | mkdirSync(path); // renaming a file onto a directory fails | |
| 84 | expect(() => saveKeptPages(path)).toThrow(); | |
| 85 | await expect(saveKeptPagesAsync(path)).rejects.toThrow(); | |
| 86 | expect(readdirSync(d).filter((f) => f.endsWith(".tmp"))).toEqual([]); | |
| 87 | chmodSync(d, 0o700); | |
| 88 | }); | |
| 89 | ||
| 90 | test("the process wiring: save writes in the background only when something new was kept, saveNow at exit, a successor loads", async () => { | |
| 91 | const d = dir(); | |
| 92 | const a = keepPagesAcrossSwaps(d); | |
| 93 | expect(a.loaded).toBe(0); | |
| 94 | a.save(); // nothing kept yet: no file | |
| 95 | await Bun.sleep(30); | |
| 96 | expect(await Bun.file(keptPagesPath(d)).exists()).toBe(false); | |
| 97 | await keepPage(KEY, html("<p>x</p>")); | |
| 98 | a.save(); | |
| 99 | for (let i = 0; i < 50 && !(await Bun.file(keptPagesPath(d)).exists()); i++) await Bun.sleep(10); | |
| 100 | expect(await Bun.file(keptPagesPath(d)).exists()).toBe(true); | |
| 101 | await keepPage(keyOf("late"), html("<p>late</p>")); | |
| 102 | await Bun.sleep(30); | |
| 103 | a.saveNow(); | |
| 104 | clearKeptPages(); | |
| 105 | const b = keepPagesAcrossSwaps(d); | |
| 106 | expect(b.loaded).toBe(2); | |
| 107 | expect(keptPage(KEY)).toBeDefined(); | |
| 108 | }); | |
| 109 | ||
| 110 | test("a file written by another build loads nothing: its pages link stylesheets and scripts this process does not serve", async () => { | |
| 111 | await keepPage(KEY, html("<p>x</p>")); | |
| 112 | const path = join(dir(), "k.bin"); | |
| 113 | saveKeptPages(path, Date.now(), "old-build"); | |
| 114 | clearKeptPages(); | |
| 115 | expect(loadKeptPages(path, Date.now(), "new-build")).toBe(0); | |
| 116 | expect(keptPage(KEY)).toBeUndefined(); | |
| 117 | // the same build, a restart or a server-only release, is carried across | |
| 118 | expect(loadKeptPages(path, Date.now(), "old-build")).toBe(1); | |
| 119 | // and by default the build is this process's own shell assets | |
| 120 | saveKeptPages(path); | |
| 121 | clearKeptPages(); | |
| 122 | expect(loadKeptPages(path, Date.now(), shellBuild())).toBe(1); | |
| 123 | clearKeptPages(); | |
| 124 | expect(loadKeptPages(path, Date.now(), "other")).toBe(0); | |
| 125 | }); | |
| 126 | ||
| 127 | test("a file bigger than FILE_BYTES is refused before it is read", async () => { | |
| 128 | await keepPage(KEY, html("<p>x</p>")); | |
| 129 | const path = join(dir(), "k.bin"); | |
| 130 | saveKeptPages(path); | |
| 131 | const full = await Bun.file(path).bytes(); | |
| 132 | clearKeptPages(); | |
| 133 | // a valid file with trailing bytes past the bound: refused whole, not parsed | |
| 134 | const big = new Uint8Array(FILE_BYTES + 1); | |
| 135 | big.set(full); | |
| 136 | writeFileSync(path, big); | |
| 137 | expect(loadKeptPages(path)).toBe(0); | |
| 138 | expect(keptPage(KEY)).toBeUndefined(); | |
| 139 | // at the bound exactly it is read (and rejected here only for the trailing zeros) | |
| 140 | writeFileSync(path, full); | |
| 141 | expect(loadKeptPages(path)).toBe(1); | |
| 142 | }); |
new above
offline
Untested wiring: add a test that proves shutdown saves and boot loads through this entry, not only through keepPagesAcrossSwaps.