Public: anyone can read this; the crew can change it.

Kept code pages cross a release swap, so the new process starts warm #9632

Closed opened this
Closed without merging

mason/forge:mason/9631-kept-code-pages-cross-a-release-swap-so → main 6 commits fork 4 threads · 4 unresolved

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
10 files changed +583 −14
bench/code-pages-at-rate.ts +32 −13
@@ -12,10 +12,20 @@
1212 * bun bench/code-pages-at-rate.ts --rate 500 --seconds 20 --addresses 4000 --crawl 3000 \
1313 * --out bench/results/9514-in-process-bypass.json
1414 *
15 * A release swap (#9631): `--data DIR` keeps the copy in DIR between runs,
16 * `--save` writes the kept pages to DIR's hand-over file when the run ends
17 * (the old process at shutdown) and `--load` reads them before serving (the
18 * new one at start). Each run is its own process, so everything else starts
19 * cold, as it does after a real swap:
20 *
21 * bun bench/code-pages-at-rate.ts --data $TMPDIR/swap --seconds 40 --save # the old release, serving
22 * bun bench/code-pages-at-rate.ts --data $TMPDIR/swap --warm 0 --load # the new one, right after
23 *
1524 * What it is not: the edge run. That one needs the real deployment and is run
1625 * with bench/load.ts against it (docs/features/load-test.md).
1726 */
18 import { mkdtemp, rm } from "node:fs/promises";
27 import { existsSync } from "node:fs";
28 import { mkdir, mkdtemp, rm } from "node:fs/promises";
1929 import { tmpdir } from "node:os";
2030 import { join } from "node:path";
2131 import { createApp } from "../src/app.ts";
@@ -23,6 +33,7 @@ import { createRepo } from "../src/actions/repos.ts";
2333 import { createUser } from "../src/core/users.ts";
2434 import { repoDir } from "../src/git/repos.ts";
2535 import { serve } from "../src/http/server.ts";
36 import { keptPagesPath, loadKeptPages, saveKeptPages } from "../src/web/pages/kept_pages_file.ts";
2637
2738 const arg = (name: string, fallback: string): string => {
2839 const at = process.argv.indexOf(`--${name}`);
@@ -33,20 +44,15 @@ const arg = (name: string, fallback: string): string => {
3344 const out = process.argv.includes("--out") ? process.argv[process.argv.indexOf("--out") + 1] : undefined;
3445 const root = join(import.meta.dir, "..");
3546
36 const dataDir = await mkdtemp(join(tmpdir(), "forge-code-rate-"));
47 const kept = process.argv.includes("--data") ? process.argv[process.argv.indexOf("--data") + 1]! : undefined;
48 if (kept !== undefined) await mkdir(kept, { recursive: true });
49 const seeded = kept !== undefined && existsSync(join(kept, "forge.db"));
50 const dataDir = kept ?? (await mkdtemp(join(tmpdir(), "forge-code-rate-")));
3751 const app = createApp({ dataDir, port: 0 });
52 if (process.argv.includes("--load")) console.error(`handover: ${loadKeptPages(keptPagesPath(dataDir))} kept pages loaded`);
3853 const servers = await serve(app);
3954 try {
40 const u = await createUser(app.db, { origin: "seed", trusted: true, username: "trevato", email: "[email protected]", password: "a-good-passphrase", isAdmin: true });
41 if (!u.ok) throw new Error(u.error);
42 const r = await createRepo(app, u.value, { name: "forge", visibility: "public" });
43 if (!r.ok) throw new Error(String(r.error));
44 // The clone's own history, fetched whole (a fetch runs no receive hook): every folder, file and commit as the forge holds them.
45 const pushed = Bun.spawnSync(["git", "--git-dir", repoDir(app.reposRoot, "trevato", "forge"), "fetch", "--quiet", root, "HEAD:refs/heads/main"], { cwd: root, stderr: "pipe" });
46 if (pushed.exitCode !== 0) throw new Error(`copying the clone failed: ${pushed.stderr.toString()}`);
47 const files = Bun.spawnSync(["git", "ls-files"], { cwd: root }).stdout.toString().split("\n").length - 1;
48 console.error(`copy: trevato/forge with ${files} files at ${Bun.spawnSync(["git", "rev-parse", "HEAD"], { cwd: root }).stdout.toString().trim()}`);
49
55 if (!seeded) await seed();
5056 const args = ["--url", servers.url, "--repo", "trevato/forge", "--bypass", "--version", "in-process"];
5157 for (const name of ["rate", "seconds", "addresses", "crawl", "warm"]) args.push(`--${name}`, arg(name, { rate: "500", seconds: "20", addresses: "4000", crawl: "3000", warm: "0" }[name]!));
5258 if (out !== undefined) args.push("--out", out);
@@ -57,8 +63,21 @@ try {
5763 stderr: "inherit",
5864 });
5965 process.exitCode = await load.exited;
66 if (process.argv.includes("--save")) console.error(`handover: ${saveKeptPages(keptPagesPath(dataDir))} kept pages saved`);
6067 } finally {
6168 servers.stop();
6269 app.close();
63 await rm(dataDir, { recursive: true, force: true });
70 if (kept === undefined) await rm(dataDir, { recursive: true, force: true });
71 }
72
73 async function seed(): Promise<void> {
74 const u = await createUser(app.db, { origin: "seed", trusted: true, username: "trevato", email: "[email protected]", password: "a-good-passphrase", isAdmin: true });
75 if (!u.ok) throw new Error(u.error);
76 const r = await createRepo(app, u.value, { name: "forge", visibility: "public" });
77 if (!r.ok) throw new Error(String(r.error));
78 // The clone's own history, fetched whole (a fetch runs no receive hook): every folder, file and commit as the forge holds them.
79 const pushed = Bun.spawnSync(["git", "--git-dir", repoDir(app.reposRoot, "trevato", "forge"), "fetch", "--quiet", root, "HEAD:refs/heads/main"], { cwd: root, stderr: "pipe" });
80 if (pushed.exitCode !== 0) throw new Error(`copying the clone failed: ${pushed.stderr.toString()}`);
81 const files = Bun.spawnSync(["git", "ls-files"], { cwd: root }).stdout.toString().split("\n").length - 1;
82 console.error(`copy: trevato/forge with ${files} files at ${Bun.spawnSync(["git", "rev-parse", "HEAD"], { cwd: root }).stdout.toString().trim()}`);
6483 }
bench/results/9631-swap-handover.md added +30
@@ -0,0 +1,30 @@
1 # Kept pages across a swap (#9631), measured against a cold start
2
3 In-process copy of this repository, Shield at its default, 500/s across the
4 `--crawl 3000` pinned links, 4000 made-up strangers, edge bypassed. Each run
5 is its own process on the same data (`--data`), so everything but the
6 hand-over file starts cold, as after a real swap. "Old release" ran 40 s and
7 saved at the end (`--save`); the new one started right after with `--warm 0`,
8 with (`--load`) or without the file.
9
10 bun bench/code-pages-at-rate.ts --data $TMPDIR/swap --seconds 40 --save
11 bun bench/code-pages-at-rate.ts --data $TMPDIR/swap --warm 0 --load # or without --load
12
13 | run | pages handed over | refused of 9999 | p95 |
14 | --- | ---: | ---: | ---: |
15 | hand-over, 4 MB file (as shipped) | 295 | 4091 | 600 ms |
16 | cold | 0 | 4487 | 659 ms |
17 | hand-over, 4 MB file, again | 279 | 4564 | 589 ms |
18 | cold, again | 0 | 4678 | 705 ms |
19 | hand-over, 64 MB file (experiment, not shipped) | 1609 (23 MB) | 5566 | 717 ms |
20
21 Per second, first hand-over run: `203/296 99/401 184/316 108/392 202/298 197/303 222/278 221/279 289/211 271/229 234/266 266/233 314/187 401/99 342/158 389/111 500/0 500/0 500/0 466/34`.
22 Per second, first cold run: `232/267 162/338 66/434 212/288 188/312 195/305 277/223 247/253 286/214 218/282 249/251 314/186 263/237 138/362 292/208 356/144 480/20 336/163 501/0 500/0`.
23
24 Reading: the hand-over refuses 2-9% fewer strangers at 4 MB, inside the
25 spread between two cold runs (4487 vs 4678); handing over five times as many
26 pages did not help at all. The cold window of #9566 is not made by the
27 missing kept pages: the pages that still miss render cold on the one main
28 thread, the loop falls behind and the stranger budgets divide all the same.
29 The issue's done condition ("refuses fewer strangers than the baseline") is
30 not met in any way that is distinguishable from noise.
src/main.ts +9
@@ -4,6 +4,7 @@
44 * environment-only — a handful of variables, all with sane defaults.
55 */
66
7 import { keepPagesAcrossSwaps } from "./web/pages/kept_pages_file.ts";
78 import { scanNightly } from "./checks/scan.ts";
89 import { nightlyDue } from "./checks/scan_store.ts";
910 import { runModelReview } from "./actions/model_review.ts";
@@ -189,6 +190,12 @@ if (problem !== null) {
189190 process.exit(1);
190191 }
191192
193 // A release swap hands the old process's kept code pages to this one before
194 // it takes traffic, so strangers' pages do not all render cold (#9631).
195 // (after the data directory is validated above)
196 const keptAcrossSwap = keepPagesAcrossSwaps(dataDir);
197 setInterval(keptAcrossSwap.save, 5_000).unref();
198

Untested wiring: add a test that proves shutdown saves and boot loads through this entry, not only through keepPagesAcrossSwaps.

192199 // FORGE_DRAIN_LINGER_S: seconds a SIGTERM'd forge keeps answering while
193200 // routers learn it is going (`http/health.ts`).
194201 const lingerMs = drainLingerMs(process.env.FORGE_DRAIN_LINGER_S);
@@ -317,6 +324,7 @@ const app = createApp({
317324 }),
318325 }),
319326 });
327 app.log.info("kept pages restored", { pages: keptAcrossSwap.loaded });
320328
321329 // A statement that holds the thread past 200 ms names its caller (#6957).
322330 timeStatements(app.db, createLog("sql"));
@@ -908,6 +916,7 @@ async function close(): Promise<void> {
908916 ssh?.stop();
909917 servers.stop();
910918 clearInterval(flagTimer);
919 keptAcrossSwap.saveNow();
911920 flushFlagUses(app.db);
912921 recordCleanExit(app.config.dataDir);
913922 app.close();
src/view/cache.ts
Binary file not shown.
src/web/asset.ts +14
@@ -243,6 +243,20 @@ export function speculationAsset(): FileAsset {
243243 return speculation;
244244 }
245245
246 /**
247 * The build a rendered page belongs to: the content-hashed shell assets every
248 * page names (stylesheet, shell, beacon, sprite, speculation rules). A process
249 * serves only its own hash of each, so a page body is servable by another
250 * process exactly when this matches; page scripts are kept on disk and served
251 * across releases (below), so they need not be in it. A page's policy names
252 * its inline styles by the hash of the text in that same body, so header and
253 * body replayed together stay consistent whatever the build.
254 */
255 export function shellBuild(): string {
256 const paths = [stylesheetAsset(), shellAsset(), beaconAsset(), spriteAsset(), speculationAsset()].map((a) => a.path);
257 return sha256(paths.join("\n")).slice(0, 32);
258 }
259
246260 // The set of page scripts is static (the code is trusted, data rides in
247261 // `data-*`), so these maps are never evicted: a cached page or an edge copy
248262 // may name a script for as long as it lives, and it must still be served.
src/web/pages/kept_page.ts +12
@@ -96,6 +96,18 @@ export async function keepPage(key: string, response: Response): Promise<Respons
9696 return new Response(body, { status: 200, headers: new Headers(headers) });
9797 }
9898
99 /** Entries of the kept pages that a successor process may start from (`kept_pages_file.ts`). */
100 export const liveKeptPages = (): { key: string; body: Uint8Array; headers: readonly [string, string][]; ttlMs: number }[] =>
101 store.live().map((e) => ({ key: e.key, ttlMs: e.ttlMs, ...(e.value as KeptPage) }));
102
103 /** Put a page read from a predecessor back, with the time it had left. */
104 export function restoreKeptPage(key: string, body: Uint8Array, headers: readonly [string, string][], ttlMs: number): void {
105 store.set(key, { body, headers }, Math.min(ttlMs, TTL_MS), body.length + 1024);
106 }
107
108 /** Changes whenever a page is kept: equal twice means nothing new to save. */
109 export const keptPagesVersion = (): number => store.version;
110
99111 export const keptPageStats = () => store.stats();
100112
101113 /** Test seam: start from nothing. */
src/web/pages/kept_pages_file.ts added +213
@@ -0,0 +1,213 @@
1 /**
2 * The kept code pages, handed across a release swap (#9631).
3 *
4 * The old and new process overlap while a release swaps, and the new one used
5 * to start with nothing kept, so every stranger's page rendered cold. The
6 * running process saves its kept pages to one file under FORGE_APP_DATA (the
7 * freshest first, up to `FILE_BYTES`), and a starting process reads it before
8 * it takes traffic. The file records when it was written; a page returns with
9 * the time it had left minus the file's age, and a page with none left is
10 * dropped, so a hand-over adds no staleness and a file left from yesterday
11 * restores nothing. The file is a cache, not truth: a missing, torn, oversized
12 * or foreign file is ignored whole. It holds page bodies, so it is mode 0600.
13 *
14 * A page names the build it was rendered by (`asset.ts` `shellBuild`: the
15 * hashed stylesheet and scripts its markup links), and a process serves only
16 * its own. So the file records that build and a process of another one loads
17 * nothing: a swap carries pages across exactly when the new release ships the
18 * same stylesheet and scripts (a server-only change, a restart, a rollback to
19 * the same assets), and starts cold when they changed. Markup a new release
20 * would render differently is otherwise served for what is left of a page's
21 * minute, the staleness the kept page already allows.
22 *
23 * Who may read a page is not in the file: a kept page is only consulted after
24 * the route loaded the repository from the database and found it public
25 * (`repo.ts`), so a repository turned private before, during or after a swap
26 * is refused at once, restored copy or not.
27 *
28 * Layout: 8 bytes savedAt (ms, float64), u32 build length, build (utf-8), then
29 * repeated: u32 header length, header JSON {key, headers, ttlMs}, u32 body
30 * length, body bytes. Written to a temp name and renamed, so a reader never
31 * sees half a file.
32 */
33
34 import { chmodSync, closeSync, fstatSync, mkdirSync, openSync, readSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
35 import { mkdir, rename, unlink, writeFile } from "node:fs/promises";
36 import { dirname, join } from "node:path";
37 import { shellBuild } from "../asset.ts";
38 import { keptPagesVersion, liveKeptPages, restoreKeptPage } from "./kept_page.ts";
39
40 /** Most bytes the file holds: ~250 pages, small enough to write in the background without a stall. */
41 export const FILE_BYTES = 4 * 1024 * 1024;
42 /** Longest header block read back; anything bigger is not ours. */
43 const MAX_HEADER = 64 * 1024;
44 /** Longest build id read back. */
45 const MAX_BUILD = 256;
46
47 export const keptPagesPath = (dataDir: string): string => join(dataDir, "cache", "kept-pages.bin");
48
outdated

Blocking multi-MB write on the event loop every 5 s under traffic, and a failed write leaks the tmp file. Async write, smaller budget, unlink on error, mode 0600.

49 /** The file's bytes for the live kept pages, and how many pages are in it. */
50 export function encodeKeptPages(now: number = Date.now(), build: string = shellBuild()): { bytes: Buffer; count: number } {
51 const id = new TextEncoder().encode(build);
52 const stamp = new DataView(new ArrayBuffer(12));
53 stamp.setFloat64(0, now);
54 stamp.setUint32(8, id.length);
55 const chunks: Uint8Array[] = [new Uint8Array(stamp.buffer), id];
56 let size = 12 + id.length;
57 let count = 0;
58 for (const page of liveKeptPages()) {
59 const header = new TextEncoder().encode(JSON.stringify({ key: page.key, headers: page.headers, ttlMs: Math.round(page.ttlMs) }));
60 const len = 8 + header.length + page.body.length;
61 if (size + len > FILE_BYTES) break;
62 const hlen = new DataView(new ArrayBuffer(4));
63 hlen.setUint32(0, header.length);
64 const blen = new DataView(new ArrayBuffer(4));
65 blen.setUint32(0, page.body.length);
66 chunks.push(new Uint8Array(hlen.buffer), header, new Uint8Array(blen.buffer), page.body);
67 size += len;
68 count++;
69 }
70 return { bytes: Buffer.concat(chunks), count };
71 }
72
73 const tmpOf = (path: string): string => `${path}.${process.pid}.tmp`;
74
75 /** Write the live kept pages, blocking; for shutdown and tests. Returns how many. */
76 export function saveKeptPages(path: string, now: number = Date.now(), build: string = shellBuild()): number {
77 const { bytes, count } = encodeKeptPages(now, build);
78 mkdirSync(dirname(path), { recursive: true });
79 const tmp = tmpOf(path);
80 try {
81 writeFileSync(tmp, bytes, { mode: 0o600 });
82 chmodSync(tmp, 0o600);
83 renameSync(tmp, path);
84 } catch (e) {
85 try {
86 unlinkSync(tmp);
87 } catch {
88 /* nothing was written */
89 }
90 throw e;
91 }
92 return count;
93 }
94
95 /** Write the live kept pages without blocking the loop on the disk. Returns how many. */
96 export async function saveKeptPagesAsync(path: string, now: number = Date.now(), build: string = shellBuild()): Promise<number> {
97 const { bytes, count } = encodeKeptPages(now, build);
outdated

Refuse the file when it is larger than FILE_BYTES before reading it (statSync), and record the release id in the header and compare it here.

98 await mkdir(dirname(path), { recursive: true });
99 const tmp = tmpOf(path);
100 try {
101 await writeFile(tmp, bytes, { mode: 0o600 });
102 await rename(tmp, path);
103 } catch (e) {
104 await unlink(tmp).catch(() => {});
105 throw e;
106 }
107 return count;
108 }
109
110 /** The file's bytes, or undefined when it is missing, unreadable or bigger than any file this writes. */
111 function readBounded(path: string): Uint8Array | undefined {
112 let fd: number;
113 try {
114 fd = openSync(path, "r");
115 } catch {
116 return undefined;
117 }
118 try {
119 const size = fstatSync(fd).size;
120 if (size > FILE_BYTES) return undefined;
121 const bytes = new Uint8Array(size);
122 let got = 0;
123 while (got < size) {
124 const n = readSync(fd, bytes, got, size - got, got);
125 if (n <= 0) return undefined;
126 got += n;
127 }
128 return bytes;
129 } catch {
130 return undefined;
131 } finally {
132 closeSync(fd);
133 }
134 }
135
136 /** Read a predecessor's pages back into the kept pages; returns how many were restored. Never throws. */
137 export function loadKeptPages(path: string, now: number = Date.now(), build: string = shellBuild()): number {
138 const bytes = readBounded(path);
139 if (bytes === undefined || bytes.length < 12) return 0;
140 const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
141 const savedAt = view.getFloat64(0);
142 if (!Number.isFinite(savedAt)) return 0;
143 const idLen = view.getUint32(8);
144 if (idLen > MAX_BUILD || 12 + idLen > bytes.length) return 0;
145 // Another build's pages link stylesheets and scripts this process does not serve.
146 if (new TextDecoder().decode(bytes.subarray(12, 12 + idLen)) !== build) return 0;
147 const age = Math.max(0, now - savedAt);
148 const staged: { key: string; headers: [string, string][]; ttlMs: number; body: Uint8Array }[] = [];
149 try {
150 let at = 12 + idLen;
151 while (at < bytes.length) {
152 if (at + 4 > bytes.length) return 0;
153 const hlen = view.getUint32(at);
154 if (hlen > MAX_HEADER || at + 8 + hlen > bytes.length) return 0;
155 const h = JSON.parse(new TextDecoder().decode(bytes.subarray(at + 4, at + 4 + hlen))) as Record<string, unknown>;
156 const blen = view.getUint32(at + 4 + hlen);
157 const start = at + 8 + hlen;
158 if (start + blen > bytes.length) return 0;
159 const { key, headers, ttlMs } = h;
160 if (typeof key !== "string" || typeof ttlMs !== "number" || !Array.isArray(headers)) return 0;
161 if (!headers.every((p) => Array.isArray(p) && p.length === 2 && typeof p[0] === "string" && typeof p[1] === "string")) return 0;
162 staged.push({ key, headers: headers as [string, string][], ttlMs: ttlMs - age, body: bytes.slice(start, start + blen) });
163 at = start + blen;
164 }
165 } catch {
166 return 0;
167 }
168 // Oldest first, so the freshest ends up most recently used.
169 let restored = 0;
170 for (const p of staged.reverse()) {
171 if (!(p.ttlMs > 0)) continue;
172 restoreKeptPage(p.key, p.body, p.headers, p.ttlMs);
173 restored++;
174 }
175 return restored;
176 }
177
178 /**
179 * Load at start. The caller ticks `save` (main.ts, every few seconds): it
180 * writes in the background, only when something new was kept and no write is
181 * still running. `saveNow` is the blocking write at shutdown, unconditional.
182 */
183 export function keepPagesAcrossSwaps(dataDir: string): { loaded: number; save: () => void; saveNow: () => void } {
184 const path = keptPagesPath(dataDir);
185 const loaded = loadKeptPages(path);
186 let saved = keptPagesVersion();
187 let writing = false;
188 return {
189 loaded,
190 save: () => {
191 const v = keptPagesVersion();
192 if (v === saved || writing) return;
193 saved = v;
194 writing = true;
195 saveKeptPagesAsync(path)
196 .catch(() => {
197 /* a cache: the next tick tries again */
198 })
199 .finally(() => {
200 writing = false;
201 });
202 },
203 // Always, even with nothing new: the successor must find this process's
204 // pages, not an older file a background write left behind.
205 saveNow: () => {
206 try {
207 saveKeptPages(path);
208 } catch {
209 /* a cache */
210 }
211 },
212 };
213 }
test/anonymous-deep.test.ts +39 −1
@@ -14,7 +14,8 @@ import { serve, type Servers } from "../src/http/server.ts";
1414 import { crawlerShare, createPerfRecord } from "../src/perf/record.ts";
1515 import { crawlReport } from "../src/perf/crawlers.ts";
1616 import { clearTreeCache, treeCacheStats } from "../src/web/pages/tree_cache.ts";
17 import { keptPageStats } from "../src/web/pages/kept_page.ts";
17 import { clearKeptPages, keptPageStats } from "../src/web/pages/kept_page.ts";
18 import { loadKeptPages, saveKeptPages } from "../src/web/pages/kept_pages_file.ts";
1819
1920 let dataDir: string;
2021 let app: App;
@@ -394,3 +395,40 @@ test("a stranger's pinned page is kept whole and encoded; a session, a theme coo
394395 await rename(join(gitDir, "objects.away"), join(gitDir, "objects"));
395396 }
396397 });
398
399 test("a page restored from a predecessor's file is refused once the repository turned private, and served while it is public (#9631)", async () => {
400 setCrawlClock(app, () => 18_000_000);
401 clearTreeCache();
402 clearKeptPages();
403 const gitDir = join(dataDir, "repos", "alice", "open.git");
404 const page = `/alice/open/blob/${sha}/README.md`;
405 const br = { "accept-encoding": "br" };
406 const first = await get(page, from("10.7.3.1", br));
407 expect(first.status).toBe(200);
408 const firstBody = await first.text();
409 // the old process saves, the new one starts with nothing and loads the file
410 const file = join(dataDir, "kept-handover.bin");
411 expect(saveKeptPages(file)).toBeGreaterThanOrEqual(1);
412 clearKeptPages();
413 clearTreeCache();
414 expect(loadKeptPages(file)).toBeGreaterThanOrEqual(1);
415 await rename(join(gitDir, "objects"), join(gitDir, "objects.away"));
416 try {
417 const hits = keptPageStats().hits;
418 // turned private during the swap: the restored copy is never handed out
419 app.db.run(`UPDATE repo SET visibility = 'private' WHERE name = 'open'`);
420 const refused = await get(page, from("10.7.3.2", br));
421 expect(refused.status).not.toBe(200);
422 expect(await refused.text()).not.toBe(firstBody);
423 expect(keptPageStats().hits).toBe(hits);
424 // public again: the restored page is served with git gone
425 app.db.run(`UPDATE repo SET visibility = 'public' WHERE name = 'open'`);
426 const warm = await get(page, from("10.7.3.3", br));
427 expect(warm.status).toBe(200);
428 expect(await warm.text()).toBe(firstBody);
429 expect(keptPageStats().hits).toBe(hits + 1);
430 } finally {
431 app.db.run(`UPDATE repo SET visibility = 'public' WHERE name = 'open'`);
432 await rename(join(gitDir, "objects.away"), join(gitDir, "objects"));
433 }
434 });
test/kept-pages-boot.test.ts added +92
@@ -0,0 +1,92 @@
1 /**
2 * The kept-pages hand-over through the real entry (#9631): a forge stopped
3 * with Ctrl-C writes its strangers' kept pages at shutdown (`close` in
4 * main.ts), and the next forge started on the same data reads them before
5 * it takes traffic, so the same page is answered with no git at all.
6 */
7
8 import { expect, test } from "bun:test";
9 import { mkdtemp, rename, rm } from "node:fs/promises";
10 import { tmpdir } from "node:os";
11 import { join } from "node:path";
12 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
13 import { createApp } from "../src/app.ts";
14 import { getRepo } from "../src/core/repos.ts";
15 import { createUser } from "../src/core/users.ts";
16 import { keptPagesPath } from "../src/web/pages/kept_pages_file.ts";
17
18 const BROWSER = "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 Chrome/130.0 Safari/537.36";
19
20 async function boot(dataDir: string, port: number) {
21 const env: Record<string, string> = {
22 ...(process.env as Record<string, string>),
23 FORGE_DATA: dataDir,
24 PORT: String(port),
25 FORGE_SSH_PORT: "0",
26 FORGE_APPS: "0",
27 };
28 delete env.FORGE_URL;
29 const proc = Bun.spawn(["bun", "run", "src/main.ts"], { env, stdout: "pipe", stderr: "pipe" });
30 const log = { text: "" };
31 const drain = async (stream: ReadableStream<Uint8Array>) => {
32 for await (const chunk of stream) log.text += new TextDecoder().decode(chunk);
33 };
34 void drain(proc.stdout);
35 void drain(proc.stderr);
36 for (let i = 0; i < 300; i++) {
37 try {
38 if ((await fetch(`http://127.0.0.1:${port}/healthz`, { signal: AbortSignal.timeout(1_000) })).ok) return { proc, log };
39 } catch {
40 await Bun.sleep(100);
41 }
42 }
43 proc.kill();
44 throw new Error(`forge did not come up:\n${log.text}`);
45 }
46
47 test("shutdown saves the kept pages and the next boot serves them with git gone", async () => {
48 const dataDir = await mkdtemp(join(tmpdir(), "forge-kept-boot-"));
49 const seed = createApp({ dataDir, port: 0 });
50 const u = await createUser(seed.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", isAdmin: true });
51 if (!u.ok) throw new Error(u.error);
52 const r = await createRepo(seed, u.value, { name: "open", visibility: "public" });
53 if (!r.ok) throw new Error(String(r.error));
54 const c = await commitToBranch(seed, u.value, getRepo(seed.db, "alice", "open")!, { message: "Start", files: [{ path: "a.txt", content: "hi\n" }] });
55 if (!c.ok) throw new Error(String(c.error));
56 seed.close();
57 const gitDir = join(dataDir, "repos", "alice", "open.git");
58 const sha = Bun.spawnSync(["git", "--git-dir", gitDir, "rev-parse", "HEAD"]).stdout.toString().trim();
59 const port = 20000 + Math.floor(Math.random() * 20000);
60 const page = `http://127.0.0.1:${port}/alice/open/blob/${sha}/a.txt`;
61 const headers = { "user-agent": BROWSER, "accept-encoding": "br", "x-forwarded-for": "10.9.0.1" };
62 let running: Awaited<ReturnType<typeof boot>> | undefined;
63 try {
64 running = await boot(dataDir, port);
65 const first = await fetch(page, { headers });
66 expect(first.status).toBe(200);
67 const body = await first.text();
68 // The background tick writes it within 5 s; take that file away, so
69 // only the shutdown write can put one back.
70 const file = keptPagesPath(dataDir);
71 for (let i = 0; i < 100 && !(await Bun.file(file).exists()); i++) await Bun.sleep(100);
72 expect(await Bun.file(file).exists()).toBe(true);
73 await Bun.sleep(200);
74 await rm(file);
75 // Ctrl-C: the drain's last step writes the file before exit
76 running.proc.kill("SIGINT");
77 await running.proc.exited;
78 expect(await Bun.file(keptPagesPath(dataDir)).exists()).toBe(true);
79
80 // the successor: no git objects, so only a restored page can answer
81 await rename(join(gitDir, "objects"), join(gitDir, "objects.away"));
82 running = await boot(dataDir, port);
83 expect(running.log.text).toMatch(/kept pages restored[^\n]*"?pages"?[":= ]+1/);
84 const warm = await fetch(page, { headers: { ...headers, "x-forwarded-for": "10.9.0.2" } });
85 expect(warm.status).toBe(200);
86 expect(await warm.text()).toBe(body);
87 } finally {
88 running?.proc.kill();
89 await running?.proc.exited;
90 await rm(dataDir, { recursive: true, force: true });
91 }
92 }, 60_000);
test/kept-pages-file.test.ts added +142
@@ -0,0 +1,142 @@
1 import { afterEach, expect, test } from "bun:test";
2 import { chmodSync, mkdirSync, mkdtempSync, readdirSync, statSync, writeFileSync } from "node:fs";
3 import { join } from "node:path";
4 import { tmpdir } from "node:os";
5 import { clearKeptPages, keepPage, keptPage } from "../src/web/pages/kept_page.ts";
6 import { shellBuild } from "../src/web/asset.ts";
7 import { FILE_BYTES, keepPagesAcrossSwaps, keptPagesPath, loadKeptPages, saveKeptPages, saveKeptPagesAsync } from "../src/web/pages/kept_pages_file.ts";
8
9 afterEach(clearKeptPages);
10 const keyOf = (n: string): string => ["gzip", "000", "r1", "http://x", `/a/${n}`, ""].join("\x00");
11 const KEY = keyOf("b");
12 const html = (body: string) => new Response(body, { headers: { "content-type": "text/html", "x-request-id": "gone" } });
13 const dir = () => mkdtempSync(join(tmpdir(), "kept-"));
14
15 test("a kept page crosses a swap: same bytes and headers, served without rendering", async () => {
16 const first = await keepPage(KEY, html("<p>hello</p>"));
17 const bytes = new Uint8Array(await first.arrayBuffer());
18 const path = join(dir(), "cache", "kept-pages.bin");
19 expect(saveKeptPages(path)).toBe(1);
20 expect((statSync(path).mode & 0o777).toString(8)).toBe("600");
21 clearKeptPages();
22 expect(keptPage(KEY)).toBeUndefined();
23 expect(loadKeptPages(path)).toBe(1);
24 const warm = keptPage(KEY)!;
25 expect(warm.headers.get("content-type")).toBe("text/html");
26 expect(warm.headers.get("x-request-id")).toBeNull();
27 expect(warm.headers.get("content-encoding")).toBe("gzip");
28 expect(new Uint8Array(await warm.arrayBuffer())).toEqual(bytes);
29 });
30
31 test("the file's age is taken off a page's time; a file older than a minute restores nothing", async () => {
32 await keepPage(KEY, html("<p>x</p>"));
33 const path = join(dir(), "k.bin");
34 const t0 = 1_000_000;
35 saveKeptPages(path, t0);
36 clearKeptPages();
37 expect(loadKeptPages(path, t0 + 3_600_000)).toBe(0);
38 expect(keptPage(KEY)).toBeUndefined();
39 expect(loadKeptPages(path, t0 + 120_000)).toBe(0);
40 // a clock that went backwards counts as no age, and the ttl is clamped to a minute
41 expect(loadKeptPages(path, t0 - 10_000)).toBe(1);
42 expect(keptPage(KEY)).toBeDefined();
43 });
44
45 test("a page with its time used up is dropped and not counted", async () => {
46 await keepPage(KEY, html("<p>x</p>"));
47 await keepPage(keyOf("c"), html("<p>y</p>"));
48 const path = join(dir(), "k.bin");
49 saveKeptPages(path, Date.now());
50 clearKeptPages();
51 expect(loadKeptPages(path, Date.now() + 61_000)).toBe(0);
52 });
53
54 test("the file holds at most FILE_BYTES; the freshest pages go first", async () => {
55 const filler = (n: number) => Array.from({ length: 1500 }, (_, i) => `${n}-${i}-${Math.random()}`).join(" ");
56 for (let i = 0; i < 40; i++) await keepPage(keyOf(String(i)), html(`<p>${filler(i)}</p>`));
57 const path = join(dir(), "k.bin");
58 const n = saveKeptPages(path);
59 expect(statSync(path).size).toBeLessThanOrEqual(FILE_BYTES);
60 expect(n).toBeGreaterThan(0);
61 expect(n).toBeLessThanOrEqual(40);
62 });
63
64 test("a missing, torn or foreign file loads nothing and does not throw", async () => {
65 const d = dir();
66 expect(loadKeptPages(join(d, "none"))).toBe(0);
67 await keepPage(KEY, html("<p>x</p>"));
68 const path = join(d, "a.bin");
69 saveKeptPages(path);
70 const full = await Bun.file(path).bytes();
71 clearKeptPages();
72 writeFileSync(path, full.subarray(0, full.length - 3));
73 expect(loadKeptPages(path)).toBe(0);
74 writeFileSync(path, "not a kept pages file at all");
75 expect(loadKeptPages(path)).toBe(0);
76 expect(keptPage(KEY)).toBeUndefined();
77 });
78
79 test("a failed write leaves no temp file behind", async () => {
80 await keepPage(KEY, html("<p>x</p>"));
81 const d = dir();
82 const path = join(d, "k.bin");
83 mkdirSync(path); // renaming a file onto a directory fails
84 expect(() => saveKeptPages(path)).toThrow();
85 await expect(saveKeptPagesAsync(path)).rejects.toThrow();
86 expect(readdirSync(d).filter((f) => f.endsWith(".tmp"))).toEqual([]);
87 chmodSync(d, 0o700);
88 });
89
90 test("the process wiring: save writes in the background only when something new was kept, saveNow at exit, a successor loads", async () => {
91 const d = dir();
92 const a = keepPagesAcrossSwaps(d);
93 expect(a.loaded).toBe(0);
94 a.save(); // nothing kept yet: no file
95 await Bun.sleep(30);
96 expect(await Bun.file(keptPagesPath(d)).exists()).toBe(false);
97 await keepPage(KEY, html("<p>x</p>"));
98 a.save();
99 for (let i = 0; i < 50 && !(await Bun.file(keptPagesPath(d)).exists()); i++) await Bun.sleep(10);
100 expect(await Bun.file(keptPagesPath(d)).exists()).toBe(true);
101 await keepPage(keyOf("late"), html("<p>late</p>"));
102 await Bun.sleep(30);
103 a.saveNow();
104 clearKeptPages();
105 const b = keepPagesAcrossSwaps(d);
106 expect(b.loaded).toBe(2);
107 expect(keptPage(KEY)).toBeDefined();
108 });
109
110 test("a file written by another build loads nothing: its pages link stylesheets and scripts this process does not serve", async () => {
111 await keepPage(KEY, html("<p>x</p>"));
112 const path = join(dir(), "k.bin");
113 saveKeptPages(path, Date.now(), "old-build");
114 clearKeptPages();
115 expect(loadKeptPages(path, Date.now(), "new-build")).toBe(0);
116 expect(keptPage(KEY)).toBeUndefined();
117 // the same build, a restart or a server-only release, is carried across
118 expect(loadKeptPages(path, Date.now(), "old-build")).toBe(1);
119 // and by default the build is this process's own shell assets
120 saveKeptPages(path);
121 clearKeptPages();
122 expect(loadKeptPages(path, Date.now(), shellBuild())).toBe(1);
123 clearKeptPages();
124 expect(loadKeptPages(path, Date.now(), "other")).toBe(0);
125 });
126
127 test("a file bigger than FILE_BYTES is refused before it is read", async () => {
128 await keepPage(KEY, html("<p>x</p>"));
129 const path = join(dir(), "k.bin");
130 saveKeptPages(path);
131 const full = await Bun.file(path).bytes();
132 clearKeptPages();
133 // a valid file with trailing bytes past the bound: refused whole, not parsed
134 const big = new Uint8Array(FILE_BYTES + 1);
135 big.set(full);
136 writeFileSync(path, big);
137 expect(loadKeptPages(path)).toBe(0);
138 expect(keptPage(KEY)).toBeUndefined();
139 // at the bound exactly it is read (and rejected here only for the trailing zeros)
140 writeFileSync(path, full);
141 expect(loadKeptPages(path)).toBe(1);
142 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs