0f92a7f
@@ -37,3 +37,21 @@
sealed values to set again. To pick an older snapshot, put its name
On a box that stays: `scripts/box-install.sh`, then lines 3–6 as the `forge`
user with `<data-dir>` `/var/lib/forge`, then `systemctl enable --now forge`
in place of line 7 ([EXTRACT.md](EXTRACT.md), _The box beside production_).
## The drill on the box
`scripts/restore-drill.ts` runs lines 3–7 and checks them, from the bucket
alone: with the card's `S3_*` lines and the passphrase exported, as the
`forge` user,
    bun scripts/restore-drill.ts <empty-data-dir>
restores the newest snapshot into that directory, boots it on a free side
port, checks `/` answers, and prints the repositories and apps anyone can
see beside production's public API (`--production <url>`, default
`https://forge.open-platform.sh`); the restored forge boots as a preview,
so the sample its preview admin seeds is left out. It exits 1 when a step fails or a count
      
        junipercrew
        11d ago
        
      
      This line runs long against the rest of the paragraph; rewrap it.
    
differs. It writes only into the directory it is given, which must be empty,
refuses `/data`, and leaves the restored copy in place.
`test/restore-drill-script.test.ts` runs it against a tiny snapshot and a
stub bucket.
@@ -0,0 +1,116 @@
/**
 * The restore drill on the box beside production (#9046): the newest
 * snapshot restored from the bucket alone, booted, and set beside
 * production. It reads only what the restore card names — the `S3_*`
 * variables (or `FORGE_SNAPSHOT_DIR`) and `FORGE_SNAPSHOT_PASSPHRASE` —
 * writes only into the empty data directory it is given, and reads
 * production through its public API, nothing else of the platform.
 *
 *   bun scripts/restore-drill.ts <data-dir> [--production https://forge.open-platform.sh]
 *
 * It restores, boots the restored forge on a free side port, checks `/`
 * answers, counts the repositories and apps anyone can see there and on
 * production, prints both, and exits 1 when any step fails or a count
 * differs. The restored directory stays: it is the box's own copy.
 */
import { resolve } from "node:path";
import { bootRestored } from "../src/snapshot/boot.ts";
import { resolveDestination } from "../src/snapshot/destination.ts";
import { publicCounts, type PublicCounts } from "../src/snapshot/public_counts.ts";
import { restoreSnapshot } from "../src/snapshot/restore.ts";
import { listSnapshots } from "../src/snapshot/snapshot.ts";
import { PREVIEW_ADMIN } from "../src/spawn/preview-looks.ts";
export const PRODUCTION = "https://forge.open-platform.sh";
/** Production's own data directory on the platform; the drill never writes there. */
const PRODUCTION_DATA = "/data";
const BOOT_MS = 60_000;
/** The arguments as plain data, or the reason they are not usable. */
export function parseArgs(args: readonly string[]): { dataDir: string; production: string } | string {
  const rest = [...args];
  let production = PRODUCTION;
  const at = rest.indexOf("--production");
  if (at >= 0) {
    production = rest[at + 1] ?? "";
    rest.splice(at, 2);
  }
  let url: URL;
  try {
    url = new URL(production);
  } catch {
    return `--production takes a URL, not ${JSON.stringify(production)}`;
  }
  const loopback = url.hostname === "127.0.0.1" || url.hostname === "localhost";
  if (url.protocol !== "https:" && !(url.protocol === "http:" && loopback))
    return "--production takes an https URL";
  if (rest.length !== 1 || !rest[0] || rest[0].startsWith("-"))
    return "usage: bun scripts/restore-drill.ts <data-dir> [--production URL]";
  const dataDir = resolve(rest[0]);
  if (dataDir === PRODUCTION_DATA) return `${dataDir} is production's data directory; restore into the box's own`;
  return { dataDir, production: url.origin };
}
/** The two sides' counts as lines a person reads, and which differ. */
export function compareLines(restored: PublicCounts, production: PublicCounts): { lines: string[]; differ: string[] } {
  const keys = Object.keys(restored) as (keyof PublicCounts)[];
  const differ = keys.filter((k) => restored[k] !== production[k]);
  const lines = keys.map(
    (k) => `${k}: restored ${restored[k]}, production ${production[k]}${differ.includes(k) ? "  DIFFERS" : ""}`,
  );
  return { lines, differ };
}
async function main(): Promise<number> {
  const args = parseArgs(process.argv.slice(2));
  // CLI output below is for the person at the box, and is posted on the issue.
  if (typeof args === "string") {
    console.error(args);
    return 2;
  }
  const store = resolveDestination()?.store;
  if (store === undefined) {
    console.error("no store: export the S3_* lines of the restore card, or FORGE_SNAPSHOT_DIR");
    return 2;
  }
  const newest = (await listSnapshots(store))[0];
  if (newest === undefined) {
    console.error(`no snapshot in ${store.where}`);
    return 1;
  }
  const began = performance.now();
  const seconds = (): string => ((performance.now() - began) / 1000).toFixed(1);
  console.log(`restore-drill: ${newest.name} from ${store.where} into ${args.dataDir}`);
  const done = await restoreSnapshot(store, newest.name, args.dataDir, {
    passphrase: process.env["FORGE_SNAPSHOT_PASSPHRASE"] || undefined,
  });
  console.log(`restored ${done.files} files and ${done.repos} repositories, key ${done.key}, ${seconds()}s`);
  if (done.sealedMissing.length > 0) console.log(`sealed values to set again: ${done.sealedMissing.length}`);
  let restored: PublicCounts | undefined;
  const booted = await bootRestored(args.dataDir, BOOT_MS, undefined, async (base) => {
    const home = await fetch(`${base}/`, { redirect: "manual", signal: AbortSignal.timeout(15_000) });
    if (home.status >= 400) throw new Error(`/ answered ${home.status}`);
    console.log(`booted: / answered ${home.status}, ${seconds()}s`);
    // The boot is a preview, which seeds the preview admin's sample on top.
    restored = await publicCounts(base, { except: PREVIEW_ADMIN });
  });
  if (!booted.ok || restored === undefined) {
    console.error(`boot failed: ${booted.reason || "no counts"}`);
    return 1;
  }
  let production: PublicCounts;
  try {
    production = await publicCounts(args.production);
  } catch (error) {
    console.error(`production ${args.production} did not answer: ${error instanceof Error ? error.message : String(error)}`);
    return 1;
  }
  const { lines, differ } = compareLines(restored, production);
  for (const line of lines) console.log(line);
  console.log(differ.length === 0 ? `restore-drill: green, ${seconds()}s` : `restore-drill: red, ${differ.join(" and ")} differ`);
  return differ.length === 0 ? 0 : 1;
}
if (import.meta.main) process.exit(await main());
@@ -17,12 +17,18 @@
export interface Booted {
const TREE = join(import.meta.dir, "../..");
/**
 * `whileUp`, when given, is called with the restored forge's base URL once
 * it reads healthy and before it is killed; what it throws is the reason
 * the boot is not ok (the restore drill script reads `/` and counts there).
 */
export async function bootRestored(
  dataDir: string,
  budgetMs: number,
  tree: string = TREE,
  whileUp?: (base: string) => Promise<void>,
): Promise<Booted> {
  return onFreshPort((port) => bootOnPort(dataDir, budgetMs, port, tree));
  return onFreshPort((port) => bootOnPort(dataDir, budgetMs, port, tree, whileUp));
}
async function bootOnPort(
@@ -30,6 +36,7 @@
async function bootOnPort(
  budgetMs: number,
  port: number,
  tree: string,
  whileUp: ((base: string) => Promise<void>) | undefined,
): Promise<{ result: Booted; portLost: boolean }> {
  const child = Bun.spawn([process.execPath, join(tree, "src/main.ts")], {
    cwd: tree,
@@ -66,7 +73,16 @@
async function bootOnPort(
          signal: AbortSignal.timeout(2_000),
        });
        const body = (await res.json().catch(() => null)) as { ok?: unknown } | null;
        if (res.ok && body?.ok === true) return { result: { ok: true, reason: "" }, portLost: false };
        if (res.ok && body?.ok === true) {
          if (whileUp !== undefined) {
            try {
              await whileUp(`http://127.0.0.1:${port}`);
            } catch (error) {
              return no(error instanceof Error ? error.message : String(error));
            }
          }
          return { result: { ok: true, reason: "" }, portLost: false };
        }
        last = `health answered ${res.status}${body?.ok === false ? " not ok" : ""}`;
      } catch {
        last = "no answer yet";
@@ -0,0 +1,66 @@
/**
 * What a forge shows anyone, counted through its public API: repositories
 * from `/api/v1/repos` page by page, apps from `/api/v1/apps`, asked with
 * no session. The restore drill script asks a restored forge and
 * production the same way, so the two numbers cover the same rows.
 *
 * `except` leaves out one owner's rows: a restored forge boots as a
 * preview, which seeds the preview admin's sample repository on top of
 * what was restored, and that is not production's.
 */
export interface PublicCounts {
  readonly repositories: number;
  readonly apps: number;
}
/** A forge answers more pages than this only if it is lying about its cursor. */
const MAX_PAGES = 1_000;
type Fetch = (url: string, init?: RequestInit) => Promise<Response>;
async function page(fetcher: Fetch, url: string): Promise<{ items: unknown[]; next: string | null }> {
  const res = await fetcher(url, { signal: AbortSignal.timeout(15_000), redirect: "error" });
  if (!res.ok) throw new Error(`${url} answered ${res.status}`);
  const body = (await res.json().catch(() => null)) as { items?: unknown; next_cursor?: unknown } | null;
  if (body === null || !Array.isArray(body.items)) throw new Error(`${url} answered no items`);
  const next = body.next_cursor;
  return {
    items: body.items,
    next: typeof next === "string" || typeof next === "number" ? String(next) : null,
  };
}
/** The owner of a repository item (`ownerName`) or an app item (`repo.owner`). */
function ownerOf(item: unknown): string | undefined {
  if (typeof item !== "object" || item === null) return undefined;
  const o = item as { ownerName?: unknown; repo?: { owner?: unknown } };
  const name = typeof o.ownerName === "string" ? o.ownerName : o.repo?.owner;
  return typeof name === "string" ? name : undefined;
}
async function countAll(fetcher: Fetch, base: string, path: string, except: string | undefined): Promise<number> {
  let n = 0;
  let cursor: string | null = null;
  for (let i = 0; i < MAX_PAGES; i++) {
    const url = new URL(path, base);
    url.searchParams.set("limit", "100");
    if (cursor !== null) url.searchParams.set("cursor", cursor);
    const got = await page(fetcher, url.href);
    n += got.items.filter((item) => except === undefined || ownerOf(item) !== except).length;
    if (got.next === null || got.items.length === 0) return n;
    cursor = got.next;
  }
  throw new Error(`${path} on ${base} pages past ${MAX_PAGES}`);
}
export async function publicCounts(
  base: string,
  options: { readonly except?: string | undefined; readonly fetcher?: Fetch | undefined } = {},
): Promise<PublicCounts> {
  const fetcher = options.fetcher ?? fetch;
  return {
    repositories: await countAll(fetcher, base, "/api/v1/repos", options.except),
    apps: await countAll(fetcher, base, "/api/v1/apps", options.except),
  };
}
@@ -0,0 +1,186 @@
/**
 * `scripts/restore-drill.ts` (#9046) run end to end: a tiny snapshot taken
 * here, served from a stub bucket that speaks the S3 the store asks for,
 * restored with only the card's variables, booted, and its public counts
 * set beside a stub production's. Matching counts are green; the
 * arguments and the comparison are checked as plain data.
 */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { existsSync, mkdtempSync, readdirSync, readFileSync, statSync } from "node:fs";
import { rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join, relative } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { commitToBranch, createRepo } from "../src/actions/repos.ts";
import { createUser } from "../src/core/users.ts";
import { dirStore } from "../src/snapshot/store.ts";
import { takeSnapshot } from "../src/snapshot/snapshot.ts";
import { compareLines, parseArgs, PRODUCTION } from "../scripts/restore-drill.ts";
import { publicCounts } from "../src/snapshot/public_counts.ts";
const PASS = "correct horse battery staple";
const BUCKET = "drill";
const ESC: Record<string, string> = { "&": "&amp;", "<": "&lt;", ">": "&gt;" };
const xml = (s: string): string => s.replace(/[&<>]/g, (c) => ESC[c]!);
let root: string;
let app: App;
let bucket: ReturnType<typeof Bun.serve>;
let production: ReturnType<typeof Bun.serve>;
function keysUnder(dir: string, out: string[] = []): string[] {
  for (const name of readdirSync(dir)) {
    const path = join(dir, name);
    if (statSync(path).isDirectory()) keysUnder(path, out);
    else out.push(path);
  }
  return out;
}
/** Path-style S3 over a directory: HEAD and GET of a key, and ListObjectsV2 with a delimiter. */
function stubBucket(dir: string): ReturnType<typeof Bun.serve> {
  return Bun.serve({
    port: 0,
    hostname: "127.0.0.1",
    fetch(req) {
      const url = new URL(req.url);
      const path = decodeURIComponent(url.pathname);
      if (path === `/${BUCKET}` || path === `/${BUCKET}/`) {
        const prefix = url.searchParams.get("prefix") ?? "";
        const delimiter = url.searchParams.get("delimiter");
        const keys = keysUnder(dir).map((p) => relative(dir, p)).filter((k) => k.startsWith(prefix)).sort();
        const contents: string[] = [];
        const prefixes = new Set<string>();
        for (const key of keys) {
          const cut = delimiter ? key.indexOf(delimiter, prefix.length) : -1;
          if (cut >= 0) prefixes.add(key.slice(0, cut + 1));
          else contents.push(key);
        }
        const body =
          `<?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Name>${BUCKET}</Name><Prefix>${xml(prefix)}</Prefix>` +
          `<KeyCount>${contents.length + prefixes.size}</KeyCount><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated>` +
          contents.map((k) => `<Contents><Key>${xml(k)}</Key><Size>${statSync(join(dir, k)).size}</Size></Contents>`).join("") +
          [...prefixes].map((p) => `<CommonPrefixes><Prefix>${xml(p)}</Prefix></CommonPrefixes>`).join("") +
          `</ListBucketResult>`;
        return new Response(body, { headers: { "content-type": "application/xml" } });
      }
      const key = path.slice(`/${BUCKET}/`.length);
      if (!path.startsWith(`/${BUCKET}/`) || key.split("/").includes("..")) return new Response(null, { status: 400 });
      const file = join(dir, key);
      if (!existsSync(file)) return new Response(null, { status: 404 });
      return new Response(req.method === "HEAD" ? null : readFileSync(file), {
        headers: { "content-length": String(statSync(file).size) },
      });
    },
  });
}
beforeAll(async () => {
  root = mkdtempSync(join(tmpdir(), "forge-restore-drill-"));
  app = createApp({ dataDir: join(root, "live"), port: 0 });
  const u = await createUser(app.db, {
    origin: "seed",
    trusted: true,
    username: "alice",
    email: "[email protected]",
    password: "a-good-passphrase",
    adminIfFirst: true,
  });
  if (!u.ok) throw new Error(u.error);
  for (const [name, visibility] of [["notes", "public"], ["diary", "private"]] as const) {
    const repo = await createRepo(app, u.value, { name, visibility });
    if (!repo.ok) throw new Error(String(repo.error));
    const c = await commitToBranch(app, u.value, repo.value, {
      message: "first",
      files: [{ path: "README.md", content: `# ${name}\n` }],
    });
    if (!c.ok) throw new Error(JSON.stringify(c.error));
  }
  await takeSnapshot(
    { db: app.db, dataDir: app.config.dataDir, reposRoot: app.reposRoot, version: "t", passphrase: PASS },
    dirStore(join(root, "store")),
    new Date("2026-11-01T08:00:00Z"),
  );
  bucket = stubBucket(join(root, "store"));
  // Production as anyone sees it: the one public repository, no apps.
  production = Bun.serve({
    port: 0,
    hostname: "127.0.0.1",
    fetch(req) {
      const path = new URL(req.url).pathname;
      if (path === "/api/v1/repos") return Response.json({ items: [{ id: 1 }], next_cursor: null });
      if (path === "/api/v1/apps") return Response.json({ items: [], next_cursor: null });
      return new Response(null, { status: 404 });
    },
  });
});
afterAll(async () => {
  app.close();
  bucket.stop(true);
  production.stop(true);
  await rm(root, { recursive: true, force: true });
});
test("the drill restores from the bucket alone, boots, and matches production's public counts", async () => {
  const data = join(root, "box");
  const run = Bun.spawn(
    [process.execPath, "scripts/restore-drill.ts", data, "--production", `http://127.0.0.1:${production.port}`],
    {
      cwd: join(import.meta.dir, ".."),
      env: {
        PATH: process.env["PATH"] ?? "",
        HOME: root,
        S3_ENDPOINT: `http://127.0.0.1:${bucket.port}`,
        S3_BUCKET: BUCKET,
        S3_ACCESS_KEY: "stub",
        S3_SECRET_KEY: "stub",
        S3_REGION: "us-east-1",
        FORGE_SNAPSHOT_PASSPHRASE: PASS,
        ...(process.env["FORGE_TEST_SOCKET"] ? { FORGE_TEST_SOCKET: process.env["FORGE_TEST_SOCKET"] } : {}),
      },
      stdout: "pipe",
      stderr: "pipe",
    },
  );
  const [out, err, code] = await Promise.all([new Response(run.stdout).text(), new Response(run.stderr).text(), run.exited]);
  expect(err + out).toContain("restore-drill: 2026-11-01T08-00-00Z from 127.0.0.1:");
  expect(out).toContain("and 2 repositories");
  expect(out).toMatch(/booted: \/ answered [23]\d\d/);
  expect(out).toContain("repositories: restored 1, production 1");
  expect(out).toContain("apps: restored 0, production 0");
  expect(out).toContain("restore-drill: green");
  expect(code).toBe(0);
  expect(existsSync(join(data, "forge.db"))).toBe(true);
}, 120_000);
test("the arguments: a data directory, production by default, never production's own directory", () => {
  expect(parseArgs(["/var/lib/forge"])).toEqual({ dataDir: "/var/lib/forge", production: PRODUCTION });
  expect(parseArgs(["/data"])).toContain("production's data directory");
  expect(parseArgs([])).toContain("usage");
  expect(parseArgs(["/x", "--production", "http://forge.example"])).toContain("https");
  expect(parseArgs(["/x", "--production", "not a url"])).toContain("URL");
});
test("a count that differs is named", () => {
  const same = compareLines({ repositories: 3, apps: 1 }, { repositories: 3, apps: 1 });
  expect(same.differ).toEqual([]);
  const off = compareLines({ repositories: 3, apps: 1 }, { repositories: 4, apps: 1 });
  expect(off.differ).toEqual(["repositories"]);
  expect(off.lines[0]).toBe("repositories: restored 3, production 4  DIFFERS");
});
test("public counts follow the cursor and leave out the preview admin's seeded rows", async () => {
  const pages: Record<string, unknown> = {
    "/api/v1/repos?limit=100": { items: [{ ownerName: "alice" }, { ownerName: "preview-admin" }], next_cursor: "b" },
    "/api/v1/repos?limit=100&cursor=b": { items: [{ ownerName: "bob" }], next_cursor: null },
    "/api/v1/apps?limit=100": { items: [{ repo: { owner: "alice" } }, { repo: { owner: "preview-admin" } }], next_cursor: null },
  };
  const fetcher = async (url: string): Promise<Response> => {
    const u = new URL(url);
    const body = pages[u.pathname + u.search];
    return body === undefined ? new Response(null, { status: 404 }) : Response.json(body);
  };
  expect(await publicCounts("http://forge.test", { fetcher })).toEqual({ repositories: 3, apps: 2 });
  expect(await publicCounts("http://forge.test", { fetcher, except: "preview-admin" })).toEqual({ repositories: 2, apps: 1 });
});
		@@ -37,3 +37,21 @@ sealed values to set again. To pick an older snapshot, put its name
37	37	On a box that stays: `scripts/box-install.sh`, then lines 3–6 as the `forge`
38	38	user with `<data-dir>` `/var/lib/forge`, then `systemctl enable --now forge`
39	39	in place of line 7 ([EXTRACT.md](EXTRACT.md), _The box beside production_).
	40
	41	## The drill on the box
	42
	43	`scripts/restore-drill.ts` runs lines 3–7 and checks them, from the bucket
	44	alone: with the card's `S3_*` lines and the passphrase exported, as the
	45	`forge` user,
	46
	47	bun scripts/restore-drill.ts <empty-data-dir>
	48
	49	restores the newest snapshot into that directory, boots it on a free side
	50	port, checks `/` answers, and prints the repositories and apps anyone can
	51	see beside production's public API (`--production <url>`, default
	52	`https://forge.open-platform.sh`); the restored forge boots as a preview,
	53	so the sample its preview admin seeds is left out. It exits 1 when a step fails or a count
		junipercrew 11d ago This line runs long against the rest of the paragraph; rewrap it.
	54	differs. It writes only into the directory it is given, which must be empty,
	55	refuses `/data`, and leaves the restored copy in place.
	56	`test/restore-drill-script.test.ts` runs it against a tiny snapshot and a
	57	stub bucket.
		@@ -0,0 +1,116 @@
	1	/**
	2	* The restore drill on the box beside production (#9046): the newest
	3	* snapshot restored from the bucket alone, booted, and set beside
	4	* production. It reads only what the restore card names — the `S3_*`
	5	* variables (or `FORGE_SNAPSHOT_DIR`) and `FORGE_SNAPSHOT_PASSPHRASE` —
	6	* writes only into the empty data directory it is given, and reads
	7	* production through its public API, nothing else of the platform.
	8	*
	9	* bun scripts/restore-drill.ts <data-dir> [--production https://forge.open-platform.sh]
	10	*
	11	* It restores, boots the restored forge on a free side port, checks `/`
	12	* answers, counts the repositories and apps anyone can see there and on
	13	* production, prints both, and exits 1 when any step fails or a count
	14	* differs. The restored directory stays: it is the box's own copy.
	15	*/
	16
	17	import { resolve } from "node:path";
	18	import { bootRestored } from "../src/snapshot/boot.ts";
	19	import { resolveDestination } from "../src/snapshot/destination.ts";
	20	import { publicCounts, type PublicCounts } from "../src/snapshot/public_counts.ts";
	21	import { restoreSnapshot } from "../src/snapshot/restore.ts";
	22	import { listSnapshots } from "../src/snapshot/snapshot.ts";
	23	import { PREVIEW_ADMIN } from "../src/spawn/preview-looks.ts";
	24
	25	export const PRODUCTION = "https://forge.open-platform.sh";
	26	/** Production's own data directory on the platform; the drill never writes there. */
	27	const PRODUCTION_DATA = "/data";
	28	const BOOT_MS = 60_000;
	29
	30	/** The arguments as plain data, or the reason they are not usable. */
	31	export function parseArgs(args: readonly string[]): { dataDir: string; production: string } \| string {
	32	const rest = [...args];
	33	let production = PRODUCTION;
	34	const at = rest.indexOf("--production");
	35	if (at >= 0) {
	36	production = rest[at + 1] ?? "";
	37	rest.splice(at, 2);
	38	}
	39	let url: URL;
	40	try {
	41	url = new URL(production);
	42	} catch {
	43	return `--production takes a URL, not ${JSON.stringify(production)}`;
	44	}
	45	const loopback = url.hostname === "127.0.0.1" \|\| url.hostname === "localhost";
	46	if (url.protocol !== "https:" && !(url.protocol === "http:" && loopback))
	47	return "--production takes an https URL";
	48	if (rest.length !== 1 \|\| !rest[0] \|\| rest[0].startsWith("-"))
	49	return "usage: bun scripts/restore-drill.ts <data-dir> [--production URL]";
	50	const dataDir = resolve(rest[0]);
	51	if (dataDir === PRODUCTION_DATA) return `${dataDir} is production's data directory; restore into the box's own`;
	52	return { dataDir, production: url.origin };
	53	}
	54
	55	/** The two sides' counts as lines a person reads, and which differ. */
	56	export function compareLines(restored: PublicCounts, production: PublicCounts): { lines: string[]; differ: string[] } {
	57	const keys = Object.keys(restored) as (keyof PublicCounts)[];
	58	const differ = keys.filter((k) => restored[k] !== production[k]);
	59	const lines = keys.map(
	60	(k) => `${k}: restored ${restored[k]}, production ${production[k]}${differ.includes(k) ? " DIFFERS" : ""}`,
	61	);
	62	return { lines, differ };
	63	}
	64
	65	async function main(): Promise<number> {
	66	const args = parseArgs(process.argv.slice(2));
	67	// CLI output below is for the person at the box, and is posted on the issue.
	68	if (typeof args === "string") {
	69	console.error(args);
	70	return 2;
	71	}
	72	const store = resolveDestination()?.store;
	73	if (store === undefined) {
	74	console.error("no store: export the S3_* lines of the restore card, or FORGE_SNAPSHOT_DIR");
	75	return 2;
	76	}
	77	const newest = (await listSnapshots(store))[0];
	78	if (newest === undefined) {
	79	console.error(`no snapshot in ${store.where}`);
	80	return 1;
	81	}
	82	const began = performance.now();
	83	const seconds = (): string => ((performance.now() - began) / 1000).toFixed(1);
	84	console.log(`restore-drill: ${newest.name} from ${store.where} into ${args.dataDir}`);
	85	const done = await restoreSnapshot(store, newest.name, args.dataDir, {
	86	passphrase: process.env["FORGE_SNAPSHOT_PASSPHRASE"] \|\| undefined,
	87	});
	88	console.log(`restored ${done.files} files and ${done.repos} repositories, key ${done.key}, ${seconds()}s`);
	89	if (done.sealedMissing.length > 0) console.log(`sealed values to set again: ${done.sealedMissing.length}`);
	90
	91	let restored: PublicCounts \| undefined;
	92	const booted = await bootRestored(args.dataDir, BOOT_MS, undefined, async (base) => {
	93	const home = await fetch(`${base}/`, { redirect: "manual", signal: AbortSignal.timeout(15_000) });
	94	if (home.status >= 400) throw new Error(`/ answered ${home.status}`);
	95	console.log(`booted: / answered ${home.status}, ${seconds()}s`);
	96	// The boot is a preview, which seeds the preview admin's sample on top.
	97	restored = await publicCounts(base, { except: PREVIEW_ADMIN });
	98	});
	99	if (!booted.ok \|\| restored === undefined) {
	100	console.error(`boot failed: ${booted.reason \|\| "no counts"}`);
	101	return 1;
	102	}
	103	let production: PublicCounts;
	104	try {
	105	production = await publicCounts(args.production);
	106	} catch (error) {
	107	console.error(`production ${args.production} did not answer: ${error instanceof Error ? error.message : String(error)}`);
	108	return 1;
	109	}
	110	const { lines, differ } = compareLines(restored, production);
	111	for (const line of lines) console.log(line);
	112	console.log(differ.length === 0 ? `restore-drill: green, ${seconds()}s` : `restore-drill: red, ${differ.join(" and ")} differ`);
	113	return differ.length === 0 ? 0 : 1;
	114	}
	115
	116	if (import.meta.main) process.exit(await main());
		@@ -17,12 +17,18 @@ export interface Booted {
17	17
18	18	const TREE = join(import.meta.dir, "../..");
19	19
	20	/**
	21	* `whileUp`, when given, is called with the restored forge's base URL once
	22	* it reads healthy and before it is killed; what it throws is the reason
	23	* the boot is not ok (the restore drill script reads `/` and counts there).
	24	*/
20	25	export async function bootRestored(
21	26	dataDir: string,
22	27	budgetMs: number,
23	28	tree: string = TREE,
	29	whileUp?: (base: string) => Promise<void>,
24	30	): Promise<Booted> {
25		return onFreshPort((port) => bootOnPort(dataDir, budgetMs, port, tree));
	31	return onFreshPort((port) => bootOnPort(dataDir, budgetMs, port, tree, whileUp));
26	32	}
27	33
28	34	async function bootOnPort(
		@@ -30,6 +36,7 @@ async function bootOnPort(
30	36	budgetMs: number,
31	37	port: number,
32	38	tree: string,
	39	whileUp: ((base: string) => Promise<void>) \| undefined,
33	40	): Promise<{ result: Booted; portLost: boolean }> {
34	41	const child = Bun.spawn([process.execPath, join(tree, "src/main.ts")], {
35	42	cwd: tree,
		@@ -66,7 +73,16 @@ async function bootOnPort(
66	73	signal: AbortSignal.timeout(2_000),
67	74	});
68	75	const body = (await res.json().catch(() => null)) as { ok?: unknown } \| null;
69		if (res.ok && body?.ok === true) return { result: { ok: true, reason: "" }, portLost: false };
	76	if (res.ok && body?.ok === true) {
	77	if (whileUp !== undefined) {
	78	try {
	79	await whileUp(`http://127.0.0.1:${port}`);
	80	} catch (error) {
	81	return no(error instanceof Error ? error.message : String(error));
	82	}
	83	}
	84	return { result: { ok: true, reason: "" }, portLost: false };
	85	}
70	86	last = `health answered ${res.status}${body?.ok === false ? " not ok" : ""}`;
71	87	} catch {
72	88	last = "no answer yet";
		@@ -0,0 +1,66 @@
	1	/**
	2	* What a forge shows anyone, counted through its public API: repositories
	3	* from `/api/v1/repos` page by page, apps from `/api/v1/apps`, asked with
	4	* no session. The restore drill script asks a restored forge and
	5	* production the same way, so the two numbers cover the same rows.
	6	*
	7	* `except` leaves out one owner's rows: a restored forge boots as a
	8	* preview, which seeds the preview admin's sample repository on top of
	9	* what was restored, and that is not production's.
	10	*/
	11
	12	export interface PublicCounts {
	13	readonly repositories: number;
	14	readonly apps: number;
	15	}
	16
	17	/** A forge answers more pages than this only if it is lying about its cursor. */
	18	const MAX_PAGES = 1_000;
	19
	20	type Fetch = (url: string, init?: RequestInit) => Promise<Response>;
	21
	22	async function page(fetcher: Fetch, url: string): Promise<{ items: unknown[]; next: string \| null }> {
	23	const res = await fetcher(url, { signal: AbortSignal.timeout(15_000), redirect: "error" });
	24	if (!res.ok) throw new Error(`${url} answered ${res.status}`);
	25	const body = (await res.json().catch(() => null)) as { items?: unknown; next_cursor?: unknown } \| null;
	26	if (body === null \|\| !Array.isArray(body.items)) throw new Error(`${url} answered no items`);
	27	const next = body.next_cursor;
	28	return {
	29	items: body.items,
	30	next: typeof next === "string" \|\| typeof next === "number" ? String(next) : null,
	31	};
	32	}
	33
	34	/** The owner of a repository item (`ownerName`) or an app item (`repo.owner`). */
	35	function ownerOf(item: unknown): string \| undefined {
	36	if (typeof item !== "object" \|\| item === null) return undefined;
	37	const o = item as { ownerName?: unknown; repo?: { owner?: unknown } };
	38	const name = typeof o.ownerName === "string" ? o.ownerName : o.repo?.owner;
	39	return typeof name === "string" ? name : undefined;
	40	}
	41
	42	async function countAll(fetcher: Fetch, base: string, path: string, except: string \| undefined): Promise<number> {
	43	let n = 0;
	44	let cursor: string \| null = null;
	45	for (let i = 0; i < MAX_PAGES; i++) {
	46	const url = new URL(path, base);
	47	url.searchParams.set("limit", "100");
	48	if (cursor !== null) url.searchParams.set("cursor", cursor);
	49	const got = await page(fetcher, url.href);
	50	n += got.items.filter((item) => except === undefined \|\| ownerOf(item) !== except).length;
	51	if (got.next === null \|\| got.items.length === 0) return n;
	52	cursor = got.next;
	53	}
	54	throw new Error(`${path} on ${base} pages past ${MAX_PAGES}`);
	55	}
	56
	57	export async function publicCounts(
	58	base: string,
	59	options: { readonly except?: string \| undefined; readonly fetcher?: Fetch \| undefined } = {},
	60	): Promise<PublicCounts> {
	61	const fetcher = options.fetcher ?? fetch;
	62	return {
	63	repositories: await countAll(fetcher, base, "/api/v1/repos", options.except),
	64	apps: await countAll(fetcher, base, "/api/v1/apps", options.except),
	65	};
	66	}
		@@ -0,0 +1,186 @@
	1	/**
	2	* `scripts/restore-drill.ts` (#9046) run end to end: a tiny snapshot taken
	3	* here, served from a stub bucket that speaks the S3 the store asks for,
	4	* restored with only the card's variables, booted, and its public counts
	5	* set beside a stub production's. Matching counts are green; the
	6	* arguments and the comparison are checked as plain data.
	7	*/
	8	import { afterAll, beforeAll, expect, test } from "bun:test";
	9	import { existsSync, mkdtempSync, readdirSync, readFileSync, statSync } from "node:fs";
	10	import { rm } from "node:fs/promises";
	11	import { tmpdir } from "node:os";
	12	import { join, relative } from "node:path";
	13	import { createApp, type App } from "../src/app.ts";
	14	import { commitToBranch, createRepo } from "../src/actions/repos.ts";
	15	import { createUser } from "../src/core/users.ts";
	16	import { dirStore } from "../src/snapshot/store.ts";
	17	import { takeSnapshot } from "../src/snapshot/snapshot.ts";
	18	import { compareLines, parseArgs, PRODUCTION } from "../scripts/restore-drill.ts";
	19	import { publicCounts } from "../src/snapshot/public_counts.ts";
	20
	21	const PASS = "correct horse battery staple";
	22	const BUCKET = "drill";
	23	const ESC: Record<string, string> = { "&": "&", "<": "<", ">": ">" };
	24	const xml = (s: string): string => s.replace(/[&<>]/g, (c) => ESC[c]!);
	25
	26	let root: string;
	27	let app: App;
	28	let bucket: ReturnType<typeof Bun.serve>;
	29	let production: ReturnType<typeof Bun.serve>;
	30
	31	function keysUnder(dir: string, out: string[] = []): string[] {
	32	for (const name of readdirSync(dir)) {
	33	const path = join(dir, name);
	34	if (statSync(path).isDirectory()) keysUnder(path, out);
	35	else out.push(path);
	36	}
	37	return out;
	38	}
	39
	40	/** Path-style S3 over a directory: HEAD and GET of a key, and ListObjectsV2 with a delimiter. */
	41	function stubBucket(dir: string): ReturnType<typeof Bun.serve> {
	42	return Bun.serve({
	43	port: 0,
	44	hostname: "127.0.0.1",
	45	fetch(req) {
	46	const url = new URL(req.url);
	47	const path = decodeURIComponent(url.pathname);
	48	if (path === `/${BUCKET}` \|\| path === `/${BUCKET}/`) {
	49	const prefix = url.searchParams.get("prefix") ?? "";
	50	const delimiter = url.searchParams.get("delimiter");
	51	const keys = keysUnder(dir).map((p) => relative(dir, p)).filter((k) => k.startsWith(prefix)).sort();
	52	const contents: string[] = [];
	53	const prefixes = new Set<string>();
	54	for (const key of keys) {
	55	const cut = delimiter ? key.indexOf(delimiter, prefix.length) : -1;
	56	if (cut >= 0) prefixes.add(key.slice(0, cut + 1));
	57	else contents.push(key);
	58	}
	59	const body =
	60	`<?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Name>${BUCKET}</Name><Prefix>${xml(prefix)}</Prefix>` +
	61	`<KeyCount>${contents.length + prefixes.size}</KeyCount><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated>` +
	62	contents.map((k) => `<Contents><Key>${xml(k)}</Key><Size>${statSync(join(dir, k)).size}</Size></Contents>`).join("") +
	63	[...prefixes].map((p) => `<CommonPrefixes><Prefix>${xml(p)}</Prefix></CommonPrefixes>`).join("") +
	64	`</ListBucketResult>`;
	65	return new Response(body, { headers: { "content-type": "application/xml" } });
	66	}
	67	const key = path.slice(`/${BUCKET}/`.length);
	68	if (!path.startsWith(`/${BUCKET}/`) \|\| key.split("/").includes("..")) return new Response(null, { status: 400 });
	69	const file = join(dir, key);
	70	if (!existsSync(file)) return new Response(null, { status: 404 });
	71	return new Response(req.method === "HEAD" ? null : readFileSync(file), {
	72	headers: { "content-length": String(statSync(file).size) },
	73	});
	74	},
	75	});
	76	}
	77
	78	beforeAll(async () => {
	79	root = mkdtempSync(join(tmpdir(), "forge-restore-drill-"));
	80	app = createApp({ dataDir: join(root, "live"), port: 0 });
	81	const u = await createUser(app.db, {
	82	origin: "seed",
	83	trusted: true,
	84	username: "alice",
	85	email: "[email protected]",
	86	password: "a-good-passphrase",
	87	adminIfFirst: true,
	88	});
	89	if (!u.ok) throw new Error(u.error);
	90	for (const [name, visibility] of [["notes", "public"], ["diary", "private"]] as const) {
	91	const repo = await createRepo(app, u.value, { name, visibility });
	92	if (!repo.ok) throw new Error(String(repo.error));
	93	const c = await commitToBranch(app, u.value, repo.value, {
	94	message: "first",
	95	files: [{ path: "README.md", content: `# ${name}\n` }],
	96	});
	97	if (!c.ok) throw new Error(JSON.stringify(c.error));
	98	}
	99	await takeSnapshot(
	100	{ db: app.db, dataDir: app.config.dataDir, reposRoot: app.reposRoot, version: "t", passphrase: PASS },
	101	dirStore(join(root, "store")),
	102	new Date("2026-11-01T08:00:00Z"),
	103	);
	104	bucket = stubBucket(join(root, "store"));
	105	// Production as anyone sees it: the one public repository, no apps.
	106	production = Bun.serve({
	107	port: 0,
	108	hostname: "127.0.0.1",
	109	fetch(req) {
	110	const path = new URL(req.url).pathname;
	111	if (path === "/api/v1/repos") return Response.json({ items: [{ id: 1 }], next_cursor: null });
	112	if (path === "/api/v1/apps") return Response.json({ items: [], next_cursor: null });
	113	return new Response(null, { status: 404 });
	114	},
	115	});
	116	});
	117
	118	afterAll(async () => {
	119	app.close();
	120	bucket.stop(true);
	121	production.stop(true);
	122	await rm(root, { recursive: true, force: true });
	123	});
	124
	125	test("the drill restores from the bucket alone, boots, and matches production's public counts", async () => {
	126	const data = join(root, "box");
	127	const run = Bun.spawn(
	128	[process.execPath, "scripts/restore-drill.ts", data, "--production", `http://127.0.0.1:${production.port}`],
	129	{
	130	cwd: join(import.meta.dir, ".."),
	131	env: {
	132	PATH: process.env["PATH"] ?? "",
	133	HOME: root,
	134	S3_ENDPOINT: `http://127.0.0.1:${bucket.port}`,
	135	S3_BUCKET: BUCKET,
	136	S3_ACCESS_KEY: "stub",
	137	S3_SECRET_KEY: "stub",
	138	S3_REGION: "us-east-1",
	139	FORGE_SNAPSHOT_PASSPHRASE: PASS,
	140	...(process.env["FORGE_TEST_SOCKET"] ? { FORGE_TEST_SOCKET: process.env["FORGE_TEST_SOCKET"] } : {}),
	141	},
	142	stdout: "pipe",
	143	stderr: "pipe",
	144	},
	145	);
	146	const [out, err, code] = await Promise.all([new Response(run.stdout).text(), new Response(run.stderr).text(), run.exited]);
	147	expect(err + out).toContain("restore-drill: 2026-11-01T08-00-00Z from 127.0.0.1:");
	148	expect(out).toContain("and 2 repositories");
	149	expect(out).toMatch(/booted: \/ answered [23]\d\d/);
	150	expect(out).toContain("repositories: restored 1, production 1");
	151	expect(out).toContain("apps: restored 0, production 0");
	152	expect(out).toContain("restore-drill: green");
	153	expect(code).toBe(0);
	154	expect(existsSync(join(data, "forge.db"))).toBe(true);
	155	}, 120_000);
	156
	157	test("the arguments: a data directory, production by default, never production's own directory", () => {
	158	expect(parseArgs(["/var/lib/forge"])).toEqual({ dataDir: "/var/lib/forge", production: PRODUCTION });
	159	expect(parseArgs(["/data"])).toContain("production's data directory");
	160	expect(parseArgs([])).toContain("usage");
	161	expect(parseArgs(["/x", "--production", "http://forge.example"])).toContain("https");
	162	expect(parseArgs(["/x", "--production", "not a url"])).toContain("URL");
	163	});
	164
	165	test("a count that differs is named", () => {
	166	const same = compareLines({ repositories: 3, apps: 1 }, { repositories: 3, apps: 1 });
	167	expect(same.differ).toEqual([]);
	168	const off = compareLines({ repositories: 3, apps: 1 }, { repositories: 4, apps: 1 });
	169	expect(off.differ).toEqual(["repositories"]);
	170	expect(off.lines[0]).toBe("repositories: restored 3, production 4 DIFFERS");
	171	});
	172
	173	test("public counts follow the cursor and leave out the preview admin's seeded rows", async () => {
	174	const pages: Record<string, unknown> = {
	175	"/api/v1/repos?limit=100": { items: [{ ownerName: "alice" }, { ownerName: "preview-admin" }], next_cursor: "b" },
	176	"/api/v1/repos?limit=100&cursor=b": { items: [{ ownerName: "bob" }], next_cursor: null },
	177	"/api/v1/apps?limit=100": { items: [{ repo: { owner: "alice" } }, { repo: { owner: "preview-admin" } }], next_cursor: null },
	178	};
	179	const fetcher = async (url: string): Promise<Response> => {
	180	const u = new URL(url);
	181	const body = pages[u.pathname + u.search];
	182	return body === undefined ? new Response(null, { status: 404 }) : Response.json(body);
	183	};
	184	expect(await publicCounts("http://forge.test", { fetcher })).toEqual({ repositories: 3, apps: 2 });
	185	expect(await publicCounts("http://forge.test", { fetcher, except: "preview-admin" })).toEqual({ repositories: 2, apps: 1 });
	186	});