7676398 · f3f2570 · 5ab5260
@@ -34,10 +34,31 @@
return the same HTML with status 503 and `Retry-After`.
apps 13 GB) plus one snapshot and headroom, so 64 GB; 4 GB of memory free for the
forge alone, 8 GB with apps running. The 16 GB droplet cannot hold it.
**Take over reads (#9657).** On a second machine of that size, with the backup
store's variables (`S3_*`, or `FORGE_SNAPSHOT_DIR`) and `FORGE_SNAPSHOT_PASSPHRASE`:
    bun scripts/standby-restore.ts latest /srv/standby      # or a snapshot name; the directory must be empty
    FORGE_READ_ONLY=1 FORGE_DATA=/srv/standby PORT=3000 bun run src/main.ts
The script restores the snapshot (`src/snapshot/restore.ts`), runs SQLite's
integrity check and confirms every repository the database lists is on disk;
it exits 1 and names what is wrong otherwise. With `FORGE_READ_ONLY=1` the forge
serves pages, the API's reads and git clone/fetch from that directory. One guard
at the front of the request path (`src/http/read_only.ts`) answers every other
method, a WebSocket upgrade and the push door (`git-receive-pack`) with a plain
503 "read-only standby", so a route added later is covered. Nothing past the
listener starts: no terminal, apps, crew, checks, releases, mirrors, snapshots
or nightly jobs, and not even the boot's reconcile or any timer. The
standby knows no one: a session or token of the main forge is read as signed
out, so it shows public pages only, and sign-in (a POST) is refused. Opening
the database still migrates it forward, on the restored copy only. Point the tunnel's origin at it for the move.
Test: `test/standby-restore.test.ts`.
## Not done
- No standby takeover: nothing replicates data to a second machine, so the
  droplet can only show the page. That needs the snapshot mirror restored onto a
  machine of the size above, and is a separate issue.
- Writes on the standby, and switching back: the standby is a copy as of its
  snapshot, so anything written on the main forge after it is not there, and
  nothing carries standby-time changes back (there are none; it takes no writes).
  Nothing replicates continuously; re-restore into a new directory to catch up.
- The page is not pushed to Cloudflare automatically; `standby.ts` is started by hand.
- The admin is told on the page, event and status; no mail or push.
- Where the kernel gives no boot id, any restart ends maintenance.
@@ -0,0 +1,54 @@
/**
 * Make a second machine a read-only standby (#9657): restore a snapshot the
 * forge's backups made into an empty data directory, and verify it.
 *
 *   bun scripts/standby-restore.ts <snapshot|latest> <data-dir>
 *
 * The store is the one the forge backs up to: the `S3_*` variables, or
 * `FORGE_SNAPSHOT_DIR`; `FORGE_SNAPSHOT_PASSPHRASE` opens sealed values.
 * Then serve reads from it:
 *
 *   FORGE_READ_ONLY=1 FORGE_DATA=<data-dir> PORT=3000 bun run src/main.ts
 *
 * Exits 0 when the database's integrity check passes and every repository
 * is on disk, 1 when it does not, 2 on a bad argument. See
 * docs/features/maintenance.md.
 */
import { resolve } from "node:path";
import { resolveDestination } from "../src/snapshot/destination.ts";
import { restoreStandby } from "../src/snapshot/standby.ts";
async function main(args: readonly string[]): Promise<number> {
  const [snapshot, dir] = args;
  // CLI output below is for the person at the machine.
  if (args.length !== 2 || !snapshot || !dir || dir.startsWith("-")) {
    console.error("usage: bun scripts/standby-restore.ts <snapshot|latest> <data-dir>");
    return 2;
  }
  const dataDir = resolve(dir);
  if (dataDir === "/data") {
    console.error("/data is the production data directory; restore into the standby's own");
    return 2;
  }
  const store = resolveDestination()?.store;
  if (store === undefined) {
    console.error("no store: export the S3_* variables, or FORGE_SNAPSHOT_DIR");
    return 2;
  }
  try {
    const done = await restoreStandby(store, snapshot, dataDir, process.env["FORGE_SNAPSHOT_PASSPHRASE"] || undefined);
    console.log(`restored ${done.name}: ${done.restored.files} files, ${done.repos} repositories, key ${done.restored.key}`);
    if (done.problems.length > 0) {
      for (const p of done.problems) console.error(`standby-restore: ${p}`);
      return 1;
    }
    console.log("verified: database integrity ok, every repository on disk");
    console.log(`serve reads: FORGE_READ_ONLY=1 FORGE_DATA=${dataDir} PORT=3000 bun run src/main.ts`);
    return 0;
  } catch (error) {
    console.error(`standby-restore: ${error instanceof Error ? error.message : String(error)}`);
    return 1;
  }
}
if (import.meta.main) process.exit(await main(process.argv.slice(2)));
@@ -205,6 +205,8 @@
export interface ForgeConfig {
   * The listener is bound with SO_REUSEPORT so the new process can share it.
   */
  selfSwap?: boolean;
  /** A standby: reads are answered, every mutation is a 503 (`FORGE_READ_ONLY=1`; `http/read_only.ts`, #9657). */
  readOnly?: boolean;
  /** How long a swapped-in release has to read healthy (`FORGE_SWAP_BUDGET_S`). @default 60000 */
  swapBudgetMs?: number;
}
@@ -0,0 +1,50 @@
/**
 * The read-only standby's one guard (#9657). A forge started with
 * `FORGE_READ_ONLY=1` answers reads and refuses everything that could
 * change state, at the front of the request path: one check on the method,
 * not a check per route, so a route added next year is covered.
 *
 * Allowed: GET, HEAD and OPTIONS (a WebSocket upgrade is a GET and is
 * refused, since a shell is a write), and the one POST that only reads,
 * git's `git-upload-pack` (clone and fetch). The push advertisement
 * (`info/refs?service=git-receive-pack`) is refused too, so a client is
 * told before it sends a pack.
 */
import { MAINTENANCE_HEADERS } from "../maintenance/page.ts";
/** True when the environment asks for a read-only standby (`FORGE_READ_ONLY=1`). */
export const readOnlyFrom = (env: Record<string, string | undefined>): boolean =>
  (env["FORGE_READ_ONLY"] ?? "").trim() === "1";
const READS = new Set(["GET", "HEAD", "OPTIONS"]);
const UPLOAD_PACK = /^\/[^/]+\/[^/]+\.git\/git-upload-pack$/;
/** Whether a request only reads: by method and path, nothing else. */
export function isRead(request: Request): boolean {
  const url = new URL(request.url);
  if (request.method === "POST" && UPLOAD_PACK.test(url.pathname)) return true;
  if (!READS.has(request.method)) return false;
  if (request.headers.get("upgrade") !== null) return false;
  return url.searchParams.get("service") !== "git-receive-pack";
}
const WORDS = "read-only standby";
/** The plain 503 a mutation gets on a standby; undefined for a read. */
export function readOnlyRefusal(request: Request): Response | undefined {
  if (isRead(request)) return undefined;
  const wantsJson =
    new URL(request.url).pathname.startsWith("/api/") ||
    (request.headers.get("accept") ?? "").includes("application/json");
  const headers = { ...MAINTENANCE_HEADERS };
  if (wantsJson) {
    headers["content-type"] = "application/json; charset=utf-8";
    return new Response(JSON.stringify({ error: "read_only", message: WORDS }), { status: 503, headers });
  }
  headers["content-type"] = "text/plain; charset=utf-8";
  return new Response(`${WORDS}: this forge is serving reads while the main one is away; nothing can be changed here.\n`, {
    status: 503,
    headers,
  });
}
@@ -80,6 +80,7 @@
import { activeEdgeToken, edgeIdentity } from "./edge.ts";
import { shieldState } from "../actions/shield.ts";
import { crawlRefused, familyOf, finishedStrangerDeep, isCrawler, isDeepPath, isShieldedPath, noteCrawler, noteDeepStranger, noteShielded, shieldRefused, takeCrawl, wasCrawler, wasDeepStranger, wasShielded } from "./crawl.ts";
import { liveResponse, readyResponse } from "./health.ts";
import { readOnlyRefusal } from "./read_only.ts";
import { running } from "../kernel/version.ts";
import { runGit } from "../git/env.ts";
import { noteRefusedInternal } from "../kernel/internal_refused.ts";
@@ -571,6 +572,9 @@
const SHELL_IDLE_MS = 30 * 60_000;
const NO_AUTH: Auth = { user: undefined, scope: undefined, via: "none" };
async function resolveAuth(app: App, request: Request): Promise<Auth> {
  // A read-only standby knows no one: a session or token of the main forge
  // must not show its private pages on a copy (#9657).
  if (app.config.readOnly === true) return NO_AUTH;
  // The platform edge, when there is one, has already signed this person
  // in; its word outranks a cookie of ours.
  const login = edgeIdentity(request, activeEdgeToken(app));
@@ -2416,6 +2420,11 @@
export async function serve(app: App): Promise<Servers> {
  ): Promise<Response | undefined> => {
      const started = performance.now();
      noteStart(request, started);
      // A read-only standby refuses every mutation here, before any route.
      if (app.config.readOnly === true) {
        const refused = readOnlyRefusal(request);
        if (refused !== undefined) return securityFloor(refused);
      }
      // An app's hostname is answered before any route: `hello.<domain>` is
      // the app, whatever its path, and the forge's own routes never apply.
      const holding = await holdAppHost(app, request);
@@ -17,6 +17,7 @@
import { drainOnHandover, handover } from "./release/swap.ts";
import { parentUrlProblem } from "./core/parent_releases.ts";
import { releaseEnv } from "./release/env.ts";
import { createApp } from "./app.ts";
import { readOnlyFrom } from "./http/read_only.ts";
import { warmFlow } from "./system/flow_cache.ts";
import { backfillCarried } from "./core/carried.ts";
import { loadTrustedHosts } from "./origins/trusted-hosts.ts";
@@ -169,7 +170,10 @@
const hosted = hostedAt !== "";
// SSH is on by default: the terminal UI is a first-class surface, not an
// add-on. `FORGE_SSH_PORT=0` turns the listener off entirely.
const sshPort = Number(process.env.FORGE_SSH_PORT ?? (hosted ? 0 : 2222));
// A read-only standby (FORGE_READ_ONLY=1, #9657) serves reads from a restored
// directory: no terminal, no apps, no crew, no jobs.
const readOnly = readOnlyFrom(process.env);
const sshPort = readOnly ? 0 : Number(process.env.FORGE_SSH_PORT ?? (hosted ? 0 : 2222));
// FORGE_URL is the address people reach the forge at when a proxy in front
// terminates TLS. Parsed once here so a typo fails the boot, not a request.
@@ -235,6 +239,7 @@
const previewKey = previewIdentityKey(process.env);
const app = createApp({
  dataDir,
  ...(readOnly ? { readOnly } : {}),
  // A new forge starts with signup = invite (FORGE_SIGNUP below still wins).
  seedSignup: true,
  ...(parentUrl === "" ? {} : { parentUrl }),
@@ -323,6 +328,19 @@
const app = createApp({
timeStatements(app.db, createLog("sql"));
loadFlags(app.db);
loadTrustedHosts(app.db);
if (readOnly) {
  const servers = await serve(app);
  // Only the listener runs: no reconcile, perf watcher, timers, apps, crew,
  // checks, releases, mirrors, snapshots or nightly jobs. Reads are answered until a signal.
  app.log.info("forge is up, read-only standby", { url: servers.url, data: app.config.dataDir });
  const stop = (): void => {
    servers.stop();
    app.close();
    process.exit(0);
  };
  for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"] as const) process.once(signal, stop);
  await new Promise<never>(() => {});
}
// FORGE_SIGNUP (checked in configProblem) sets who may sign up; kept like any change.
const bootSignup = signupFromEnv(process.env.FORGE_SIGNUP);
if (bootSignup.ok && bootSignup.value !== null)
@@ -0,0 +1,65 @@
/**
 * A snapshot restored for a read-only standby (#9657): the restore of
 * `restore.ts` into an empty directory, then checked — the database passes
 * SQLite's integrity check and every repository the database lists is a
 * directory on disk. Plain data out; the script prints it.
 */
import { existsSync } from "node:fs";
import { join } from "node:path";
import { repoDir } from "../git/repos.ts";
import { restoreSnapshot, type Restored } from "./restore.ts";
import { listSnapshots } from "./snapshot.ts";
import type { Store } from "./store.ts";
export interface Standby {
  readonly name: string;
  readonly restored: Restored;
  /** Repositories the database lists. */
  readonly repos: number;
  /** What failed the verification, one line each; empty is verified. */
  readonly problems: readonly string[];
}
/** The database's integrity and its repositories, checked in a restored directory. */
export async function verifyRestored(dataDir: string): Promise<{ repos: number; problems: string[] }> {
  const { Database } = await import("bun:sqlite");
  const file = join(dataDir, "forge.db");
  if (!existsSync(file)) return { repos: 0, problems: ["forge.db is missing"] };
  const db = new Database(file, { readonly: true });
  try {
    const problems: string[] = [];
    const rows = db.query<{ integrity_check: string }, []>("PRAGMA integrity_check").all();
    if (rows.length !== 1 || rows[0]?.integrity_check !== "ok")
      problems.push(`integrity check: ${rows.map((r) => r.integrity_check).slice(0, 3).join("; ")}`);
    const repos = db
      .query<{ owner: string; name: string }, []>(
        "SELECT u.username AS owner, r.name AS name FROM repo r JOIN user u ON u.id = r.owner_id",
      )
      .all();
    const missing = repos.filter((r) => !existsSync(repoDir(join(dataDir, "repos"), r.owner, r.name)));
    for (const r of missing.slice(0, 5)) problems.push(`repository ${r.owner}/${r.name} is not on disk`);
    if (missing.length > 5) problems.push(`and ${missing.length - 5} more repositories not on disk`);
    return { repos: repos.length, problems };
  } finally {
    db.close();
  }
}
/** `name` is a snapshot name or `latest`; the directory must be empty. */
export async function restoreStandby(
  store: Store,
  name: string,
  dataDir: string,
  passphrase?: string,
): Promise<Standby> {
  let chosen = name;
  if (name === "latest") {
    const newest = (await listSnapshots(store))[0];
    if (newest === undefined) throw new Error(`no snapshot in ${store.where}`);
    chosen = newest.name;
  }
  const restored = await restoreSnapshot(store, chosen, dataDir, { passphrase });
  const checked = await verifyRestored(dataDir);
  return { name: chosen, restored, repos: checked.repos, problems: checked.problems };
}
@@ -0,0 +1,136 @@
/**
 * The read-only standby (#9657): a small snapshot restored by
 * `scripts/standby-restore.ts`, the restored directory booted with
 * FORGE_READ_ONLY=1, `/` answering and git reading, and every mutation a
 * plain 503. The guard is also checked as plain data over the methods.
 */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { existsSync, mkdtempSync } from "node:fs";
import { rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { commitToBranch, createRepo } from "../src/actions/repos.ts";
import { createUser } from "../src/core/users.ts";
import { createSession } from "../src/core/sessions.ts";
import { isRead, readOnlyFrom, readOnlyRefusal } from "../src/http/read_only.ts";
import { lostPort, onFreshPort } from "../src/kernel/port.ts";
import { dirStore } from "../src/snapshot/store.ts";
import { takeSnapshot } from "../src/snapshot/snapshot.ts";
const TREE = join(import.meta.dir, "..");
let root: string;
let app: App;
let session = "";
beforeAll(async () => {
  root = mkdtempSync(join(tmpdir(), "forge-standby-"));
  app = createApp({ dataDir: join(root, "live"), port: 0 });
  const u = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
  if (!u.ok) throw new Error(u.error);
  const repo = await createRepo(app, u.value, { name: "notes", visibility: "public" });
  if (!repo.ok) throw new Error(String(repo.error));
  const c = await commitToBranch(app, u.value, repo.value, { message: "first", files: [{ path: "README.md", content: "# notes\n" }] });
  if (!c.ok) throw new Error(JSON.stringify(c.error));
  const secret = await createRepo(app, u.value, { name: "diary", visibility: "private" });
  if (!secret.ok) throw new Error(String(secret.error));
  const d = await commitToBranch(app, u.value, secret.value, { message: "first", files: [{ path: "README.md", content: "# diary\n" }] });
  if (!d.ok) throw new Error(JSON.stringify(d.error));
  session = createSession(app.db, u.value.id).token;
  await takeSnapshot(
    { db: app.db, dataDir: app.config.dataDir, reposRoot: app.reposRoot, version: "t" },
    dirStore(join(root, "store")),
    new Date("2026-11-01T08:00:00Z"),
  );
});
afterAll(async () => {
  app.close();
  await rm(root, { recursive: true, force: true });
});
const run = (args: string[], env: Record<string, string>) =>
  Bun.spawn([process.execPath, ...args], {
    cwd: TREE,
    env: { PATH: process.env["PATH"] ?? "", HOME: root, ...(process.env["FORGE_TEST_SOCKET"] ? { FORGE_TEST_SOCKET: process.env["FORGE_TEST_SOCKET"] } : {}), ...env },
    stdout: "pipe",
    stderr: "pipe",
  });
test("the guard lets reads through and refuses every other method and the push door", () => {
  const req = (method: string, path: string, headers: Record<string, string> = {}) =>
    new Request(`http://forge.test${path}`, { method, headers });
  expect(isRead(req("GET", "/"))).toBe(true);
  expect(isRead(req("HEAD", "/alice/notes"))).toBe(true);
  expect(isRead(req("POST", "/alice/notes.git/git-upload-pack"))).toBe(true);
  expect(isRead(req("POST", "/alice/notes.git/git-receive-pack"))).toBe(false);
  expect(isRead(req("GET", "/alice/notes.git/info/refs?service=git-receive-pack"))).toBe(false);
  expect(isRead(req("GET", "/alice/notes.git/info/refs?service=git-upload-pack"))).toBe(true);
  expect(isRead(req("GET", "/x/shell", { upgrade: "websocket" }))).toBe(false);
  for (const m of ["POST", "PUT", "PATCH", "DELETE"]) expect(readOnlyRefusal(req(m, "/anything"))?.status).toBe(503);
  expect(readOnlyFrom({ FORGE_READ_ONLY: "1" })).toBe(true);
  expect(readOnlyFrom({})).toBe(false);
});
test("a restored snapshot boots read-only: / answers, git reads, a POST is a 503", async () => {
  const data = join(root, "standby");
  const restore = run(["scripts/standby-restore.ts", "latest", data], { FORGE_SNAPSHOT_DIR: join(root, "store") });
  const [out, code] = [await new Response(restore.stdout).text(), await restore.exited];
  expect({ code, out }).toMatchObject({ code: 0 });
  expect(out).toContain("verified");
  // Not empty now: a second restore is refused.
  const again = run(["scripts/standby-restore.ts", "latest", data], { FORGE_SNAPSHOT_DIR: join(root, "store") });
  const [refusal, againCode] = [await new Response(again.stderr).text(), await again.exited];
  expect(againCode).toBe(1);
  expect(refusal).toContain(`${data} is not empty`);
  await onFreshPort(async (port) => {
      
        sentinelcrew
        3d ago
        outdated
      
      Only the ref advertisement is checked. Add a real clone through git-upload-pack.
    
    const child = run(["src/main.ts"], { FORGE_READ_ONLY: "1", FORGE_DATA: data, PORT: String(port), FORGE_SSH_PORT: "2222" });
    let stderr = "";
    let logged = "";
    void (async () => { for await (const c of child.stderr) stderr = (stderr + new TextDecoder().decode(c)).slice(-4096); })();
    void (async () => { for await (const c of child.stdout) logged += new TextDecoder().decode(c); })();
    const base = `http://127.0.0.1:${port}`;
    try {
      const until = Date.now() + 45_000;
      let up = false;
      while (Date.now() < until && child.exitCode === null && !up) {
        up = await fetch(`${base}/`, { signal: AbortSignal.timeout(2000) }).then((r) => r.status === 200, () => false);
        if (!up) await Bun.sleep(200);
      }
      expect({ up, stderr: up ? "" : stderr }).toEqual({ up: true, stderr: "" });
      const home = await fetch(`${base}/`);
      expect(home.status).toBe(200);
      expect((await fetch(`${base}/alice/notes`)).status).toBe(200);
      const refs = await fetch(`${base}/alice/notes.git/info/refs?service=git-upload-pack`);
      expect(refs.status).toBe(200);
      // A real clone, through git-upload-pack.
      const clone = join(root, "clone");
      const cloned = Bun.spawn(["git", "clone", "--quiet", `${base}/alice/notes.git`, clone], { env: { PATH: process.env["PATH"] ?? "", HOME: root }, stdout: "pipe", stderr: "pipe" });
      const cloneErr = await new Response(cloned.stderr).text();
      expect({ code: await cloned.exited, cloneErr }).toEqual({ code: 0, cloneErr: "" });
      expect(await Bun.file(join(clone, "README.md")).text()).toBe("# notes\n");
      // A session of the main forge is not an identity here: the private repository stays private.
      const cookie = { cookie: `forge_session=${session}` };
      expect((await fetch(`${base}/alice/diary`, { headers: cookie, redirect: "manual" })).status).not.toBe(200);
      expect((await fetch(`${base}/alice/diary`, { redirect: "manual" })).status).not.toBe(200);
      // Only the listener started: no boot record, no timers' log lines.
      expect(logged).toContain("read-only standby");
      expect(logged).not.toMatch(/processes|apps are up|crew is|snapshot|nightly|reconcil/i);
      expect(existsSync(join(data, "boots.json"))).toBe(false);
      for (const [method, path] of [["POST", "/signin"], ["POST", "/new"], ["POST", "/alice/notes.git/git-receive-pack"], ["DELETE", "/alice/notes"]] as const) {
        const res = await fetch(`${base}${path}`, { method, headers: { origin: base }, body: method === "DELETE" ? null : "x=1" });
        expect({ path, status: res.status }).toEqual({ path, status: 503 });
        expect(await res.text()).toContain("read-only standby");
        expect(res.headers.get("x-content-type-options")).toBe("nosniff");
      }
      expect((await fetch(`${base}/alice/notes.git/info/refs?service=git-receive-pack`)).status).toBe(503);
      expect((await fetch(`${base}/api/v1/repos`, { method: "POST", headers: { "content-type": "application/json" }, body: "{}" })).status).toBe(503);
    } finally {
      child.kill();
      await child.exited;
    }
    return { result: undefined, portLost: lostPort(stderr) };
  });
}, 90_000);
		@@ -34,10 +34,31 @@ return the same HTML with status 503 and `Retry-After`.
34	34	apps 13 GB) plus one snapshot and headroom, so 64 GB; 4 GB of memory free for the
35	35	forge alone, 8 GB with apps running. The 16 GB droplet cannot hold it.
36	36
	37	Take over reads (#9657). On a second machine of that size, with the backup
	38	store's variables (`S3_*`, or `FORGE_SNAPSHOT_DIR`) and `FORGE_SNAPSHOT_PASSPHRASE`:
	39
	40	bun scripts/standby-restore.ts latest /srv/standby # or a snapshot name; the directory must be empty
	41	FORGE_READ_ONLY=1 FORGE_DATA=/srv/standby PORT=3000 bun run src/main.ts
	42
	43	The script restores the snapshot (`src/snapshot/restore.ts`), runs SQLite's
	44	integrity check and confirms every repository the database lists is on disk;
	45	it exits 1 and names what is wrong otherwise. With `FORGE_READ_ONLY=1` the forge
	46	serves pages, the API's reads and git clone/fetch from that directory. One guard
	47	at the front of the request path (`src/http/read_only.ts`) answers every other
	48	method, a WebSocket upgrade and the push door (`git-receive-pack`) with a plain
	49	503 "read-only standby", so a route added later is covered. Nothing past the
	50	listener starts: no terminal, apps, crew, checks, releases, mirrors, snapshots
	51	or nightly jobs, and not even the boot's reconcile or any timer. The
	52	standby knows no one: a session or token of the main forge is read as signed
	53	out, so it shows public pages only, and sign-in (a POST) is refused. Opening
	54	the database still migrates it forward, on the restored copy only. Point the tunnel's origin at it for the move.
	55	Test: `test/standby-restore.test.ts`.
	56
37	57	## Not done
38		- No standby takeover: nothing replicates data to a second machine, so the
39		droplet can only show the page. That needs the snapshot mirror restored onto a
40		machine of the size above, and is a separate issue.
	58	- Writes on the standby, and switching back: the standby is a copy as of its
	59	snapshot, so anything written on the main forge after it is not there, and
	60	nothing carries standby-time changes back (there are none; it takes no writes).
	61	Nothing replicates continuously; re-restore into a new directory to catch up.
41	62	- The page is not pushed to Cloudflare automatically; `standby.ts` is started by hand.
42	63	- The admin is told on the page, event and status; no mail or push.
43	64	- Where the kernel gives no boot id, any restart ends maintenance.
		@@ -0,0 +1,54 @@
	1	/**
	2	* Make a second machine a read-only standby (#9657): restore a snapshot the
	3	* forge's backups made into an empty data directory, and verify it.
	4	*
	5	* bun scripts/standby-restore.ts <snapshot\|latest> <data-dir>
	6	*
	7	* The store is the one the forge backs up to: the `S3_*` variables, or
	8	* `FORGE_SNAPSHOT_DIR`; `FORGE_SNAPSHOT_PASSPHRASE` opens sealed values.
	9	* Then serve reads from it:
	10	*
	11	* FORGE_READ_ONLY=1 FORGE_DATA=<data-dir> PORT=3000 bun run src/main.ts
	12	*
	13	* Exits 0 when the database's integrity check passes and every repository
	14	* is on disk, 1 when it does not, 2 on a bad argument. See
	15	* docs/features/maintenance.md.
	16	*/
	17	import { resolve } from "node:path";
	18	import { resolveDestination } from "../src/snapshot/destination.ts";
	19	import { restoreStandby } from "../src/snapshot/standby.ts";
	20
	21	async function main(args: readonly string[]): Promise<number> {
	22	const [snapshot, dir] = args;
	23	// CLI output below is for the person at the machine.
	24	if (args.length !== 2 \|\| !snapshot \|\| !dir \|\| dir.startsWith("-")) {
	25	console.error("usage: bun scripts/standby-restore.ts <snapshot\|latest> <data-dir>");
	26	return 2;
	27	}
	28	const dataDir = resolve(dir);
	29	if (dataDir === "/data") {
	30	console.error("/data is the production data directory; restore into the standby's own");
	31	return 2;
	32	}
	33	const store = resolveDestination()?.store;
	34	if (store === undefined) {
	35	console.error("no store: export the S3_* variables, or FORGE_SNAPSHOT_DIR");
	36	return 2;
	37	}
	38	try {
	39	const done = await restoreStandby(store, snapshot, dataDir, process.env["FORGE_SNAPSHOT_PASSPHRASE"] \|\| undefined);
	40	console.log(`restored ${done.name}: ${done.restored.files} files, ${done.repos} repositories, key ${done.restored.key}`);
	41	if (done.problems.length > 0) {
	42	for (const p of done.problems) console.error(`standby-restore: ${p}`);
	43	return 1;
	44	}
	45	console.log("verified: database integrity ok, every repository on disk");
	46	console.log(`serve reads: FORGE_READ_ONLY=1 FORGE_DATA=${dataDir} PORT=3000 bun run src/main.ts`);
	47	return 0;
	48	} catch (error) {
	49	console.error(`standby-restore: ${error instanceof Error ? error.message : String(error)}`);
	50	return 1;
	51	}
	52	}
	53
	54	if (import.meta.main) process.exit(await main(process.argv.slice(2)));
		@@ -205,6 +205,8 @@ export interface ForgeConfig {
205	205	* The listener is bound with SO_REUSEPORT so the new process can share it.
206	206	*/
207	207	selfSwap?: boolean;
	208	/** A standby: reads are answered, every mutation is a 503 (`FORGE_READ_ONLY=1`; `http/read_only.ts`, #9657). */
	209	readOnly?: boolean;
208	210	/** How long a swapped-in release has to read healthy (`FORGE_SWAP_BUDGET_S`). @default 60000 */
209	211	swapBudgetMs?: number;
210	212	}
		@@ -0,0 +1,50 @@
	1	/**
	2	* The read-only standby's one guard (#9657). A forge started with
	3	* `FORGE_READ_ONLY=1` answers reads and refuses everything that could
	4	* change state, at the front of the request path: one check on the method,
	5	* not a check per route, so a route added next year is covered.
	6	*
	7	* Allowed: GET, HEAD and OPTIONS (a WebSocket upgrade is a GET and is
	8	* refused, since a shell is a write), and the one POST that only reads,
	9	* git's `git-upload-pack` (clone and fetch). The push advertisement
	10	* (`info/refs?service=git-receive-pack`) is refused too, so a client is
	11	* told before it sends a pack.
	12	*/
	13
	14	import { MAINTENANCE_HEADERS } from "../maintenance/page.ts";
	15
	16	/** True when the environment asks for a read-only standby (`FORGE_READ_ONLY=1`). */
	17	export const readOnlyFrom = (env: Record<string, string \| undefined>): boolean =>
	18	(env["FORGE_READ_ONLY"] ?? "").trim() === "1";
	19
	20	const READS = new Set(["GET", "HEAD", "OPTIONS"]);
	21	const UPLOAD_PACK = /^\/[^/]+\/[^/]+\.git\/git-upload-pack$/;
	22
	23	/** Whether a request only reads: by method and path, nothing else. */
	24	export function isRead(request: Request): boolean {
	25	const url = new URL(request.url);
	26	if (request.method === "POST" && UPLOAD_PACK.test(url.pathname)) return true;
	27	if (!READS.has(request.method)) return false;
	28	if (request.headers.get("upgrade") !== null) return false;
	29	return url.searchParams.get("service") !== "git-receive-pack";
	30	}
	31
	32	const WORDS = "read-only standby";
	33
	34	/** The plain 503 a mutation gets on a standby; undefined for a read. */
	35	export function readOnlyRefusal(request: Request): Response \| undefined {
	36	if (isRead(request)) return undefined;
	37	const wantsJson =
	38	new URL(request.url).pathname.startsWith("/api/") \|\|
	39	(request.headers.get("accept") ?? "").includes("application/json");
	40	const headers = { ...MAINTENANCE_HEADERS };
	41	if (wantsJson) {
	42	headers["content-type"] = "application/json; charset=utf-8";
	43	return new Response(JSON.stringify({ error: "read_only", message: WORDS }), { status: 503, headers });
	44	}
	45	headers["content-type"] = "text/plain; charset=utf-8";
	46	return new Response(`${WORDS}: this forge is serving reads while the main one is away; nothing can be changed here.\n`, {
	47	status: 503,
	48	headers,
	49	});
	50	}
		@@ -80,6 +80,7 @@ import { activeEdgeToken, edgeIdentity } from "./edge.ts";
80	80	import { shieldState } from "../actions/shield.ts";
81	81	import { crawlRefused, familyOf, finishedStrangerDeep, isCrawler, isDeepPath, isShieldedPath, noteCrawler, noteDeepStranger, noteShielded, shieldRefused, takeCrawl, wasCrawler, wasDeepStranger, wasShielded } from "./crawl.ts";
82	82	import { liveResponse, readyResponse } from "./health.ts";
	83	import { readOnlyRefusal } from "./read_only.ts";
83	84	import { running } from "../kernel/version.ts";
84	85	import { runGit } from "../git/env.ts";
85	86	import { noteRefusedInternal } from "../kernel/internal_refused.ts";
		@@ -571,6 +572,9 @@ const SHELL_IDLE_MS = 30 * 60_000;
571	572	const NO_AUTH: Auth = { user: undefined, scope: undefined, via: "none" };
572	573
573	574	async function resolveAuth(app: App, request: Request): Promise<Auth> {
	575	// A read-only standby knows no one: a session or token of the main forge
	576	// must not show its private pages on a copy (#9657).
	577	if (app.config.readOnly === true) return NO_AUTH;
574	578	// The platform edge, when there is one, has already signed this person
575	579	// in; its word outranks a cookie of ours.
576	580	const login = edgeIdentity(request, activeEdgeToken(app));
		@@ -2416,6 +2420,11 @@ export async function serve(app: App): Promise<Servers> {
2416	2420	): Promise<Response \| undefined> => {
2417	2421	const started = performance.now();
2418	2422	noteStart(request, started);
	2423	// A read-only standby refuses every mutation here, before any route.
	2424	if (app.config.readOnly === true) {
	2425	const refused = readOnlyRefusal(request);
	2426	if (refused !== undefined) return securityFloor(refused);
	2427	}
2419	2428	// An app's hostname is answered before any route: `hello.<domain>` is
2420	2429	// the app, whatever its path, and the forge's own routes never apply.
2421	2430	const holding = await holdAppHost(app, request);
		@@ -17,6 +17,7 @@ import { drainOnHandover, handover } from "./release/swap.ts";
17	17	import { parentUrlProblem } from "./core/parent_releases.ts";
18	18	import { releaseEnv } from "./release/env.ts";
19	19	import { createApp } from "./app.ts";
	20	import { readOnlyFrom } from "./http/read_only.ts";
20	21	import { warmFlow } from "./system/flow_cache.ts";
21	22	import { backfillCarried } from "./core/carried.ts";
22	23	import { loadTrustedHosts } from "./origins/trusted-hosts.ts";
		@@ -169,7 +170,10 @@ const hosted = hostedAt !== "";
169	170
170	171	// SSH is on by default: the terminal UI is a first-class surface, not an
171	172	// add-on. `FORGE_SSH_PORT=0` turns the listener off entirely.
172		const sshPort = Number(process.env.FORGE_SSH_PORT ?? (hosted ? 0 : 2222));
	173	// A read-only standby (FORGE_READ_ONLY=1, #9657) serves reads from a restored
	174	// directory: no terminal, no apps, no crew, no jobs.
	175	const readOnly = readOnlyFrom(process.env);
	176	const sshPort = readOnly ? 0 : Number(process.env.FORGE_SSH_PORT ?? (hosted ? 0 : 2222));
173	177
174	178	// FORGE_URL is the address people reach the forge at when a proxy in front
175	179	// terminates TLS. Parsed once here so a typo fails the boot, not a request.
		@@ -235,6 +239,7 @@ const previewKey = previewIdentityKey(process.env);
235	239
236	240	const app = createApp({
237	241	dataDir,
	242	...(readOnly ? { readOnly } : {}),
238	243	// A new forge starts with signup = invite (FORGE_SIGNUP below still wins).
239	244	seedSignup: true,
240	245	...(parentUrl === "" ? {} : { parentUrl }),
		@@ -323,6 +328,19 @@ const app = createApp({
323	328	timeStatements(app.db, createLog("sql"));
324	329	loadFlags(app.db);
325	330	loadTrustedHosts(app.db);
	331	if (readOnly) {
	332	const servers = await serve(app);
	333	// Only the listener runs: no reconcile, perf watcher, timers, apps, crew,
	334	// checks, releases, mirrors, snapshots or nightly jobs. Reads are answered until a signal.
	335	app.log.info("forge is up, read-only standby", { url: servers.url, data: app.config.dataDir });
	336	const stop = (): void => {
	337	servers.stop();
	338	app.close();
	339	process.exit(0);
	340	};
	341	for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"] as const) process.once(signal, stop);
	342	await new Promise<never>(() => {});
	343	}
326	344	// FORGE_SIGNUP (checked in configProblem) sets who may sign up; kept like any change.
327	345	const bootSignup = signupFromEnv(process.env.FORGE_SIGNUP);
328	346	if (bootSignup.ok && bootSignup.value !== null)
		@@ -0,0 +1,65 @@
	1	/**
	2	* A snapshot restored for a read-only standby (#9657): the restore of
	3	* `restore.ts` into an empty directory, then checked — the database passes
	4	* SQLite's integrity check and every repository the database lists is a
	5	* directory on disk. Plain data out; the script prints it.
	6	*/
	7
	8	import { existsSync } from "node:fs";
	9	import { join } from "node:path";
	10	import { repoDir } from "../git/repos.ts";
	11	import { restoreSnapshot, type Restored } from "./restore.ts";
	12	import { listSnapshots } from "./snapshot.ts";
	13	import type { Store } from "./store.ts";
	14
	15	export interface Standby {
	16	readonly name: string;
	17	readonly restored: Restored;
	18	/** Repositories the database lists. */
	19	readonly repos: number;
	20	/** What failed the verification, one line each; empty is verified. */
	21	readonly problems: readonly string[];
	22	}
	23
	24	/** The database's integrity and its repositories, checked in a restored directory. */
	25	export async function verifyRestored(dataDir: string): Promise<{ repos: number; problems: string[] }> {
	26	const { Database } = await import("bun:sqlite");
	27	const file = join(dataDir, "forge.db");
	28	if (!existsSync(file)) return { repos: 0, problems: ["forge.db is missing"] };
	29	const db = new Database(file, { readonly: true });
	30	try {
	31	const problems: string[] = [];
	32	const rows = db.query<{ integrity_check: string }, []>("PRAGMA integrity_check").all();
	33	if (rows.length !== 1 \|\| rows[0]?.integrity_check !== "ok")
	34	problems.push(`integrity check: ${rows.map((r) => r.integrity_check).slice(0, 3).join("; ")}`);
	35	const repos = db
	36	.query<{ owner: string; name: string }, []>(
	37	"SELECT u.username AS owner, r.name AS name FROM repo r JOIN user u ON u.id = r.owner_id",
	38	)
	39	.all();
	40	const missing = repos.filter((r) => !existsSync(repoDir(join(dataDir, "repos"), r.owner, r.name)));
	41	for (const r of missing.slice(0, 5)) problems.push(`repository ${r.owner}/${r.name} is not on disk`);
	42	if (missing.length > 5) problems.push(`and ${missing.length - 5} more repositories not on disk`);
	43	return { repos: repos.length, problems };
	44	} finally {
	45	db.close();
	46	}
	47	}
	48
	49	/** `name` is a snapshot name or `latest`; the directory must be empty. */
	50	export async function restoreStandby(
	51	store: Store,
	52	name: string,
	53	dataDir: string,
	54	passphrase?: string,
	55	): Promise<Standby> {
	56	let chosen = name;
	57	if (name === "latest") {
	58	const newest = (await listSnapshots(store))[0];
	59	if (newest === undefined) throw new Error(`no snapshot in ${store.where}`);
	60	chosen = newest.name;
	61	}
	62	const restored = await restoreSnapshot(store, chosen, dataDir, { passphrase });
	63	const checked = await verifyRestored(dataDir);
	64	return { name: chosen, restored, repos: checked.repos, problems: checked.problems };
	65	}
		@@ -0,0 +1,136 @@
	1	/**
	2	* The read-only standby (#9657): a small snapshot restored by
	3	* `scripts/standby-restore.ts`, the restored directory booted with
	4	* FORGE_READ_ONLY=1, `/` answering and git reading, and every mutation a
	5	* plain 503. The guard is also checked as plain data over the methods.
	6	*/
	7	import { afterAll, beforeAll, expect, test } from "bun:test";
	8	import { existsSync, mkdtempSync } from "node:fs";
	9	import { rm } from "node:fs/promises";
	10	import { tmpdir } from "node:os";
	11	import { join } from "node:path";
	12	import { createApp, type App } from "../src/app.ts";
	13	import { commitToBranch, createRepo } from "../src/actions/repos.ts";
	14	import { createUser } from "../src/core/users.ts";
	15	import { createSession } from "../src/core/sessions.ts";
	16	import { isRead, readOnlyFrom, readOnlyRefusal } from "../src/http/read_only.ts";
	17	import { lostPort, onFreshPort } from "../src/kernel/port.ts";
	18	import { dirStore } from "../src/snapshot/store.ts";
	19	import { takeSnapshot } from "../src/snapshot/snapshot.ts";
	20
	21	const TREE = join(import.meta.dir, "..");
	22	let root: string;
	23	let app: App;
	24	let session = "";
	25
	26	beforeAll(async () => {
	27	root = mkdtempSync(join(tmpdir(), "forge-standby-"));
	28	app = createApp({ dataDir: join(root, "live"), port: 0 });
	29	const u = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
	30	if (!u.ok) throw new Error(u.error);
	31	const repo = await createRepo(app, u.value, { name: "notes", visibility: "public" });
	32	if (!repo.ok) throw new Error(String(repo.error));
	33	const c = await commitToBranch(app, u.value, repo.value, { message: "first", files: [{ path: "README.md", content: "# notes\n" }] });
	34	if (!c.ok) throw new Error(JSON.stringify(c.error));
	35	const secret = await createRepo(app, u.value, { name: "diary", visibility: "private" });
	36	if (!secret.ok) throw new Error(String(secret.error));
	37	const d = await commitToBranch(app, u.value, secret.value, { message: "first", files: [{ path: "README.md", content: "# diary\n" }] });
	38	if (!d.ok) throw new Error(JSON.stringify(d.error));
	39	session = createSession(app.db, u.value.id).token;
	40	await takeSnapshot(
	41	{ db: app.db, dataDir: app.config.dataDir, reposRoot: app.reposRoot, version: "t" },
	42	dirStore(join(root, "store")),
	43	new Date("2026-11-01T08:00:00Z"),
	44	);
	45	});
	46
	47	afterAll(async () => {
	48	app.close();
	49	await rm(root, { recursive: true, force: true });
	50	});
	51
	52	const run = (args: string[], env: Record<string, string>) =>
	53	Bun.spawn([process.execPath, ...args], {
	54	cwd: TREE,
	55	env: { PATH: process.env["PATH"] ?? "", HOME: root, ...(process.env["FORGE_TEST_SOCKET"] ? { FORGE_TEST_SOCKET: process.env["FORGE_TEST_SOCKET"] } : {}), ...env },
	56	stdout: "pipe",
	57	stderr: "pipe",
	58	});
	59
	60	test("the guard lets reads through and refuses every other method and the push door", () => {
	61	const req = (method: string, path: string, headers: Record<string, string> = {}) =>
	62	new Request(`http://forge.test${path}`, { method, headers });
	63	expect(isRead(req("GET", "/"))).toBe(true);
	64	expect(isRead(req("HEAD", "/alice/notes"))).toBe(true);
	65	expect(isRead(req("POST", "/alice/notes.git/git-upload-pack"))).toBe(true);
	66	expect(isRead(req("POST", "/alice/notes.git/git-receive-pack"))).toBe(false);
	67	expect(isRead(req("GET", "/alice/notes.git/info/refs?service=git-receive-pack"))).toBe(false);
	68	expect(isRead(req("GET", "/alice/notes.git/info/refs?service=git-upload-pack"))).toBe(true);
	69	expect(isRead(req("GET", "/x/shell", { upgrade: "websocket" }))).toBe(false);
	70	for (const m of ["POST", "PUT", "PATCH", "DELETE"]) expect(readOnlyRefusal(req(m, "/anything"))?.status).toBe(503);
	71	expect(readOnlyFrom({ FORGE_READ_ONLY: "1" })).toBe(true);
	72	expect(readOnlyFrom({})).toBe(false);
	73	});
	74
	75	test("a restored snapshot boots read-only: / answers, git reads, a POST is a 503", async () => {
	76	const data = join(root, "standby");
	77	const restore = run(["scripts/standby-restore.ts", "latest", data], { FORGE_SNAPSHOT_DIR: join(root, "store") });
	78	const [out, code] = [await new Response(restore.stdout).text(), await restore.exited];
	79	expect({ code, out }).toMatchObject({ code: 0 });
	80	expect(out).toContain("verified");
	81
	82	// Not empty now: a second restore is refused.
	83	const again = run(["scripts/standby-restore.ts", "latest", data], { FORGE_SNAPSHOT_DIR: join(root, "store") });
	84	const [refusal, againCode] = [await new Response(again.stderr).text(), await again.exited];
	85	expect(againCode).toBe(1);
	86	expect(refusal).toContain(`${data} is not empty`);
	87
	88	await onFreshPort(async (port) => {
		sentinelcrew 3d ago outdated Only the ref advertisement is checked. Add a real clone through git-upload-pack.
	89	const child = run(["src/main.ts"], { FORGE_READ_ONLY: "1", FORGE_DATA: data, PORT: String(port), FORGE_SSH_PORT: "2222" });
	90	let stderr = "";
	91	let logged = "";
	92	void (async () => { for await (const c of child.stderr) stderr = (stderr + new TextDecoder().decode(c)).slice(-4096); })();
	93	void (async () => { for await (const c of child.stdout) logged += new TextDecoder().decode(c); })();
	94	const base = `http://127.0.0.1:${port}`;
	95	try {
	96	const until = Date.now() + 45_000;
	97	let up = false;
	98	while (Date.now() < until && child.exitCode === null && !up) {
	99	up = await fetch(`${base}/`, { signal: AbortSignal.timeout(2000) }).then((r) => r.status === 200, () => false);
	100	if (!up) await Bun.sleep(200);
	101	}
	102	expect({ up, stderr: up ? "" : stderr }).toEqual({ up: true, stderr: "" });
	103	const home = await fetch(`${base}/`);
	104	expect(home.status).toBe(200);
	105	expect((await fetch(`${base}/alice/notes`)).status).toBe(200);
	106	const refs = await fetch(`${base}/alice/notes.git/info/refs?service=git-upload-pack`);
	107	expect(refs.status).toBe(200);
	108	// A real clone, through git-upload-pack.
	109	const clone = join(root, "clone");
	110	const cloned = Bun.spawn(["git", "clone", "--quiet", `${base}/alice/notes.git`, clone], { env: { PATH: process.env["PATH"] ?? "", HOME: root }, stdout: "pipe", stderr: "pipe" });
	111	const cloneErr = await new Response(cloned.stderr).text();
	112	expect({ code: await cloned.exited, cloneErr }).toEqual({ code: 0, cloneErr: "" });
	113	expect(await Bun.file(join(clone, "README.md")).text()).toBe("# notes\n");
	114	// A session of the main forge is not an identity here: the private repository stays private.
	115	const cookie = { cookie: `forge_session=${session}` };
	116	expect((await fetch(`${base}/alice/diary`, { headers: cookie, redirect: "manual" })).status).not.toBe(200);
	117	expect((await fetch(`${base}/alice/diary`, { redirect: "manual" })).status).not.toBe(200);
	118	// Only the listener started: no boot record, no timers' log lines.
	119	expect(logged).toContain("read-only standby");
	120	expect(logged).not.toMatch(/processes\|apps are up\|crew is\|snapshot\|nightly\|reconcil/i);
	121	expect(existsSync(join(data, "boots.json"))).toBe(false);
	122	for (const [method, path] of [["POST", "/signin"], ["POST", "/new"], ["POST", "/alice/notes.git/git-receive-pack"], ["DELETE", "/alice/notes"]] as const) {
	123	const res = await fetch(`${base}${path}`, { method, headers: { origin: base }, body: method === "DELETE" ? null : "x=1" });
	124	expect({ path, status: res.status }).toEqual({ path, status: 503 });
	125	expect(await res.text()).toContain("read-only standby");
	126	expect(res.headers.get("x-content-type-options")).toBe("nosniff");
	127	}
	128	expect((await fetch(`${base}/alice/notes.git/info/refs?service=git-receive-pack`)).status).toBe(503);
	129	expect((await fetch(`${base}/api/v1/repos`, { method: "POST", headers: { "content-type": "application/json" }, body: "{}" })).status).toBe(503);
	130	} finally {
	131	child.kill();
	132	await child.exited;
	133	}
	134	return { result: undefined, portLost: lostPort(stderr) };
	135	});
	136	}, 90_000);