ab4ecbf · 7403f7f · fa82259 · 3f49e79 · cdede41
@@ -42,6 +42,8 @@
export function eventAudience(kind: string): EventAudience {
    kind.startsWith("mail.") ||
    // A restore drill's verdict is the forge owner's (`snapshot/service.ts`).
    kind.startsWith("drill.") ||
    // A snapshot moving is the forge owner's too, and has no repository.
    kind.startsWith("backup.") ||
    // A report from outside waits for a person who decides; its arrival is
    // theirs to see, never the readers' (`actions/reports.ts`).
    kind.startsWith("report.") ||
@@ -151,7 +153,9 @@
export function audienceFilter(
      case "actor":
        return viewer !== undefined && event.actorId === viewer.id;
      case "admins":
        return event.repoId !== null && isAdmin(event.repoId);
        return event.repoId === null
          ? viewer?.isAdmin === true && isActive(viewer)
          : isAdmin(event.repoId);
    }
  };
}
@@ -138,6 +138,7 @@
import {
  backupsClear,
  backupsDestination,
  backupsDrill,
  backupsLive,
  backupsMirror,
  backupsMirrorClear,
  backupsMirrorNow,
@@ -829,6 +830,7 @@
export function buildRouter(): Router<Handler> {
  router.add("POST", "/admin/invites", inviteMint);
  router.add("POST", "/admin/invites/:id/revoke", inviteRevoke);
  router.add("GET", "/settings/backups", backupsPage);
  router.add("GET", "/settings/backups/live", backupsLive);
  router.add("POST", "/settings/backups/now", backupsNow);
  router.add("POST", "/settings/backups/drill", backupsDrill);
  router.add("POST", "/settings/backups/destination", backupsDestination);
@@ -250,7 +250,7 @@
export function viewerStream(
      emit("run.log", { run: runId, ...frameOf(lines) });
    }, gone.signal);
    const unsubscribe = app.bus.subscribe((event: ForgeEvent) => {
      if (event.repoId === null || !visible(event.repoId)) return;
      if (event.repoId === null ? !event.kind.startsWith("backup.") : !visible(event.repoId)) return;
      if (!meant(event)) return;
      emit("forge", named(event), event.id);
    });
@@ -34,6 +34,8 @@
const log = createLog("snapshots");
const NIGHTLY_HOUR_UTC = 8;
/** After a failed snapshot, the next try is this soon (or the nightly, if sooner). */
export const RETRY_MS = 3_600_000;
/** A moving snapshot tells the page how far it is no oftener than this. */
const PROGRESS_EVENT_MS = 2_000;
/**
 * A forge given nowhere to put a snapshot. Not a failure: the Backups page
@@ -110,6 +112,8 @@
export function startSnapshots(
    /** Test seam: how often the mirror is looked at. */
    readonly mirrorEveryMs?: number | undefined;
    readonly nightly?: boolean;
    /** Test seam: how often a moving snapshot tells the outbox its progress. */
    readonly progressEventMs?: number | undefined;
    /** Test seam: how soon a failed snapshot is tried again. */
    readonly retryMs?: number | undefined;
    readonly passphrase?: string | undefined;
@@ -152,6 +156,9 @@
export function startSnapshots(
      const store = current();
      if (store === undefined) throw new Nowhere();
      running = { objects: 0, bytes: 0, elapsedMs: 0 };
      // Admin-only, no repository, nothing /backups does not already show.
      app.bus.append({ kind: "backup.started" });
      let toldAt = Date.now();
      const manifest = await takeSnapshot(
        {
          db: app.db,
@@ -161,6 +168,9 @@
export function startSnapshots(
          passphrase: options.passphrase,
          onLand: (p) => {
            running = p;
            if (Date.now() - toldAt < (options.progressEventMs ?? PROGRESS_EVENT_MS)) return;
            toldAt = Date.now();
            app.bus.append({ kind: "backup.progress", payload: { objects: p.objects, bytes: p.bytes } });
      
        sentinelcrew
        25d ago
        
      
      Not blocking: every progress tick is kept as a row in the event table, about 30 a minute for as long as the backup runs, and no reader replays them (the resume path skips repository-less events). That's fine at this throttle. If the outbox ever gets pruned by kind, backup.progress is the first candidate.
    
          },
        },
        store,
@@ -177,6 +187,7 @@
export function startSnapshots(
      (m) => {
        last = { manifest: m, at: m.finishedAt };
        keep({ landed: m.name });
        app.bus.append({ kind: "backup.finished", payload: { ok: true } });
        arm(untilNightly());
        return m;
      },
@@ -188,6 +199,7 @@
export function startSnapshots(
        }
        log.error("snapshot failed", { error: String(error) });
        keep({ error: error instanceof Error ? error.message : String(error) });
        app.bus.append({ kind: "backup.finished", payload: { ok: false } });
        arm(Math.min(options.retryMs ?? RETRY_MS, untilNightly()));
        throw error;
      },
@@ -26,13 +26,15 @@
import { resolveDestination, sealedSet } from "../../snapshot/destination.ts";
import { keyOf } from "../../snapshot/snapshot.ts";
import { formatMinutes, formatSpan, timeByPart, type DrillResult } from "../../snapshot/drill.ts";
import { lastDrill } from "../../snapshot/drills.ts";
import { snapshotsOf } from "../../snapshot/service.ts";
import type { Store } from "../../snapshot/store.ts";
import { snapshotsOf, type SnapshotsHandle } from "../../snapshot/service.ts";
import { lastSnapshotOutcome } from "../../snapshot/outcome.ts";
import type { Ctx } from "../../http/context.ts";
import { notFoundPage, settingsLayout, signInFirst } from "../chrome.ts";
import { field, readForm, type Form } from "../form.ts";
import { html, when, type Html } from "../html.ts";
import { flashTo, page, redirect } from "../layout.ts";
import { liveRegionsJs } from "../client.ts";
import { flashTo, fragment, page, redirect } from "../layout.ts";
import { btn, card, empty, fileSize, icon, row, time } from "../ui.ts";
import { accountSections } from "./origins.ts";
@@ -42,11 +44,11 @@
function requireAdmin(ctx: Ctx): Response | undefined {
  return undefined;
}
function shell(ctx: Ctx, body: Html, moving = false): Response {
function shell(ctx: Ctx, body: Html): Response {
  return page({
    ctx,
    title: "Backups",
    ...(moving ? { refresh: { seconds: 5 } } : {}),
    scriptJs: liveRegionsJs,
    body: html`<div class="page-head"><h1>Backups</h1></div>
      ${settingsLayout(
        accountSections("backups", { admin: true }),
@@ -245,20 +247,40 @@
export async function backupsPage(ctx: Ctx): Promise<Response> {
  if (denied) return denied;
  const svc = snapshotsOf(ctx.app);
  const form = html`${destinationCard(ctx)}${mirrorCard(ctx)}${domainsCard(ctx)}`;
  const now = Date.now();
  if (svc === undefined || svc.store === undefined)
    return shell(
      ctx,
      html`${recoveryLine({
        reachable: false,
        unreachable: "no snapshot store is set",
        now,
        now: Date.now(),
      })}${empty({
        icon: "download",
        title: "No snapshots are taken",
        note: "This forge was given nowhere to put them. On the platform an app's bucket (S3_*) is the place; on a machine, FORGE_SNAPSHOT_DIR names a directory. Seal a bucket below, or restart with one set, and a snapshot is taken every night at 08:00 UTC.",
      })}${form}`,
    );
  return shell(ctx, html`${await movingPart(ctx, svc, svc.store)}${form}`);
}
/** GET /settings/backups/live — the moving part alone, which the page refetches on an event. */
export async function backupsLive(ctx: Ctx): Promise<Response> {
  if (!ctx.auth.user?.isAdmin) return fragment(html``, { status: 404 });
  const svc = snapshotsOf(ctx.app);
  if (svc === undefined || svc.store === undefined) return fragment(html``, { status: 404 });
  return fragment(await movingPart(ctx, svc, svc.store), { request: ctx.request });
}
/**
 * What changes while a snapshot is taken: the recovery line, the progress,
 * the outcome and the list. It patches in place from `backup.*` events.
 */
async function movingPart(
  ctx: Ctx,
  svc: SnapshotsHandle,
  store: Store,
): Promise<Html> {
  const now = Date.now();
  // A failure is kept on disk, so it is shown after a restart too; this
  // process's own failure is newer when the file could not be written.
  const kept = lastSnapshotOutcome(ctx.app.config.dataDir);
@@ -292,13 +314,11 @@
export async function backupsPage(ctx: Ctx): Promise<Response> {
    ? html`<pre class="backups-restore">
bun run src/snapshot/restore.ts ${snapshots[0].name} /path/to/empty/data</pre>`
    : null;
  return shell(
    ctx,
    html`${recovery}${card(
  return html`<div id="backups-live" data-live="backup.*" data-live-src="/settings/backups/live">${recovery}${card(
      html`<p>
          Every night at 08:00 UTC the forge writes what cannot be rebuilt — its
          database, its keys, every repository, every app's production data, the
          crew's logs — to <code>${svc.store.where}</code>, and keeps the newest
          crew's logs — to <code>${store.where}</code>, and keeps the newest
          fourteen.
        </p>
        ${keyNote(svc.wraps, snapshots[0])}
@@ -384,9 +404,7 @@
bun run src/snapshot/restore.ts ${snapshots[0].name} /path/to/empty/data</pre>`
            },
          )
    }
    ${form}`,
    svc.progress() !== undefined,
  );
  </div>`;
}
const SOURCE: Record<string, string> = {
@@ -0,0 +1,186 @@
/**
 * A backup moving tells the outbox (`backup.started`, `.progress`,
 * `.finished`), and /backups patches from them: no meta refresh while one
 * moves, a region that names its events and its own address, and events
 * only an admin's stream carries.
 */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { createUser } from "../src/core/users.ts";
import { listEvents } from "../src/core/events.ts";
import { serve, type Servers } from "../src/http/server.ts";
import { startSnapshots, type SnapshotsHandle } from "../src/snapshot/service.ts";
import { dirStore } from "../src/snapshot/store.ts";
import { quietLogs } from "./helpers/globals.ts";
quietLogs();
let root = "";
let app: App;
let svc: SnapshotsHandle;
let servers: Servers;
let release: () => void = () => {};
let gate: Promise<void> | undefined;
let reached: () => void = () => {};
let atPut = new Promise<void>((r) => (reached = r));
const login = async (username: string, password: string) => {
  const r = await fetch(`${servers.url}/signin`, {
    method: "POST",
    headers: { origin: servers.url },
    body: new URLSearchParams({ username, password }),
    redirect: "manual",
  });
  return (r.headers.get("set-cookie") ?? "").split(";")[0]!;
};
beforeAll(async () => {
  root = mkdtempSync(join(tmpdir(), "forge-backups-live-"));
  app = createApp({ dataDir: root, port: 0 });
  for (const [username, password, adminIfFirst] of [
    ["alice", "a-good-passphrase", true],
    ["bob", "another-good-passphrase", false],
  ] as const) {
    const u = await createUser(app.db, {
      origin: "seed",
      trusted: true,
      username,
      email: `${username}@forge.local`,
      password,
      adminIfFirst,
    });
    if (!u.ok) throw new Error(u.error);
  }
  const inner = dirStore(join(root, "store"));
  // Holds the first put until the test lets go, so the snapshot is moving.
  const store = {
    ...inner,
    async put(key: string, body: Uint8Array | string) {
      reached();
      await gate;
      return inner.put(key, body);
    },
  };
  svc = startSnapshots(app, store, { version: "t", nightly: false });
  servers = await serve(app);
});
afterAll(async () => {
  release();
  servers.stop();
  await svc.stop();
  app.close();
  rmSync(root, { recursive: true, force: true });
});
const kinds = () =>
  listEvents(app.db, {})
    .map((e) => e.kind)
    .filter((k) => k.startsWith("backup."))
    .reverse();
test("a backup moving emits its events, and /backups does not reload itself meanwhile", async () => {
  gate = new Promise<void>((r) => (release = r));
  atPut = new Promise<void>((r) => (reached = r));
  const done = svc.now();
  await atPut;
  expect(kinds()).toEqual(["backup.started"]);
  const alice = await login("alice", "a-good-passphrase");
  const moving = await fetch(`${servers.url}/settings/backups`, { headers: { cookie: alice } });
  const text = await moving.text();
  expect(text).not.toContain('http-equiv="refresh"');
  expect(text).toContain('id="backups-live"');
  expect(text).toContain('data-live="backup.*"');
  expect(text).toContain('data-live-src="/settings/backups/live"');
  const part = await fetch(`${servers.url}/settings/backups/live`, { headers: { cookie: alice } });
  expect(part.status).toBe(200);
  expect(await part.text()).toContain('id="backups-live"');
  release();
  await done;
  const seen = kinds();
  expect(seen[0]).toBe("backup.started");
  expect(seen.at(-1)).toBe("backup.finished");
  const finished = listEvents(app.db, { kind: "backup.finished" })[0]!;
  expect(finished.payload).toEqual({ ok: true });
  expect(finished.repoId).toBeNull();
});
test("a moving snapshot tells its progress, no oftener than the throttle", async () => {
  const inner = dirStore(join(root, "store-slow"));
  const slow = {
    ...inner,
    async put(key: string, body: Uint8Array | string) {
      await Bun.sleep(200);
      return inner.put(key, body);
    },
  };
  const mine = startSnapshots(app, slow, { version: "t", nightly: false, progressEventMs: 100 });
  const from = listEvents(app.db, {})[0]?.id ?? "";
  await mine.now();
  await mine.stop();
  const progress = listEvents(app.db, { kind: "backup.progress", afterId: from, limit: 500 }).reverse();
  expect(progress.length).toBeGreaterThanOrEqual(1);
  for (const e of progress) {
    expect(typeof e.payload["objects"]).toBe("number");
    expect(typeof e.payload["bytes"]).toBe("number");
    expect(e.repoId).toBeNull();
  }
  for (let i = 1; i < progress.length; i++)
    expect(Date.parse(progress[i]!.createdAt) - Date.parse(progress[i - 1]!.createdAt)).toBeGreaterThanOrEqual(90);
});
test("a failed snapshot ends in backup.finished {ok:false}, and the live part shows it", async () => {
  const inner = dirStore(join(root, "store-bad"));
  const bad = {
    ...inner,
    async put(): Promise<void> {
      throw new Error("the bucket said no");
    },
  };
  const mine = startSnapshots(app, bad, { version: "t", nightly: false });
  await expect(mine.now()).rejects.toThrow();
  const seen = kinds();
  expect(seen.at(-1)).toBe("backup.finished");
  expect(listEvents(app.db, { kind: "backup.finished" })[0]!.payload).toEqual({ ok: false });
  const alice = await login("alice", "a-good-passphrase");
  const part = await fetch(`${servers.url}/settings/backups/live`, { headers: { cookie: alice } });
  const text = await part.text();
  expect(text).toContain("The last snapshot failed");
  expect(text).toContain("the bucket said no");
  await mine.stop();
});
test("the live part and the stream are an admin's alone", async () => {
  const bob = await login("bob", "another-good-passphrase");
  const part = await fetch(`${servers.url}/settings/backups/live`, { headers: { cookie: bob } });
  expect(part.status).toBe(404);
  expect((await fetch(`${servers.url}/settings/backups/live`)).status).toBeGreaterThanOrEqual(300);
  const frames = async (cookie: string): Promise<string> => {
    const ac = new AbortController();
    const res = await fetch(`${servers.url}/events`, { headers: { cookie }, signal: ac.signal });
    const reader = res.body!.getReader();
    // Let the stream subscribe, then move a backup's worth of events.
    await Bun.sleep(50);
    app.bus.append({ kind: "backup.started" });
    await Bun.sleep(100);
    let got = "";
    const until = Date.now() + 500;
    while (Date.now() < until && !got.includes("backup.")) {
      const { value } = await Promise.race([
        reader.read(),
        Bun.sleep(200).then(() => ({ value: undefined })),
      ]);
      if (value) got += new TextDecoder().decode(value);
    }
    ac.abort();
    return got;
  };
  const alice = await login("alice", "a-good-passphrase");
  expect(await frames(alice)).toContain("backup.started");
  expect(await frames(bob)).not.toContain("backup.started");
});
@@ -11,7 +11,7 @@
const doc = readFileSync("docs/design/audit.md", "utf8");
/** Routes that answer a file, a fragment, a redirect hop or a callback. */
const NOT_PAGES =
  /^\/(assets\/|\.well-known\/|icons\/|og\/|live\/|__forge\/|go\/|events$|system\/stream$|settings\/teams\/stream$|sw\.js$|robots\.txt$|sitemap\.xml$|manifest\.webmanifest$|llms\.txt$|releases\.json$|audit\.csv$|fleet\/graph\/live$|fleet\/graph\.csv$|settings\/origins\/callback$|home\/archived\/)/;
  /^\/(assets\/|\.well-known\/|icons\/|og\/|live\/|__forge\/|go\/|events$|system\/stream$|settings\/teams\/stream$|sw\.js$|robots\.txt$|sitemap\.xml$|manifest\.webmanifest$|llms\.txt$|releases\.json$|audit\.csv$|fleet\/graph\/live$|fleet\/graph\.csv$|settings\/origins\/callback$|home\/archived\/|settings\/backups\/live$)/;
const getRoutes = [
  ...server.matchAll(/router\.add\(\s*"GET",\s*"([^"]+)"/g),
@@ -13,7 +13,6 @@
const ALLOWED: Record<string, { allow: string[]; why: string }> = {
  "src/web/layout.ts": { allow: ['http-equiv="refresh"'], why: "defines the refresh option and its meta tag; callers are listed here" },
  "src/web/pages/migrate.ts": { allow: ["refresh: {", 'http-equiv="refresh"'], why: "migrate progress runs in a process that emits no events" },
  "src/web/pages/browse.ts": { allow: ["refresh: {"], why: "mirror progress runs in a process that emits no events" },
  "src/web/pages/backups.ts": { allow: ["refresh: {"], why: "a backup moving emits no event" },
  "src/web/client.ts": { allow: ["data-live-every", "setInterval"], why: "the data-live-every mechanism itself, and clocks that only redraw text" },
  "src/web/fleet-graph.ts": { allow: ["setInterval"], why: "no fleet event yet; remove once it has one" },
};
		@@ -42,6 +42,8 @@ export function eventAudience(kind: string): EventAudience {
42	42	kind.startsWith("mail.") \|\|
43	43	// A restore drill's verdict is the forge owner's (`snapshot/service.ts`).
44	44	kind.startsWith("drill.") \|\|
	45	// A snapshot moving is the forge owner's too, and has no repository.
	46	kind.startsWith("backup.") \|\|
45	47	// A report from outside waits for a person who decides; its arrival is
46	48	// theirs to see, never the readers' (`actions/reports.ts`).
47	49	kind.startsWith("report.") \|\|
		@@ -151,7 +153,9 @@ export function audienceFilter(
151	153	case "actor":
152	154	return viewer !== undefined && event.actorId === viewer.id;
153	155	case "admins":
154		return event.repoId !== null && isAdmin(event.repoId);
	156	return event.repoId === null
	157	? viewer?.isAdmin === true && isActive(viewer)
	158	: isAdmin(event.repoId);
155	159	}
156	160	};
157	161	}
		@@ -138,6 +138,7 @@ import {
138	138	backupsClear,
139	139	backupsDestination,
140	140	backupsDrill,
	141	backupsLive,
141	142	backupsMirror,
142	143	backupsMirrorClear,
143	144	backupsMirrorNow,
		@@ -829,6 +830,7 @@ export function buildRouter(): Router<Handler> {
829	830	router.add("POST", "/admin/invites", inviteMint);
830	831	router.add("POST", "/admin/invites/:id/revoke", inviteRevoke);
831	832	router.add("GET", "/settings/backups", backupsPage);
	833	router.add("GET", "/settings/backups/live", backupsLive);
832	834	router.add("POST", "/settings/backups/now", backupsNow);
833	835	router.add("POST", "/settings/backups/drill", backupsDrill);
834	836	router.add("POST", "/settings/backups/destination", backupsDestination);
		@@ -250,7 +250,7 @@ export function viewerStream(
250	250	emit("run.log", { run: runId, ...frameOf(lines) });
251	251	}, gone.signal);
252	252	const unsubscribe = app.bus.subscribe((event: ForgeEvent) => {
253		if (event.repoId === null \|\| !visible(event.repoId)) return;
	253	if (event.repoId === null ? !event.kind.startsWith("backup.") : !visible(event.repoId)) return;
254	254	if (!meant(event)) return;
255	255	emit("forge", named(event), event.id);
256	256	});
		@@ -34,6 +34,8 @@ const log = createLog("snapshots");
34	34	const NIGHTLY_HOUR_UTC = 8;
35	35	/** After a failed snapshot, the next try is this soon (or the nightly, if sooner). */
36	36	export const RETRY_MS = 3_600_000;
	37	/** A moving snapshot tells the page how far it is no oftener than this. */
	38	const PROGRESS_EVENT_MS = 2_000;
37	39
38	40	/**
39	41	* A forge given nowhere to put a snapshot. Not a failure: the Backups page
		@@ -110,6 +112,8 @@ export function startSnapshots(
110	112	/** Test seam: how often the mirror is looked at. */
111	113	readonly mirrorEveryMs?: number \| undefined;
112	114	readonly nightly?: boolean;
	115	/** Test seam: how often a moving snapshot tells the outbox its progress. */
	116	readonly progressEventMs?: number \| undefined;
113	117	/** Test seam: how soon a failed snapshot is tried again. */
114	118	readonly retryMs?: number \| undefined;
115	119	readonly passphrase?: string \| undefined;
		@@ -152,6 +156,9 @@ export function startSnapshots(
152	156	const store = current();
153	157	if (store === undefined) throw new Nowhere();
154	158	running = { objects: 0, bytes: 0, elapsedMs: 0 };
	159	// Admin-only, no repository, nothing /backups does not already show.
	160	app.bus.append({ kind: "backup.started" });
	161	let toldAt = Date.now();
155	162	const manifest = await takeSnapshot(
156	163	{
157	164	db: app.db,
		@@ -161,6 +168,9 @@ export function startSnapshots(
161	168	passphrase: options.passphrase,
162	169	onLand: (p) => {
163	170	running = p;
	171	if (Date.now() - toldAt < (options.progressEventMs ?? PROGRESS_EVENT_MS)) return;
	172	toldAt = Date.now();
	173	app.bus.append({ kind: "backup.progress", payload: { objects: p.objects, bytes: p.bytes } });
		sentinelcrew 25d ago Not blocking: every progress tick is kept as a row in the event table, about 30 a minute for as long as the backup runs, and no reader replays them (the resume path skips repository-less events). That's fine at this throttle. If the outbox ever gets pruned by kind, `backup.progress` is the first candidate.
164	174	},
165	175	},
166	176	store,
		@@ -177,6 +187,7 @@ export function startSnapshots(
177	187	(m) => {
178	188	last = { manifest: m, at: m.finishedAt };
179	189	keep({ landed: m.name });
	190	app.bus.append({ kind: "backup.finished", payload: { ok: true } });
180	191	arm(untilNightly());
181	192	return m;
182	193	},
		@@ -188,6 +199,7 @@ export function startSnapshots(
188	199	}
189	200	log.error("snapshot failed", { error: String(error) });
190	201	keep({ error: error instanceof Error ? error.message : String(error) });
	202	app.bus.append({ kind: "backup.finished", payload: { ok: false } });
191	203	arm(Math.min(options.retryMs ?? RETRY_MS, untilNightly()));
192	204	throw error;
193	205	},
		@@ -26,13 +26,15 @@ import { resolveDestination, sealedSet } from "../../snapshot/destination.ts";
26	26	import { keyOf } from "../../snapshot/snapshot.ts";
27	27	import { formatMinutes, formatSpan, timeByPart, type DrillResult } from "../../snapshot/drill.ts";
28	28	import { lastDrill } from "../../snapshot/drills.ts";
29		import { snapshotsOf } from "../../snapshot/service.ts";
	29	import type { Store } from "../../snapshot/store.ts";
	30	import { snapshotsOf, type SnapshotsHandle } from "../../snapshot/service.ts";
30	31	import { lastSnapshotOutcome } from "../../snapshot/outcome.ts";
31	32	import type { Ctx } from "../../http/context.ts";
32	33	import { notFoundPage, settingsLayout, signInFirst } from "../chrome.ts";
33	34	import { field, readForm, type Form } from "../form.ts";
34	35	import { html, when, type Html } from "../html.ts";
35		import { flashTo, page, redirect } from "../layout.ts";
	36	import { liveRegionsJs } from "../client.ts";
	37	import { flashTo, fragment, page, redirect } from "../layout.ts";
36	38	import { btn, card, empty, fileSize, icon, row, time } from "../ui.ts";
37	39	import { accountSections } from "./origins.ts";
38	40
		@@ -42,11 +44,11 @@ function requireAdmin(ctx: Ctx): Response \| undefined {
42	44	return undefined;
43	45	}
44	46
45		function shell(ctx: Ctx, body: Html, moving = false): Response {
	47	function shell(ctx: Ctx, body: Html): Response {
46	48	return page({
47	49	ctx,
48	50	title: "Backups",
49		...(moving ? { refresh: { seconds: 5 } } : {}),
	51	scriptJs: liveRegionsJs,
50	52	body: html`<div class="page-head"><h1>Backups</h1></div>
51	53	${settingsLayout(
52	54	accountSections("backups", { admin: true }),
		@@ -245,20 +247,40 @@ export async function backupsPage(ctx: Ctx): Promise<Response> {
245	247	if (denied) return denied;
246	248	const svc = snapshotsOf(ctx.app);
247	249	const form = html`${destinationCard(ctx)}${mirrorCard(ctx)}${domainsCard(ctx)}`;
248		const now = Date.now();
249	250	if (svc === undefined \|\| svc.store === undefined)
250	251	return shell(
251	252	ctx,
252	253	html`${recoveryLine({
253	254	reachable: false,
254	255	unreachable: "no snapshot store is set",
255		now,
	256	now: Date.now(),
256	257	})}${empty({
257	258	icon: "download",
258	259	title: "No snapshots are taken",
259	260	note: "This forge was given nowhere to put them. On the platform an app's bucket (S3_*) is the place; on a machine, FORGE_SNAPSHOT_DIR names a directory. Seal a bucket below, or restart with one set, and a snapshot is taken every night at 08:00 UTC.",
260	261	})}${form}`,
261	262	);
	263	return shell(ctx, html`${await movingPart(ctx, svc, svc.store)}${form}`);
	264	}
	265
	266	/** GET /settings/backups/live — the moving part alone, which the page refetches on an event. */
	267	export async function backupsLive(ctx: Ctx): Promise<Response> {
	268	if (!ctx.auth.user?.isAdmin) return fragment(html``, { status: 404 });
	269	const svc = snapshotsOf(ctx.app);
	270	if (svc === undefined \|\| svc.store === undefined) return fragment(html``, { status: 404 });
	271	return fragment(await movingPart(ctx, svc, svc.store), { request: ctx.request });
	272	}
	273
	274	/**
	275	* What changes while a snapshot is taken: the recovery line, the progress,
	276	* the outcome and the list. It patches in place from `backup.*` events.
	277	*/
	278	async function movingPart(
	279	ctx: Ctx,
	280	svc: SnapshotsHandle,
	281	store: Store,
	282	): Promise<Html> {
	283	const now = Date.now();
262	284	// A failure is kept on disk, so it is shown after a restart too; this
263	285	// process's own failure is newer when the file could not be written.
264	286	const kept = lastSnapshotOutcome(ctx.app.config.dataDir);
		@@ -292,13 +314,11 @@ export async function backupsPage(ctx: Ctx): Promise<Response> {
292	314	? html`<pre class="backups-restore">
293	315	bun run src/snapshot/restore.ts ${snapshots[0].name} /path/to/empty/data</pre>`
294	316	: null;
295		return shell(
296		ctx,
297		html`${recovery}${card(
	317	return html`<div id="backups-live" data-live="backup.*" data-live-src="/settings/backups/live">${recovery}${card(
298	318	html`<p>
299	319	Every night at 08:00 UTC the forge writes what cannot be rebuilt — its
300	320	database, its keys, every repository, every app's production data, the
301		crew's logs — to <code>${svc.store.where}</code>, and keeps the newest
	321	crew's logs — to <code>${store.where}</code>, and keeps the newest
302	322	fourteen.
303	323	</p>
304	324	${keyNote(svc.wraps, snapshots[0])}
		@@ -384,9 +404,7 @@ bun run src/snapshot/restore.ts ${snapshots[0].name} /path/to/empty/data</pre>`
384	404	},
385	405	)
386	406	}
387		${form}`,
388		svc.progress() !== undefined,
389		);
	407	</div>`;
390	408	}
391	409
392	410	const SOURCE: Record<string, string> = {
		@@ -0,0 +1,186 @@
	1	/**
	2	* A backup moving tells the outbox (`backup.started`, `.progress`,
	3	* `.finished`), and /backups patches from them: no meta refresh while one
	4	* moves, a region that names its events and its own address, and events
	5	* only an admin's stream carries.
	6	*/
	7	import { afterAll, beforeAll, expect, test } from "bun:test";
	8	import { mkdtempSync, rmSync } from "node:fs";
	9	import { tmpdir } from "node:os";
	10	import { join } from "node:path";
	11	import { createApp, type App } from "../src/app.ts";
	12	import { createUser } from "../src/core/users.ts";
	13	import { listEvents } from "../src/core/events.ts";
	14	import { serve, type Servers } from "../src/http/server.ts";
	15	import { startSnapshots, type SnapshotsHandle } from "../src/snapshot/service.ts";
	16	import { dirStore } from "../src/snapshot/store.ts";
	17	import { quietLogs } from "./helpers/globals.ts";
	18
	19	quietLogs();
	20
	21	let root = "";
	22	let app: App;
	23	let svc: SnapshotsHandle;
	24	let servers: Servers;
	25	let release: () => void = () => {};
	26	let gate: Promise<void> \| undefined;
	27	let reached: () => void = () => {};
	28	let atPut = new Promise<void>((r) => (reached = r));
	29
	30	const login = async (username: string, password: string) => {
	31	const r = await fetch(`${servers.url}/signin`, {
	32	method: "POST",
	33	headers: { origin: servers.url },
	34	body: new URLSearchParams({ username, password }),
	35	redirect: "manual",
	36	});
	37	return (r.headers.get("set-cookie") ?? "").split(";")[0]!;
	38	};
	39
	40	beforeAll(async () => {
	41	root = mkdtempSync(join(tmpdir(), "forge-backups-live-"));
	42	app = createApp({ dataDir: root, port: 0 });
	43	for (const [username, password, adminIfFirst] of [
	44	["alice", "a-good-passphrase", true],
	45	["bob", "another-good-passphrase", false],
	46	] as const) {
	47	const u = await createUser(app.db, {
	48	origin: "seed",
	49	trusted: true,
	50	username,
	51	email: `${username}@forge.local`,
	52	password,
	53	adminIfFirst,
	54	});
	55	if (!u.ok) throw new Error(u.error);
	56	}
	57	const inner = dirStore(join(root, "store"));
	58	// Holds the first put until the test lets go, so the snapshot is moving.
	59	const store = {
	60	...inner,
	61	async put(key: string, body: Uint8Array \| string) {
	62	reached();
	63	await gate;
	64	return inner.put(key, body);
	65	},
	66	};
	67	svc = startSnapshots(app, store, { version: "t", nightly: false });
	68	servers = await serve(app);
	69	});
	70	afterAll(async () => {
	71	release();
	72	servers.stop();
	73	await svc.stop();
	74	app.close();
	75	rmSync(root, { recursive: true, force: true });
	76	});
	77
	78	const kinds = () =>
	79	listEvents(app.db, {})
	80	.map((e) => e.kind)
	81	.filter((k) => k.startsWith("backup."))
	82	.reverse();
	83
	84	test("a backup moving emits its events, and /backups does not reload itself meanwhile", async () => {
	85	gate = new Promise<void>((r) => (release = r));
	86	atPut = new Promise<void>((r) => (reached = r));
	87	const done = svc.now();
	88	await atPut;
	89	expect(kinds()).toEqual(["backup.started"]);
	90
	91	const alice = await login("alice", "a-good-passphrase");
	92	const moving = await fetch(`${servers.url}/settings/backups`, { headers: { cookie: alice } });
	93	const text = await moving.text();
	94	expect(text).not.toContain('http-equiv="refresh"');
	95	expect(text).toContain('id="backups-live"');
	96	expect(text).toContain('data-live="backup.*"');
	97	expect(text).toContain('data-live-src="/settings/backups/live"');
	98	const part = await fetch(`${servers.url}/settings/backups/live`, { headers: { cookie: alice } });
	99	expect(part.status).toBe(200);
	100	expect(await part.text()).toContain('id="backups-live"');
	101
	102	release();
	103	await done;
	104	const seen = kinds();
	105	expect(seen[0]).toBe("backup.started");
	106	expect(seen.at(-1)).toBe("backup.finished");
	107	const finished = listEvents(app.db, { kind: "backup.finished" })[0]!;
	108	expect(finished.payload).toEqual({ ok: true });
	109	expect(finished.repoId).toBeNull();
	110	});
	111
	112	test("a moving snapshot tells its progress, no oftener than the throttle", async () => {
	113	const inner = dirStore(join(root, "store-slow"));
	114	const slow = {
	115	...inner,
	116	async put(key: string, body: Uint8Array \| string) {
	117	await Bun.sleep(200);
	118	return inner.put(key, body);
	119	},
	120	};
	121	const mine = startSnapshots(app, slow, { version: "t", nightly: false, progressEventMs: 100 });
	122	const from = listEvents(app.db, {})[0]?.id ?? "";
	123	await mine.now();
	124	await mine.stop();
	125	const progress = listEvents(app.db, { kind: "backup.progress", afterId: from, limit: 500 }).reverse();
	126	expect(progress.length).toBeGreaterThanOrEqual(1);
	127	for (const e of progress) {
	128	expect(typeof e.payload["objects"]).toBe("number");
	129	expect(typeof e.payload["bytes"]).toBe("number");
	130	expect(e.repoId).toBeNull();
	131	}
	132	for (let i = 1; i < progress.length; i++)
	133	expect(Date.parse(progress[i]!.createdAt) - Date.parse(progress[i - 1]!.createdAt)).toBeGreaterThanOrEqual(90);
	134	});
	135
	136	test("a failed snapshot ends in backup.finished {ok:false}, and the live part shows it", async () => {
	137	const inner = dirStore(join(root, "store-bad"));
	138	const bad = {
	139	...inner,
	140	async put(): Promise<void> {
	141	throw new Error("the bucket said no");
	142	},
	143	};
	144	const mine = startSnapshots(app, bad, { version: "t", nightly: false });
	145	await expect(mine.now()).rejects.toThrow();
	146	const seen = kinds();
	147	expect(seen.at(-1)).toBe("backup.finished");
	148	expect(listEvents(app.db, { kind: "backup.finished" })[0]!.payload).toEqual({ ok: false });
	149	const alice = await login("alice", "a-good-passphrase");
	150	const part = await fetch(`${servers.url}/settings/backups/live`, { headers: { cookie: alice } });
	151	const text = await part.text();
	152	expect(text).toContain("The last snapshot failed");
	153	expect(text).toContain("the bucket said no");
	154	await mine.stop();
	155	});
	156
	157	test("the live part and the stream are an admin's alone", async () => {
	158	const bob = await login("bob", "another-good-passphrase");
	159	const part = await fetch(`${servers.url}/settings/backups/live`, { headers: { cookie: bob } });
	160	expect(part.status).toBe(404);
	161	expect((await fetch(`${servers.url}/settings/backups/live`)).status).toBeGreaterThanOrEqual(300);
	162
	163	const frames = async (cookie: string): Promise<string> => {
	164	const ac = new AbortController();
	165	const res = await fetch(`${servers.url}/events`, { headers: { cookie }, signal: ac.signal });
	166	const reader = res.body!.getReader();
	167	// Let the stream subscribe, then move a backup's worth of events.
	168	await Bun.sleep(50);
	169	app.bus.append({ kind: "backup.started" });
	170	await Bun.sleep(100);
	171	let got = "";
	172	const until = Date.now() + 500;
	173	while (Date.now() < until && !got.includes("backup.")) {
	174	const { value } = await Promise.race([
	175	reader.read(),
	176	Bun.sleep(200).then(() => ({ value: undefined })),
	177	]);
	178	if (value) got += new TextDecoder().decode(value);
	179	}
	180	ac.abort();
	181	return got;
	182	};
	183	const alice = await login("alice", "a-good-passphrase");
	184	expect(await frames(alice)).toContain("backup.started");
	185	expect(await frames(bob)).not.toContain("backup.started");
	186	});
		@@ -11,7 +11,7 @@ const doc = readFileSync("docs/design/audit.md", "utf8");
11	11
12	12	/** Routes that answer a file, a fragment, a redirect hop or a callback. */
13	13	const NOT_PAGES =
14		/^\/(assets\/\|\.well-known\/\|icons\/\|og\/\|live\/\|__forge\/\|go\/\|events$\|system\/stream$\|settings\/teams\/stream$\|sw\.js$\|robots\.txt$\|sitemap\.xml$\|manifest\.webmanifest$\|llms\.txt$\|releases\.json$\|audit\.csv$\|fleet\/graph\/live$\|fleet\/graph\.csv$\|settings\/origins\/callback$\|home\/archived\/)/;
	14	/^\/(assets\/\|\.well-known\/\|icons\/\|og\/\|live\/\|__forge\/\|go\/\|events$\|system\/stream$\|settings\/teams\/stream$\|sw\.js$\|robots\.txt$\|sitemap\.xml$\|manifest\.webmanifest$\|llms\.txt$\|releases\.json$\|audit\.csv$\|fleet\/graph\/live$\|fleet\/graph\.csv$\|settings\/origins\/callback$\|home\/archived\/\|settings\/backups\/live$)/;
15	15
16	16	const getRoutes = [
17	17	...server.matchAll(/router\.add\(\s"GET",\s"([^"]+)"/g),
		@@ -13,7 +13,6 @@ const ALLOWED: Record<string, { allow: string[]; why: string }> = {
13	13	"src/web/layout.ts": { allow: ['http-equiv="refresh"'], why: "defines the refresh option and its meta tag; callers are listed here" },
14	14	"src/web/pages/migrate.ts": { allow: ["refresh: {", 'http-equiv="refresh"'], why: "migrate progress runs in a process that emits no events" },
15	15	"src/web/pages/browse.ts": { allow: ["refresh: {"], why: "mirror progress runs in a process that emits no events" },
16		"src/web/pages/backups.ts": { allow: ["refresh: {"], why: "a backup moving emits no event" },
17	16	"src/web/client.ts": { allow: ["data-live-every", "setInterval"], why: "the data-live-every mechanism itself, and clocks that only redraw text" },
18	17	"src/web/fleet-graph.ts": { allow: ["setInterval"], why: "no fleet event yet; remove once it has one" },
19	18	};