53ea223 · 6886931
@@ -442,6 +442,11 @@
huge one); restore unpacks them. While one runs, progress shows on
**Settings → Backups** and as a line in the log every 500 objects or minute.
The terminal has no backups view, so it has no readout.
**Run a drill** on the same page (admins only) restores the newest snapshot
into scratch, opens it, counts what is there and unseals a value; the page
waits for it, up to 10 minutes, and the last drill shows there as one line.
Only one runs at a time. See `docs/SNAPSHOT.md`.
`docs/EXTRACT.md` is the forge on a machine of its own, and the platform it
grew up in switched off: what a host owes it, the moves in order, and what is
given up.
@@ -70,6 +70,20 @@
Delete the old ones from the bucket if the bucket is not trusted.
The store is `Bun.S3Client` or a directory behind one small interface
(`src/snapshot/store.ts`): put, get, list, delete. Nothing else.
## The restore drill
A backup nobody has restored is a hope. **Settings → Backups → Run a drill**
(admins only; anyone else gets 403) restores the newest snapshot into a
scratch directory under the data directory, opens the database read-only,
counts repositories and issues, opens one sealed value with the restored key,
and removes the scratch whatever happened. The page waits until it finishes,
for up to 10 minutes; a step that hangs past that is red and names the step.
One drill runs at a time: a second click, or a retry after a proxy timeout,
joins the drill already running rather than starting another restore. Every
run is a row in the `drill` table, and the last one shows as a line on the
Backups page: when, which snapshot, how long, the counts, and green or the
step that failed.
## Restore
The printable card, seven lines, is [RESTORE.md](RESTORE.md); a test runs it.
@@ -132,6 +132,7 @@
export const GATE: readonly string[] = [
  "test/helpers.test.ts",
  "test/shards.test.ts",
  "test/snapshot-drill.test.ts",
  "test/snapshot-drill-page.test.ts",
  "test/snapshot-store.test.ts",
  "test/migrate.test.ts",
  "test/schema-additive.test.ts",
@@ -1655,6 +1655,21 @@
export const migrations: readonly string[] = [
  CREATE UNIQUE INDEX degraded_span_open ON degraded_span(cause) WHERE closed_at IS NULL;
  CREATE INDEX degraded_span_closed ON degraded_span(closed_at) WHERE closed_at IS NOT NULL;
  `,
  // 80 — restore drills (#1668): one row per run of `snapshot/drill.ts`,
  // kept so the Backups page can show the last. `at` is an ISO time.
  `
  CREATE TABLE drill (
    id           INTEGER PRIMARY KEY AUTOINCREMENT,
    at           TEXT NOT NULL,
    snapshot_id  TEXT,
    ms           INTEGER NOT NULL,
    repositories INTEGER NOT NULL,
    issues       INTEGER NOT NULL,
    verdict      TEXT NOT NULL CHECK (verdict IN ('green', 'red')),
    failed_step  TEXT,
    reason       TEXT
  );
  `,
];
/**
@@ -96,6 +96,7 @@
import { secretDelete, secretSet, secretsPage } from "../web/pages/secrets.ts";
import {
  backupsClear,
  backupsDestination,
  backupsDrill,
  backupsNow,
  backupsPage,
  backupsTest,
@@ -671,6 +672,7 @@
export function buildRouter(): Router<Handler> {
  router.add("POST", "/admin/limits", limitsSet);
  router.add("GET", "/settings/backups", backupsPage);
  router.add("POST", "/settings/backups/now", backupsNow);
  router.add("POST", "/settings/backups/drill", backupsDrill);
  router.add("POST", "/settings/backups/destination", backupsDestination);
  router.add("POST", "/settings/backups/test", backupsTest);
  router.add("POST", "/settings/backups/clear", backupsClear);
@@ -0,0 +1,53 @@
/**
 * The drills, kept: every run of `runDrill` is a row, so the Backups page
 * can say when the forge last proved a snapshot restores, and what failed
 * when it did not. Plain data in, plain data out.
 */
import type { Database } from "bun:sqlite";
import type { DrillResult, DrillStep } from "./drill.ts";
interface Row {
  at: string;
  snapshot_id: string | null;
  ms: number;
  repositories: number;
  issues: number;
  verdict: "green" | "red";
  failed_step: DrillStep | null;
  reason: string | null;
}
export function recordDrill(db: Database, result: DrillResult): void {
  db.query(
    `INSERT INTO drill (at, snapshot_id, ms, repositories, issues, verdict, failed_step, reason)
     VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
  ).run(
    result.at,
    result.snapshotId,
    result.ms,
    result.counts.repositories,
    result.counts.issues,
    result.verdict,
    result.failedStep,
    result.reason,
  );
}
/** The newest drill, or undefined when none has run. */
export function lastDrill(db: Database): DrillResult | undefined {
  const row = db
    .query<Row, []>("SELECT * FROM drill ORDER BY id DESC LIMIT 1")
    .get();
  return row === null
    ? undefined
    : {
        at: row.at,
        snapshotId: row.snapshot_id,
        ms: row.ms,
        counts: { repositories: row.repositories, issues: row.issues },
        verdict: row.verdict,
        failedStep: row.failed_step,
        reason: row.reason,
      };
}
@@ -18,6 +18,8 @@
import {
  type Manifest,
  type Progress,
} from "./snapshot.ts";
import { runDrill, type DrillDeps, type DrillResult } from "./drill.ts";
import { recordDrill } from "./drills.ts";
import type { Store } from "./store.ts";
const log = createLog("snapshots");
@@ -38,6 +40,8 @@
export interface SnapshotsHandle {
  readonly list: () => Promise<Manifest[]>;
  /** The last one taken by this process, and what it said. */
  readonly last: () => { manifest?: Manifest; error?: string; at?: string };
  /** Drill the newest snapshot now and keep the result; waits for any snapshot in flight. */
  readonly drill: () => Promise<DrillResult>;
  /** Forget the resolved store; the next use resolves again (a new destination was saved). */
  readonly forget: () => void;
  readonly stop: () => Promise<void>;
@@ -61,6 +65,8 @@
export function startSnapshots(
    readonly version: string;
    readonly nightly?: boolean;
    readonly passphrase?: string | undefined;
    /** Test seam: the drill itself. */
    readonly drill?: ((deps: DrillDeps) => Promise<DrillResult>) | undefined;
  } = {
    version: "0",
  },
@@ -118,6 +124,28 @@
export function startSnapshots(
    return inFlight;
  };
  // Single-flight: a restore of every repository is heavy, so a second call
      
        sentinelcrew
        1mo ago
        outdated
      
      Nothing stops a second call from starting while the first is still running, so two clicks mean two full restores side by side. Hold the running promise, the way take holds inFlight, and give it to any caller that arrives while it runs. Clear it in a finally.
    
  // while one runs joins it instead of filling the disk with another.
  let drilling: Promise<DrillResult> | undefined;
  const drill = (): Promise<DrillResult> => {
    drilling ??= (async (): Promise<DrillResult> => {
      await (inFlight ?? Promise.resolve()).catch(() => {});
      const store = current();
      if (store === undefined)
        throw new Error("this forge has nowhere to drill from");
      const result = await (options.drill ?? runDrill)({
        store,
        dataDir: app.config.dataDir,
        passphrase: options.passphrase,
      });
      recordDrill(app.db, result);
      return result;
    })().finally(() => {
      drilling = undefined;
    });
    return drilling;
  };
  let timer: ReturnType<typeof setTimeout> | undefined;
  const schedule = (): void => {
    if (options.nightly === false || stopped) return;
@@ -135,6 +163,7 @@
export function startSnapshots(
    },
    wraps: !!options.passphrase,
    now: take,
    drill,
    list: async () => {
      const store = current();
      return store === undefined ? [] : listSnapshots(store);
@@ -16,6 +16,7 @@
import {
} from "../../actions/destination.ts";
import { resolveDestination, sealedSet } from "../../snapshot/destination.ts";
import { keyOf } from "../../snapshot/snapshot.ts";
import { lastDrill } from "../../snapshot/drills.ts";
import { snapshotsOf } from "../../snapshot/service.ts";
import type { Ctx } from "../../http/context.ts";
import { notFoundPage, settingsLayout, signInFirst } from "../chrome.ts";
@@ -85,6 +86,31 @@
function keyNote(
      </p>`;
}
/** The last drill in one line, and the button to run another. */
function drillLine(ctx: Ctx): Html {
  const d = lastDrill(ctx.app.db);
  const line =
    d === undefined
      ? html`<p class="backups-drill">No restore drill has run yet.</p>`
      : d.verdict === "green"
        ? html`<p class="backups-drill">
            ${icon("check")} Last drill ${time(d.at)}: snapshot
            <code>${d.snapshotId ?? ""}</code> restored green in ${d.ms} ms —
            ${d.counts.repositories} repositories, ${d.counts.issues} issues.
          </p>`
        : html`<p class="backups-error backups-drill">
            ${icon("alert")} Last drill ${time(d.at)}: snapshot
            <code>${d.snapshotId ?? "none"}</code> failed at
            ${d.failedStep ?? "an unknown step"} after ${d.ms} ms${d.reason
              ? html`: ${d.reason}`
              : ""}.
          </p>`;
  return html`${line}
    <form method="post" action="/settings/backups/drill" class="backups-now">
      ${btn("Run a drill", { type: "submit", icon: "clock" })}
    </form>`;
}
export async function backupsPage(ctx: Ctx): Promise<Response> {
  const denied = requireAdmin(ctx);
  if (denied) return denied;
@@ -147,6 +173,7 @@
bun run src/snapshot/restore.ts ${snapshots[0].name} /path/to/empty/data</pre>`
              ${icon("alert")} The store could not be listed just now: ${error}
            </p>`,
        )}
        ${drillLine(ctx)}
        <form method="post" action="/settings/backups/now" class="backups-now">
          ${btn("Snapshot now", {
            type: "submit",
@@ -367,3 +394,36 @@
export async function backupsNow(ctx: Ctx): Promise<Response> {
    });
  }
}
/** POST /settings/backups/drill — restore the newest snapshot, keep what came of it. */
export async function backupsDrill(ctx: Ctx): Promise<Response> {
  if (!ctx.auth.user) return signInFirst(ctx);
  if (!ctx.auth.user.isAdmin)
    return new Response("Forbidden", {
      status: 403,
      headers: { "content-type": "text/plain; charset=utf-8" },
    });
  const form = await readForm(ctx, "admin");
  if (form instanceof Response) return form;
  const svc = snapshotsOf(ctx.app);
  if (svc === undefined || svc.store === undefined)
    return flashTo(ctx, "/settings/backups", {
      tone: "danger",
      message: "This forge has no snapshots to drill.",
    });
  try {
    const d = await svc.drill();
      
        sentinelcrew
        1mo ago
        
      
      This waits up to DRILL_TIMEOUT_MS (10 minutes) inside the request. The proxy will likely time out first, the admin will likely click again, and with no single-flight that starts a second restore. Once drill() is single-flight, the retry joins the drill already running, which is acceptable for this step.
    
    return flashTo(
      ctx,
      "/settings/backups",
      d.verdict === "green"
        ? { message: "The drill restored green." }
        : { tone: "danger", message: `The drill failed at ${d.failedStep}.` },
    );
  } catch (error) {
    return flashTo(ctx, "/settings/backups", {
      tone: "danger",
      message: `The drill could not run: ${String(error)}`,
    });
  }
}
@@ -0,0 +1,140 @@
/**
 * The drill on the Backups page: an admin's POST runs a (stubbed) drill and
 * keeps it as a row, and the page shows the last one in a line; a non-admin
 * gets 403 and nothing runs; a cross-origin POST is refused.
 */
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { createUser } from "../src/core/users.ts";
import { serve, type Servers } from "../src/http/server.ts";
import type { DrillResult } from "../src/snapshot/drill.ts";
import { startSnapshots, type SnapshotsHandle } from "../src/snapshot/service.ts";
import { dirStore } from "../src/snapshot/store.ts";
import { quietLogs } from "./helpers/globals.ts";
quietLogs();
describe("the drill on the Backups page", () => {
  let root = "";
  let app: App;
  let svc: SnapshotsHandle;
  let servers: Servers;
  let ran = 0;
  let gate: Promise<void> | undefined;
  let verdict: "green" | "red" = "green";
  const login = async (username: string, password: string) => {
    const r = await fetch(`${servers.url}/signin`, {
      method: "POST",
      headers: { origin: servers.url },
      body: new URLSearchParams({ username, password }),
      redirect: "manual",
    });
    return (r.headers.get("set-cookie") ?? "").split(";")[0]!;
  };
  beforeAll(async () => {
    root = mkdtempSync(join(tmpdir(), "forge-drillpage-"));
    app = createApp({ dataDir: root, port: 0 });
    for (const [username, password, adminIfFirst] of [
      ["alice", "a-good-passphrase", true],
      ["bob", "another-good-passphrase", false],
    ] as const) {
      const u = await createUser(app.db, {
        trusted: true,
        username,
        email: `${username}@forge.local`,
        password,
        adminIfFirst,
      });
      if (!u.ok) throw new Error(u.error);
    }
    svc = startSnapshots(app, dirStore(join(root, "store")), {
      version: "t",
      nightly: false,
      drill: async (): Promise<DrillResult> => {
        ran++;
        await gate;
        return {
          at: new Date().toISOString(),
          snapshotId: "snap-stub-1",
          ms: 1234,
          counts: { repositories: 7, issues: 42 },
          verdict,
          failedStep: verdict === "green" ? null : "unseal",
          reason: verdict === "green" ? null : "a sealed value does not open",
        };
      },
    });
    servers = await serve(app);
  });
  afterAll(async () => {
    servers.stop();
    await svc.stop();
    app.close();
    rmSync(root, { recursive: true, force: true });
  });
  const post = (cookie: string, origin = servers.url) =>
    fetch(`${servers.url}/settings/backups/drill`, {
      method: "POST",
      headers: { cookie, origin },
      redirect: "manual",
    });
  const page = async (cookie: string) =>
    (await fetch(`${servers.url}/settings/backups`, { headers: { cookie } })).text();
  test("an admin runs a drill and the line shows it", async () => {
    const cookie = await login("alice", "a-good-passphrase");
    const before = await page(cookie);
    expect(before).toContain("No restore drill has run yet");
    expect(before).toContain("Run a drill");
    expect((await post(cookie)).status).toBe(303);
    expect(ran).toBe(1);
    const html = await page(cookie);
    expect(html).toContain("snap-stub-1");
    expect(html).toContain("7 repositories, 42 issues");
    expect(html).toContain("1234 ms");
    expect(app.db.query("SELECT count(*) AS n FROM drill").get()).toEqual({ n: 1 });
  });
  test("a red drill names the failing step", async () => {
    const cookie = await login("alice", "a-good-passphrase");
    verdict = "red";
    await post(cookie);
    const html = await page(cookie);
    expect(html).toContain("failed at");
    expect(html).toContain("unseal");
    expect(app.db.query("SELECT count(*) AS n FROM drill").get()).toEqual({ n: 2 });
  });
  test("a non-admin is refused, and so is a cross-origin post", async () => {
    const before = ran;
    const bob = await login("bob", "another-good-passphrase");
    expect((await post(bob)).status).toBe(403);
    const alice = await login("alice", "a-good-passphrase");
    expect((await post(alice, "https://evil.example")).status).toBeGreaterThanOrEqual(400);
    expect(ran).toBe(before);
  });
  test("two drills at once are one restore", async () => {
    const alice = await login("alice", "a-good-passphrase");
    let release!: () => void;
    gate = new Promise<void>((r) => (release = r));
    const before = ran;
    const rows = () =>
      (app.db.query("SELECT count(*) AS n FROM drill").get() as { n: number }).n;
    const rowsBefore = rows();
    const both = Promise.all([post(alice), post(alice)]);
    await Bun.sleep(200);
    release();
    const [a, b] = await both;
    gate = undefined;
    expect([a.status, b.status]).toEqual([303, 303]);
    expect(ran - before).toBe(1);
    expect(rows() - rowsBefore).toBe(1);
  });
});
		@@ -442,6 +442,11 @@ huge one); restore unpacks them. While one runs, progress shows on
442	442	Settings → Backups and as a line in the log every 500 objects or minute.
443	443	The terminal has no backups view, so it has no readout.
444	444
	445	Run a drill on the same page (admins only) restores the newest snapshot
	446	into scratch, opens it, counts what is there and unseals a value; the page
	447	waits for it, up to 10 minutes, and the last drill shows there as one line.
	448	Only one runs at a time. See `docs/SNAPSHOT.md`.
	449
445	450	`docs/EXTRACT.md` is the forge on a machine of its own, and the platform it
446	451	grew up in switched off: what a host owes it, the moves in order, and what is
447	452	given up.
		@@ -70,6 +70,20 @@ Delete the old ones from the bucket if the bucket is not trusted.
70	70	The store is `Bun.S3Client` or a directory behind one small interface
71	71	(`src/snapshot/store.ts`): put, get, list, delete. Nothing else.
72	72
	73	## The restore drill
	74
	75	A backup nobody has restored is a hope. Settings → Backups → Run a drill
	76	(admins only; anyone else gets 403) restores the newest snapshot into a
	77	scratch directory under the data directory, opens the database read-only,
	78	counts repositories and issues, opens one sealed value with the restored key,
	79	and removes the scratch whatever happened. The page waits until it finishes,
	80	for up to 10 minutes; a step that hangs past that is red and names the step.
	81	One drill runs at a time: a second click, or a retry after a proxy timeout,
	82	joins the drill already running rather than starting another restore. Every
	83	run is a row in the `drill` table, and the last one shows as a line on the
	84	Backups page: when, which snapshot, how long, the counts, and green or the
	85	step that failed.
	86
73	87	## Restore
74	88
75	89	The printable card, seven lines, is [RESTORE.md](RESTORE.md); a test runs it.
		@@ -132,6 +132,7 @@ export const GATE: readonly string[] = [
132	132	"test/helpers.test.ts",
133	133	"test/shards.test.ts",
134	134	"test/snapshot-drill.test.ts",
	135	"test/snapshot-drill-page.test.ts",
135	136	"test/snapshot-store.test.ts",
136	137	"test/migrate.test.ts",
137	138	"test/schema-additive.test.ts",
		@@ -1655,6 +1655,21 @@ export const migrations: readonly string[] = [
1655	1655	CREATE UNIQUE INDEX degraded_span_open ON degraded_span(cause) WHERE closed_at IS NULL;
1656	1656	CREATE INDEX degraded_span_closed ON degraded_span(closed_at) WHERE closed_at IS NOT NULL;
1657	1657	`,
	1658	// 80 — restore drills (#1668): one row per run of `snapshot/drill.ts`,
	1659	// kept so the Backups page can show the last. `at` is an ISO time.
	1660	`
	1661	CREATE TABLE drill (
	1662	id INTEGER PRIMARY KEY AUTOINCREMENT,
	1663	at TEXT NOT NULL,
	1664	snapshot_id TEXT,
	1665	ms INTEGER NOT NULL,
	1666	repositories INTEGER NOT NULL,
	1667	issues INTEGER NOT NULL,
	1668	verdict TEXT NOT NULL CHECK (verdict IN ('green', 'red')),
	1669	failed_step TEXT,
	1670	reason TEXT
	1671	);
	1672	`,
1658	1673	];
1659	1674
1660	1675	/**
		@@ -96,6 +96,7 @@ import { secretDelete, secretSet, secretsPage } from "../web/pages/secrets.ts";
96	96	import {
97	97	backupsClear,
98	98	backupsDestination,
	99	backupsDrill,
99	100	backupsNow,
100	101	backupsPage,
101	102	backupsTest,
		@@ -671,6 +672,7 @@ export function buildRouter(): Router<Handler> {
671	672	router.add("POST", "/admin/limits", limitsSet);
672	673	router.add("GET", "/settings/backups", backupsPage);
673	674	router.add("POST", "/settings/backups/now", backupsNow);
	675	router.add("POST", "/settings/backups/drill", backupsDrill);
674	676	router.add("POST", "/settings/backups/destination", backupsDestination);
675	677	router.add("POST", "/settings/backups/test", backupsTest);
676	678	router.add("POST", "/settings/backups/clear", backupsClear);
		@@ -0,0 +1,53 @@
	1	/**
	2	* The drills, kept: every run of `runDrill` is a row, so the Backups page
	3	* can say when the forge last proved a snapshot restores, and what failed
	4	* when it did not. Plain data in, plain data out.
	5	*/
	6
	7	import type { Database } from "bun:sqlite";
	8	import type { DrillResult, DrillStep } from "./drill.ts";
	9
	10	interface Row {
	11	at: string;
	12	snapshot_id: string \| null;
	13	ms: number;
	14	repositories: number;
	15	issues: number;
	16	verdict: "green" \| "red";
	17	failed_step: DrillStep \| null;
	18	reason: string \| null;
	19	}
	20
	21	export function recordDrill(db: Database, result: DrillResult): void {
	22	db.query(
	23	`INSERT INTO drill (at, snapshot_id, ms, repositories, issues, verdict, failed_step, reason)
	24	VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
	25	).run(
	26	result.at,
	27	result.snapshotId,
	28	result.ms,
	29	result.counts.repositories,
	30	result.counts.issues,
	31	result.verdict,
	32	result.failedStep,
	33	result.reason,
	34	);
	35	}
	36
	37	/** The newest drill, or undefined when none has run. */
	38	export function lastDrill(db: Database): DrillResult \| undefined {
	39	const row = db
	40	.query<Row, []>("SELECT * FROM drill ORDER BY id DESC LIMIT 1")
	41	.get();
	42	return row === null
	43	? undefined
	44	: {
	45	at: row.at,
	46	snapshotId: row.snapshot_id,
	47	ms: row.ms,
	48	counts: { repositories: row.repositories, issues: row.issues },
	49	verdict: row.verdict,
	50	failedStep: row.failed_step,
	51	reason: row.reason,
	52	};
	53	}
		@@ -18,6 +18,8 @@ import {
18	18	type Manifest,
19	19	type Progress,
20	20	} from "./snapshot.ts";
	21	import { runDrill, type DrillDeps, type DrillResult } from "./drill.ts";
	22	import { recordDrill } from "./drills.ts";
21	23	import type { Store } from "./store.ts";
22	24
23	25	const log = createLog("snapshots");
		@@ -38,6 +40,8 @@ export interface SnapshotsHandle {
38	40	readonly list: () => Promise<Manifest[]>;
39	41	/** The last one taken by this process, and what it said. */
40	42	readonly last: () => { manifest?: Manifest; error?: string; at?: string };
	43	/** Drill the newest snapshot now and keep the result; waits for any snapshot in flight. */
	44	readonly drill: () => Promise<DrillResult>;
41	45	/** Forget the resolved store; the next use resolves again (a new destination was saved). */
42	46	readonly forget: () => void;
43	47	readonly stop: () => Promise<void>;
		@@ -61,6 +65,8 @@ export function startSnapshots(
61	65	readonly version: string;
62	66	readonly nightly?: boolean;
63	67	readonly passphrase?: string \| undefined;
	68	/** Test seam: the drill itself. */
	69	readonly drill?: ((deps: DrillDeps) => Promise<DrillResult>) \| undefined;
64	70	} = {
65	71	version: "0",
66	72	},
		@@ -118,6 +124,28 @@ export function startSnapshots(
118	124	return inFlight;
119	125	};
120	126
	127	// Single-flight: a restore of every repository is heavy, so a second call
		sentinelcrew 1mo ago outdated Nothing stops a second call from starting while the first is still running, so two clicks mean two full restores side by side. Hold the running promise, the way `take` holds `inFlight`, and give it to any caller that arrives while it runs. Clear it in a `finally`.
	128	// while one runs joins it instead of filling the disk with another.
	129	let drilling: Promise<DrillResult> \| undefined;
	130	const drill = (): Promise<DrillResult> => {
	131	drilling ??= (async (): Promise<DrillResult> => {
	132	await (inFlight ?? Promise.resolve()).catch(() => {});
	133	const store = current();
	134	if (store === undefined)
	135	throw new Error("this forge has nowhere to drill from");
	136	const result = await (options.drill ?? runDrill)({
	137	store,
	138	dataDir: app.config.dataDir,
	139	passphrase: options.passphrase,
	140	});
	141	recordDrill(app.db, result);
	142	return result;
	143	})().finally(() => {
	144	drilling = undefined;
	145	});
	146	return drilling;
	147	};
	148
121	149	let timer: ReturnType<typeof setTimeout> \| undefined;
122	150	const schedule = (): void => {
123	151	if (options.nightly === false \|\| stopped) return;
		@@ -135,6 +163,7 @@ export function startSnapshots(
135	163	},
136	164	wraps: !!options.passphrase,
137	165	now: take,
	166	drill,
138	167	list: async () => {
139	168	const store = current();
140	169	return store === undefined ? [] : listSnapshots(store);
		@@ -16,6 +16,7 @@ import {
16	16	} from "../../actions/destination.ts";
17	17	import { resolveDestination, sealedSet } from "../../snapshot/destination.ts";
18	18	import { keyOf } from "../../snapshot/snapshot.ts";
	19	import { lastDrill } from "../../snapshot/drills.ts";
19	20	import { snapshotsOf } from "../../snapshot/service.ts";
20	21	import type { Ctx } from "../../http/context.ts";
21	22	import { notFoundPage, settingsLayout, signInFirst } from "../chrome.ts";
		@@ -85,6 +86,31 @@ function keyNote(
85	86	</p>`;
86	87	}
87	88
	89	/** The last drill in one line, and the button to run another. */
	90	function drillLine(ctx: Ctx): Html {
	91	const d = lastDrill(ctx.app.db);
	92	const line =
	93	d === undefined
	94	? html`<p class="backups-drill">No restore drill has run yet.</p>`
	95	: d.verdict === "green"
	96	? html`<p class="backups-drill">
	97	${icon("check")} Last drill ${time(d.at)}: snapshot
	98	<code>${d.snapshotId ?? ""}</code> restored green in ${d.ms} ms —
	99	${d.counts.repositories} repositories, ${d.counts.issues} issues.
	100	</p>`
	101	: html`<p class="backups-error backups-drill">
	102	${icon("alert")} Last drill ${time(d.at)}: snapshot
	103	<code>${d.snapshotId ?? "none"}</code> failed at
	104	${d.failedStep ?? "an unknown step"} after ${d.ms} ms${d.reason
	105	? html`: ${d.reason}`
	106	: ""}.
	107	</p>`;
	108	return html`${line}
	109	<form method="post" action="/settings/backups/drill" class="backups-now">
	110	${btn("Run a drill", { type: "submit", icon: "clock" })}
	111	</form>`;
	112	}
	113
88	114	export async function backupsPage(ctx: Ctx): Promise<Response> {
89	115	const denied = requireAdmin(ctx);
90	116	if (denied) return denied;
		@@ -147,6 +173,7 @@ bun run src/snapshot/restore.ts ${snapshots[0].name} /path/to/empty/data</pre>`
147	173	${icon("alert")} The store could not be listed just now: ${error}
148	174	</p>`,
149	175	)}
	176	${drillLine(ctx)}
150	177	<form method="post" action="/settings/backups/now" class="backups-now">
151	178	${btn("Snapshot now", {
152	179	type: "submit",
		@@ -367,3 +394,36 @@ export async function backupsNow(ctx: Ctx): Promise<Response> {
367	394	});
368	395	}
369	396	}
	397
	398	/** POST /settings/backups/drill — restore the newest snapshot, keep what came of it. */
	399	export async function backupsDrill(ctx: Ctx): Promise<Response> {
	400	if (!ctx.auth.user) return signInFirst(ctx);
	401	if (!ctx.auth.user.isAdmin)
	402	return new Response("Forbidden", {
	403	status: 403,
	404	headers: { "content-type": "text/plain; charset=utf-8" },
	405	});
	406	const form = await readForm(ctx, "admin");
	407	if (form instanceof Response) return form;
	408	const svc = snapshotsOf(ctx.app);
	409	if (svc === undefined \|\| svc.store === undefined)
	410	return flashTo(ctx, "/settings/backups", {
	411	tone: "danger",
	412	message: "This forge has no snapshots to drill.",
	413	});
	414	try {
	415	const d = await svc.drill();
		sentinelcrew 1mo ago This waits up to `DRILL_TIMEOUT_MS` (10 minutes) inside the request. The proxy will likely time out first, the admin will likely click again, and with no single-flight that starts a second restore. Once `drill()` is single-flight, the retry joins the drill already running, which is acceptable for this step.
	416	return flashTo(
	417	ctx,
	418	"/settings/backups",
	419	d.verdict === "green"
	420	? { message: "The drill restored green." }
	421	: { tone: "danger", message: `The drill failed at ${d.failedStep}.` },
	422	);
	423	} catch (error) {
	424	return flashTo(ctx, "/settings/backups", {
	425	tone: "danger",
	426	message: `The drill could not run: ${String(error)}`,
	427	});
	428	}
	429	}
		@@ -0,0 +1,140 @@
	1	/**
	2	* The drill on the Backups page: an admin's POST runs a (stubbed) drill and
	3	* keeps it as a row, and the page shows the last one in a line; a non-admin
	4	* gets 403 and nothing runs; a cross-origin POST is refused.
	5	*/
	6	import { afterAll, beforeAll, describe, expect, test } from "bun:test";
	7	import { mkdtempSync, rmSync } from "node:fs";
	8	import { tmpdir } from "node:os";
	9	import { join } from "node:path";
	10	import { createApp, type App } from "../src/app.ts";
	11	import { createUser } from "../src/core/users.ts";
	12	import { serve, type Servers } from "../src/http/server.ts";
	13	import type { DrillResult } from "../src/snapshot/drill.ts";
	14	import { startSnapshots, type SnapshotsHandle } from "../src/snapshot/service.ts";
	15	import { dirStore } from "../src/snapshot/store.ts";
	16	import { quietLogs } from "./helpers/globals.ts";
	17
	18	quietLogs();
	19
	20	describe("the drill on the Backups page", () => {
	21	let root = "";
	22	let app: App;
	23	let svc: SnapshotsHandle;
	24	let servers: Servers;
	25	let ran = 0;
	26	let gate: Promise<void> \| undefined;
	27	let verdict: "green" \| "red" = "green";
	28
	29	const login = async (username: string, password: string) => {
	30	const r = await fetch(`${servers.url}/signin`, {
	31	method: "POST",
	32	headers: { origin: servers.url },
	33	body: new URLSearchParams({ username, password }),
	34	redirect: "manual",
	35	});
	36	return (r.headers.get("set-cookie") ?? "").split(";")[0]!;
	37	};
	38
	39	beforeAll(async () => {
	40	root = mkdtempSync(join(tmpdir(), "forge-drillpage-"));
	41	app = createApp({ dataDir: root, port: 0 });
	42	for (const [username, password, adminIfFirst] of [
	43	["alice", "a-good-passphrase", true],
	44	["bob", "another-good-passphrase", false],
	45	] as const) {
	46	const u = await createUser(app.db, {
	47	trusted: true,
	48	username,
	49	email: `${username}@forge.local`,
	50	password,
	51	adminIfFirst,
	52	});
	53	if (!u.ok) throw new Error(u.error);
	54	}
	55	svc = startSnapshots(app, dirStore(join(root, "store")), {
	56	version: "t",
	57	nightly: false,
	58	drill: async (): Promise<DrillResult> => {
	59	ran++;
	60	await gate;
	61	return {
	62	at: new Date().toISOString(),
	63	snapshotId: "snap-stub-1",
	64	ms: 1234,
	65	counts: { repositories: 7, issues: 42 },
	66	verdict,
	67	failedStep: verdict === "green" ? null : "unseal",
	68	reason: verdict === "green" ? null : "a sealed value does not open",
	69	};
	70	},
	71	});
	72	servers = await serve(app);
	73	});
	74	afterAll(async () => {
	75	servers.stop();
	76	await svc.stop();
	77	app.close();
	78	rmSync(root, { recursive: true, force: true });
	79	});
	80
	81	const post = (cookie: string, origin = servers.url) =>
	82	fetch(`${servers.url}/settings/backups/drill`, {
	83	method: "POST",
	84	headers: { cookie, origin },
	85	redirect: "manual",
	86	});
	87	const page = async (cookie: string) =>
	88	(await fetch(`${servers.url}/settings/backups`, { headers: { cookie } })).text();
	89
	90	test("an admin runs a drill and the line shows it", async () => {
	91	const cookie = await login("alice", "a-good-passphrase");
	92	const before = await page(cookie);
	93	expect(before).toContain("No restore drill has run yet");
	94	expect(before).toContain("Run a drill");
	95	expect((await post(cookie)).status).toBe(303);
	96	expect(ran).toBe(1);
	97	const html = await page(cookie);
	98	expect(html).toContain("snap-stub-1");
	99	expect(html).toContain("7 repositories, 42 issues");
	100	expect(html).toContain("1234 ms");
	101	expect(app.db.query("SELECT count(*) AS n FROM drill").get()).toEqual({ n: 1 });
	102	});
	103
	104	test("a red drill names the failing step", async () => {
	105	const cookie = await login("alice", "a-good-passphrase");
	106	verdict = "red";
	107	await post(cookie);
	108	const html = await page(cookie);
	109	expect(html).toContain("failed at");
	110	expect(html).toContain("unseal");
	111	expect(app.db.query("SELECT count(*) AS n FROM drill").get()).toEqual({ n: 2 });
	112	});
	113
	114	test("a non-admin is refused, and so is a cross-origin post", async () => {
	115	const before = ran;
	116	const bob = await login("bob", "another-good-passphrase");
	117	expect((await post(bob)).status).toBe(403);
	118	const alice = await login("alice", "a-good-passphrase");
	119	expect((await post(alice, "https://evil.example")).status).toBeGreaterThanOrEqual(400);
	120	expect(ran).toBe(before);
	121	});
	122
	123	test("two drills at once are one restore", async () => {
	124	const alice = await login("alice", "a-good-passphrase");
	125	let release!: () => void;
	126	gate = new Promise<void>((r) => (release = r));
	127	const before = ran;
	128	const rows = () =>
	129	(app.db.query("SELECT count(*) AS n FROM drill").get() as { n: number }).n;
	130	const rowsBefore = rows();
	131	const both = Promise.all([post(alice), post(alice)]);
	132	await Bun.sleep(200);
	133	release();
	134	const [a, b] = await both;
	135	gate = undefined;
	136	expect([a.status, b.status]).toEqual([303, 303]);
	137	expect(ran - before).toBe(1);
	138	expect(rows() - rowsBefore).toBe(1);
	139	});
	140	});