trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRuns19g rEventsFlows

restoreSnapshot reports which phase it is in and how far it has got #2619

Merged scoutcrew opened this

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

scout/forge:scout/2616-restoresnapshot-reports-which-phase-it-i → main 1 commit fork

  • Merged. scout/2616-restoresnapshot-reports-which-phase-it-i was merged into main as d61b359.

Review

Sign in to review this change.
2 files changed +109 −4
src/snapshot/restore.ts +48 −4
@@ -56,6 +56,14 @@ export interface Restored {
5656 readonly sealedMissing: readonly string[];
5757 }
5858
59 /** Where a restore is, as plain data: `done` of `total` in `phase`, `bytes` so far in it. */
60 export interface Progress {
61 readonly phase: "manifest" | "files" | "unpack" | "bundles" | "key";
62 readonly done: number;
63 readonly total: number;
64 readonly bytes: number;
65 }
66
5967 /** Repositories cloned at once while restoring. */
6068 export const CLONES = 4;
6169
@@ -144,6 +152,8 @@ export async function restoreSnapshot(
144152 readonly inFlightBytes?: number | undefined;
145153 /** Test seam: the git runner the clones go through. */
146154 readonly git?: typeof runGit | undefined;
155 /** Told when each phase starts and after each file or bundle finishes. */
156 readonly onProgress?: ((p: Progress) => void) | undefined;
147157 } = {},
148158 ): Promise<Restored> {
149159 if (!/^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}-[0-9]{2}-[0-9]{2}Z$/.test(name))
@@ -152,11 +162,15 @@ export async function restoreSnapshot(
152162 throw new Error(
153163 `${dataDir} is not empty; a restore needs an empty data directory`,
154164 );
165 const say = (phase: Progress["phase"], done: number, total: number, bytes: number): void =>
166 options.onProgress?.({ phase, done, total, bytes });
155167 const prefix = `${SNAPSHOT_PREFIX}${name}/`;
168 say("manifest", 0, 1, 0);
156169 const raw = await store.get(`${prefix}manifest.json`);
157170 if (raw === undefined)
158171 throw new Error(`no snapshot ${name} in ${store.where}`);
159172 const manifest = JSON.parse(new TextDecoder().decode(raw)) as Manifest;
173 say("manifest", 1, 1, raw.byteLength);
160174 const wrapped = manifest.files.some((f) => f.path === WRAPPED_KEY)
161175 ? await store.get(prefix + WRAPPED_KEY)
162176 : undefined;
@@ -183,7 +197,13 @@ export async function restoreSnapshot(
183197 let files = 0;
184198 let repos = 0;
185199 const bundles: { at: number; owner: string; name: string; file: string }[] = [];
200 let fetched = 0;
201 let fetchedBytes = 0;
202 say("files", 0, manifest.files.length, 0);
186203 await fetchAll(store, prefix, manifest.files, async (entry, body, at) => {
204 fetched++;
205 fetchedBytes += entry.bytes;
206 say("files", fetched, manifest.files.length, fetchedBytes);
187207 if (entry.path === WRAPPED_KEY) return;
188208 // A part waits for the rest of its file.
189209 const part = partOf.get(entry.path);
@@ -220,6 +240,9 @@ export async function restoreSnapshot(
220240 }, name, options.inFlightBytes ?? IN_FLIGHT_BYTES);
221241 // Reads land in any order; repositories come back in the manifest's.
222242 bundles.sort((a, b) => a.at - b.at);
243 let unpacked = 0;
244 let unpackedBytes = 0;
245 say("unpack", 0, packed.length, 0);
223246 for (const file of packed) {
224247 const got = gathered.get(file.path) ?? [];
225248 const parts = partPaths(file).map((path, n) => {
@@ -231,13 +254,21 @@ export async function restoreSnapshot(
231254 gathered.delete(file.path);
232255 const target = join(dataDir, file.path);
233256 mkdirSync(dirname(target), { recursive: true });
234 await Bun.write(target, await unpack(file, parts));
257 const whole = await unpack(file, parts);
258 await Bun.write(target, whole);
235259 files++;
260 unpacked++;
261 unpackedBytes += whole.byteLength;
262 say("unpack", unpacked, packed.length, unpackedBytes);
236263 }
237264 // Each repository is its own clone, so several at once: a forge of
238265 // hundreds of them spent the drill's minutes here, one after another.
239266 const git = options.git ?? runGit;
240267 let next = 0;
268 let cloned = 0;
269 let clonedBytes = 0;
270 const sizes = new Map(manifest.files.map((f) => [f.path, f.bytes]));
271 say("bundles", 0, bundles.length, 0);
241272 // Clones finish in any order; a refusal names the earliest bundle in the
242273 // manifest that failed, so it reads the same however the clones raced.
243274 const failed: { at: number; error: string }[] = [];
@@ -246,18 +277,18 @@ export async function restoreSnapshot(
246277 for (let b = bundles[next++]; b !== undefined; b = bundles[next++]) {
247278 const dir = repoDir(join(dataDir, "repos"), b.owner, b.name);
248279 mkdirSync(dirname(dir), { recursive: true });
249 const cloned = await git([
280 const result = await git([
250281 "clone",
251282 "--quiet",
252283 "--mirror",
253284 b.file,
254285 dir,
255286 ]);
256 if (!cloned.ok) {
287 if (!result.ok) {
257288 next = bundles.length;
258289 failed.push({
259290 at: b.at,
260 error: `could not restore ${b.owner}/${b.name}: ${cloned.error.stderr}`,
291 error: `could not restore ${b.owner}/${b.name}: ${result.error.stderr}`,
261292 });
262293 return;
263294 }
@@ -267,6 +298,9 @@ export async function restoreSnapshot(
267298 await git(["config", "core.logAllRefUpdates", "true"], { cwd: dir });
268299 await installHooks(dir);
269300 repos++;
301 cloned++;
302 clonedBytes += sizes.get(`repos/${b.owner}/${b.name}.bundle`) ?? 0;
303 say("bundles", cloned, bundles.length, clonedBytes);
270304 }
271305 }),
272306 );
@@ -274,11 +308,13 @@ export async function restoreSnapshot(
274308 throw new Error(failed.sort((a, b) => a.at - b.at)[0]!.error);
275309 const { rmSync } = await import("node:fs");
276310 rmSync(join(dataDir, "restore-bundles"), { recursive: true, force: true });
311 say("key", 0, key === undefined ? 0 : 1, 0);
277312 if (key !== undefined) {
278313 const target = join(dataDir, "secret.key");
279314 await Bun.write(target, key);
280315 chmodSync(target, 0o600);
281316 files++;
317 say("key", 1, 1, key.byteLength);
282318 }
283319 const kind = keyOf(manifest);
284320 const have =
@@ -313,8 +349,16 @@ if (import.meta.main) {
313349 console.error(`no snapshot in ${store.where}`);
314350 process.exit(1);
315351 }
352 // One line per phase when it ends: its count, bytes and seconds.
353 let began = performance.now();
316354 const done = await restoreSnapshot(store, newest?.name ?? name, dataDir, {
317355 passphrase: process.env["FORGE_SNAPSHOT_PASSPHRASE"] || undefined,
356 onProgress: (p) => {
357 if (p.done < p.total) return;
358 const seconds = ((performance.now() - began) / 1000).toFixed(1);
359 console.log(`${p.phase}: ${p.done}/${p.total}, ${p.bytes} bytes, ${seconds}s`);
360 began = performance.now();
361 },
318362 });
319363 console.log(
320364 `restored ${done.manifest.name} (forge ${done.manifest.version}): ${done.files} files, ${done.repos} repositories into ${dataDir}`,
test/snapshot-restore-progress.test.ts added +61
@@ -0,0 +1,61 @@
1 /**
2 * A restore says where it is (#2616): the drill that timed out at ten
3 * minutes could not tell fetching from unpacking from cloning.
4 */
5
6 import { expect, test } from "bun:test";
7 import { mkdtempSync } from "node:fs";
8 import { rm } from "node:fs/promises";
9 import { tmpdir } from "node:os";
10 import { join } from "node:path";
11 import { ok } from "../src/kernel/result.ts";
12 import { restoreSnapshot, type Progress } from "../src/snapshot/restore.ts";
13 import type { Store } from "../src/snapshot/store.ts";
14
15 const NAME = "2026-10-19T08-00-00Z";
16 const prefix = `snapshots/${NAME}/`;
17
18 function memory(objects: Map<string, Uint8Array>): Store {
19 return {
20 where: "memory",
21 get: async (path: string) => objects.get(path),
22 } as unknown as Store;
23 }
24
25 test("progress runs through the phases with done rising to total and bytes adding up", async () => {
26 const objects = new Map<string, Uint8Array>();
27 const enc = new TextEncoder();
28 const paths = ["a.txt", "b.txt", "c.txt", "repos/u/one.bundle", "repos/u/two.bundle"];
29 const files = paths.map((path, n) => {
30 const body = enc.encode("x".repeat(10 * (n + 1)));
31 objects.set(prefix + path, body);
32 return { path, bytes: body.byteLength };
33 });
34 objects.set(
35 `${prefix}manifest.json`,
36 enc.encode(
37 JSON.stringify({ name: NAME, startedAt: "t", finishedAt: "t", version: "t", repos: 2, apps: 0, files, bytes: 150, key: "raw" }),
38 ),
39 );
40 const root = mkdtempSync(join(tmpdir(), "forge-restore-progress-"));
41 const seen: Progress[] = [];
42 const git = (async () => ok("")) as unknown as typeof import("../src/git/env.ts").runGit;
43 try {
44 await restoreSnapshot(memory(objects), NAME, join(root, "data"), {
45 git,
46 onProgress: (p) => seen.push(p),
47 });
48 } finally {
49 await rm(root, { recursive: true, force: true });
50 }
51 const order = [...new Set(seen.map((p) => p.phase))];
52 expect(order).toEqual(["manifest", "files", "unpack", "bundles", "key"]);
53 const of = (phase: string) => seen.filter((p) => p.phase === phase);
54 expect(of("files").map((p) => p.done)).toEqual([0, 1, 2, 3, 4, 5]);
55 expect(of("files").every((p) => p.total === 5)).toBe(true);
56 expect(of("files").at(-1)!.bytes).toBe(files.reduce((s, f) => s + f.bytes, 0));
57 expect(of("bundles").map((p) => p.done)).toEqual([0, 1, 2]);
58 expect(of("bundles").at(-1)).toEqual({ phase: "bundles", done: 2, total: 2, bytes: 40 + 50 });
59 expect(of("unpack")).toEqual([{ phase: "unpack", done: 0, total: 0, bytes: 0 }]);
60 expect(of("key")).toEqual([{ phase: "key", done: 0, total: 0, bytes: 0 }]);
61 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs