Public: anyone can read this; the crew can change it.
Sentinel: drill was degraded 175 minutes today: The restore drill failed at restore: restore timed out: no answer in 596591.192678988 ms. Snapshot 2026-10-14T08-00-00Z… #2544
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/2543-sentinel-drill-was-degraded-175-minutes → main 2 commits fork 2 threads · 2 unresolved
3 files
changed +147
−15
docs/SNAPSHOT.md +1
| @@ -133,6 +133,7 @@ The restore reads several objects at a time, under the same limits as the | ||
| 133 | 133 | upload (`IN_FLIGHT` gets, `IN_FLIGHT_BYTES` held): read one after another, a |
| 134 | 134 | snapshot of thousands of objects spent the drill's whole 10 minutes in the |
| 135 | 135 | restore step (#2539). |
| 136 | Repositories come back four clones at a time, not one after another (#2543). | |
| 136 | 137 | One drill runs at a time: a second click, or a retry after a proxy timeout, |
| 137 | 138 | joins the drill already running rather than starting another restore. Every |
| 138 | 139 | run is a row in the `drill` table, and the last one shows as a line on the |
src/snapshot/restore.ts +36 −15
| @@ -56,6 +56,9 @@ export interface Restored { | ||
| 56 | 56 | readonly sealedMissing: readonly string[]; |
| 57 | 57 | } |
| 58 | 58 | |
| 59 | /** Repositories cloned at once while restoring. */ | |
| 60 | export const CLONES = 4; | |
| 61 | ||
| 59 | 62 | /** Names of every sealed value in a restored database; never the values. */ |
| 60 | 63 | async function sealedNames(dbFile: string): Promise<string[]> { |
| 61 | 64 | const { Database } = await import("bun:sqlite"); |
| @@ -139,6 +142,8 @@ export async function restoreSnapshot( | ||
| 139 | 142 | readonly passphrase?: string | undefined; |
| 140 | 143 | /** Bytes held by reads in flight; `IN_FLIGHT_BYTES` unless a test shrinks it. */ |
| 141 | 144 | readonly inFlightBytes?: number | undefined; |
| 145 | /** Test seam: the git runner the clones go through. */ | |
| 146 | readonly git?: typeof runGit | undefined; | |
| 142 | 147 | } = {}, |
| 143 | 148 | ): Promise<Restored> { |
| 144 | 149 | if (!/^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}-[0-9]{2}-[0-9]{2}Z$/.test(name)) |
| @@ -229,21 +234,37 @@ export async function restoreSnapshot( | ||
| 229 | 234 | await Bun.write(target, await unpack(file, parts)); |
| 230 | 235 | files++; |
| 231 | 236 | } |
| 232 | for (const b of bundles) { | |
| 233 | const dir = repoDir(join(dataDir, "repos"), b.owner, b.name); | |
| 234 | mkdirSync(dirname(dir), { recursive: true }); | |
| 235 | const cloned = await runGit(["clone", "--quiet", "--mirror", b.file, dir]); | |
| 236 | if (!cloned.ok) | |
| 237 | throw new Error( | |
| 238 | `could not restore ${b.owner}/${b.name}: ${cloned.error.stderr}`, | |
| 239 | ); | |
| 240 | // A mirror clone remembers the bundle as its origin; the forge's | |
| 241 | // repositories have no origin of their own. | |
| 242 | await runGit(["remote", "remove", "origin"], { cwd: dir }); | |
| 243 | await runGit(["config", "core.logAllRefUpdates", "true"], { cwd: dir }); | |
| 244 | await installHooks(dir); | |
| 245 | repos++; | |
| 246 | } | |
| 237 | // Each repository is its own clone, so several at once: a forge of | |
| 238 | // hundreds of them spent the drill's minutes here, one after another. | |
| 239 | const git = options.git ?? runGit; | |
| 240 | let next = 0; | |
| 241 | await Promise.all( | |
| 242 | Array.from({ length: Math.min(CLONES, bundles.length) }, async () => { | |
| 243 | for (let b = bundles[next++]; b !== undefined; b = bundles[next++]) { | |
| 244 | const dir = repoDir(join(dataDir, "repos"), b.owner, b.name); | |
| 245 | mkdirSync(dirname(dir), { recursive: true }); | |
| 246 | const cloned = await git([ | |
| 247 | "clone", | |
| 248 | "--quiet", | |
| 249 | "--mirror", | |
| 250 | b.file, | |
| 251 | dir, | |
| 252 | ]); | |
| 253 | if (!cloned.ok) { | |
| 254 | next = bundles.length; | |
| 255 | throw new Error( | |
| 256 | `could not restore ${b.owner}/${b.name}: ${cloned.error.stderr}`, | |
| 257 | ); | |
| 258 | } | |
| 259 | // A mirror clone remembers the bundle as its origin; the forge's | |
| 260 | // repositories have no origin of their own. | |
| 261 | await git(["remote", "remove", "origin"], { cwd: dir }); | |
| 262 | await git(["config", "core.logAllRefUpdates", "true"], { cwd: dir }); | |
| 263 | await installHooks(dir); | |
| 264 | repos++; | |
| 265 | } | |
| 266 | }), | |
| 267 | ); | |
| 247 | 268 | const { rmSync } = await import("node:fs"); |
| 248 | 269 | rmSync(join(dataDir, "restore-bundles"), { recursive: true, force: true }); |
| 249 | 270 | if (key !== undefined) { |
test/snapshot-restore-clones.test.ts added +110
| @@ -0,0 +1,110 @@ | ||
| 1 | /** | |
| 2 | * A restore clones its repositories several at a time (#2543): a forge of | |
| 3 | * hundreds spent the drill's minutes cloning one after another. Every | |
| 4 | * repository still comes back, and one bundle git refuses fails the | |
| 5 | * restore by name. | |
| 6 | */ | |
| 7 | ||
| 8 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 9 | import { existsSync, mkdtempSync, readFileSync } from "node:fs"; | |
| 10 | import { rm } from "node:fs/promises"; | |
| 11 | import { tmpdir } from "node:os"; | |
| 12 | import { join } from "node:path"; | |
| 13 | import { runGit } from "../src/git/env.ts"; | |
| 14 | import { CLONES, restoreSnapshot } from "../src/snapshot/restore.ts"; | |
| 15 | import { dirStore, type Store } from "../src/snapshot/store.ts"; | |
| 16 | ||
| 17 | const NAME = "2026-10-18T08-00-00Z"; | |
| 18 | const prefix = `snapshots/${NAME}/`; | |
| 19 | const REPOS = 11; | |
| 20 | ||
| 21 | async function seed(store: Store, root: string, broken?: number): Promise<void> { | |
| 22 | const src = join(root, "src"); | |
| 23 | if (!existsSync(src)) { | |
| 24 | await runGit(["init", "--quiet", src]); | |
| 25 | await Bun.write(join(src, "a.txt"), "hello\n"); | |
| 26 | await runGit(["add", "."], { cwd: src }); | |
| 27 | await runGit(["-c", "user.name=t", "-c", "user.email=t@t", "commit", "--quiet", "-m", "one"], { cwd: src }); | |
| 28 | await runGit(["bundle", "create", join(root, "ok.bundle"), "--all"], { cwd: src }); | |
| 29 | } | |
| 30 | const good = new Uint8Array(await Bun.file(join(root, "ok.bundle")).arrayBuffer()); | |
| 31 | const files: { path: string; bytes: number }[] = []; | |
| 32 | for (let n = 0; n < REPOS; n++) { | |
| 33 | const path = `repos/alice/r${n}.bundle`; | |
| 34 | const body = n === broken ? new TextEncoder().encode("not a bundle") : good; | |
| 35 | await store.put(prefix + path, body); | |
| 36 | files.push({ path, bytes: body.byteLength }); | |
| 37 | } | |
| 38 | await store.put( | |
| 39 | `${prefix}manifest.json`, | |
| 40 | JSON.stringify({ | |
| 41 | name: NAME, | |
| 42 | startedAt: "2026-10-18T08:00:00.000Z", | |
| 43 | finishedAt: "2026-10-18T08:01:00.000Z", | |
| 44 | version: "t", | |
| 45 | repos: REPOS, | |
| 46 | apps: 0, | |
| 47 | files, | |
| 48 | bytes: files.reduce((s, f) => s + f.bytes, 0), | |
| 49 | key: "raw", | |
| 50 | }), | |
| 51 | ); | |
| 52 | } | |
| 53 | ||
| 54 | describe("a restore clones in parallel", () => { | |
| 55 | let root: string; | |
| 56 | beforeAll(() => { | |
| 57 | root = mkdtempSync(join(tmpdir(), "forge-restore-clones-")); | |
| 58 | }); | |
| 59 | afterAll(async () => { | |
| 60 | await rm(root, { recursive: true, force: true }); | |
| 61 | }); | |
| 62 | ||
| 63 | test("every repository comes back with its history", async () => { | |
| 64 | const store = dirStore(join(root, "store-1")); | |
| 65 | await seed(store, root); | |
| 66 | const dataDir = join(root, "data-1"); | |
| 67 | const done = await restoreSnapshot(store, NAME, dataDir); | |
| 68 | expect(done.repos).toBe(REPOS); | |
This test passes with sequential cloning too, so it cannot catch a regression to one at a time. Add an assertion on peak concurrency (clones in flight at once), failing if it is 1 or above 4. |
||
| 69 | for (let n = 0; n < REPOS; n++) { | |
| 70 | const dir = join(dataDir, "repos", "alice", `r${n}.git`); | |
| 71 | expect(existsSync(dir)).toBe(true); | |
| 72 | const log = await runGit(["log", "--format=%s"], { cwd: dir }); | |
| 73 | expect(log.ok && log.value.trim()).toBe("one"); | |
| 74 | } | |
| 75 | expect(existsSync(join(dataDir, "restore-bundles"))).toBe(false); | |
| 76 | expect(readFileSync(join(dataDir, "repos", "alice", "r0.git", "HEAD"), "utf8")).toContain("ref:"); | |
| 77 | }); | |
| 78 | ||
| 79 | test("several clones run at once, never more than CLONES", async () => { | |
| 80 | const store = dirStore(join(root, "store-3")); | |
| 81 | await seed(store, root); | |
| 82 | let open = 0; | |
| 83 | let most = 0; | |
| 84 | const git: typeof runGit = async (args, options) => { | |
| 85 | const cloning = args[0] === "clone"; | |
| 86 | if (cloning) { | |
| 87 | open++; | |
| 88 | most = Math.max(most, open); | |
| 89 | await Bun.sleep(30); | |
| 90 | } | |
| 91 | try { | |
| 92 | return await runGit(args, options); | |
| 93 | } finally { | |
| 94 | if (cloning) open--; | |
| 95 | } | |
| 96 | }; | |
| 97 | const done = await restoreSnapshot(store, NAME, join(root, "data-3"), { git }); | |
| 98 | expect(done.repos).toBe(REPOS); | |
| 99 | expect(CLONES).toBe(4); | |
| 100 | expect(most).toBe(4); | |
| 101 | }); | |
| 102 | ||
| 103 | test("a bundle git refuses fails the restore, by name", async () => { | |
| 104 | const store = dirStore(join(root, "store-2")); | |
| 105 | await seed(store, root, 6); | |
| 106 | await expect(restoreSnapshot(store, NAME, join(root, "data-2"))).rejects.toThrow( | |
| 107 | "could not restore alice/r6", | |
| 108 | ); | |
| 109 | }); | |
| 110 | }); |
new above
offline
This inserts CLONES between the
Names of every sealed valuedoc comment and sealedNames, so that comment now sits on the wrong declaration. Put CLONES and its comment above the sealedNames doc.