Public: anyone can read this; the crew can change it.
docs/RECOVERY.md says what to do for each phase a red drill can name #2629
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/2625-docs-recovery-md-says-what-to-do-for-eac → main 1 commit fork
3 files
changed +68
−2
docs/RECOVERY.md +41
| @@ -69,6 +69,47 @@ a repository with `git clone --mirror <file> <data-dir>/repos/<owner>/<repo>.git | ||
| 69 | 69 | scrypt and AES-256-GCM (`src/snapshot/wrap.ts`): only `restore.ts` opens it, |
| 70 | 70 | so by hand the sealed values are set again. |
| 71 | 71 | |
| 72 | ## When the drill is red | |
| 73 | ||
| 74 | The weekly drill restores the newest snapshot into scratch and names where it | |
| 75 | stopped: a step, and for `restore` a phase inside it. Each entry says what it | |
| 76 | was doing, the likely cause, and the first thing to check. Whatever the step, | |
| 77 | the next move is the same: run Route 1 by hand on a new directory and watch | |
| 78 | which line it stops at. | |
| 79 | ||
| 80 | ### Steps | |
| 81 | ||
| 82 | - **fetch** — listing the store for the newest snapshot. Likely the store is | |
| 83 | unreachable or its keys changed, or no snapshot exists. Check the `S3_*` | |
| 84 | variables and that the store lists the snapshot objects. | |
| 85 | - **restore** — running the restore command; the phase below says where. | |
| 86 | Likely a bad snapshot or a slow store. Check the phase, then Route 1. | |
| 87 | - **open** — opening the restored `forge.db`. Likely the file is truncated or | |
| 88 | corrupt. Check its size against the snapshot's manifest. | |
| 89 | - **count** — counting rows and comparing with the counts the snapshot kept. | |
| 90 | Likely the snapshot was cut mid-write or rows were lost. Check which table | |
| 91 | the message names and its count in the snapshot. | |
| 92 | - **unseal** — opening one sealed value with the restored key. Likely the key | |
| 93 | is missing or from another forge, or the passphrase is wrong. Check | |
| 94 | `FORGE_SNAPSHOT_PASSPHRASE` and that the snapshot holds the key. | |
| 95 | - **health** — booting the restored forge and asking it for health. Likely a | |
| 96 | migration or config problem, or a boot slower than the limit. Boot it by | |
| 97 | hand as in Route 1 and read its log. | |
| 98 | ||
| 99 | ### Restore phases | |
| 100 | ||
| 101 | - **manifest** — reading the snapshot's list of files and bundles. Likely the | |
| 102 | manifest object is missing or damaged. Check the object exists in the store. | |
| 103 | - **files** — fetching the plain files. Likely a slow or failing store. Check | |
| 104 | how far it got against the manifest's file count, then retry. | |
| 105 | - **unpack** — unpacking the fetched files into the data directory. Likely the | |
| 106 | disk is full or the archive is damaged. Check free space. | |
| 107 | - **bundles** — cloning each repository from its bundle. Likely a slow store | |
| 108 | or many repositories. Compare its counts with the snapshot's and retry the | |
| 109 | restore by hand with Route 1. | |
| 110 | - **key** — restoring the secret key, opened with the passphrase. Likely the | |
| 111 | passphrase is missing or wrong. Check `FORGE_SNAPSHOT_PASSPHRASE`. | |
| 112 | ||
| 72 | 113 | ## What a snapshot does NOT hold |
| 73 | 114 | |
| 74 | 115 | - Sealed values, unless the passphrase was set: they stay sealed and are set |
test/recovery-doc.test.ts +2 −2
| @@ -62,8 +62,8 @@ test("what a snapshot lacks is what snapshot.ts leaves out", () => { | ||
| 62 | 62 | expect(doc).toContain("/api/v1/health"); |
| 63 | 63 | }); |
| 64 | 64 | |
| 65 | test("the doc fits one printed side", () => { | |
| 66 | expect(doc.split("\n").length).toBeLessThanOrEqual(80); | |
| 65 | test("the doc fits two printed sides", () => { | |
| 66 | expect(doc.split("\n").length).toBeLessThanOrEqual(125); | |
| 67 | 67 | }); |
| 68 | 68 | |
| 69 | 69 | test("the doc says where the drill's verdicts are posted, and names its steps", () => { |
test/recovery-red-drill.test.ts added +25
| @@ -0,0 +1,25 @@ | ||
| 1 | import { expect, test } from "bun:test"; | |
| 2 | import { readFileSync } from "node:fs"; | |
| 3 | import { join } from "node:path"; | |
| 4 | ||
| 5 | const root = join(import.meta.dir, ".."); | |
| 6 | const read = (p: string) => readFileSync(join(root, p), "utf8"); | |
| 7 | const doc = read("docs/RECOVERY.md"); | |
| 8 | const section = doc.split("## When the drill is red")[1]?.split(/\n## /)[0] ?? ""; | |
| 9 | ||
| 10 | const names = (union: string | undefined): string[] => | |
| 11 | [...(union ?? "").matchAll(/"([a-z]+)"/g)].map((m) => m[1]!); | |
| 12 | ||
| 13 | const steps = names(read("src/snapshot/drill.ts").match(/type DrillStep =([^;]+);/)?.[1]); | |
| 14 | const phases = names(read("src/snapshot/restore.ts").match(/phase: ([^;\n]+);/)?.[1]); | |
| 15 | ||
| 16 | test("the code names steps and phases to look for", () => { | |
| 17 | expect(steps).toContain("fetch"); | |
| 18 | expect(phases).toContain("bundles"); | |
| 19 | }); | |
| 20 | ||
| 21 | test("every drill step and restore phase has an entry in 'When the drill is red'", () => { | |
| 22 | expect(section).not.toBe(""); | |
| 23 | for (const name of [...steps, ...phases]) | |
| 24 | expect(section, `no entry for ${name}`).toMatch(new RegExp(`^- \\*\\*${name}\\*\\* — `, "m")); | |
| 25 | }); |
new above
offline