trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRuns16g rEventsFlows

Schema migrations are identified by name, so two pulls can each add one without colliding #2560

Merged masoncrew opened this in answer to trevatoowner

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

mason/forge:mason/783-schema-migrations-are-identified-by-name → main 3 commits fork

  • Merged. mason/783-schema-migrations-are-identified-by-name was merged into main as abebc75.

Review

Sign in to review this change.
15 files changed +460 −47
CLAUDE.md +6 −3
@@ -254,9 +254,12 @@ flag exists only in the stdio dev runner and is never reachable over SSH.
254254 whose default export is the SQL — never the next number: `0001`–`0091`
255255 are the numbered history, and a shared next number is what parallel pulls
256256 collided on (#1837). Files run in name order, once each, recognised by a
257 hash of their text in `schema_migration` (`kernel/migrate.ts`), so two
258 pulls' migrations land in either order. Never edit an existing migration
259 — an edited one is a new one and runs again. It must only add
257 hash of their text in `schema_migration` (`kernel/migrate.ts`, which also
258 records each name), so two pulls' migrations land in either order, and a
259 database with one this build does not know boots and names it. Never edit
260 an existing migration — an edited one is a new one and runs again; once
261 pinned in `core/migration-pins.json` (`bun scripts/pin-migrations.ts`, run
262 by the release agent, never by a pull) the schema test fails on an edit. It must only add
260263 (`test/schema-additive.test.ts`); the release agent lists its id in
261264 `core/releases.json`.
262265 - **New git read:** put it in `git/read.ts`; reuse the long-lived `cat-file`
docs/ARCHITECTURE.md +3 −2
@@ -176,8 +176,9 @@ NOT NULL column without a default, and `test/schema-additive.test.ts`
176176 refuses one in any migration not shipped by both of the two newest releases
177177 recorded in `src/core/releases.json` (by id: a count, and the stamped ids),
178178 which the release agent bumps when it cuts one.
179 An old process that restarts mid-roll refuses a database ahead of it
180 (`user_version`), as it always has.
179 An old process that restarts mid-roll boots on a database ahead of it and
180 logs, by name, the migrations it does not know (`schema_migration` records
181 each name, #783) — they only add, so it serves as it did.
181182
182183 State transitions use compare-and-swap (`UPDATE … WHERE state = ?`), and every
183184 transition writes its event row in the same transaction. On boot, a
scripts/pin-migrations.ts added +33
@@ -0,0 +1,33 @@
1 /**
2 * `bun scripts/pin-migrations.ts`: pin every migration in
3 * `src/core/migrations/` that `src/core/migration-pins.json` does not
4 * name yet, by its id (`migrationId`). An existing pin is never rewritten:
5 * once pinned, a migration's text is history, and
6 * `test/schema-migrations.test.ts` fails if it changes (#783).
7 *
8 * The release agent runs it when it cuts a release, beside bumping
9 * `src/core/releases.json`, so pulls never touch the file and two of them
10 * adding a migration each never meet in it.
11 */
12
13 import { readFileSync, writeFileSync } from "node:fs";
14 import { join } from "node:path";
15 import { migrationFiles } from "../src/core/schema.ts";
16 import { type MigrationPins, pinBreaks, pinned } from "../src/kernel/migrate.ts";
17
18 const file = join(import.meta.dir, "../src/core/migration-pins.json");
19 const pins = JSON.parse(readFileSync(file, "utf8")) as MigrationPins;
20 const breaks = pinBreaks(migrationFiles, pins);
21 if (breaks.length > 0) {
22 for (const line of breaks) console.error(`pin-migrations: ${line}`);
23 process.exit(1);
24 }
25 const next = pinned(migrationFiles, pins);
26 const added = Object.keys(next).filter((name) => !(name in pins));
27 writeFileSync(file, `${JSON.stringify(next, null, 2)}\n`);
28 // CLI answer for the person at the terminal.
29 console.log(
30 added.length === 0
31 ? "pin-migrations: every migration is already pinned"
32 : `pin-migrations: pinned ${added.join(", ")}`,
33 );
src/app.ts +2 −2
@@ -11,7 +11,7 @@ import type { VerifyHuman } from "./core/turnstile.ts";
1111 import { randomBytes } from "node:crypto";
1212 import { mkdirSync } from "node:fs";
1313 import { join, resolve } from "node:path";
14 import { migrations } from "./core/schema.ts";
14 import { migrationFiles } from "./core/schema.ts";
1515 import { dropLegacyNewAccountGuards } from "./core/schema.ts";
1616 import { EventBus } from "./core/events.ts";
1717 import { follow } from "./core/following.ts";
@@ -212,7 +212,7 @@ export function createApp(config: ForgeConfig): App {
212212 mkdirSync(reposRoot, { recursive: true });
213213 const socketPath = join(dataDir, `forge-${id()}.sock`);
214214
215 const db = openDb(join(dataDir, "forge.db"), migrations);
215 const db = openDb(join(dataDir, "forge.db"), migrationFiles);
216216 // Hand-installed new-account guards become the `guard_new_accounts`
217217 // setting (default off): drop the legacy triggers once, here, on boot.
218218 // Idempotent — a forge without them changes nothing.
src/chaos/worker.ts +2 −2
@@ -4,12 +4,12 @@
44 */
55
66 import { openDb } from "../kernel/db.ts";
7 import { migrations } from "../core/schema.ts";
7 import { migrationFiles } from "../core/schema.ts";
88 import { claimRun, createRun, heartbeatRuns } from "../core/runs.ts";
99
1010 const [dbPath, repoId] = process.argv.slice(2);
1111 if (dbPath === undefined || repoId === undefined) process.exit(2);
12 const db = openDb(dbPath, migrations);
12 const db = openDb(dbPath, migrationFiles);
1313 console.log("ready");
1414 for (;;) {
1515 const run = createRun(db, { repoId, kind: "check" });
src/core/migration-pins.json added +119
@@ -0,0 +1,119 @@
1 {
2 "0001-foundation": "b6c433603a3fe1c2e79b9d66fe9bc3bc67f403d0f815a9f81dc16a00ef245da5",
3 "0002-conversations": "5af40c718d7c91c0340d2e1609ecc2edd69a91373e33d78ea561f4ef4de700e1",
4 "0003-pull-requests": "6a53fb594a0bc023a80948c2cde3500a37a95aafe30a896b157794aee1998094",
5 "0004-forks": "9c323d30f8afb5b5e0b26dece297d88e41c758f21bf8155c9cc32d9df7d61609",
6 "0005-organizations-and-teams": "90ac9619b147b5818762c74e97da40fba00fdfe77071b642ddf4bb45b279f927",
7 "0006-ssh-public-keys": "f5bcf371354ab8e32c71e33c220a5d1e9554b4a6bb5b202922a47a07aa645979",
8 "0007-per-user-interface-preferences": "97d11a5e8f8d14bd1b4c8027201f6d286bf480d61c7d116ef3abcfa87d4824e8",
9 "0008-origins": "ae3db8d70c0890e72362e4c4e246eb4cf34e66354de622338015b5276b22eefb",
10 "0009-oauth-for-origins": "5e8796b669bca4271907987f88fab4f37e230247b5f49465a00ce14fedd1666a",
11 "0010-a-short-link-for-a": "37d793621278ccb06b0f0e3c9c4f5996fddc10ac4532a24c5ed5fdee5273a931",
12 "0011-counts-on-a-list": "a8d5bac07bdbd8eaa77a73e0344dd3b8233fea0791084b247a4d5c85288d5344",
13 "0012-the-indexes-a-busy-forge": "6aa333646f3751566cab4892f957e4aa7ffbfd8750fa3e1b76f514eaa46a6e57",
14 "0013-review-states": "54c0f87c1f3bbbabf131b1722d811f915c536a752d2ebbdcec3a0fccbe22b6d7",
15 "0014-apps": "3508216ce82bf466b5fe9e07bc69dd10c5ddf1d3cbd7e95c1daa7027ab480747",
16 "0015-an-app-s-environment": "8949dbd4c0c38431d02cc9818827b68a3f196265624df9d100afc9a0974f84ad",
17 "0016-a-paused-run": "af46f93cd74f18e640fd0a91e02dac0c31a5a52c4d354bcdb2856e21b23e234e",
18 "0017-checks": "69ca4560ae58d18ee836d835555d436a5ea2a48e79a1467e51b1f937172b7173",
19 "0018-a-run-s-lease": "eca89f69fee4c278d27e8764d08ac49a3e5bdeefc54a29e22098a7110daaac2a",
20 "0019-how-many-turns-an-agent": "1217cffaad87574c9b786de2cae43e6638a3803b6df75f0e4df5ea16bb895376",
21 "0020-issues-that-depend-on-issues": "5f8a0b647e9060606e21eb4a982d1815225ffc365e145eb9471c297c3343af2f",
22 "0021-a-repository-can-run-on": "85a05853ae9e636049597fe9036ac536ba3efac6bca66496b6cd75e52a70fc25",
23 "0022-the-decision-ledger": "b5e6e4d873686180dd9e76bab6ce0f9ef402a3444d96f2706d62c07cc18e80a3",
24 "0023-the-look": "86890cbc353427bd6abe535a5ea2cad2661d437ed285407099b73d484e72071a",
25 "0024-a-variable-has-a-scope": "1f9049bb4d0a0843dee7975f6b6201d396ac2e6bc4a155b8c66d65dff47efad5",
26 "0025-what-a-running-app-said": "77a8887bfb3309ca024bbd9430240d716454fae860383a6d02e95d8e47223cfa",
27 "0026-subscriptions": "85b5a7e228cefb1689c48f6e7862b9539f80d9c4240263a7e428716ccacf123b",
28 "0027-the-moment-a-deploy-started": "e43e6b3d4fadba7e03dfb6bb54406ac90326823e9fb8f56b127a551bf4f8ae5f",
29 "0028-a-failed-check-s-own": "8b3ff15d07875635f1a5d378f09b0a432fa5650272b621bf1f1ba47c352c2d79",
30 "0029-a-third-width": "b4d8527a8a9e5002d2a2b3012fe064c365c180b680a59bb2d19db718f3ec9ad0",
31 "0030-take-it-back": "727ee2c2371a095738a3df1079d8f4f98234b695f84d63466d98444a681f0f31",
32 "0031-a-look-knows-which-page": "a62ab5826175af737f7b56c2d209ecf61d1bcad1707d0db625ee152bea8cc70f",
33 "0032-an-app-s-own-domains": "fa420e16fc6a7723092944018cea038e29bfed19d1cb4c7729fbcb708bf96b69",
34 "0033-where-a-repository-s-next": "321bf75d987ac60b6a0dd58b12d461c1758be11a14e84edc8d071ca49ad3ee0d",
35 "0034-forge-level-secrets": "2c31a3db69e0e7cb1167ed3efa2d2a496532da896bc90e39979f4793cb70cd49",
36 "0035-what-a-viewer-did-with": "fe01b3e75c1f9d4cae068fdb696949ac2691cde2796edb513a175eed3ef2162c",
37 "0036-a-share": "dc1c385f26c13547e79eecafaa830a4adc0f4ff0d4856b95e6a50d075f0c94e9",
38 "0037-the-usage-ledger": "d5d815aa3bef5e1f4cceb114771cf1a3e57931fdc21235da9208c0122ab93d7b",
39 "0038-a-subscription-s-standing": "2445f7d024399cc6f27248ad91ac6693369f6dfbcbd96702e91f1cf7ff022f59",
40 "0039-whether-the-agent-s-shell": "cc3a57567f75490d5607e43097ecefc00bf3dc9529304be8874ae697a6231ee7",
41 "0040-a-fleet-operation-s-receipt": "ebe243edc22f30a29b6950e0f8cfce4c7ba346d11f7a4a4c61e4a08298b14965",
42 "0041-the-meter": "15b995bb9dcfcb16aa9dbee69372ede8e1dcee56857dfa06e5deb2a583520569",
43 "0042-every-credential-is-a-subscription": "f56c85102d0ce700a5a3eb958a0134f47a3a0f0cfa1e60c37e0167d2be802012",
44 "0043-an-interrupted-run-is-not": "d9949e799843d1221644af17544e4f80fd4e60232c101186432dae0b7d1d2057",
45 "0044-why-an-issue-closed": "4f24a266d54eae3910da18f620e396b29b406ac4d993c13158875ce7590687c9",
46 "0045-oauth-clients-registered-dynamically": "ce0cf0b16b9094766362a288b9b984a00e8394b5b4821f370acf863b308ace61",
47 "0046-which-address-the-forge-s": "96917061cb4b064779a15dba9cbac82c4cba9ff2aefd71acd229e6aca4273190",
48 "0047-a-report-from-outside": "be5336004ba3141e71809194b65eb333866a678c5f2bb65c16be8681c65b63e7",
49 "0048-an-account-the-owner-suspended": "21c01ec3f30ef0e7f55a293601e95db31c1437f3da33ecf46036d9a425e5a96d",
50 "0049-the-asks-ledger": "a29d814f26ba38e871e7271cd955f14022ff923299b7a3af1ba85866f07221e2",
51 "0050-outsiders-may-trigger-the-crew": "28be9094e981e547f7f3bd1eaa4b0fd87acf346ccc1188469341c262901de488",
52 "0051-the-front-page-s-picks": "e3e9f27841b17e3951e627680cdfcca30e366bc687d8fd6f4c3b56a9661a5b2a",
53 "0052-a-whole-life-budget-on": "922ee458bbc52f0d6a9938e36ff09494066a8cb557bd7788495d259c9b335f39",
54 "0053-whether-an-account-s-email": "1e64971a56d0c0e16ec513cc506bc3a637efe843092d8a8cc99753a6c77a32e8",
55 "0054-the-people-a-private-app": "7670ecd442c0f1a1e23e878fe5e8accb4fd29a58c6bf3b27841a1c2a062352e6",
56 "0055-a-hold-on-a-pull": "8a0da478bdb878d0b0bf29d5824735ac3e67a6a504877abaff81bcf36d282597",
57 "0056-what-an-account-has-spent": "be08ec2eab632260848ed4c9951291907ace5bd81c514c4e3e53cd6cd0ab85cb",
58 "0057-spend-reported-from-outside": "15000df57c5e4f686dc91bb0c937d9484a4466bc424be5b7570c22d4f820c7d4",
59 "0058-a-signed-in-browser-has": "32b86a5f3074b1bce792e99aadc1f432b8ccff508703c9fcf6fa3ae537ad4eb4",
60 "0059-every-message-the-forge-sent": "4b7829c5fc97384cb4d5d9005168288eeab9419003552d8051ec5561bc9ef242",
61 "0060-who-a-picture-was-taken": "7d2319f66c78239cc662477d4ceda2ac50fd670342ebd69db1b7736041c1ed47",
62 "0061-an-agent-never-owns-an": "7d4b174818752723caaef32300a0a21783e1db9aca6b72334a44dd7ca092fd1f",
63 "0062-only-an-agent-is-ever": "57c2dc3400fc1e2db9acf86f2eb0231561e8e6ed87b54770a9d79493137cfe0e",
64 "0063-the-forge-s-capacity": "7721b2146736a958a08eb3e6f77b5ea6e179fd920dfc94e9a9ae98c88ce4eaba",
65 "0064-the-front-page-s-settings": "e15834cc1c7e45a58f4849cbad2ee5b627c3ad895c716030431b4961a47f30e9",
66 "0065-a-sealed-issue": "46d20691683748274ff0aeab0cd8d2640bb3f0427a4a3235e6a8b828ab0d2396",
67 "0066-a-milestone-a-person-watches": "782bfde34aac27ea01782ae22bbf3152e6f32925443e5ce212594f29402af0ff",
68 "0067-every-fork-a-person-made": "c2f5bf838522142e42b047e400b6725f711bd0280536a8c143aa5f30a8f24ad9",
69 "0068-the-run-a-comment-was": "1a92482c52858a1a326539c0aa1ed9c94d52dd9a2f274c7645a44c4a8840b2a2",
70 "0069-an-event-s-place-in": "b266f8f36f9a9a2382e7c5107dc9c5ce70a0c097ad8f5dc0c3f53f893c7a3476",
71 "0070-a-sealed-repository-is-never": "be9c7af107676fee4a652dc45580c817309a917211dc90435eb94fa5e2a46e45",
72 "0071-an-optional-anthropic-compatible-api": "f5e6a4df5bd7cca6371d164a7e254ec19d5a5db9986c8b120b2f398d2af0fc05",
73 "0072-when-a-mirror-lost-its": "34867bc4869e7d8928fa284027179adb6b86680ae540cd31564d4bb941c6eb94",
74 "0073-authorization-codes-for-the-mcp": "a1d0617595b908c81a4aba107a00b600c980b8048ffeea98975b6e724dd385e6",
75 "0074-a-look-a-person-attached": "4fd216ce408ee877ffbb8b38cab0234c22242da61377e9011f7ccf3189096d74",
76 "0075-a-limb-s-place-among": "bc117e5988500afdbc7b4406a232449d1881cd1e78a6f9e9edbabb09cff9c53f",
77 "0076-the-sentinel-s-memory": "ff8882d41e1702253a32d09324b1d9f00f29b21a22511779c914dbd7adfd7a01",
78 "0077-openid-connect-over-the-same": "1507c0bc5b5168e4b440d9963be59dcb10b94b14d18da1182ca0da6787f22090",
79 "0078-variants": "8c983ce83e8cad50f1e015ee5f4eefe77e503471d46bc46a43e5e3b71180d06d",
80 "0079-degraded-history": "6c58267cfafaa53117a3945013e9591ae98a8fcc5f1f3998bc9113e4ea62d892",
81 "0080-restore-drills": "35147ba9da81cda49cc77346c5c2d5eb71c72afc108676480237bd8d01f98f06",
82 "0081-thoughts": "b8fca6511f61669d33d34eeb180a64767b08b744e41496b2968b19943f4a6574",
83 "0082-passkeys": "074a5d91370d9d0ec073d9abbeded908a6541a743dcf55456e758422381614e4",
84 "0083-releases": "c0e3680642b226841bf039c48b5ae806c97ac0ce18fc8d71fc8eec49aeefe27c",
85 "0084-placements": "60036ee7281d17aad4a034409daaaab027ca6c524297da7d29618ef9eaa957df",
86 "0085-a-standing-rule-per-repository": "8c47857982bb540589b1b418926218475036050ef9449f3a5f80ef2ed3f5c8f0",
87 "0086-what-a-production-deploy-carries": "62379dffd5a0273355fcd1e5e353e046283c65a8ab7c068958efaa866d5db37d",
88 "0087-supersession-and-repeats": "bc2647e3fcb0f32b7dda800cb6a38eb0d5e340243e07bd6ad7cf8291d48efbcb",
89 "0088-what-the-machine-used": "afc9a4d24ead17763ccd7bdb4aeae80b9bafc3af769c5ded89bef921c7f622ad",
90 "0089-a-parked-pull": "7b564005fff642a38dc60cd58c805e97888845f89ad884af9baf79fa615cf77f",
91 "0090-a-green-check-proven": "21f33d1dabaf9722611db3a1630bc1c81aca6535a32398b7bbe7ddc47259994f",
92 "0091-where-a-check-ran": "8ac84fa10e0079815a48cb7e57dfbf3e5f67bcf820695e57b7841374d570f943",
93 "20261015000100-plans-need-a-nod": "754eb716bfe65056915e5f01bcbd6f15cf6b73b413da952c0a9b6ea1990b26ba",
94 "20261015090000-a-repository-put-away": "692ca4ed5c82f4c9a281a74ee0716182fd4067a148a2652575c33e9e2a0055c4",
95 "20261015090000-feed-chips": "a6057329aa558e6d25916f1edb88fffb30383eac87e24e966994631051029bfb",
96 "20261015120000-check-base-tip": "14e4279c9da82e3f7f9a5e163c0f8c75dcaa436e628586af9cab9c25e2db653d",
97 "20261015230000-a-domain-drill": "8db343e16d0bfacda42598908cb7ec113485b5aff30c7f663dffe0198cfbeb5c",
98 "20261015230532-telemetry-read": "8106ad063b8b13c22c0115dfc23141dab081cb2a23168a1eb891b2fe30b4cc65",
99 "20261016090000-sentinel-kept-settings": "03c30ac6d4e9d3164cc0f32af1c0f9d1f69a02e1147dc25bd1fe1838c077d5aa",
100 "20261016120000-kept-is-a-person-s": "d12558f4a7395c91e46b6abddf836719509e0531630a046b6114926c5aad4a91",
101 "20261016120000-proposal-state": "f24b95f42c1ceee8ed0f69093500365c218be3ec2c00be47a60ab77383a833f7",
102 "20261016130000-sign-in-with-the-parent": "cd2a2f60afac864446fee454e746ca3dac0a29848446bd27697a5dc0ba5a7124",
103 "20261016130000-usage-count": "1e715bfd5e4e28709903ee3a380fd1e8bf6b2be0a0321186ffa803676e6d5906",
104 "20261016140000-a-model-grant": "7ec2dbdfe5e782eb98b5c42f2ae79a686631c3416c9787ab3708adab8fec0777",
105 "20261016140000-drill-counts": "2ec120b90753826a5cbb6c7416b84a563566a0995ccd6d0350e00cf63ea16d45",
106 "20261016140000-flag-use": "a3375653f892f6eccff8493aebd2b42084d68b4612e80b595cd1f09dcda988da",
107 "20261016140000-machine-temperature": "74754feca643996c3b214f799a57868b0ab298a6ac9d9fa85905cd16e8907e9d",
108 "20261016150000-memory-hour": "41077b8c7b9b0421255f82b6c14954b4abad249cb51472e064cb130bc0aa0128",
109 "20261016160000-cloud-audit": "a64b2356735e172b21a8c462418b84065de9a54f121fa8371e812d97264d00d1",
110 "20261016160000-proposal-tokens": "335e0558843d8c103213ef8b9a1202e52cc8bd0674dc0e5db73f3429d38cc7c9",
111 "20261016170000-pull-run-grant": "dca1fa79e48ece7ba962b6e2eabeda513b7b364ed68e41a814ca73e123a9b811",
112 "20261016170000-upstream-proposals": "ea8c2b809705a5ac38dd94470e3024a560e61f44f01e4f51a84f7d9699df3338",
113 "20261016180000-account-ceiling": "0077901b08d41b6af61fe4b7ad7bf3bb16c923e755b5898c2f6d86837e9a42d0",
114 "20261016180000-signin-only-code": "c69edd5906a2f3425d6ce867ef9c7b6d59c40fb582ef6cbea0f4c86a67eb0030",
115 "20261016190000-member": "9076848fe032dea6905927a1b454c2381a3129acb3ce357c2726e82f1c161bc4",
116 "20261016200000-invite": "72bc7e9be6a7223048d1d1cf0226bc93ee4dbea9f00dbb20a6d457305886cbf9",
117 "20261017195831-ceiling-reserved": "1e6756752ae06c78a5920ae839e911e837a14cff3ac766cbc38e117c0a40186b",
118 "20261018100000-incident-remembered": "61fefa1d0750f3be617fdc23c1e6e17abc1ba610d980d85aff8824f14bf3e253"
119 }
src/core/releases.json +1 −1
@@ -1,5 +1,5 @@
11 {
2 "note": "The migrations each release applied, newest first. The release agent adds the new release here when it cuts one: `migrations` counts the migration ids it applied (every file in src/core/migrations/), and `ids` names the stamped ones among them — every file past 0090, its name without .ts (`ls src/core/migrations | grep -v ^0 | sed s/.ts$//`). A release from before #1837 has a count only: the first that many numbered files. Every migration that is not in both of the two newest releases must be additive (test/schema-additive.test.ts, notAdditive in src/kernel/migrate.ts): the old process keeps serving on the same database while the new one rolls in.",
2 "note": "The migrations each release applied, newest first. The release agent adds the new release here when it cuts one: `migrations` counts the migration ids it applied (every file in src/core/migrations/), and `ids` names the stamped ones among them — every file past 0090, its name without .ts (`ls src/core/migrations | grep -v ^0 | sed s/.ts$//`) — and runs `bun scripts/pin-migrations.ts`, which pins each new migration's text in src/core/migration-pins.json. A release from before #1837 has a count only: the first that many numbered files. Every migration that is not in both of the two newest releases must be additive (test/schema-additive.test.ts, notAdditive in src/kernel/migrate.ts): the old process keeps serving on the same database while the new one rolls in.",
33 "releases": [
44 {
55 "version": "0.7.165",
src/core/schema.ts +3 −1
@@ -9,7 +9,9 @@
99 * new one is named by the UTC second it was written
1010 * (`date -u +%Y%m%d%H%M%S`, e.g. `20261015093000-deploy-notes.ts`), so two
1111 * pulls that each add one add two files and never touch the same line
12 * (#1837). Each file is one `export default` of SQL.
12 * (#1837). Each file is one `export default` of SQL. The database records
13 * each applied one by name (#783); `migration-pins.json` holds what each
14 * said when it was released, and a test fails if one says something else.
1315 */
1416
1517 import type { Database } from "bun:sqlite";
src/kernel/db.ts +14 −5
@@ -7,7 +7,10 @@
77
88 import { Database } from "bun:sqlite";
99 import { existsSync, writeFileSync } from "node:fs";
10 import { migrate } from "./migrate.ts";
10 import { createLog } from "./log.ts";
11 import { type Migration, migrate } from "./migrate.ts";
12
13 const log = createLog("db");
1114
1215 /**
1316 * How long a writer waits for another writer before giving up. Long by
@@ -34,7 +37,7 @@ const CACHE_KIB = 64 * 1024;
3437 */
3538 const MMAP_BYTES = 512 * 1024 * 1024;
3639
37 export function openDb(path: string, migrations: readonly string[]): Database {
40 export function openDb(path: string, migrations: readonly Migration[]): Database {
3841 const memory = path === ":memory:";
3942 const fresh = memory || !existsSync(path);
4043 const image = fresh ? migratedEmpty.get(migrations) : undefined;
@@ -47,8 +50,14 @@ export function openDb(path: string, migrations: readonly string[]): Database {
4750 }
4851 applyPragmas(db);
4952 // On a copy every migration is already recorded, so this is a no-op —
50 // and a list that grew since the copy was taken still runs what is new.
51 migrate(db, migrations);
53 // and a list that grew since the copy was taken still runs what is new;
54 // one behind the copy boots and says what it does not know (#783).
55 const { unknown } = migrate(db, migrations);
56 if (unknown.length > 0)
57 log.warn("database has migrations this build does not know; left as they are", {
58 count: unknown.length,
59 unknown: unknown.join(","),
60 });
5261 if (fresh && image === undefined) migratedEmpty.set(migrations, db.serialize());
5362 return db;
5463 }
@@ -63,7 +72,7 @@ export function openDb(path: string, migrations: readonly string[]): Database {
6372 * bytes however it was made: the migrations that draw randomness only
6473 * rewrite rows an empty database does not have.
6574 */
66 const migratedEmpty = new WeakMap<readonly string[], Uint8Array>();
75 const migratedEmpty = new WeakMap<readonly Migration[], Uint8Array>();
6776
6877 /**
6978 * The image with its header saying "rollback journal" (bytes 18 and 19 of a
src/kernel/migrate.ts +123 −15
@@ -11,7 +11,8 @@
1111 * migration is a row in `schema_migration`, keyed by a hash of its text, and
1212 * a boot runs exactly the ones whose text it has not seen. A second boot is
1313 * a no-op; a migration that moved is still recognised; a sibling's migration
14 * this build does not know is left alone.
14 * this build does not know is left alone and named, never a refusal to boot
15 * (#783) — each row also carries its migration's file name.
1516 *
1617 * Each statement is also run so that meeting its own result is a no-op —
1718 * `CREATE` gains `IF NOT EXISTS`, an `ADD COLUMN` whose column is there is
@@ -77,10 +78,73 @@ export function migrationId(sql: string): string {
7778 return sha256(sql.replace(/\s+/g, " ").trim());
7879 }
7980
80 export function migrate(db: Database, migrations: readonly string[]): void {
81 /** Each pinned migration's id, by name: what it said when it was pinned. */
82 export type MigrationPins = Readonly<Record<string, string>>;
83
84 /**
85 * How `files` differ from what was pinned: one line for each pinned
86 * migration whose text says something else now, or that is gone. A
87 * migration on main is history — an edited one is a new one, and a
88 * database that ran the old text runs the new one again on top of it.
89 */
90 export function pinBreaks(
91 files: readonly MigrationFile[],
92 pins: MigrationPins,
93 ): string[] {
94 const now = new Map(files.map((m) => [m.name, migrationId(m.sql)]));
95 const breaks: string[] = [];
96 for (const [name, id] of Object.entries(pins)) {
97 const current = now.get(name);
98 if (current === undefined) breaks.push(`${name} was pinned and is gone`);
99 else if (current !== id)
100 breaks.push(`${name} was edited after it was pinned — add a new migration instead`);
101 }
102 return breaks;
103 }
104
105 /**
106 * `pins` with every migration in `files` it lacks added; a pin already
107 * there is never rewritten. Sorted by name, so the file it is written to
108 * changes only where a migration was added.
109 */
110 export function pinned(
111 files: readonly MigrationFile[],
112 pins: MigrationPins,
113 ): MigrationPins {
114 const next: Record<string, string> = { ...pins };
115 for (const m of files) next[m.name] ??= migrationId(m.sql);
116 return Object.fromEntries(
117 Object.entries(next).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)),
118 );
119 }
120
121 /**
122 * A migration as `migrate` takes it: its file, which names it, or bare SQL —
123 * a test's own list — which is known by its position alone.
124 */
125 export type Migration = string | MigrationFile;
126
127 /** What a boot did to the database, by migration name. */
128 export interface Migrated {
129 /** The migrations this boot applied, in the order it applied them. */
130 readonly applied: string[];
131 /**
132 * What the database has applied that this build does not know: a newer
133 * build's or a sibling branch's, on data copied for a preview. Each by
134 * the name its build recorded; one recorded with none by its position and
135 * the start of its id. Left alone, never undone.
136 */
137 readonly unknown: string[];
138 }
139
140 export function migrate(db: Database, list: readonly Migration[]): Migrated {
141 const migrations = list.map((m) => (typeof m === "string" ? m : m.sql));
142 const names = list.map((m, i) =>
143 typeof m === "string" ? `migration ${i + 1}` : `migration ${m.name}`,
144 );
81145 const ids = migrations.map((sql, i) => {
82146 if (sql.trim() === "")
83 defect("db.migration_missing", `migration ${i + 1} is empty`);
147 defect("db.migration_missing", `${names[i]} is empty`);
84148 return migrationId(sql);
85149 });
86150 const seen = new Set<string>();
@@ -88,22 +152,19 @@ export function migrate(db: Database, migrations: readonly string[]): void {
88152 if (seen.has(id))
89153 defect(
90154 "db.migration_duplicate",
91 `migration ${i + 1} repeats an earlier one word for word — it would never run`,
155 `${names[i]} repeats an earlier one word for word — it would never run`,
92156 );
93157 seen.add(id);
94158 });
95159
96160 const applied = appliedIds(db, migrations, ids);
97 if (applied.size > migrations.length) {
98 defect(
99 "db.future_schema",
100 `database has applied ${applied.size} migrations, this build only knows ${migrations.length} — refusing to run`,
101 );
102 }
161 recordNames(db, list, ids);
162 const ran: string[] = [];
103163 for (let i = 0; i < migrations.length; i++) {
104164 const id = ids[i]!;
105165 if (applied.has(id)) continue;
106166 const sql = migrations[i]!;
167 const entry = list[i]!;
107168 db.transaction(() => {
108169 for (const statement of statements(sql)) {
109170 try {
@@ -111,19 +172,63 @@ export function migrate(db: Database, migrations: readonly string[]): void {
111172 } catch (error) {
112173 defect(
113174 "db.migration_failed",
114 `migration ${i + 1} failed at ${quote(statement)}: ${messageOf(error)}`,
175 `${names[i]} failed at ${quote(statement)}: ${messageOf(error)}`,
115176 error,
116177 );
117178 }
118179 }
119180 db.query(
120 "INSERT INTO schema_migration (id, position, applied_at) VALUES (?1, ?2, ?3)",
121 ).run(id, i + 1, new Date().toISOString());
181 "INSERT INTO schema_migration (id, position, applied_at, name) VALUES (?1, ?2, ?3, ?4)",
182 ).run(
183 id,
184 i + 1,
185 new Date().toISOString(),
186 typeof entry === "string" ? null : entry.name,
187 );
122188 applied.add(id);
123189 // PRAGMA can't be parameterized; the value is a count, not input.
124190 db.exec(`PRAGMA user_version = ${applied.size}`);
125191 })();
192 ran.push(typeof entry === "string" ? String(i + 1) : entry.name);
126193 }
194 const known = new Set(ids);
195 const unknown = db
196 .query<{ id: string; position: number; name: string | null }, []>(
197 "SELECT id, position, name FROM schema_migration ORDER BY position, id",
198 )
199 .all()
200 .filter((row) => !known.has(row.id))
201 .map((row) => row.name ?? `#${row.position} (${row.id.slice(0, 12)})`);
202 return { applied: ran, unknown };
203 }
204
205 /**
206 * Write each known migration's name beside its id, where a build from
207 * before names were recorded left none — so the database says which
208 * migrations it has by name, and a later build that does not know one can
209 * say which. A database written by test lists keeps its blanks.
210 */
211 function recordNames(
212 db: Database,
213 list: readonly Migration[],
214 ids: readonly string[],
215 ): void {
216 if (!hasColumn(db, "schema_migration", "name"))
217 db.exec("ALTER TABLE schema_migration ADD COLUMN name TEXT");
218 const blank = db
219 .query<{ n: number }, []>(
220 "SELECT COUNT(*) AS n FROM schema_migration WHERE name IS NULL",
221 )
222 .get()!.n;
223 if (blank === 0) return;
224 const name = db.query(
225 "UPDATE schema_migration SET name = ?2 WHERE id = ?1 AND name IS NULL",
226 );
227 db.transaction(() => {
228 list.forEach((m, i) => {
229 if (typeof m !== "string") name.run(ids[i]!, m.name);
230 });
231 })();
127232 }
128233
129234 /**
@@ -132,7 +237,9 @@ export function migrate(db: Database, migrations: readonly string[]): void {
132237 * taken as applied — what that number always meant — unless one plainly
133238 * did not run (a table it creates or a column it adds is missing: a
134239 * branch's migration sat at that number instead). They are written down
135 * once, after which positions never matter again.
240 * once, after which positions never matter again. Such a database ahead of
241 * this build is still refused: a count names nothing, so there is no
242 * telling which of its migrations this build lacks.
136243 */
137244 function appliedIds(
138245 db: Database,
@@ -153,7 +260,8 @@ function appliedIds(
153260 db.exec(`CREATE TABLE schema_migration (
154261 id TEXT PRIMARY KEY,
155262 position INTEGER NOT NULL,
156 applied_at TEXT NOT NULL
263 applied_at TEXT NOT NULL,
264 name TEXT
157265 )`);
158266 const insert = db.query(
159267 "INSERT INTO schema_migration (id, position, applied_at) VALUES (?1, ?2, ?3)",
src/kernel/preflight.ts +9 −4
@@ -24,7 +24,7 @@ import { join } from "node:path";
2424 import { parseCidrs } from "./cidr.ts";
2525 import { dataHomeRefusal } from "./data_home.ts";
2626 import { applyPragmas } from "./db.ts";
27 import { migrate } from "./migrate.ts";
27 import { type Migration, migrate } from "./migrate.ts";
2828
2929 /** A scratch directory older than this was left by a killed run. */
3030 const STALE_MS = 60 * 60 * 1000;
@@ -81,13 +81,14 @@ export function configProblem(
8181 export function preflight(
8282 env: Record<string, string | undefined>,
8383 treeDir: string,
84 migrations: readonly string[],
84 migrations: readonly Migration[],
8585 ): Preflight {
8686 const bad = configProblem(env, treeDir);
8787 if (bad !== null) return { ok: false, line: `preflight failed: ${bad}` };
8888
8989 const dataDir = dataDirOf(env);
9090 let scratch: string | undefined;
91 let unknown: string[] = [];
9192 try {
9293 mkdirSync(dataDir, { recursive: true });
9394 sweepStale(dataDir);
@@ -101,13 +102,17 @@ export function preflight(
101102 const db = new Database(copy, { create: true, strict: true });
102103 try {
103104 applyPragmas(db);
104 migrate(db, migrations);
105 unknown = migrate(db, migrations).unknown;
105106 } finally {
106107 db.close();
107108 }
108109 return {
109110 ok: true,
110 line: `preflight ok: ${migrations.length} migrations apply to a copy of the database`,
111 line:
112 `preflight ok: ${migrations.length} migrations apply to a copy of the database` +
113 (unknown.length > 0
114 ? `; it also has ${unknown.length} this build does not know (${unknown.join(", ")})`
115 : ""),
111116 };
112117 } catch (error) {
113118 const message = (error instanceof Error ? error.message : String(error))
src/main.ts +2 −2
@@ -73,8 +73,8 @@ if (process.argv[2] === "hook") {
7373 // config parsed, migrations run on a copy of the database. No socket, no
7474 // timers, and the live database is never opened.
7575 if (process.argv.includes("--preflight")) {
76 const { migrations } = await import("./core/schema.ts");
77 const result = preflight(process.env, import.meta.dir, migrations);
76 const { migrationFiles } = await import("./core/schema.ts");
77 const result = preflight(process.env, import.meta.dir, migrationFiles);
7878 // CLI answer on the stream a person or a script reads.
7979 (result.ok ? console.log : console.error)(result.line);
8080 process.exit(result.ok ? 0 : 1);
src/release/rollback.ts +3 −2
@@ -4,8 +4,9 @@
44 * each take one line: the release is not kept, or its schema is behind the
55 * live database's (a down-migration is not something this forge has).
66 * A release whose schema is ahead of the live database is fine: it migrates
7 * forward on boot, and the database's own refusal of a future schema is
8 * untouched.
7 * forward on boot. A release behind it is refused here, not by the
8 * database: a build boots on migrations it does not know and only names
9 * them (#783).
910 */
1011
1112 import { Database } from "bun:sqlite";
test/kernel.test.ts +14 −3
@@ -2,7 +2,9 @@ import { describe, expect, test } from "bun:test";
22 import { mkdtempSync, rmSync } from "node:fs";
33 import { tmpdir } from "node:os";
44 import { join } from "node:path";
5 import { Database } from "bun:sqlite";
56 import { openDb } from "../src/kernel/db.ts";
7 import { migrate } from "../src/kernel/migrate.ts";
68 import { createUser, verifyPasswordThrottled } from "../src/core/users.ts";
79 import { migrations } from "../src/core/schema.ts";
810 import { CHEAP, cheapened, realHash } from "./helpers/cheap-hash.ts";
@@ -139,13 +141,22 @@ describe("db", () => {
139141 ).toThrow();
140142 });
141143
142 test("future schema refuses to run", () => {
144 test("a schema ahead of this build boots and names what it does not know (#783)", () => {
143145 const dir = mkdtempSync(join(tmpdir(), "forge-kernel-"));
144146 try {
145147 const path = join(dir, "forge.db");
146 const db = openDb(path, ["CREATE TABLE a (x INTEGER)"]);
148 const a = { name: "20261015110000-a", sql: "CREATE TABLE a (x INTEGER)" };
149 openDb(path, [a]).close();
150 const db = openDb(path, []);
151 expect(migrate(db, []).unknown).toEqual(["20261015110000-a"]);
147152 db.close();
148 expect(() => openDb(path, [])).toThrow(ForgeError);
153 // Data from before the tracking table has only a count, which names
154 // nothing: ahead of the build, it is still refused.
155 const legacy = join(dir, "legacy.db");
156 const old = new Database(legacy, { create: true });
157 old.exec("CREATE TABLE a (x INTEGER); PRAGMA user_version = 1");
158 old.close();
159 expect(() => openDb(legacy, [])).toThrow(ForgeError);
149160 } finally {
150161 rmSync(dir, { recursive: true, force: true });
151162 }
test/schema-migrations.test.ts +126 −5
@@ -24,9 +24,18 @@ import { Database } from "bun:sqlite";
2424 import { openDb } from "../src/kernel/db.ts";
2525 import { ForgeError } from "../src/kernel/error.ts";
2626 import { sha256 } from "../src/kernel/hash.ts";
27 import { migrationId, readMigrations, statements } from "../src/kernel/migrate.ts";
27 import {
28 type Migration,
29 migrate,
30 migrationId,
31 pinBreaks,
32 pinned,
33 readMigrations,
34 statements,
35 } from "../src/kernel/migrate.ts";
2836 import { migrationFiles, migrations, NUMBERED } from "../src/core/schema.ts";
2937 import { failureReason } from "../src/apps/process.ts";
38 import pins from "../src/core/migration-pins.json" with { type: "json" };
3039
3140 const root = mkdtempSync(join(tmpdir(), "forge-schema-"));
3241 afterAll(() => rmSync(root, { recursive: true, force: true }));
@@ -35,7 +44,7 @@ let n = 0;
3544 const fresh = (): string => join(root, `db-${++n}`);
3645
3746 /** Production's data, closed and checkpointed, ready to be copied. */
38 function production(list: readonly string[]): string {
47 function production(list: readonly Migration[]): string {
3948 const path = fresh();
4049 const db = openDb(path, list);
4150 db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
@@ -50,7 +59,7 @@ function copyOf(path: string): string {
5059 return target;
5160 }
5261
53 function boot(path: string, list: readonly string[]): void {
62 function boot(path: string, list: readonly Migration[]): void {
5463 openDb(path, list).close();
5564 }
5665
@@ -155,12 +164,78 @@ describe("a preview booting on copied data", () => {
155164 db.close();
156165 });
157166
158 test("data ahead of this build is still refused", () => {
167 test("data with one more migration than its code boots and names it (#783)", () => {
168 const a = { name: "20261015110000-a", sql: "CREATE TABLE a (x INTEGER)" };
169 const b = { name: "20261015120000-b", sql: "CREATE TABLE b (y INTEGER)" };
170 const path = copyOf(production([a, b]));
171 const db = new Database(path, { strict: true });
172 expect(migrate(db, [a])).toEqual({ applied: [], unknown: ["20261015120000-b"] });
173 // What it does not know is left as it is, and so is its record.
174 expect(has(db, "b")).toBe(true);
175 expect(migrate(db, [a]).unknown).toEqual(["20261015120000-b"]);
176 db.close();
177 // openDb boots on it too: a warning, not a refusal.
178 boot(path, [a]);
179 });
180
181 test("data with one fewer migration than its code takes the missing one", () => {
182 const a = { name: "20261015110000-a", sql: "CREATE TABLE a (x INTEGER)" };
183 const b = { name: "20261015120000-b", sql: "CREATE TABLE b (y INTEGER)" };
184 const path = copyOf(production([a]));
185 const db = new Database(path, { strict: true });
186 expect(migrate(db, [a, b])).toEqual({ applied: ["20261015120000-b"], unknown: [] });
187 expect(has(db, "b")).toBe(true);
188 db.close();
189 });
190
191 test("a migration recorded without a name is named by its position and id", () => {
159192 const path = production(["CREATE TABLE a (x INTEGER)", "CREATE TABLE b (y INTEGER)"]);
193 const db = new Database(path, { strict: true });
194 const [unknown] = migrate(db, ["CREATE TABLE a (x INTEGER)"]).unknown;
195 expect(unknown).toBe(`#2 (${migrationId("CREATE TABLE b (y INTEGER)").slice(0, 12)})`);
196 db.close();
197 });
198
199 test("data from before the tracking table that is ahead is still refused: a count names nothing", () => {
200 const path = fresh();
201 const old = new Database(path, { create: true, strict: true });
202 old.exec("CREATE TABLE a (x INTEGER); CREATE TABLE b (y INTEGER); PRAGMA user_version = 2");
203 old.close();
160204 expect(() => openDb(path, ["CREATE TABLE a (x INTEGER)"])).toThrow(ForgeError);
161205 });
162206 });
163207
208 describe("the database records migrations by name (#783)", () => {
209 const names = (path: string): (string | null)[] => {
210 const db = new Database(path, { readonly: true, strict: true });
211 try {
212 return db
213 .query<{ name: string | null }, []>("SELECT name FROM schema_migration ORDER BY position")
214 .all()
215 .map((row) => row.name);
216 } finally {
217 db.close();
218 }
219 };
220
221 test("a fresh database applies every migration in order and records each one's name", () => {
222 expect(names(production(migrationFiles))).toEqual(migrationFiles.map((m) => m.name));
223 });
224
225 test("a database recorded before names were kept has them written in, and nothing else changes", () => {
226 const path = copyOf(production(migrations));
227 // As a build from before #783 left it: no name column at all.
228 const old = new Database(path, { strict: true });
229 old.exec("ALTER TABLE schema_migration DROP COLUMN name");
230 const before = old.query("SELECT id, position, applied_at FROM schema_migration ORDER BY id").all();
231 old.close();
232 const db = openDb(path, migrationFiles);
233 expect(db.query("SELECT id, position, applied_at FROM schema_migration ORDER BY id").all()).toEqual(before);
234 db.close();
235 expect(names(path)).toEqual(migrationFiles.map((m) => m.name));
236 });
237 });
238
164239 describe("a migration that fails", () => {
165240 test("names the statement that failed, in one line, and applies nothing", () => {
166241 const path = production(["CREATE TABLE a (x INTEGER)"]);
@@ -238,7 +313,7 @@ describe("a migration is a file named by a sortable id (#1837)", () => {
238313 return dir;
239314 }
240315
241 const listIn = (dir: string): string[] => readMigrations(dir).map((m) => m.sql);
316 const listIn = readMigrations;
242317
243318 /** Everything a schema is but the record of how it got there. */
244319 function shape(path: string): unknown[] {
@@ -285,6 +360,17 @@ describe("a migration is a file named by a sortable id (#1837)", () => {
285360 expect(recorded(aFirst)).toEqual(recorded(together));
286361 expect(recorded(bFirst)).toEqual(recorded(together));
287362 expect(recorded(together).length).toBe(migrations.length + 2);
363 // Each database names both, whichever it took first.
364 for (const path of [aFirst, bFirst, together]) {
365 const db = new Database(path, { readonly: true, strict: true });
366 const names = db
367 .query<{ name: string }, []>("SELECT name FROM schema_migration WHERE name LIKE '2026101511%' OR name LIKE '2026101512%' ORDER BY name")
368 .all()
369 .map((row) => row.name);
370 db.close();
371 expect(names).toContain("20261015110000-user-motto");
372 expect(names).toContain("20261015120000-deploy-note");
373 }
288374 const db = openDb(together, both);
289375 expect(has(db, "deploy_note")).toBe(true);
290376 db.close();
@@ -333,3 +419,38 @@ describe("a migration is a file named by a sortable id (#1837)", () => {
333419 }
334420 });
335421 });
422
423 describe("a migration on main is never edited (#783)", () => {
424 test("every pinned migration still says what it said when it was pinned", () => {
425 expect(pinBreaks(migrationFiles, pins)).toEqual([]);
426 });
427
428 test("every numbered migration is pinned", () => {
429 for (const m of migrationFiles.slice(0, NUMBERED)) expect(Object.hasOwn(pins, m.name)).toBe(true);
430 });
431
432 test("an edit after its pin is named, a removal too; re-indenting is not an edit", () => {
433 const files = [
434 { name: "20261015110000-a", sql: "CREATE TABLE a (x INTEGER)" },
435 { name: "20261015120000-b", sql: "CREATE TABLE b (y INTEGER)" },
436 ];
437 const pin = pinned(files, {});
438 expect(pinBreaks(files, pin)).toEqual([]);
439 expect(pinBreaks([files[0]!, { name: "20261015120000-b", sql: "CREATE TABLE b (y TEXT)" }], pin)).toEqual([
440 "20261015120000-b was edited after it was pinned — add a new migration instead",
441 ]);
442 expect(pinBreaks([files[1]!], pin)).toEqual(["20261015110000-a was pinned and is gone"]);
443 expect(pinBreaks([{ name: "20261015110000-a", sql: "\n CREATE TABLE a\n (x INTEGER)\n" }, files[1]!], pin)).toEqual([]);
444 });
445
446 test("pinning adds what is new and never rewrites a pin", () => {
447 const a = { name: "20261015110000-a", sql: "CREATE TABLE a (x INTEGER)" };
448 const b = { name: "20261015120000-b", sql: "CREATE TABLE b (y INTEGER)" };
449 const pin = pinned([b], {});
450 const edited = { ...b, sql: "CREATE TABLE b (y TEXT)" };
451 const next = pinned([a, edited], pin);
452 expect(Object.keys(next)).toEqual([a.name, b.name]);
453 expect(next[b.name]).toBe(pin[b.name]!);
454 expect(next[a.name]).toBe(migrationId(a.sql));
455 });
456 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs