Public: anyone can read this; the crew can change it.

A database import refuses CREATE TABLE expressions that can build a huge value, by allowlist not denylist #14922

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/14898-a-database-import-refuses-create-table-e → main 4 commits fork 2 threads · 2 unresolved

  • Merged. scout/14898-a-database-import-refuses-create-table-e was merged into main as 2566ee0.

Review

Sign in to review this change.
5 files changed +327 −7
docs/features/platform-app-move.md +8 −2
@@ -52,8 +52,14 @@ admins; the repository must be an app. The dump is SQL text up to 16 MB (413 pas
5252 the bytes read), the kind `sqlite3 .dump` writes. A Postgres dump is not run as it is:
5353 `pg_dump --inserts --no-owner` output that SQLite accepts works, nothing else (#11698).
5454
55 What runs is narrower than a statement's first words. `CREATE TABLE` (never `AS`), `CREATE [UNIQUE]
56 INDEX`, `BEGIN`, `COMMIT`, `PRAGMA foreign_keys`, and `INSERT INTO <table> [(columns)] VALUES (...)[, (...)]`
55 What runs is narrower than a statement's first words. `CREATE TABLE` is read against an
56 allowlist (#14898): columns with a type, `PRIMARY KEY`, `NOT NULL`, `UNIQUE`, `REFERENCES` and a
57 `DEFAULT` of a plain literal (a number, signed or not, a string, `X'..'`, NULL, TRUE, FALSE,
58 `CURRENT_TIMESTAMP` and its kin, bare or in parentheses), and table-level `PRIMARY KEY`, `UNIQUE`
59 and `FOREIGN KEY`. `CHECK`, generated columns (`AS`, `GENERATED ALWAYS`), `COLLATE`, `CREATE TABLE
60 ... AS` and any function call are refused with the statement's number: a computed default such as
61 `DEFAULT (datetime('now'))` no longer imports, so remove it from the dump (or write it as
62 `DEFAULT CURRENT_TIMESTAMP`) and import again. Also run: `CREATE [UNIQUE] INDEX`, `BEGIN`, `COMMIT`, `PRAGMA foreign_keys`, and `INSERT INTO <table> [(columns)] VALUES (...)[, (...)]`
5763 whose values are literals: numbers, strings, `X'..'`, NULL, TRUE, FALSE. `SELECT` and `WITH` outside
5864 quotes, a function call in a `VALUES` list, and the functions that make a big value from a small
5965 statement (`randomblob`, `zeroblob`, `printf`, `replace`, `hex`...) are refused with the statement's
src/actions/migrate.ts +1 −1
@@ -1011,7 +1011,7 @@ export function importDataMessage(error: ImportDataError): string {
10111011 case "empty":
10121012 return "The dump has no statements.";
10131013 case "refused":
1014 return `Statement ${error.statement} (${error.kind}) is not allowed: a dump may only CREATE TABLE (not AS), CREATE INDEX, INSERT INTO ... VALUES of plain values, BEGIN, COMMIT and PRAGMA foreign_keys; no SELECT, WITH or function that builds a large value.`;
1014 return `Statement ${error.statement} (${error.kind}) is not allowed: a dump may only CREATE TABLE (not AS), CREATE INDEX, INSERT INTO ... VALUES of plain values, BEGIN, COMMIT and PRAGMA foreign_keys; no SELECT, WITH or function that builds a large value. A CREATE TABLE's columns may carry only plain literal DEFAULTs: a CHECK constraint, generated column (AS), COLLATE or function call is refused, so drop it from the dump and load again.`;
10151015 case "invalid":
10161016 return `Statement ${error.statement} failed: ${error.message}`;
10171017 case "mismatch":
src/apps/dump_create_table.ts added +268
@@ -0,0 +1,268 @@
1 /**
2 * Whether one `CREATE TABLE` from an uploaded dump has the shape a dump may run (#14898).
3 *
4 * An allowlist, read token by token: a table name, then columns and table constraints, and
5 * nothing that is an expression. A column is a name, a type (words, with at most two signed
6 * numbers in parentheses: `DECIMAL(10,2)`), and constraints from a short list: `PRIMARY KEY`,
7 * `NOT NULL`, `NULL`, `UNIQUE`, `REFERENCES`, and `DEFAULT` of a plain literal (a number, a
8 * string, a blob, `NULL`, `TRUE`, `FALSE` or `CURRENT_TIME`/`DATE`/`TIMESTAMP`, bare or in
9 * parentheses). `CHECK`, generated columns (`AS`, `GENERATED ALWAYS`), `COLLATE` and every other
10 * word are refused, and so is any name followed by `(` outside a type or a column list, quoted
11 * or not: a default or check that calls a function would run it once per row and could build a
12 * large value out of a small dump. Plain text in, a boolean out.
13 */
14
15 type Token =
16 | { readonly t: "word"; readonly v: string } // unquoted, lowercased
17 | { readonly t: "name" } // quoted identifier
18 | { readonly t: "str" } // 'text' or X'blob'
19 | { readonly t: "num" }
20 | { readonly t: "punct"; readonly v: string };
21
22 const QUOTE_CLOSE: Readonly<Record<string, string>> = { '"': '"', "`": "`", "[": "]" };
23
24 /** The statement's tokens, or undefined when it holds a character no column definition needs. */
25 function tokens(sql: string): Token[] | undefined {
26 const out: Token[] = [];
27 let i = 0;
28 const n = sql.length;
29 while (i < n) {
30 const c = sql[i]!;
31 if (/\s/.test(c)) {
32 i++;
33 continue;
34 }
35 const blob = (c === "x" || c === "X") && sql[i + 1] === "'";
36 if (c === "'" || blob || c in QUOTE_CLOSE) {
37 const open = blob ? i + 1 : i;
38 const close = c === "'" || blob ? "'" : QUOTE_CLOSE[c]!;
39 let j = open + 1;
40 for (;;) {
41 if (j >= n) return undefined;
42 if (sql[j] === close) {
43 if (close !== "]" && sql[j + 1] === close) j += 2;
44 else break;
45 } else j++;
46 }
47 out.push(c === "'" || blob ? { t: "str" } : { t: "name" });
48 i = j + 1;
49 continue;
50 }
51 const num = /^(?:0[xX][0-9a-fA-F]+|(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?)/.exec(sql.slice(i, i + 400));
52 if (num) {
53 if (/[\w$]/.test(sql[i + num[0].length] ?? "")) return undefined;
54 out.push({ t: "num" });
55 i += num[0].length;
56 continue;
57 }
58 const word = /^[A-Za-z_][\w$]*/.exec(sql.slice(i, i + 400));
59 if (word) {
60 out.push({ t: "word", v: word[0].toLowerCase() });
61 i += word[0].length;
62 continue;
63 }
64 if ("(),.+-".includes(c)) {
65 out.push({ t: "punct", v: c });
66 i++;
67 continue;
68 }
69 return undefined;
70 }
71 return out;
72 }
73
74 /** Words that end a column's type and start a constraint. */
75 const CONSTRAINT_WORDS = new Set([
76 "constraint", "primary", "not", "null", "unique", "check", "default", "collate", "references", "generated", "as",
77 ]);
78 const LITERAL_WORDS = new Set(["null", "true", "false", "current_time", "current_date", "current_timestamp"]);
79 const CONFLICT_ACTIONS = new Set(["rollback", "abort", "fail", "ignore", "replace"]);
80
81 class Refused extends Error {}
82
83 /** A cursor over the tokens; every `expect` that misses throws `Refused`. */
84 class Reader {
85 private i = 0;
86 constructor(private readonly toks: readonly Token[]) {}
87 peek(): Token | undefined {
88 return this.toks[this.i];
89 }
90 done(): boolean {
91 return this.i >= this.toks.length;
92 }
93 isWord(...vs: string[]): boolean {
94 const t = this.peek();
95 return t?.t === "word" && vs.includes(t.v);
96 }
97 /** Whether the token after the next one is the word `v`. */
98 nextIsWord(v: string): boolean {
99 const t = this.toks[this.i + 1];
100 return t?.t === "word" && t.v === v;
101 }
102 isPunct(v: string): boolean {
103 const t = this.peek();
104 return t?.t === "punct" && t.v === v;
105 }
106 word(...vs: string[]): boolean {
107 if (!this.isWord(...vs)) return false;
108 this.i++;
109 return true;
110 }
111 punct(v: string): boolean {
112 if (!this.isPunct(v)) return false;
113 this.i++;
114 return true;
115 }
116 must(ok: boolean): void {
117 if (!ok) throw new Refused();
118 }
119 /** A name: a quoted identifier, or a word that is not a constraint keyword. */
120 name(): void {
121 this.must(this.isName());
122 this.i++;
123 }
124 isName(): boolean {
125 const t = this.peek();
126 return t?.t === "name" || (t?.t === "word" && !CONSTRAINT_WORDS.has(t.v));
127 }
128 signedNumber(): boolean {
129 const at = this.i;
130 if (!this.punct("+")) this.punct("-");
131 if (this.peek()?.t === "num") {
132 this.i++;
133 return true;
134 }
135 this.i = at;
136 return false;
137 }
138 literal(): boolean {
139 if (this.signedNumber()) return true;
140 const t = this.peek();
141 if (t?.t === "str" || (t?.t === "word" && LITERAL_WORDS.has(t.v))) {
142 this.i++;
143 return true;
144 }
145 return false;
146 }
147 }
148
149 function nameList(r: Reader, ordered: boolean): void {
150 r.must(r.punct("("));
151 do {
152 r.name();
153 if (ordered) r.word("asc", "desc");
outdated

Consume the matched word directly instead of re-calling word() with a ternary.

154 } while (r.punct(","));
155 r.must(r.punct(")"));
156 }
157
158 function conflictClause(r: Reader): void {
159 if (!r.word("on")) return;
160 r.must(r.word("conflict"));
161 r.must(r.word(...CONFLICT_ACTIONS));
162 }
163
164 function foreignKeyClause(r: Reader): void {
165 r.name();
166 if (r.isPunct("(")) nameList(r, false);
167 for (;;) {
168 if (r.word("on")) {
169 r.must(r.word("delete", "update"));
170 if (r.word("set")) r.must(r.word("null", "default"));
171 else if (r.word("no")) r.must(r.word("action"));
172 else r.must(r.word("cascade", "restrict"));
173 } else if (r.word("match")) r.name();
174 else if (r.isWord("deferrable") || (r.isWord("not") && r.nextIsWord("deferrable"))) {
175 r.word("not");
176 r.word("deferrable");
outdated

Split this crammed else-if/else onto separate lines.

177 if (r.word("initially")) r.must(r.word("deferred", "immediate"));
178 } else return;
179 }
180 }
181
182 function columnConstraint(r: Reader): void {
183 if (r.word("constraint")) r.name();
184 if (r.word("primary")) {
185 r.must(r.word("key"));
186 r.word("asc", "desc");
187 conflictClause(r);
188 r.word("autoincrement");
189 } else if (r.word("not")) {
190 r.must(r.word("null"));
191 conflictClause(r);
192 } else if (r.word("null")) conflictClause(r);
193 else if (r.word("unique")) conflictClause(r);
194 else if (r.word("default")) {
195 if (r.punct("(")) {
196 r.must(r.literal());
197 r.must(r.punct(")"));
198 } else r.must(r.literal());
199 } else if (r.word("references")) {
200 foreignKeyClause(r);
201 } else {
202 throw new Refused(); // CHECK, COLLATE, AS, GENERATED, or any other word
203 }
204 }
205
206 function columnDef(r: Reader): void {
207 r.name();
208 if (r.isName()) {
209 while (r.isName()) r.name();
210 if (r.punct("(")) {
211 r.must(r.signedNumber());
212 if (r.punct(",")) r.must(r.signedNumber());
213 r.must(r.punct(")"));
214 }
215 }
216 while (!r.done() && !r.isPunct(",") && !r.isPunct(")")) columnConstraint(r);
217 }
218
219 function tableConstraint(r: Reader): void {
220 if (r.word("constraint")) r.name();
221 if (r.word("primary")) {
222 r.must(r.word("key"));
223 nameList(r, true);
224 conflictClause(r);
225 } else if (r.word("unique")) {
226 nameList(r, true);
227 conflictClause(r);
228 } else if (r.word("foreign")) {
229 r.must(r.word("key"));
230 nameList(r, false);
231 r.must(r.word("references"));
232 foreignKeyClause(r);
233 } else throw new Refused(); // CHECK and anything else
234 }
235
236 /**
237 * True when `sql` is a `CREATE TABLE` of plain columns and constraints, with nothing that
238 * evaluates.
239 */
240 export function createTableAllowed(sql: string): boolean {
241 const toks = tokens(sql);
242 if (!toks) return false;
243 const r = new Reader(toks);
244 try {
245 r.must(r.word("create"));
246 r.word("temp", "temporary");
247 r.must(r.word("table"));
248 if (r.word("if")) r.must(r.word("not") && r.word("exists"));
249 r.name();
250 if (r.punct(".")) r.name();
251 r.must(r.punct("("));
252 do {
253 if (r.isWord("constraint", "primary", "unique", "foreign", "check")) tableConstraint(r);
254 else columnDef(r);
255 } while (r.punct(","));
256 r.must(r.punct(")"));
257 if (!r.done()) {
258 do {
259 if (r.word("without")) r.must(r.word("rowid"));
260 else r.must(r.word("strict"));
261 } while (r.punct(","));
262 }
263 return r.done();
264 } catch (e) {
265 if (e instanceof Refused) return false;
266 throw e;
267 }
268 }
src/apps/dump_import.ts +6 −1
@@ -11,7 +11,10 @@
1111 * The kinds are narrower than their first words. An `INSERT` is only
1212 * `INSERT INTO <table> [(columns)] VALUES (...)[, (...)]`, its values
1313 * literals (numbers, strings, `X'..'` blobs, NULL, TRUE, FALSE): no function
14 * call, no `SELECT`, no `WITH`. A `CREATE TABLE` is never `AS`. `SELECT` and
14 * call, no `SELECT`, no `WITH`. A `CREATE TABLE` is never `AS`, and its columns
15 * are read by an allowlist (`dump_create_table.ts`): types, keys, `NOT NULL`,
16 * `UNIQUE`, `REFERENCES` and literal `DEFAULT`s only, so no `CHECK`, generated
17 * column, `COLLATE` or function call, quoted or not, runs per row. `SELECT` and
1518 * `WITH` are refused anywhere outside quotes, and so are the functions that
1619 * make a big value out of a small statement (`randomblob`, `zeroblob`,
1720 * `printf`, `replace`...). What is left costs time and disk in proportion to
@@ -34,6 +37,7 @@
3437 import { Database } from "bun:sqlite";
3538 import { existsSync, linkSync, mkdirSync, renameSync, rmSync } from "node:fs";
3639 import { join } from "node:path";
40 import { createTableAllowed } from "./dump_create_table.ts";
3741
3842 /** A table's rows as one side holds them. */
3943 export interface TableFingerprint {
@@ -233,6 +237,7 @@ export function firstRefused(
233237 continue;
234238 }
235239 if (!ALLOWED.some((re) => re.test(s)) || CREATE_TABLE_AS.test(s) || HEAVY.test(bare(s))) return refuse;
240 if (/^CREATE\s+TABLE\s/i.test(s) && !createTableAllowed(s)) return refuse;
236241 }
237242 return undefined;
238243 }
test/app-dump-import.test.ts +44 −3
@@ -1,13 +1,13 @@
11 /** A dump in, loaded and proved per table on both sides (#11697). */
22 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
33 import { Database } from "bun:sqlite";
4 import { existsSync, readdirSync, mkdirSync, writeFileSync } from "node:fs";
4 import { existsSync, readdirSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
55 import { mkdtemp, rm } from "node:fs/promises";
66 import { tmpdir } from "node:os";
77 import { join } from "node:path";
88 import { createHash } from "node:crypto";
99 import { createApp, type App } from "../src/app.ts";
10 import { importAppData } from "../src/actions/migrate.ts";
10 import { importAppData, importDataMessage } from "../src/actions/migrate.ts";
1111 import { envDataDir } from "../src/apps/data.ts";
1212 import { compareTables, firstRefused, importDump, insertedRows, splitSql } from "../src/apps/dump_import.ts";
1313 import { createApp as createAppRecord } from "../src/core/apps.ts";
@@ -55,10 +55,51 @@ describe("what a dump may do (#11697 review)", () => {
5555 expect(refused("CREATE INDEX i ON t(a) WHERE a IN (SELECT 1)")?.statement).toBe(1);
5656 });
5757 test("plain dumps still pass, quoted words and column names that look like functions included", () => {
58 expect(refused(`CREATE TABLE "t" (format TEXT, "select" TEXT, price DECIMAL(10,2) DEFAULT (datetime('now')));
58 expect(refused(`CREATE TABLE "t" (format TEXT, "select" TEXT, price DECIMAL(10,2) DEFAULT (-1.5), at TEXT DEFAULT CURRENT_TIMESTAMP);
5959 INSERT INTO "t" VALUES('select * from x; replace(1)', 'with', 1.5e3), ('b', NULL, -2);
6060 INSERT INTO "t"(format) VALUES (X'00ff'), (TRUE);`)).toBeUndefined();
6161 });
62 test("a CREATE TABLE evaluates nothing: quoted calls, nested calls, generated columns, CHECK and COLLATE are refused (#14898)", () => {
63 // each of these slipped past the old word list
64 expect(refused(`CREATE TABLE t (a BLOB DEFAULT ("randomblob"(1e9)))`)?.statement).toBe(1);
65 expect(refused("CREATE TABLE t (a TEXT DEFAULT (quote(quote(quote(quote('x'))))))")?.statement).toBe(1);
66 expect(refused("CREATE TABLE t (a TEXT, b TEXT AS (a || a || a || a), c TEXT GENERATED ALWAYS AS (b || b) STORED)")?.statement).toBe(1);
67 expect(refused("BEGIN; CREATE TABLE t (n INTEGER CHECK (n > 0))")).toEqual({ statement: 2, kind: "CREATE TABLE" });
68 expect(refused("CREATE TABLE t (n INTEGER, CONSTRAINT pos CHECK (n > 0))")?.statement).toBe(1);
69 expect(refused("CREATE TABLE t (s TEXT COLLATE NOCASE)")?.statement).toBe(1);
70 expect(refused("CREATE TABLE t (s TEXT DEFAULT (datetime('now')))")?.statement).toBe(1);
71 expect(refused("CREATE TABLE t (s TEXT DEFAULT [length](1))")?.statement).toBe(1);
72 expect(refused("CREATE TABLE t (s TEXT DEFAULT abc)")?.statement).toBe(1);
73 expect(refused("CREATE TABLE t (s TEXT); garbage")?.statement).toBe(2);
74 });
75 test("an ordinary schema with keys, references and literal defaults still passes", () => {
76 expect(refused(DUMP)).toBeUndefined();
77 expect(refused(`CREATE TABLE IF NOT EXISTS "main"."users" (
78 id INTEGER PRIMARY KEY AUTOINCREMENT,
79 name character varying(40) NOT NULL ON CONFLICT ABORT,
80 [e-mail] TEXT UNIQUE DEFAULT '',
81 flag BOOLEAN DEFAULT FALSE, score REAL DEFAULT -0.5e2, pic BLOB DEFAULT X'00', n INT DEFAULT NULL,
82 team INTEGER CONSTRAINT fk REFERENCES teams(id) ON DELETE SET NULL ON UPDATE NO ACTION DEFERRABLE INITIALLY DEFERRED NOT NULL,
83 UNIQUE (name, team DESC),
84 FOREIGN KEY (team) REFERENCES "teams" ("id") NOT DEFERRABLE
85 ) WITHOUT ROWID, STRICT`)).toBeUndefined();
86 });
87 test("a CHECK in a dump is refused through importDump, writes nothing, and the words say what to drop", () => {
88 const dir = join(tmpdir(), `forge-dump-check-${process.pid}`);
89 const done = importDump(dir, "c.db", "CREATE TABLE t (n INTEGER CHECK (n > 0)); INSERT INTO t VALUES (1)", false);
90 expect(done).toMatchObject({ ok: false, error: { error: "refused", statement: 1 } });
91 expect(existsSync(join(dir, "c.db"))).toBe(false);
92 expect(importDataMessage({ error: "refused", statement: 1, kind: "CREATE TABLE" })).toContain("a CHECK constraint, generated column (AS), COLLATE or function call is refused");
93 });
94 test("signed and parenthesised literal defaults import end to end through importDump", () => {
95 const dir = join(tmpdir(), `forge-dump-defaults-${process.pid}`);
96 const done = importDump(dir, "d.db", "CREATE TABLE t (id INTEGER PRIMARY KEY, a REAL DEFAULT -1.5, b INT DEFAULT (+7), c TEXT DEFAULT ('x')); INSERT INTO t (id) VALUES (1)", false);
97 expect(done.ok).toBe(true);
98 const db = new Database(join(dir, "d.db"), { readonly: true });
99 expect(db.query("SELECT a, b, c FROM t").all()).toEqual([{ a: -1.5, b: 7, c: "x" }]);
100 db.close();
101 rmSync(dir, { recursive: true, force: true });
102 });
62103 test("an unbounded or mixed VALUES list is refused", () => {
63104 expect(refused("INSERT INTO t VALUES (1) (2)")?.statement).toBe(1);
64105 expect(refused("INSERT INTO t VALUES (1), (2) ON CONFLICT DO NOTHING")?.statement).toBe(1);

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs