Public: anyone can read this; the crew can change it.

A malformed JSON body is answered invalid_json, not the first field's validation error #14291

opened this

Check · Review · Land · Live — Its check is red. Next: builder, its check is red.

Ready to land

scout/forge:scout/14269-a-malformed-json-body-is-answered-invali → main 2 commits fork

  • Ready to merge — no conflicts with main. 2 commits will land on main.

    Check failed: a person who cannot write, and a repository no longer stuck, get nothing doneCheck of de31989checked against main at 59903fafast tier: red in 6m15s

    You need write access to merge.

Review

Sign in to review this change.
5 files changed +180 −19
src/contract/openapi.ts +15 −3
@@ -43,6 +43,14 @@ export const pagesByCursor = (r: Pick<Route, "method" | "query">): boolean => r.
4343
4444 const PAGE_REFUSAL = "page or offset was sent: this list pages by cursor, pass next_cursor as ?cursor= (400)";
4545
46 /**
47 * A write that declares a body reads JSON, and a body that does not parse is refused at the
48 * boundary (`http/json_body.ts`).
49 */
50 export const readsJsonBody = (r: Pick<Route, "method" | "body">): boolean => r.method !== "get" && r.method !== "delete" && r.body !== undefined;
51
52 const JSON_REFUSAL = "400 { error, message, hint }: the body is not JSON; the message names the first bad token and its byte offset";
53
4654 /** A route's operationId, and the name of the file that holds it. */
4755 export function operationId(r: Pick<Route, "method" | "path">): string {
4856 return `${r.method}${r.path.replace(/[^A-Za-z0-9]+(.)?/g, (_, c: string | undefined) => (c ?? "").toUpperCase())}`;
@@ -90,7 +98,11 @@ export function openapi(origin: string, version: string, role: Role): Record<str
9098 const paths: Record<string, Record<string, unknown>> = {};
9199 const rows = routesFor(role);
92100 for (const r of rows) {
93 const refusals = pagesByCursor(r) ? { invalid_query: PAGE_REFUSAL, ...r.refusals } : r.refusals;
101 const refusals = {
102 ...(pagesByCursor(r) ? { invalid_query: PAGE_REFUSAL } : {}),
103 ...(readsJsonBody(r) ? { invalid_json: JSON_REFUSAL } : {}),
104 ...r.refusals,
105 };
94106 const params = [...r.path.matchAll(/\{(\w+)\}/g)].map((m) => ({
95107 name: m[1],
96108 in: "path",
@@ -155,12 +167,12 @@ export function openapi(origin: string, version: string, role: Role): Record<str
155167 }),
156168 },
157169 ...Object.fromEntries((r.alsoStatus ?? []).map((c) => [String(c), { description: "Done" }])),
170 ...(pagesByCursor(r) ? { "400": { description: PAGE_REFUSAL } } : readsJsonBody(r) ? { "400": { description: JSON_REFUSAL } } : {}),
158171 ...Object.fromEntries(Object.entries(r.errorStatus ?? {}).map(([c, d]) => [c, { description: d }])),
159 ...(pagesByCursor(r) ? { "400": { description: PAGE_REFUSAL } } : {}),
160172 default: {
161173 description:
162174 "An error: { error, message }; `error` is a stable code" +
163 (refusals === undefined
175 (Object.keys(refusals).length === 0
164176 ? ""
165177 : `. Refusals: ${Object.entries(refusals).map(([c, w]) => `\`${c}\` (${w})`).join("; ")}`),
166178 },
src/http/json_body.ts added +105
@@ -0,0 +1,105 @@
1 /**
2 * A JSON body that does not parse, found at the boundary (#14269). Before,
3 * `{bad` read as `{}` and the route answered its first field's validation
4 * error — an agent then hunted for a field it had sent. Now the body reader
5 * throws `InvalidJson` and the server's door answers 400 `invalid_json` in
6 * the `{ error, message, hint }` shape, naming the byte offset of the first
7 * bad token. An empty body is still `{}`; valid JSON that is not an object
8 * is still `{}` (the route's own validation says what it lacked).
9 */
10
11 export class InvalidJson extends Error {
12 constructor(readonly offset: number, readonly token: string) {
13 super(`not JSON at byte ${offset}`);
14 }
15 }
16
17 /** Byte offset of the first character that cannot continue a JSON text, or -1 when it is whole. */
18 export function firstBadOffset(text: string): number {
19 let i = 0;
20 const ws = () => {
21 while (i < text.length && " \t\n\r".includes(text[i] as string)) i++;
22 };
23 // Each reader returns true when it read a value; on false, `i` is the bad spot.
24 const literal = (word: string) => {
25 if (text.startsWith(word, i)) { i += word.length; return true; }
26 let k = 0;
27 while (i + k < text.length && text[i + k] === word[k]) k++;
28 i += k;
29 return false;
30 };
31 const string = (): boolean => {
32 i++;
33 while (i < text.length) {
34 const c = text[i] as string;
35 if (c === '"') { i++; return true; }
36 if (c < " ") return false;
37 if (c === "\\") {
38 const e = text[i + 1];
39 if (e === "u") {
40 if (!/^[0-9a-fA-F]{4}$/.test(text.slice(i + 2, i + 6))) return false;
41 i += 6;
42 } else if (e !== undefined && '"\\/bfnrt'.includes(e)) i += 2;
43 else { i++; return false; }
44 } else i++;
45 }
46 return false;
47 };
48 const number = (): boolean => {
49 const m = /^-?(0|[1-9]\d*)(\.\d+)?([eE][+-]?\d+)?/.exec(text.slice(i));
50 if (m === null) return false;
51 i += m[0].length;
52 return true;
53 };
54 const value = (depth: number): boolean => {
55 ws();
56 const c = text[i];
57 if (c === undefined || depth > 512) return false;
58 if (c === '"') return string();
59 if (c === "{" || c === "[") {
60 const close = c === "{" ? "}" : "]";
61 i++; ws();
62 if (text[i] === close) { i++; return true; }
63 for (;;) {
64 ws();
65 if (c === "{") {
66 if (text[i] !== '"' || !string()) return false;
67 ws();
68 if (text[i] !== ":") return false;
69 i++;
70 }
71 if (!value(depth + 1)) return false;
72 ws();
73 if (text[i] === ",") { i++; continue; }
74 if (text[i] === close) { i++; return true; }
75 return false;
76 }
77 }
78 if (c === "t") return literal("true");
79 if (c === "f") return literal("false");
80 if (c === "n") return literal("null");
81 return number();
82 };
83 if (!value(0)) return i;
84 ws();
85 return i < text.length ? i : -1;
86 }
87
88 /** Bytes as a JSON object; empty is `{}`; a parse failure throws `InvalidJson`. */
89 export function objectOf(bytes: Uint8Array): Record<string, unknown> {
90 const text = new TextDecoder().decode(bytes);
91 if (text.trim() === "") return {};
92 let value: unknown;
93 try {
94 value = JSON.parse(text);
95 } catch {
96 // The offset is in characters of the decoded text; for the bytes, re-encode what precedes it.
97 const at = Math.max(0, firstBadOffset(text));
98 const byte = new TextEncoder().encode(text.slice(0, at)).length;
99 const token = text.slice(at).match(/^\S{1,12}/u)?.[0] ?? "end of body";
100 throw new InvalidJson(byte, token);
101 }
102 return typeof value === "object" && value !== null && !Array.isArray(value)
103 ? (value as Record<string, unknown>)
104 : {};
105 }
src/http/respond.ts +13 −16
@@ -4,6 +4,7 @@
44 * before it reaches an action.
55 */
66
7 import { InvalidJson, objectOf } from "./json_body.ts";
78 import { bodyCapFor, readBounded } from "./limits.ts";
89 import { scopeAllows, type TokenScope } from "../core/tokens.ts";
910 import { ARCHIVED_WHY } from "../core/repos.ts";
@@ -399,30 +400,17 @@ export function respondAccess(error: string, status: number): Response {
399400 return json({ error }, status);
400401 }
401402
402 /** Bytes as a JSON object, or `{}` for anything else — malformed input is not a 500. */
403 function asObject(bytes: Uint8Array): Record<string, unknown> {
404 let value: unknown;
405 try {
406 value = JSON.parse(new TextDecoder().decode(bytes));
407 } catch {
408 return {};
409 }
410 return typeof value === "object" && value !== null && !Array.isArray(value)
411 ? (value as Record<string, unknown>)
412 : {};
413 }
414
415403 /**
416404 * A JSON object body, at most \`BODY_MAX\` bytes for a signed-in caller and
417405 * \`STRANGER_MAX\` for anyone else (\`http/limits.ts\`) — a route reads its
418406 * body before it asks who is calling, so the cap cannot wait for that. Past
419407 * the cap it throws \`BodyTooLarge\` (answered 413) and the connection closes;
420 * malformed, \`{}\`.
408 * malformed, it throws \`InvalidJson\` (answered 400 \`invalid_json\`).
421409 */
422410 export async function body(request: Request): Promise<Record<string, unknown>> {
423411 const bytes = await readBounded(request, bodyCapFor(request));
424412 if (bytes === "too_large") throw new BodyTooLarge(bodyCapFor(request));
425 return asObject(bytes);
413 return objectOf(bytes);
426414 }
427415
428416 /**
@@ -437,6 +425,15 @@ export class BodyTooLarge extends Error {
437425 }
438426 }
439427
428 /** The 400 an \`InvalidJson\` is answered with: where the body stopped being JSON. */
429 export const invalidJson = (error: InvalidJson): Response =>
430 apiError(
431 "invalid_json",
432 400,
433 `the body is not valid JSON: bad token ${JSON.stringify(error.token)} at byte ${error.offset}`,
434 "send a JSON object with content-type application/json, e.g. {\"name\":\"value\"}",
435 );
436
440437 /** The 413 a \`BodyTooLarge\` is answered with. */
441438 export const bodyTooLarge = (error: BodyTooLarge): Response =>
442439 apiError("too_large", 413, `at most ${error.max} bytes`);
@@ -452,7 +449,7 @@ export async function boundedBody(
452449 max: number,
453450 ): Promise<Record<string, unknown> | "too_large"> {
454451 const bytes = await readBounded(request, max);
455 return bytes === "too_large" ? "too_large" : asObject(bytes);
452 return bytes === "too_large" ? "too_large" : objectOf(bytes);
456453 }
457454
458455 export const str = (v: unknown): string => (typeof v === "string" ? v : "");
src/http/server.ts +3
@@ -6,6 +6,7 @@
66 */
77
88 import { standingFragment } from "../web/pages/fork-standing.ts";
9 import { InvalidJson } from "./json_body.ts";
910 import { answeringFor, parseableRequest } from "./parseable_request.ts";
1011 import { expireCookie } from "../auth/cookies.ts";
1112 import { checkText } from "./check_text.ts";
@@ -648,6 +649,7 @@ import {
648649 securityFloor,
649650 BodyTooLarge,
650651 bodyTooLarge,
652 invalidJson,
651653 privateWhenSignedIn,
652654 tooManyAttempts,
653655 tryAgain,
@@ -2556,6 +2558,7 @@ export interface Servers {
25562558 */
25572559 function failure(app: App, error: unknown, request?: Request): Response {
25582560 if (error instanceof BodyTooLarge) return securityFloor(bodyTooLarge(error));
2561 if (error instanceof InvalidJson) return securityFloor(invalidJson(error));
25592562 if (isFullError(error)) {
25602563 noteFull(app.config.dataDir);
25612564 app.log.error("disk full: forge is read-only until space returns", { error: String(error) });
test/api-errors.test.ts +44
@@ -136,3 +136,47 @@ describe("API errors are distinguishable and self-describing", () => {
136136 expect(nope.error).toBe("no_route");
137137 });
138138 });
139
140 // #14269: a body that is not JSON used to read as `{}`, and the route answered
141 // its first field's validation error. One reader now refuses it as itself.
142 describe("a malformed JSON body is invalid_json", () => {
143 test("three routes name the bad token and its byte offset", async () => {
144 const dataDir = await mkdtemp(join(tmpdir(), "forge-badjson-"));
145 const app = createApp({ dataDir, port: 0 });
146 const servers = await serve(app);
147 try {
148 for (const path of ["/api/v1/start", "/api/v1/users", "/api/v1/first-admin"]) {
149 const res = await fetch(`${servers.url}${path}`, {
150 method: "POST",
151 headers: { "content-type": "application/json" },
152 body: '{"a":1, bad}',
153 });
154 const said = (await res.json()) as { error: string; message: string; hint: string };
155 expect(res.status).toBe(400);
156 expect(said.error).toBe("invalid_json");
157 expect(said.message).toContain("byte 8");
158 expect(said.hint).toBeTruthy();
159 }
160 const empty = await fetch(`${servers.url}/api/v1/start`, { method: "POST", body: "" });
161 expect(((await empty.json()) as { error: string }).error).not.toBe("invalid_json");
162 } finally {
163 servers.stop();
164 await rm(dataDir, { recursive: true, force: true });
165 }
166 });
167
168 test("the offset of the first bad token", async () => {
169 const { firstBadOffset } = await import("../src/http/json_body.ts");
170 expect(firstBadOffset("{bad")).toBe(1);
171 expect(firstBadOffset('{"a":')).toBe(5);
172 expect(firstBadOffset('{"a":1,}')).toBe(7);
173 expect(firstBadOffset('{"a":1} x')).toBe(8);
174 expect(firstBadOffset('{"a":[1,2,{"b":null}],"c":"\\u00e9"}')).toBe(-1);
175 });
176
177 test("the contract lists it on a route that reads a body", async () => {
178 const { openapi } = await import("../src/contract/openapi.ts");
179 const doc = JSON.stringify(openapi("http://x", "1", "admin"));
180 expect(doc).toContain("`invalid_json`");
181 });
182 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs