Public: anyone can read this; the crew can change it.
A malformed JSON body is answered invalid_json, not the first field's validation error #14291
Check · Review · Land · Live — Its check is red.
scout/forge:scout/14269-a-malformed-json-body-is-answered-invali → main 2 commits fork
5 files
changed +180
−19
src/contract/openapi.ts +15 −3
| @@ -43,6 +43,14 @@ export const pagesByCursor = (r: Pick<Route, "method" | "query">): boolean => r. | ||
| 43 | 43 | |
| 44 | 44 | const PAGE_REFUSAL = "page or offset was sent: this list pages by cursor, pass next_cursor as ?cursor= (400)"; |
| 45 | 45 | |
| 46 | /** | |
| 47 | * A write that declares a body reads JSON, and a body that does not parse is refused at the | |
| 48 | * boundary (`http/json_body.ts`). | |
| 49 | */ | |
| 50 | export const readsJsonBody = (r: Pick<Route, "method" | "body">): boolean => r.method !== "get" && r.method !== "delete" && r.body !== undefined; | |
| 51 | ||
| 52 | const JSON_REFUSAL = "400 { error, message, hint }: the body is not JSON; the message names the first bad token and its byte offset"; | |
| 53 | ||
| 46 | 54 | /** A route's operationId, and the name of the file that holds it. */ |
| 47 | 55 | export function operationId(r: Pick<Route, "method" | "path">): string { |
| 48 | 56 | return `${r.method}${r.path.replace(/[^A-Za-z0-9]+(.)?/g, (_, c: string | undefined) => (c ?? "").toUpperCase())}`; |
| @@ -90,7 +98,11 @@ export function openapi(origin: string, version: string, role: Role): Record<str | ||
| 90 | 98 | const paths: Record<string, Record<string, unknown>> = {}; |
| 91 | 99 | const rows = routesFor(role); |
| 92 | 100 | for (const r of rows) { |
| 93 | const refusals = pagesByCursor(r) ? { invalid_query: PAGE_REFUSAL, ...r.refusals } : r.refusals; | |
| 101 | const refusals = { | |
| 102 | ...(pagesByCursor(r) ? { invalid_query: PAGE_REFUSAL } : {}), | |
| 103 | ...(readsJsonBody(r) ? { invalid_json: JSON_REFUSAL } : {}), | |
| 104 | ...r.refusals, | |
| 105 | }; | |
| 94 | 106 | const params = [...r.path.matchAll(/\{(\w+)\}/g)].map((m) => ({ |
| 95 | 107 | name: m[1], |
| 96 | 108 | in: "path", |
| @@ -155,12 +167,12 @@ export function openapi(origin: string, version: string, role: Role): Record<str | ||
| 155 | 167 | }), |
| 156 | 168 | }, |
| 157 | 169 | ...Object.fromEntries((r.alsoStatus ?? []).map((c) => [String(c), { description: "Done" }])), |
| 170 | ...(pagesByCursor(r) ? { "400": { description: PAGE_REFUSAL } } : readsJsonBody(r) ? { "400": { description: JSON_REFUSAL } } : {}), | |
| 158 | 171 | ...Object.fromEntries(Object.entries(r.errorStatus ?? {}).map(([c, d]) => [c, { description: d }])), |
| 159 | ...(pagesByCursor(r) ? { "400": { description: PAGE_REFUSAL } } : {}), | |
| 160 | 172 | default: { |
| 161 | 173 | description: |
| 162 | 174 | "An error: { error, message }; `error` is a stable code" + |
| 163 | (refusals === undefined | |
| 175 | (Object.keys(refusals).length === 0 | |
| 164 | 176 | ? "" |
| 165 | 177 | : `. Refusals: ${Object.entries(refusals).map(([c, w]) => `\`${c}\` (${w})`).join("; ")}`), |
| 166 | 178 | }, |
src/http/json_body.ts added +105
| @@ -0,0 +1,105 @@ | ||
| 1 | /** | |
| 2 | * A JSON body that does not parse, found at the boundary (#14269). Before, | |
| 3 | * `{bad` read as `{}` and the route answered its first field's validation | |
| 4 | * error — an agent then hunted for a field it had sent. Now the body reader | |
| 5 | * throws `InvalidJson` and the server's door answers 400 `invalid_json` in | |
| 6 | * the `{ error, message, hint }` shape, naming the byte offset of the first | |
| 7 | * bad token. An empty body is still `{}`; valid JSON that is not an object | |
| 8 | * is still `{}` (the route's own validation says what it lacked). | |
| 9 | */ | |
| 10 | ||
| 11 | export class InvalidJson extends Error { | |
| 12 | constructor(readonly offset: number, readonly token: string) { | |
| 13 | super(`not JSON at byte ${offset}`); | |
| 14 | } | |
| 15 | } | |
| 16 | ||
| 17 | /** Byte offset of the first character that cannot continue a JSON text, or -1 when it is whole. */ | |
| 18 | export function firstBadOffset(text: string): number { | |
| 19 | let i = 0; | |
| 20 | const ws = () => { | |
| 21 | while (i < text.length && " \t\n\r".includes(text[i] as string)) i++; | |
| 22 | }; | |
| 23 | // Each reader returns true when it read a value; on false, `i` is the bad spot. | |
| 24 | const literal = (word: string) => { | |
| 25 | if (text.startsWith(word, i)) { i += word.length; return true; } | |
| 26 | let k = 0; | |
| 27 | while (i + k < text.length && text[i + k] === word[k]) k++; | |
| 28 | i += k; | |
| 29 | return false; | |
| 30 | }; | |
| 31 | const string = (): boolean => { | |
| 32 | i++; | |
| 33 | while (i < text.length) { | |
| 34 | const c = text[i] as string; | |
| 35 | if (c === '"') { i++; return true; } | |
| 36 | if (c < " ") return false; | |
| 37 | if (c === "\\") { | |
| 38 | const e = text[i + 1]; | |
| 39 | if (e === "u") { | |
| 40 | if (!/^[0-9a-fA-F]{4}$/.test(text.slice(i + 2, i + 6))) return false; | |
| 41 | i += 6; | |
| 42 | } else if (e !== undefined && '"\\/bfnrt'.includes(e)) i += 2; | |
| 43 | else { i++; return false; } | |
| 44 | } else i++; | |
| 45 | } | |
| 46 | return false; | |
| 47 | }; | |
| 48 | const number = (): boolean => { | |
| 49 | const m = /^-?(0|[1-9]\d*)(\.\d+)?([eE][+-]?\d+)?/.exec(text.slice(i)); | |
| 50 | if (m === null) return false; | |
| 51 | i += m[0].length; | |
| 52 | return true; | |
| 53 | }; | |
| 54 | const value = (depth: number): boolean => { | |
| 55 | ws(); | |
| 56 | const c = text[i]; | |
| 57 | if (c === undefined || depth > 512) return false; | |
| 58 | if (c === '"') return string(); | |
| 59 | if (c === "{" || c === "[") { | |
| 60 | const close = c === "{" ? "}" : "]"; | |
| 61 | i++; ws(); | |
| 62 | if (text[i] === close) { i++; return true; } | |
| 63 | for (;;) { | |
| 64 | ws(); | |
| 65 | if (c === "{") { | |
| 66 | if (text[i] !== '"' || !string()) return false; | |
| 67 | ws(); | |
| 68 | if (text[i] !== ":") return false; | |
| 69 | i++; | |
| 70 | } | |
| 71 | if (!value(depth + 1)) return false; | |
| 72 | ws(); | |
| 73 | if (text[i] === ",") { i++; continue; } | |
| 74 | if (text[i] === close) { i++; return true; } | |
| 75 | return false; | |
| 76 | } | |
| 77 | } | |
| 78 | if (c === "t") return literal("true"); | |
| 79 | if (c === "f") return literal("false"); | |
| 80 | if (c === "n") return literal("null"); | |
| 81 | return number(); | |
| 82 | }; | |
| 83 | if (!value(0)) return i; | |
| 84 | ws(); | |
| 85 | return i < text.length ? i : -1; | |
| 86 | } | |
| 87 | ||
| 88 | /** Bytes as a JSON object; empty is `{}`; a parse failure throws `InvalidJson`. */ | |
| 89 | export function objectOf(bytes: Uint8Array): Record<string, unknown> { | |
| 90 | const text = new TextDecoder().decode(bytes); | |
| 91 | if (text.trim() === "") return {}; | |
| 92 | let value: unknown; | |
| 93 | try { | |
| 94 | value = JSON.parse(text); | |
| 95 | } catch { | |
| 96 | // The offset is in characters of the decoded text; for the bytes, re-encode what precedes it. | |
| 97 | const at = Math.max(0, firstBadOffset(text)); | |
| 98 | const byte = new TextEncoder().encode(text.slice(0, at)).length; | |
| 99 | const token = text.slice(at).match(/^\S{1,12}/u)?.[0] ?? "end of body"; | |
| 100 | throw new InvalidJson(byte, token); | |
| 101 | } | |
| 102 | return typeof value === "object" && value !== null && !Array.isArray(value) | |
| 103 | ? (value as Record<string, unknown>) | |
| 104 | : {}; | |
| 105 | } |
src/http/respond.ts +13 −16
| @@ -4,6 +4,7 @@ | ||
| 4 | 4 | * before it reaches an action. |
| 5 | 5 | */ |
| 6 | 6 | |
| 7 | import { InvalidJson, objectOf } from "./json_body.ts"; | |
| 7 | 8 | import { bodyCapFor, readBounded } from "./limits.ts"; |
| 8 | 9 | import { scopeAllows, type TokenScope } from "../core/tokens.ts"; |
| 9 | 10 | import { ARCHIVED_WHY } from "../core/repos.ts"; |
| @@ -399,30 +400,17 @@ export function respondAccess(error: string, status: number): Response { | ||
| 399 | 400 | return json({ error }, status); |
| 400 | 401 | } |
| 401 | 402 | |
| 402 | /** Bytes as a JSON object, or `{}` for anything else — malformed input is not a 500. */ | |
| 403 | function asObject(bytes: Uint8Array): Record<string, unknown> { | |
| 404 | let value: unknown; | |
| 405 | try { | |
| 406 | value = JSON.parse(new TextDecoder().decode(bytes)); | |
| 407 | } catch { | |
| 408 | return {}; | |
| 409 | } | |
| 410 | return typeof value === "object" && value !== null && !Array.isArray(value) | |
| 411 | ? (value as Record<string, unknown>) | |
| 412 | : {}; | |
| 413 | } | |
| 414 | ||
| 415 | 403 | /** |
| 416 | 404 | * A JSON object body, at most \`BODY_MAX\` bytes for a signed-in caller and |
| 417 | 405 | * \`STRANGER_MAX\` for anyone else (\`http/limits.ts\`) — a route reads its |
| 418 | 406 | * body before it asks who is calling, so the cap cannot wait for that. Past |
| 419 | 407 | * the cap it throws \`BodyTooLarge\` (answered 413) and the connection closes; |
| 420 | * malformed, \`{}\`. | |
| 408 | * malformed, it throws \`InvalidJson\` (answered 400 \`invalid_json\`). | |
| 421 | 409 | */ |
| 422 | 410 | export async function body(request: Request): Promise<Record<string, unknown>> { |
| 423 | 411 | const bytes = await readBounded(request, bodyCapFor(request)); |
| 424 | 412 | if (bytes === "too_large") throw new BodyTooLarge(bodyCapFor(request)); |
| 425 | return asObject(bytes); | |
| 413 | return objectOf(bytes); | |
| 426 | 414 | } |
| 427 | 415 | |
| 428 | 416 | /** |
| @@ -437,6 +425,15 @@ export class BodyTooLarge extends Error { | ||
| 437 | 425 | } |
| 438 | 426 | } |
| 439 | 427 | |
| 428 | /** The 400 an \`InvalidJson\` is answered with: where the body stopped being JSON. */ | |
| 429 | export const invalidJson = (error: InvalidJson): Response => | |
| 430 | apiError( | |
| 431 | "invalid_json", | |
| 432 | 400, | |
| 433 | `the body is not valid JSON: bad token ${JSON.stringify(error.token)} at byte ${error.offset}`, | |
| 434 | "send a JSON object with content-type application/json, e.g. {\"name\":\"value\"}", | |
| 435 | ); | |
| 436 | ||
| 440 | 437 | /** The 413 a \`BodyTooLarge\` is answered with. */ |
| 441 | 438 | export const bodyTooLarge = (error: BodyTooLarge): Response => |
| 442 | 439 | apiError("too_large", 413, `at most ${error.max} bytes`); |
| @@ -452,7 +449,7 @@ export async function boundedBody( | ||
| 452 | 449 | max: number, |
| 453 | 450 | ): Promise<Record<string, unknown> | "too_large"> { |
| 454 | 451 | const bytes = await readBounded(request, max); |
| 455 | return bytes === "too_large" ? "too_large" : asObject(bytes); | |
| 452 | return bytes === "too_large" ? "too_large" : objectOf(bytes); | |
| 456 | 453 | } |
| 457 | 454 | |
| 458 | 455 | export const str = (v: unknown): string => (typeof v === "string" ? v : ""); |
src/http/server.ts +3
| @@ -6,6 +6,7 @@ | ||
| 6 | 6 | */ |
| 7 | 7 | |
| 8 | 8 | import { standingFragment } from "../web/pages/fork-standing.ts"; |
| 9 | import { InvalidJson } from "./json_body.ts"; | |
| 9 | 10 | import { answeringFor, parseableRequest } from "./parseable_request.ts"; |
| 10 | 11 | import { expireCookie } from "../auth/cookies.ts"; |
| 11 | 12 | import { checkText } from "./check_text.ts"; |
| @@ -648,6 +649,7 @@ import { | ||
| 648 | 649 | securityFloor, |
| 649 | 650 | BodyTooLarge, |
| 650 | 651 | bodyTooLarge, |
| 652 | invalidJson, | |
| 651 | 653 | privateWhenSignedIn, |
| 652 | 654 | tooManyAttempts, |
| 653 | 655 | tryAgain, |
| @@ -2556,6 +2558,7 @@ export interface Servers { | ||
| 2556 | 2558 | */ |
| 2557 | 2559 | function failure(app: App, error: unknown, request?: Request): Response { |
| 2558 | 2560 | if (error instanceof BodyTooLarge) return securityFloor(bodyTooLarge(error)); |
| 2561 | if (error instanceof InvalidJson) return securityFloor(invalidJson(error)); | |
| 2559 | 2562 | if (isFullError(error)) { |
| 2560 | 2563 | noteFull(app.config.dataDir); |
| 2561 | 2564 | app.log.error("disk full: forge is read-only until space returns", { error: String(error) }); |
test/api-errors.test.ts +44
| @@ -136,3 +136,47 @@ describe("API errors are distinguishable and self-describing", () => { | ||
| 136 | 136 | expect(nope.error).toBe("no_route"); |
| 137 | 137 | }); |
| 138 | 138 | }); |
| 139 | ||
| 140 | // #14269: a body that is not JSON used to read as `{}`, and the route answered | |
| 141 | // its first field's validation error. One reader now refuses it as itself. | |
| 142 | describe("a malformed JSON body is invalid_json", () => { | |
| 143 | test("three routes name the bad token and its byte offset", async () => { | |
| 144 | const dataDir = await mkdtemp(join(tmpdir(), "forge-badjson-")); | |
| 145 | const app = createApp({ dataDir, port: 0 }); | |
| 146 | const servers = await serve(app); | |
| 147 | try { | |
| 148 | for (const path of ["/api/v1/start", "/api/v1/users", "/api/v1/first-admin"]) { | |
| 149 | const res = await fetch(`${servers.url}${path}`, { | |
| 150 | method: "POST", | |
| 151 | headers: { "content-type": "application/json" }, | |
| 152 | body: '{"a":1, bad}', | |
| 153 | }); | |
| 154 | const said = (await res.json()) as { error: string; message: string; hint: string }; | |
| 155 | expect(res.status).toBe(400); | |
| 156 | expect(said.error).toBe("invalid_json"); | |
| 157 | expect(said.message).toContain("byte 8"); | |
| 158 | expect(said.hint).toBeTruthy(); | |
| 159 | } | |
| 160 | const empty = await fetch(`${servers.url}/api/v1/start`, { method: "POST", body: "" }); | |
| 161 | expect(((await empty.json()) as { error: string }).error).not.toBe("invalid_json"); | |
| 162 | } finally { | |
| 163 | servers.stop(); | |
| 164 | await rm(dataDir, { recursive: true, force: true }); | |
| 165 | } | |
| 166 | }); | |
| 167 | ||
| 168 | test("the offset of the first bad token", async () => { | |
| 169 | const { firstBadOffset } = await import("../src/http/json_body.ts"); | |
| 170 | expect(firstBadOffset("{bad")).toBe(1); | |
| 171 | expect(firstBadOffset('{"a":')).toBe(5); | |
| 172 | expect(firstBadOffset('{"a":1,}')).toBe(7); | |
| 173 | expect(firstBadOffset('{"a":1} x')).toBe(8); | |
| 174 | expect(firstBadOffset('{"a":[1,2,{"b":null}],"c":"\\u00e9"}')).toBe(-1); | |
| 175 | }); | |
| 176 | ||
| 177 | test("the contract lists it on a route that reads a body", async () => { | |
| 178 | const { openapi } = await import("../src/contract/openapi.ts"); | |
| 179 | const doc = JSON.stringify(openapi("http://x", "1", "admin")); | |
| 180 | expect(doc).toContain("`invalid_json`"); | |
| 181 | }); | |
| 182 | }); |
new above
offline