Public: anyone can read this; the crew can change it.
Every route that reads a request body has a size cap, and a test names any that does not #14688
Check · Review · Land · Live done — Landed and live in production.
mason/forge:mason/14677-every-route-that-reads-a-request-body-ha → main 5 commits fork 3 threads · 3 unresolved
20 files
changed +490
−43
AGENTS.md +2
Diff not shown — a line 3,551 characters wide (limit 2,000). +2
View filesrc/core/conversation_files.ts +2 −1
| @@ -14,9 +14,10 @@ import { join } from "node:path"; | ||
| 14 | 14 | import { id } from "../kernel/id.ts"; |
| 15 | 15 | import { err, ok, type Result } from "../kernel/result.ts"; |
| 16 | 16 | import { openConversation } from "./conversations.ts"; |
| 17 | import { BODY_CAPS } from "../kernel/body_caps.ts"; | |
| 17 | 18 | |
| 18 | 19 | /** The limits, stated to the person in every refusal. */ |
| 19 | export const IMAGE_MAX = 5 * 1024 * 1024; | |
| 20 | export const IMAGE_MAX = BODY_CAPS.talkImage.bytes; | |
| 20 | 21 | export const TEXT_MAX = 100 * 1024; |
| 21 | 22 | export const FILES_PER_MESSAGE = 4; |
| 22 | 23 | /** Bytes one person may keep in all; past it the oldest sent files are dropped to make room. */ |
src/core/timers.ts +2 −1
| @@ -9,6 +9,7 @@ | ||
| 9 | 9 | import type { Database } from "bun:sqlite"; |
| 10 | 10 | import { nextRun, parseCron } from "../apps/cron.ts"; |
| 11 | 11 | import type { ScheduleSpec } from "../apps/schedule_spec.ts"; |
| 12 | import { BODY_CAPS } from "../kernel/body_caps.ts"; | |
| 12 | 13 | |
| 13 | 14 | /** "production" or the preview environment it belongs to (`pull-12`). */ |
| 14 | 15 | export type TimerEnv = string; |
| @@ -204,7 +205,7 @@ export function syncSchedules(db: Database, appId: string, env: TimerEnv, specs: | ||
| 204 | 205 | })(); |
| 205 | 206 | } |
| 206 | 207 | |
| 207 | export const ONCE_BODY_MAX = 64 * 1024; | |
| 208 | export const ONCE_BODY_MAX = BODY_CAPS.timerBody.bytes; | |
| 208 | 209 | export const ONCE_KEY = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,99}$/; |
| 209 | 210 | |
| 210 | 211 | export interface Arm { |
src/devices/print.ts +2 −1
| @@ -13,8 +13,9 @@ import { RateLimiter } from "../core/ratelimit.ts"; | ||
| 13 | 13 | import { err, ok, type Result } from "../kernel/result.ts"; |
| 14 | 14 | import { printDocument, printPdf, probeRasterSupport, type PrintJob } from "./ipp.ts"; |
| 15 | 15 | import { PWG_MAX_BYTES, readPwg, unsupportedBy, type PwgError } from "./pwg.ts"; |
| 16 | import { BODY_CAPS } from "../kernel/body_caps.ts"; | |
| 16 | 17 | |
| 17 | export const PDF_MAX_BYTES = 10 * 1024 * 1024; | |
| 18 | export const PDF_MAX_BYTES = BODY_CAPS.printPdf.bytes; | |
| 18 | 19 | export const PDF_MAX_PAGES = 20; |
| 19 | 20 | |
| 20 | 21 | /** Prints one app may start in a minute, and at once: each holds the forge up to the printer's own bound. */ |
src/devices/pwg.ts +2 −1
| @@ -6,9 +6,10 @@ | ||
| 6 | 6 | * Plain bytes in, plain data out. |
| 7 | 7 | */ |
| 8 | 8 | |
| 9 | import { BODY_CAPS } from "../kernel/body_caps.ts"; | |
| 9 | 10 | import { err, ok, type Result } from "../kernel/result.ts"; |
| 10 | 11 | |
| 11 | export const PWG_MAX_BYTES = 80 * 1024 * 1024; | |
| 12 | export const PWG_MAX_BYTES = BODY_CAPS.printRaster.bytes; | |
| 12 | 13 | export const PWG_MAX_PAGES = 20; |
| 13 | 14 | export const PWG_HEADER_BYTES = 1796; |
| 14 | 15 | /** Sides and line length no real page reaches; keeps a forged header from making a long walk. */ |
src/http/api.ts +2 −2
| @@ -43,7 +43,7 @@ import { MILESTONE_WATCH_PERSON_ONLY, VOTE_NOT_RECORDED } from "../actions/board | ||
| 43 | 43 | import { recordDrill } from "../core/domain_drills.ts"; |
| 44 | 44 | import { drillVerdict } from "../kernel/domain_drill.ts"; |
| 45 | 45 | import { DOMAINS } from "../kernel/domains.ts"; |
| 46 | import { readBounded } from "./limits.ts"; | |
| 46 | import { BODY_CAPS, readBounded } from "./limits.ts"; | |
| 47 | 47 | import { askCheck, askedWords } from "../checks/again.ts"; |
| 48 | 48 | import { nextUp, queueJson } from "../core/next_up.ts"; |
| 49 | 49 | import { notStartable } from "../crew/not_startable.ts"; |
| @@ -3812,7 +3812,7 @@ export function apiRoutes(router: Router<Handler>, isPage: (path: string) => boo | ||
| 3812 | 3812 | // A child's signed heartbeat (docs/SPAWN.md): counts and version, never |
| 3813 | 3813 | // data. Every refusal is the same 404, so nothing says who is a child. |
| 3814 | 3814 | router.add("POST", "/api/v1/heartbeat", async (ctx) => { |
| 3815 | const bytes = await readBounded(ctx.request, 1024); | |
| 3815 | const bytes = await readBounded(ctx.request, BODY_CAPS.heartbeat.bytes); | |
| 3816 | 3816 | if (bytes === "too_large") return apiError("not_found", 404, "no such door"); |
| 3817 | 3817 | const raw = new TextDecoder().decode(bytes); |
| 3818 | 3818 | const kept = await receiveBeat(ctx.app, raw, ctx.request.headers.get(SIGNATURE_HEADER)); |
src/http/api/build_shots.ts +2 −2
| @@ -17,11 +17,11 @@ import { mayFetchLook } from "../../web/pages/looks.ts"; | ||
| 17 | 17 | import { loadRepo } from "../api.ts"; |
| 18 | 18 | import type { Handler } from "../context.ts"; |
| 19 | 19 | import { holds, weakEtag } from "../etag.ts"; |
| 20 | import { readBounded } from "../limits.ts"; | |
| 20 | import { BODY_CAPS, readBounded } from "../limits.ts"; | |
| 21 | 21 | import { apiError, body, json, notFound, requireScope, respondAccess } from "../respond.ts"; |
| 22 | 22 | import type { Router } from "../router.ts"; |
| 23 | 23 | |
| 24 | const UPLOAD_MAX = 3 * 1024 * 1024; | |
| 24 | const UPLOAD_MAX = BODY_CAPS.buildShot.bytes; | |
| 25 | 25 | const STATUS: Record<string, number> = { too_large: 413, moved: 409, too_many: 409, mirrored: 409, no_such_shot: 404 }; |
| 26 | 26 | |
| 27 | 27 | const items = (repo: Repo, label: string, shots: Shots) => ({ |
src/http/api/timers.ts +2 −3
| @@ -4,9 +4,8 @@ | ||
| 4 | 4 | */ |
| 5 | 5 | |
| 6 | 6 | import { armFor, cancelFor, type TimerError } from "../../actions/timers.ts"; |
| 7 | import { ONCE_BODY_MAX } from "../../core/timers.ts"; | |
| 8 | 7 | import type { Handler } from "../context.ts"; |
| 9 | import { readBounded } from "../limits.ts"; | |
| 8 | import { BODY_CAPS, readBounded } from "../limits.ts"; | |
| 10 | 9 | import { apiError, json, unauthorized } from "../respond.ts"; |
| 11 | 10 | import type { Router } from "../router.ts"; |
| 12 | 11 | import { tokenSecret } from "../token_header.ts"; |
| @@ -18,7 +17,7 @@ export function timersRoutes(router: Router<Handler>): void { | ||
| 18 | 17 | const secret = tokenSecret(ctx.request.headers.get("authorization") ?? ""); |
| 19 | 18 | if (secret === null) return unauthorized(); |
| 20 | 19 | // The body is a JSON envelope around a body of up to 64 KB. |
| 21 | const raw = await readBounded(ctx.request, ONCE_BODY_MAX + 4096); | |
| 20 | const raw = await readBounded(ctx.request, BODY_CAPS.timerArm.bytes); | |
| 22 | 21 | if (raw === "too_large") return apiError("invalid", 422, "`body` is at most 64 KB."); |
| 23 | 22 | let input: unknown; |
| 24 | 23 | try { |
src/http/csp_report.ts +2 −2
| @@ -18,11 +18,11 @@ import type { Database } from "bun:sqlite"; | ||
| 18 | 18 | import type { App } from "../app.ts"; |
| 19 | 19 | import { RateLimiter } from "../core/ratelimit.ts"; |
| 20 | 20 | import type { Ctx } from "./context.ts"; |
| 21 | import { readBounded } from "./limits.ts"; | |
| 21 | import { BODY_CAPS, readBounded } from "./limits.ts"; | |
| 22 | 22 | import { clientBucket } from "./peer.ts"; |
| 23 | 23 | |
| 24 | 24 | export const CSP_REPORT_PATH = "/csp-report"; |
| 25 | export const CSP_REPORT_MAX = 8 * 1024; | |
| 25 | export const CSP_REPORT_MAX = BODY_CAPS.cspReport.bytes; | |
| 26 | 26 | /** The Reporting API endpoint name a policy's `report-to` names. */ |
| 27 | 27 | export const CSP_REPORT_GROUP = "csp"; |
| 28 | 28 | /** What a page's policy appends: both the old and the new way to report. */ |
src/http/funnel.ts +2 −2
| @@ -13,7 +13,7 @@ import { getPlans } from "../settings/plans.ts"; | ||
| 13 | 13 | import { optedOut } from "../web/measure.ts"; |
| 14 | 14 | import { noteStep } from "../perf/funnel.ts"; |
| 15 | 15 | import type { Ctx } from "./context.ts"; |
| 16 | import { readBounded } from "./limits.ts"; | |
| 16 | import { BODY_CAPS, readBounded } from "./limits.ts"; | |
| 17 | 17 | import { clientAddress, clientBucket } from "./peer.ts"; |
| 18 | 18 | |
| 19 | 19 | /** What the home's buttons call the free start when no plan is free. */ |
| @@ -34,7 +34,7 @@ export async function funnelSubmit(ctx: Ctx): Promise<Response> { | ||
| 34 | 34 | } |
| 35 | 35 | const verdict = limiter.take(clientBucket(request), "ro"); |
| 36 | 36 | if (!verdict.allowed) return refuse(429, { "retry-after": String(Math.max(1, verdict.retryAfter)) }); |
| 37 | const bytes = await readBounded(request, 200); | |
| 37 | const bytes = await readBounded(request, BODY_CAPS.funnel.bytes); | |
| 38 | 38 | if (bytes === "too_large") return refuse(413); |
| 39 | 39 | let raw: unknown; |
| 40 | 40 | try { |
src/http/limits.ts +18 −17
| @@ -15,26 +15,27 @@ | ||
| 15 | 15 | * closed — left alone, the runtime would drain it for as long as the client |
| 16 | 16 | * sends. |
| 17 | 17 | * |
| 18 | * The caps: | |
| 19 | * REQUEST_MAX the listener's own ceiling (\`maxRequestBodySize\`): the | |
| 20 | * largest thing anyone may send, a git push. The runtime | |
| 21 | * enforces it on a declared length only. | |
| 22 | * BODY_MAX any JSON or form body read by the shared readers. | |
| 23 | * STRANGER_MAX a body someone without a token may send: signing up or | |
| 24 | * in, a report, a client registration. | |
| 25 | * FETCH_MAX a git fetch's negotiation (wants and haves), decompressed. | |
| 26 | * GZIP_MAX that negotiation as sent, when the client compressed it. | |
| 27 | * PUSH_MAX a git push's pack, as it streams to git. | |
| 28 | * DRAIN_MAX what is read and dropped past a cap before giving up. | |
| 18 | * Every cap is a row of `BODY_CAPS` (`kernel/body_caps.ts`, re-exported | |
| 19 | * here), with the reason it is that size; the module that reads the body | |
| 20 | * names its row, and | |
| 21 | * `test/body-caps.test.ts` walks the route table and names any route that | |
| 22 | * reads a body past every cap here. | |
| 29 | 23 | */ |
| 30 | 24 | |
| 31 | export const PUSH_MAX = 512 * 1024 * 1024; | |
| 25 | export { BODY_CAPS, type BodyCap, type BodyCapName } from "../kernel/body_caps.ts"; | |
| 26 | import { BODY_CAPS } from "../kernel/body_caps.ts"; | |
| 27 | ||
| 28 | export const PUSH_MAX = BODY_CAPS.push.bytes; | |
| 29 | /** | |
| 30 | * The listener's own ceiling (`maxRequestBodySize`); the runtime enforces it on a declared length | |
| 31 | * only. | |
| 32 | */ | |
| 32 | 33 | export const REQUEST_MAX = PUSH_MAX; |
| 33 | export const BODY_MAX = 16 * 1024 * 1024; | |
| 34 | export const STRANGER_MAX = 64 * 1024; | |
| 35 | export const FETCH_MAX = 16 * 1024 * 1024; | |
| 36 | export const DRAIN_MAX = 1024 * 1024; | |
| 37 | export const GZIP_MAX = 1024 * 1024; | |
| 34 | export const BODY_MAX = BODY_CAPS.body.bytes; | |
| 35 | export const STRANGER_MAX = BODY_CAPS.stranger.bytes; | |
| 36 | export const FETCH_MAX = BODY_CAPS.fetch.bytes; | |
| 37 | export const DRAIN_MAX = BODY_CAPS.drain.bytes; | |
| 38 | export const GZIP_MAX = BODY_CAPS.gzip.bytes; | |
| 38 | 39 | |
| 39 | 40 | const closing = new WeakSet<Request>(); |
| 40 | 41 | const signedIn = new WeakSet<Request>(); |
src/http/script_errors.ts +2 −2
| @@ -15,13 +15,13 @@ import type { App } from "../app.ts"; | ||
| 15 | 15 | import { RateLimiter } from "../core/ratelimit.ts"; |
| 16 | 16 | import { optedOut } from "../web/measure.ts"; |
| 17 | 17 | import type { Ctx } from "./context.ts"; |
| 18 | import { readBounded } from "./limits.ts"; | |
| 18 | import { BODY_CAPS, readBounded } from "./limits.ts"; | |
| 19 | 19 | import { clientBucket } from "./peer.ts"; |
| 20 | 20 | import type { RouteOf } from "./measure.ts"; |
| 21 | 21 | import { windowStart } from "./csp_report.ts"; |
| 22 | 22 | |
| 23 | 23 | export const SCRIPT_ERROR_PATH = "/_script-error"; |
| 24 | export const SCRIPT_ERROR_MAX = 2 * 1024; | |
| 24 | export const SCRIPT_ERROR_MAX = BODY_CAPS.scriptError.bytes; | |
| 25 | 25 | export const MESSAGE_MAX = 120; |
| 26 | 26 | const PATH_MAX = 200; |
| 27 | 27 | const DAY_ROWS = 500; |
src/http/server.ts +10 −1
| @@ -77,6 +77,7 @@ import { | ||
| 77 | 77 | import { defect } from "../kernel/error.ts"; |
| 78 | 78 | import { |
| 79 | 79 | armClose, |
| 80 | BODY_CAPS, | |
| 80 | 81 | BODY_MAX, |
| 81 | 82 | carryNotes, |
| 82 | 83 | closeAfter, |
| @@ -2337,7 +2338,15 @@ function hookCallerAllowed(app: App, request: Request): boolean { | ||
| 2337 | 2338 | } |
| 2338 | 2339 | |
| 2339 | 2340 | async function handleHook(app: App, request: Request): Promise<Response> { |
| 2340 | const parsed = (await request.json()) as HookRequest; | |
| 2341 | // The internal listener is still a door: its body is counted like any other. | |
| 2342 | const bytes = await readBounded(request, BODY_CAPS.gitHook.bytes); | |
| 2343 | if (bytes === "too_large") return new Response("hook call too large", { status: 413 }); | |
| 2344 | let parsed: HookRequest; | |
| 2345 | try { | |
| 2346 | parsed = JSON.parse(new TextDecoder().decode(bytes)) as HookRequest; | |
| 2347 | } catch { | |
Non-blocking: JSON.parse may return null or a non-object, which makes the next line throw. Answer 400 unless the value is an object. |
||
| 2348 | return new Response("hook call is not JSON", { status: 400 }); | |
| 2349 | } | |
| 2341 | 2350 | const input: HookRequest = { ...parsed, gitEnv: forwardedGitEnv(parsed.gitEnv) }; |
| 2342 | 2351 | const repo = getRepoById(app.db, input.repoId); |
| 2343 | 2352 | if (!repo) return new Response("unknown repository", { status: 400 }); |
src/http/stripe_hook.ts +2 −2
| @@ -22,10 +22,10 @@ import { subscriptionIdOf } from "../billing/subscription.ts"; | ||
| 22 | 22 | import { verifyStripeSignature } from "../billing/signature.ts"; |
| 23 | 23 | import { RateLimiter } from "../core/ratelimit.ts"; |
| 24 | 24 | import type { Ctx } from "./context.ts"; |
| 25 | import { readBounded } from "./limits.ts"; | |
| 25 | import { BODY_CAPS, readBounded } from "./limits.ts"; | |
| 26 | 26 | import { clientBucket } from "./peer.ts"; |
| 27 | 27 | |
| 28 | export const STRIPE_HOOK_MAX = 256 * 1024; | |
| 28 | export const STRIPE_HOOK_MAX = BODY_CAPS.stripeHook.bytes; | |
| 29 | 29 | const PER_CLIENT = 120; |
| 30 | 30 | const WHOLE_FORGE = 1200; |
| 31 | 31 | const EVENT_ID = /^evt_[A-Za-z0-9_]{1,200}$/; |
src/kernel/body_caps.ts added +114
| @@ -0,0 +1,114 @@ | ||
| 1 | /** | |
| 2 | * Every request-body cap the forge reads with, one row each with its | |
| 3 | * reason (#14677). Plain numbers, so the modules that state a limit to a | |
| 4 | * person (`core/`, `devices/`, `web/`) read it here without depending on | |
| 5 | * the HTTP layer; `http/limits.ts` re-exports the table and holds the | |
| 6 | * readers that enforce it. | |
| 7 | */ | |
| 8 | ||
| 9 | /** One cap: the most bytes of a body a reader keeps, and why that many. */ | |
| 10 | export interface BodyCap { | |
| 11 | readonly bytes: number; | |
| 12 | readonly reason: string; | |
| 13 | } | |
| 14 | ||
| 15 | const KIB = 1024; | |
| 16 | const MIB = 1024 * KIB; | |
| 17 | ||
| 18 | /** Every request-body cap the forge's routes read with, one row each. */ | |
| 19 | export const BODY_CAPS = { | |
| 20 | push: { | |
| 21 | bytes: 512 * MIB, | |
| 22 | reason: | |
| 23 | "a git push's pack, as it streams to git: the largest thing anyone may send, and the listener's own ceiling", | |
| 24 | }, | |
| 25 | fetch: { | |
| 26 | bytes: 16 * MIB, | |
| 27 | reason: "a git fetch's negotiation (wants and haves), decompressed: a huge history's haves fit", | |
| 28 | }, | |
| 29 | gzip: { | |
| 30 | bytes: 1 * MIB, | |
| 31 | reason: "that negotiation as sent, when the client compressed it: a sixteenth of what it may grow to", | |
| 32 | }, | |
| 33 | body: { | |
| 34 | bytes: 16 * MIB, | |
| 35 | reason: "any JSON or form body a signed-in caller sends through the shared readers: an issue, a file edit, a board", | |
| 36 | }, | |
| 37 | stranger: { | |
| 38 | bytes: 64 * KIB, | |
| 39 | reason: | |
| 40 | "a body someone without a token may send: signing up or in, a report, a client registration; read before the route asks who is calling", | |
| 41 | }, | |
| 42 | drain: { | |
| 43 | bytes: 1 * MIB, | |
| 44 | reason: "read and dropped past a cap so the connection stays framed; past it the connection closes", | |
| 45 | }, | |
| 46 | dropForm: { | |
| 47 | bytes: 32 * MIB, | |
| 48 | reason: "a Drop upload sent as a plain form: read whole to parse; bigger files go by the script or the API, which stream", | |
| 49 | }, | |
| 50 | beacon: { | |
| 51 | bytes: 512, | |
| 52 | reason: "a page-view beacon: a few dozen bytes of numbers, so this is already generous", | |
| 53 | }, | |
| 54 | scriptError: { | |
| 55 | bytes: 2 * KIB, | |
| 56 | reason: "a page's script error: a message and a path, both cut short before they are kept", | |
| 57 | }, | |
| 58 | cspReport: { | |
| 59 | bytes: 8 * KIB, | |
| 60 | reason: "a browser's Content-Security-Policy report: one violation, its URLs and a sample", | |
| 61 | }, | |
| 62 | stripeHook: { | |
| 63 | bytes: 256 * KIB, | |
| 64 | reason: "a Stripe event delivery, signed over its exact bytes: one event, never a batch", | |
| 65 | }, | |
| 66 | originHook: { | |
| 67 | bytes: 256 * KIB, | |
| 68 | reason: "an origin's webhook delivery: it names a pull and little else", | |
| 69 | }, | |
| 70 | brokerCall: { | |
| 71 | bytes: 16 * KIB, | |
| 72 | reason: "a hosted child's signed call to its parent's origin broker, an HMAC over its exact bytes: one call, not a payload", | |
| 73 | }, | |
| 74 | timerBody: { | |
| 75 | bytes: 64 * KIB, | |
| 76 | reason: "the body a one-shot timer delivers to its app, stated to the app when it is refused", | |
| 77 | }, | |
| 78 | timerArm: { | |
| 79 | bytes: 68 * KIB, | |
| 80 | reason: "a call that arms a one-shot timer: the timer's body (`timerBody`) and 4 KiB for the JSON envelope around it", | |
| 81 | }, | |
| 82 | heartbeat: { | |
| 83 | bytes: 1 * KIB, | |
| 84 | reason: "a spawned child's signed heartbeat: a version and a few counts, never data", | |
| 85 | }, | |
| 86 | funnel: { | |
| 87 | bytes: 200, | |
| 88 | reason: "a funnel step a page reports: a step name and a path, nothing more", | |
| 89 | }, | |
| 90 | gitHook: { | |
| 91 | bytes: 16 * MIB, | |
| 92 | reason: | |
| 93 | "a git hook's call on the internal listener: a push's ref updates, a line each; a mirror of a huge repository's refs fits", | |
| 94 | }, | |
| 95 | talkImage: { | |
| 96 | bytes: 5 * MIB, | |
| 97 | reason: "a picture attached to a conversation: a screenshot or a photo; the limit is stated to the person in every refusal", | |
| 98 | }, | |
| 99 | buildShot: { | |
| 100 | bytes: 3 * MIB, | |
| 101 | reason: "one screenshot a build uploads of itself: a single picture", | |
| 102 | }, | |
| 103 | printPdf: { | |
| 104 | bytes: 10 * MIB, | |
| 105 | reason: "a PDF sent to a printer: a long document fits, and the refusal names the limit", | |
| 106 | }, | |
| 107 | printRaster: { | |
| 108 | bytes: 80 * MIB, | |
| 109 | reason: "PWG raster sent to a printer: uncompressed pages run large; every page is walked before anything is sent", | |
| 110 | }, | |
| 111 | } as const satisfies Record<string, BodyCap>; | |
| 112 | ||
| 113 | /** The name of one row of {@link BODY_CAPS}. */ | |
| 114 | export type BodyCapName = keyof typeof BODY_CAPS; |
src/origins/hook.ts +3 −2
| @@ -19,11 +19,12 @@ import { providerForOrigin } from "./credential.ts"; | ||
| 19 | 19 | import { HOOK_SIGNATURE_HEADER, hookSecret, hookUrl, signatureOf } from "./hook-secret.ts"; |
| 20 | 20 | import { refreshOriginItem } from "./item-check.ts"; |
| 21 | 21 | import { refreshOriginPull } from "./pull-check.ts"; |
| 22 | import { BODY_CAPS } from "../kernel/body_caps.ts"; | |
| 22 | 23 | |
| 23 | 24 | export { HOOK_SIGNATURE_HEADER, hookSecret, hookUrl, signatureOf }; |
| 24 | 25 | |
| 25 | /** The most a delivery may say; it names a pull and little else. */ | |
| 26 | export const HOOK_BODY_MAX = 256 * 1024; | |
| 26 | /** The most a delivery may say; it names a pull and little else (`kernel/body_caps.ts`). */ | |
| 27 | export const HOOK_BODY_MAX = BODY_CAPS.originHook.bytes; | |
| 27 | 28 | |
| 28 | 29 | /** The pull number a delivery names: ours, Gitea's or GitHub's shape. */ |
| 29 | 30 | export function numberIn(body: unknown): number | undefined { |
src/spawn/origin_broker.ts +2 −1
| @@ -84,6 +84,7 @@ import { | ||
| 84 | 84 | type BrokerOffer, |
| 85 | 85 | } from "./origin_broker_wire.ts"; |
| 86 | 86 | import { parentOrigin } from "./placement.ts"; |
| 87 | import { BODY_CAPS } from "../kernel/body_caps.ts"; | |
| 87 | 88 | |
| 88 | 89 | /** A token waits for its child this long, and not a second more. */ |
| 89 | 90 | export const CODE_TTL_MS = 60_000; |
| @@ -93,7 +94,7 @@ export const CONSENT_TTL_MS = 10 * 60_000; | ||
| 93 | 94 | export const START_SKEW_MS = 5 * 60_000; |
| 94 | 95 | /** A signed call is good this far either side of this forge's clock. */ |
| 95 | 96 | export const CALL_SKEW_MS = 2 * 60_000; |
| 96 | export const CALL_BODY_MAX = 16 * 1024; | |
| 97 | export const CALL_BODY_MAX = BODY_CAPS.brokerCall.bytes; | |
| 97 | 98 | /** Unfinished sign-ins one child may have waiting here. */ |
| 98 | 99 | const PENDING_MAX = 20; |
| 99 | 100 | /** Calls one child may make a minute. */ |
src/web/measure.ts +3 −2
| @@ -16,9 +16,10 @@ import type { Database } from "bun:sqlite"; | ||
| 16 | 16 | import { quietly } from "../core/write_clock.ts"; |
| 17 | 17 | import { sha256 } from "../kernel/hash.ts"; |
| 18 | 18 | import { dayOf } from "../perf/day.ts"; |
| 19 | import { BODY_CAPS } from "../kernel/body_caps.ts"; | |
| 19 | 20 | |
| 20 | /** A report is a few dozen bytes; this is already generous. */ | |
| 21 | export const REPORT_MAX = 512; | |
| 21 | /** A report is a few dozen bytes (`kernel/body_caps.ts`, `beacon`). */ | |
| 22 | export const REPORT_MAX = BODY_CAPS.beacon.bytes; | |
| 22 | 23 | /** Times in milliseconds: ten minutes is longer than any real page view. */ |
| 23 | 24 | export const TIME_MAX = 600_000; |
| 24 | 25 | export const CLS_MAX = 100; |
src/web/studio/drop.ts +2 −1
| @@ -19,12 +19,13 @@ import type { Flash } from "../flash.ts"; | ||
| 19 | 19 | import { html } from "../html.ts"; |
| 20 | 20 | import { page, redirect } from "../layout.ts"; |
| 21 | 21 | import { badge, btn, empty, fileSize } from "../ui.ts"; |
| 22 | import { BODY_CAPS } from "../../kernel/body_caps.ts"; | |
| 22 | 23 | |
| 23 | 24 | /** |
| 24 | 25 | * The most a form upload may hold: the body is read whole to parse it; bigger files use the script |
| 25 | 26 | * or the API. |
| 26 | 27 | */ |
| 27 | export const FORM_MAX = 32 * 1024 * 1024; | |
| 28 | export const FORM_MAX = BODY_CAPS.dropForm.bytes; | |
| 28 | 29 | |
| 29 | 30 | /** GET /:owner/:name/drop */ |
| 30 | 31 | export function dropPageGet(ctx: Ctx): Response { |
test/body-caps.test.ts added +314
| @@ -0,0 +1,314 @@ | ||
| 1 | /** | |
| 2 | * Every route that reads a request body stops at a cap (#14677). The caps | |
| 3 | * are one table, `BODY_CAPS` in `src/kernel/body_caps.ts`, each row with its | |
| 4 | * reason. The routes come from both routers (`buildRouter`, | |
| 5 | * `buildApiRouter`): every route that is not a GET is asked in-process with | |
| 6 | * an endless body — once as the site's admin with an admin token, once as a | |
| 7 | * stranger — and the bytes it pulls are counted. A route that reads the | |
| 8 | * whole probe, longer than any cap in the table allows, has no cap, and the | |
| 9 | * test names it. A 413 under `/api/` must be the standard | |
| 10 | * `{ error: "too_large" }`. | |
| 11 | * | |
| 12 | * A route whose path parameters name nothing seeded answers 404 before it | |
| 13 | * reads, so the probe cannot see past it; the source scan below covers that | |
| 14 | * blind side: no module buffers a request body whole (`request.json()` and | |
| 15 | * the like), and no cap is a number written at the call instead of a row. | |
| 16 | */ | |
| 17 | ||
| 18 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 19 | import { mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs"; | |
| 20 | import { tmpdir } from "node:os"; | |
| 21 | import { join } from "node:path"; | |
| 22 | import { createRepo } from "../src/actions/repos.ts"; | |
| 23 | import { createApp, type App } from "../src/app.ts"; | |
| 24 | import { createIssue } from "../src/core/issues.ts"; | |
| 25 | import { getRepo } from "../src/core/repos.ts"; | |
| 26 | import { createToken } from "../src/core/tokens.ts"; | |
| 27 | import { createUser, type User } from "../src/core/users.ts"; | |
| 28 | import { BODY_CAPS, DRAIN_MAX, REQUEST_MAX } from "../src/http/limits.ts"; | |
| 29 | import { body, BodyTooLarge, bodyTooLarge } from "../src/http/respond.ts"; | |
| 30 | import { buildApiRouter, buildRouter, forgeHttp, serve, type ForgeHttp } from "../src/http/server.ts"; | |
| 31 | import { HOOK_SECRET_HEADER } from "../src/git/hook-protocol.ts"; | |
| 32 | import { quietLogs } from "./helpers/globals.ts"; | |
| 33 | import { proveAll } from "./helpers/verified.ts"; | |
| 34 | ||
| 35 | quietLogs(); | |
| 36 | ||
| 37 | const MIB = 1024 * 1024; | |
| 38 | /** One chunk of the probe body: spaces, so a reader that kept them all would parse `{}`. */ | |
| 39 | const CHUNK = new Uint8Array(MIB).fill(0x20); | |
| 40 | ||
| 41 | /** | |
| 42 | * The largest cap a route reads whole with: every row but the push, which | |
| 43 | * streams to git and is never kept. | |
| 44 | */ | |
| 45 | const LARGEST_KEPT = Math.max( | |
| 46 | ...Object.entries(BODY_CAPS) | |
| 47 | .filter(([name]) => name !== "push") | |
| 48 | .map(([, cap]) => cap.bytes), | |
| 49 | ); | |
| 50 | /** | |
| 51 | * Past the largest cap, the drain budget and a chunk either side: a route that reads it all has no | |
| 52 | * cap. | |
| 53 | */ | |
| 54 | const PROBE = LARGEST_KEPT + DRAIN_MAX + 2 * CHUNK.byteLength; | |
| 55 | ||
| 56 | /** | |
| 57 | * What each path parameter is filled with: the seeded admin, its public repository and its first | |
| 58 | * issue. | |
| 59 | */ | |
| 60 | const FILL: Readonly<Record<string, string>> = { | |
| 61 | owner: "alice", username: "alice", user: "alice", org: "alice", scope: "alice", | |
| 62 | name: "notes", repo: "notes", number: "1", n: "1", id: "1", job: "1", | |
| 63 | ref: "main", branch: "main", oid: "0".repeat(40), sha: "0".repeat(40), commit: "0".repeat(40), | |
| 64 | }; | |
| 65 | ||
| 66 | const pathOf = (pattern: string): string => | |
| 67 | pattern | |
| 68 | .replace(/:([A-Za-z_]+)/g, (_whole, k: string) => FILL[k] ?? "x") | |
| 69 | .replace(/\*[A-Za-z_]+$/, "README.md"); | |
| 70 | ||
| 71 | interface Probed { | |
| 72 | route: string; | |
| 73 | pulled: number; | |
| 74 | status: number; | |
| 75 | /** The answer's body, when it was a 413. */ | |
| 76 | refusal: string; | |
| 77 | } | |
| 78 | ||
| 79 | /** | |
| 80 | * Ask `handle` once with an endless body and count what it pulls. The body | |
| 81 | * ends at PROBE, so a reader with no cap finishes instead of hanging; the | |
| 82 | * answer is read through, since a streamed route (a git push) may go on | |
| 83 | * pulling as it answers. | |
| 84 | */ | |
| 85 | async function probe( | |
| 86 | handle: (request: Request) => Promise<Response>, | |
| 87 | method: string, | |
| 88 | path: string, | |
| 89 | headers: Record<string, string>, | |
| 90 | ): Promise<Omit<Probed, "route">> { | |
| 91 | let pulled = 0; | |
| 92 | const stream = new ReadableStream<Uint8Array>( | |
| 93 | { | |
| 94 | pull(controller) { | |
| 95 | if (pulled >= PROBE) return controller.close(); | |
| 96 | pulled += CHUNK.byteLength; | |
| 97 | controller.enqueue(CHUNK); | |
| 98 | }, | |
| 99 | }, | |
| 100 | { highWaterMark: 0 }, | |
| 101 | ); | |
| 102 | const request = new Request(`http://localhost${path}`, { | |
| 103 | method, | |
| 104 | body: stream, | |
| 105 | headers: { "content-type": "application/json", ...headers }, | |
| 106 | duplex: "half", | |
| 107 | } as RequestInit); | |
| 108 | let response: Response; | |
| 109 | try { | |
| 110 | response = await within(handle(request), 5000); | |
| 111 | } catch (error) { | |
| 112 | // The listener answers this one itself (`http/server.ts`, `failure`). | |
| 113 | if (!(error instanceof BodyTooLarge)) throw error; | |
| 114 | response = bodyTooLarge(error); | |
| 115 | } | |
| 116 | const text = await within(response.text(), 3000).catch(() => ""); | |
| 117 | return { pulled, status: response.status, refusal: response.status === 413 ? text : "" }; | |
| 118 | } | |
| 119 | ||
| 120 | const within = <T>(work: Promise<T>, ms: number): Promise<T> => | |
| 121 | Promise.race([ | |
| 122 | work, | |
| 123 | new Promise<T>((_, reject) => setTimeout(() => reject(new Error(`no answer in ${ms} ms`)), ms)), | |
| 124 | ]); | |
| 125 | ||
| 126 | const bodyRoutes = (): { method: string; pattern: string }[] => { | |
| 127 | const seen = new Set<string>(); | |
| 128 | return [...buildRouter().routes(), ...buildApiRouter().routes()].filter((r) => { | |
| 129 | const key = `${r.method} ${r.pattern}`; | |
| 130 | if (r.method === "GET" || r.method === "HEAD" || seen.has(key)) return false; | |
| 131 | seen.add(key); | |
| 132 | return true; | |
| 133 | }); | |
| 134 | }; | |
| 135 | ||
| 136 | let root = ""; | |
| 137 | let app: App; | |
| 138 | let http: ForgeHttp; | |
| 139 | let admin: User; | |
| 140 | ||
| 141 | /** The public repository and its issue; a route that deletes it is followed by a new one. */ | |
| 142 | async function seedRepo(): Promise<void> { | |
| 143 | if (getRepo(app.db, "alice", "notes") !== undefined) return; | |
| 144 | const made = await createRepo(app, admin, { name: "notes", visibility: "public" }); | |
| 145 | if (!made.ok) throw new Error(made.error); | |
| 146 | const issue = createIssue(app.db, { repoId: made.value.id, authorId: admin.id, title: "First", body: "" }); | |
| 147 | if (!issue.ok) throw new Error(issue.error); | |
| 148 | } | |
| 149 | ||
| 150 | /** | |
| 151 | * Every non-GET route asked once; `headers` says who is asking (a fresh token each, so none | |
| 152 | * outlives a route that ends it). | |
| 153 | */ | |
| 154 | async function probeAll(headers: () => Record<string, string>): Promise<Probed[]> { | |
| 155 | const out: Probed[] = []; | |
| 156 | for (const { method, pattern } of bodyRoutes()) { | |
| 157 | const route = `${method} ${pattern}`; | |
| 158 | out.push({ route, ...(await probe(http.handle, method, pathOf(pattern), headers())) }); | |
| 159 | await seedRepo(); | |
| 160 | } | |
| 161 | return out; | |
| 162 | } | |
| 163 | ||
| 164 | const uncapped = (probed: Probed[]): string[] => | |
| 165 | probed.filter((p) => p.pulled >= PROBE).map((p) => `${p.route} read ${p.pulled} bytes`); | |
| 166 | ||
| 167 | const nonstandard413 = (probed: Probed[]): string[] => | |
| 168 | probed | |
| 169 | .filter((p) => p.status === 413 && p.route.includes(" /api/")) | |
| 170 | .filter((p) => { | |
| 171 | try { | |
| 172 | return (JSON.parse(p.refusal) as { error?: unknown }).error !== "too_large"; | |
| 173 | } catch { | |
| 174 | return true; | |
| 175 | } | |
| 176 | }) | |
| 177 | .map((p) => `${p.route} answered 413 ${JSON.stringify(p.refusal.slice(0, 80))}`); | |
| 178 | ||
| 179 | let asAdmin: Probed[] = []; | |
| 180 | let asStranger: Probed[] = []; | |
| 181 | ||
| 182 | beforeAll(async () => { | |
| 183 | root = mkdtempSync(join(tmpdir(), "forge-body-caps-")); | |
| 184 | app = createApp({ dataDir: root, port: 0 }); | |
| 185 | const made = await createUser(app.db, { | |
| 186 | origin: "seed", trusted: true, username: "alice", email: "[email protected]", | |
| 187 | password: "a-good-passphrase", adminIfFirst: true, | |
| 188 | }); | |
| 189 | if (!made.ok) throw new Error(made.error); | |
| 190 | admin = made.value; | |
| 191 | proveAll(app); | |
| 192 | await seedRepo(); | |
| 193 | http = forgeHttp(app); | |
| 194 | let n = 0; | |
| 195 | asAdmin = await probeAll(() => ({ | |
| 196 | authorization: `Bearer ${createToken(app.db, admin.id, `probe-${++n}`, "admin").token}`, | |
| 197 | })); | |
| 198 | asStranger = await probeAll(() => ({})); | |
| 199 | }, 120_000); | |
| 200 | ||
| 201 | afterAll(() => { | |
| 202 | http.close(); | |
| 203 | app.close(); | |
| 204 | rmSync(root, { recursive: true, force: true }); | |
| 205 | }); | |
| 206 | ||
| 207 | describe("the cap table", () => { | |
| 208 | test("every row is a whole number of bytes the listener can take, with a reason", () => { | |
| 209 | for (const [name, cap] of Object.entries(BODY_CAPS)) { | |
| 210 | expect({ name, whole: Number.isSafeInteger(cap.bytes) && cap.bytes > 0 }).toEqual({ name, whole: true }); | |
| 211 | expect({ name, fits: cap.bytes <= REQUEST_MAX }).toEqual({ name, fits: true }); | |
| 212 | expect({ name, reason: cap.reason.trim().length > 10 }).toEqual({ name, reason: true }); | |
| 213 | } | |
| 214 | }); | |
| 215 | }); | |
| 216 | ||
| 217 | describe("the probe", () => { | |
| 218 | test("names a reader with no cap, and passes one that reads through the shared reader", async () => { | |
| 219 | const unbounded = await probe(async (r) => new Response(String((await r.text()).length)), "POST", "/x", {}); | |
| 220 | expect(unbounded.pulled).toBeGreaterThanOrEqual(PROBE); | |
| 221 | const shared = await probe(async (r) => Response.json(await body(r)), "POST", "/x", {}).catch(() => undefined); | |
| 222 | // A stranger's body through `body` stops at STRANGER_MAX, then drains. | |
| 223 | expect(shared?.pulled ?? 0).toBeLessThan(PROBE); | |
| 224 | }); | |
| 225 | ||
| 226 | test("reached the routes: most refuse the admin's probe with a 413", () => { | |
| 227 | // A seed that broke (the token refused, the repository gone) would | |
| 228 | // leave every route answering 401 or 404 and the walk proving nothing. | |
| 229 | expect(asAdmin.filter((p) => p.status === 413).length).toBeGreaterThan(300); | |
| 230 | }); | |
| 231 | }); | |
| 232 | ||
| 233 | describe("every route that reads a body", () => { | |
| 234 | test("stops at a cap for the site's admin", () => { | |
| 235 | expect(uncapped(asAdmin)).toEqual([]); | |
| 236 | }); | |
| 237 | ||
| 238 | test("stops at a cap for a stranger", () => { | |
| 239 | expect(uncapped(asStranger)).toEqual([]); | |
| 240 | }); | |
| 241 | ||
| 242 | test("under /api/, refuses past its cap in the standard shape", () => { | |
| 243 | expect([...nonstandard413(asAdmin), ...nonstandard413(asStranger)]).toEqual([]); | |
| 244 | }); | |
| 245 | }); | |
| 246 | ||
| 247 | describe("the internal listener", () => { | |
| 248 | /** POST to this forge's own hook listener, with its boot secret. */ | |
| 249 | const hook = async (on: App, body: Uint8Array): Promise<Response> => { | |
| 250 | const at = on.hookAddress.current; | |
| 251 | const init = { method: "POST", body, headers: { [HOOK_SECRET_HEADER]: on.hookSecret } }; | |
| 252 | return at.startsWith("tcp:") | |
| 253 | ? fetch(`http://${at.slice("tcp:".length)}/internal/hook`, init) | |
outdated
UNCAPPED names an uncapped body reader as acceptable. Cap handleHook (add a BODY_CAPS row) and drop this entry; also scan for literal numeric caps passed to readBounded. |
||
| 254 | : fetch("http://localhost/internal/hook", { ...init, unix: at } as RequestInit); | |
| 255 | }; | |
| 256 | ||
| 257 | test("refuses a git hook call past its cap, and one that is not JSON", async () => { | |
| 258 | const dir = mkdtempSync(join(tmpdir(), "forge-hook-cap-")); | |
| 259 | const own = createApp({ dataDir: dir, port: 0 }); | |
| 260 | const servers = await serve(own); | |
| 261 | try { | |
| 262 | const over = await hook(own, new Uint8Array(BODY_CAPS.gitHook.bytes + 1).fill(0x20)); | |
| 263 | expect(over.status).toBe(413); | |
| 264 | await over.text(); | |
| 265 | const garbled = await hook(own, new TextEncoder().encode("{not json")); | |
| 266 | expect(garbled.status).toBe(400); | |
| 267 | await garbled.text(); | |
| 268 | } finally { | |
| 269 | servers.stop(); | |
| 270 | own.close(); | |
| 271 | rmSync(dir, { recursive: true, force: true }); | |
| 272 | } | |
| 273 | }); | |
| 274 | }); | |
| 275 | ||
| 276 | /** A request's body read whole by the runtime, with nothing counting it. */ | |
| 277 | const BUFFERED = | |
| 278 | /\b(?:request|req|ctx\.request)\.(?:json|text|arrayBuffer|formData|blob|bytes)\(\)|new Response\((?:ctx\.)?req(?:uest)?\.body\)|readableStreamTo\w+\((?:ctx\.)?req(?:uest)?\.body\b/; | |
| 279 | ||
| 280 | const sources = (dir: string): string[] => | |
| 281 | readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { | |
| 282 | const path = join(dir, entry.name); | |
| 283 | return entry.isDirectory() ? sources(path) : path.endsWith(".ts") ? [path] : []; | |
| 284 | }); | |
| 285 | ||
| 286 | /** | |
| 287 | * A cap passed to `readBounded` or `counted` that is written as a number | |
| 288 | * there, not named from `BODY_CAPS`: a call on one line, its second argument | |
| 289 | * holding a digit. | |
| 290 | */ | |
| 291 | const LITERAL_CAP = /\b(?:readBounded|counted)\([^,()]+,\s*[^)]*\b\d/; | |
| 292 | ||
| 293 | const matching = (pattern: RegExp): string[] => | |
| 294 | sources("src").flatMap((path) => | |
| 295 | readFileSync(path, "utf8") | |
| 296 | .split("\n") | |
| 297 | .flatMap((line, i) => (pattern.test(line) ? [`${path}:${i + 1}: ${line.trim()}`] : [])), | |
| 298 | ); | |
| 299 | ||
| 300 | describe("the source", () => { | |
| 301 | test("buffers no request body whole", () => { | |
| 302 | expect(matching(BUFFERED)).toEqual([]); | |
| 303 | }); | |
| 304 | ||
| 305 | test("names every cap from the table, never a number at the call", () => { | |
| 306 | expect(matching(LITERAL_CAP)).toEqual([]); | |
| 307 | }); | |
| 308 | ||
| 309 | test("the literal-cap scan names a number written at the call", () => { | |
| 310 | expect(LITERAL_CAP.test("await readBounded(ctx.request, 1024);")).toBe(true); | |
| 311 | expect(LITERAL_CAP.test("await readBounded(request, ONCE_BODY_MAX + 4096);")).toBe(true); | |
| 312 | expect(LITERAL_CAP.test("await readBounded(request, BODY_CAPS.heartbeat.bytes);")).toBe(false); | |
| 313 | }); | |
| 314 | }); |
new above
offline
core/ importing http/limits.ts inverts the layering; put the table in a neutral module.