Public: anyone can read this; the crew can change it.
Every 429 and 503 the API and MCP answer carries Retry-After, and one test walks them all #10781
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/10762-every-429-and-503-the-api-and-mcp-answer → main 5 commits fork 2 threads · 2 unresolved
21 files
changed +384
−111
docs/MCP.md +7
| @@ -15,6 +15,13 @@ for a tunnel and a forge that sleeps. | ||
| 15 | 15 | - **The first call wakes it.** A forge scaled to zero is woken by the request |
| 16 | 16 | like any page; a client's retry on a slow first answer is safe, because |
| 17 | 17 | every write is an ordinary forge action and the reads change nothing. |
| 18 | - **Slowing down is told in seconds.** Every 429 and 503 from the API and | |
| 19 | `/mcp` carries an integer `Retry-After` header, and a JSON body names the | |
| 20 | same seconds as `retryAfter`; wait that long. The answers that are not JSON | |
| 21 | (`/healthz`, `/readyz`, a refused shell, the plain-text standby page) carry | |
| 22 | the header alone. `test/retry-after.test.ts` checks every site in the code | |
| 23 | and drives the token, stranger, code-search, sign-up and OAuth limits and | |
| 24 | `/mcp` itself. | |
| 18 | 25 | - **A token, never a cookie.** `Authorization: Bearer <forge token>`. A |
| 19 | 26 | browser session is refused even when signed in: a cookie is what a |
| 20 | 27 | cross-site page could ride, and this door has no page. An `ro` token reads; |
src/contract/openapi.ts +1 −1
| @@ -127,7 +127,7 @@ export function openapi(origin: string, version: string): Record<string, unknown | ||
| 127 | 127 | title: "forge", |
| 128 | 128 | version, |
| 129 | 129 | description: |
| 130 | "The routes the forge names to an agent. `not_found` means missing or not yours to see.", | |
| 130 | "The routes the forge names to an agent. `not_found` means missing or not yours to see. Every 429 and 503 carries an integer `Retry-After` and `retryAfter` (the same seconds) in a JSON body.", | |
| 131 | 131 | }, |
| 132 | 132 | servers: [{ url: origin }], |
| 133 | 133 | // The default: a bearer token. Open routes say `security: []`. |
src/http/api.ts +13 −17
| @@ -327,6 +327,8 @@ import { | ||
| 327 | 327 | respondAccess, |
| 328 | 328 | str, |
| 329 | 329 | unauthorized, |
| 330 | tryAgain, | |
| 331 | retryAfterOf, | |
| 330 | 332 | } from "./respond.ts"; |
| 331 | 333 | import { |
| 332 | 334 | explorePage, |
| @@ -1174,11 +1176,7 @@ export function apiRoutes(router: Router<Handler>): void { | ||
| 1174 | 1176 | if (crossOrigin(ctx.request, ctx.url, ctx.app.log)) return forbidden(); |
| 1175 | 1177 | const slot = signupSlot(ctx); |
| 1176 | 1178 | if (typeof slot === "number") |
| 1177 | return json( | |
| 1178 | { error: "rate_limited", message: "too many sign-ups; wait and send again" }, | |
| 1179 | 429, | |
| 1180 | { "retry-after": String(slot) }, | |
| 1181 | ); | |
| 1179 | return tryAgain("rate_limited", 429, "too many sign-ups; wait and send again", slot); | |
| 1182 | 1180 | // Anyone may call this, so its body is a stranger's (`http/limits.ts`). |
| 1183 | 1181 | const input = await boundedBody(ctx.request, STRANGER_MAX); |
| 1184 | 1182 | if (input === "too_large") |
| @@ -1386,7 +1384,7 @@ export function apiRoutes(router: Router<Handler>): void { | ||
| 1386 | 1384 | if (!minted.ok && minted.error === "invalid_via") |
| 1387 | 1385 | return apiError("invalid_conversation", 422, "not a conversation of yours"); |
| 1388 | 1386 | if (!minted.ok && minted.error === "rate_limited") |
| 1389 | return json({ error: "rate_limited", message: "too many conversation tokens are alive; wait for one to expire" }, 429, { "retry-after": "60" }); | |
| 1387 | return tryAgain("rate_limited", 429, "too many conversation tokens are alive; wait for one to expire", 60); | |
| 1390 | 1388 | if (!minted.ok) return unauthorized(); |
| 1391 | 1389 | return json(minted.value, 201); |
| 1392 | 1390 | }); |
| @@ -1430,7 +1428,7 @@ export function apiRoutes(router: Router<Handler>): void { | ||
| 1430 | 1428 | ); |
| 1431 | 1429 | // The throttle is in the action, before the code is read (H2's shape). |
| 1432 | 1430 | if (!made.ok && made.error === "rate_limited") |
| 1433 | return json({ error: made.error, message: FIRST_ADMIN_WORDS[made.error] }, 429, { "retry-after": "60" }); | |
| 1431 | return tryAgain(made.error, 429, FIRST_ADMIN_WORDS[made.error], 60); | |
| 1434 | 1432 | if (!made.ok) |
| 1435 | 1433 | return apiError(made.error, made.error === "refused" ? 403 : 422, FIRST_ADMIN_WORDS[made.error]); |
| 1436 | 1434 | return json( |
| @@ -1465,7 +1463,7 @@ export function apiRoutes(router: Router<Handler>): void { | ||
| 1465 | 1463 | address: clientAddress(ctx.request), |
| 1466 | 1464 | }); |
| 1467 | 1465 | if (!recovered.ok && recovered.error === "rate_limited") |
| 1468 | return json({ error: recovered.error, message: FIRST_ADMIN_WORDS.rate_limited }, 429, { "retry-after": "60" }); | |
| 1466 | return tryAgain(recovered.error, 429, FIRST_ADMIN_WORDS.rate_limited, 60); | |
| 1469 | 1467 | if (!recovered.ok) return apiError("refused", 403, FIRST_ADMIN_WORDS.refused); |
| 1470 | 1468 | const { user, id, token, expiresAt } = recovered.value; |
| 1471 | 1469 | return json({ user: publicUser(user), scope, id, token, expiresAt, expiresIn }, 201); |
| @@ -1529,9 +1527,7 @@ export function apiRoutes(router: Router<Handler>): void { | ||
| 1529 | 1527 | cleanQuery(query) !== ""; |
| 1530 | 1528 | const held = greps ? strangerCodeSearch(ctx.app, ctx.request) : undefined; |
| 1531 | 1529 | if (held !== undefined && !held.allowed) { |
| 1532 | const refused = apiError("rate_limited", 429, `${held.why}; sign in, or retry in ${held.retryAfter}s`); | |
| 1533 | refused.headers.set("retry-after", String(held.retryAfter)); | |
| 1534 | return refused; | |
| 1530 | return tryAgain("rate_limited", 429, `${held.why}; sign in, or retry in ${retryAfterOf(held.retryAfter)}s`, held.retryAfter); | |
| 1535 | 1531 | } |
| 1536 | 1532 | try { |
| 1537 | 1533 | const input = { |
| @@ -3469,13 +3465,13 @@ export function apiRoutes(router: Router<Handler>): void { | ||
| 3469 | 3465 | }); |
| 3470 | 3466 | if (!made.ok) |
| 3471 | 3467 | return made.error.kind === "limit" |
| 3472 | ? json( | |
| 3473 | { | |
| 3474 | error: "limit", | |
| 3475 | message: `at most ${COPIES_PER_DAY} copies a day; another after ${made.error.resetsAt}`, | |
| 3476 | resetsAt: made.error.resetsAt, | |
| 3477 | }, | |
| 3468 | ? tryAgain( | |
| 3469 | "limit", | |
| 3478 | 3470 | 429, |
| 3471 | `at most ${COPIES_PER_DAY} copies a day; another after ${made.error.resetsAt}`, | |
| 3472 | (Date.parse(String(made.error.resetsAt)) - Date.now()) / 1000, | |
| 3473 | {}, | |
| 3474 | { resetsAt: made.error.resetsAt }, | |
| 3479 | 3475 | ) |
| 3480 | 3476 | : made.error.kind === "repository_limit" |
| 3481 | 3477 | ? apiError("repository_limit", 403, made.error.message) |
src/http/api/app.ts +3 −2
| @@ -18,7 +18,8 @@ import type { Result } from "../../kernel/result.ts"; | ||
| 18 | 18 | import { appJson, appLimitResponse, loadRepo } from "../api.ts"; |
| 19 | 19 | import type { Ctx, Handler } from "../context.ts"; |
| 20 | 20 | import { holds, weakEtag } from "../etag.ts"; |
| 21 | import { apiError, body, json, notFound, requireScope, respondAccess, str, unauthorized } from "../respond.ts"; | |
| 21 | import { BUILD_ASKS_WINDOW_MS } from "../../actions/build_preview.ts"; | |
| 22 | import { apiError, body, json, notFound, requireScope, respondAccess, str, tryAgain, unauthorized } from "../respond.ts"; | |
| 22 | 23 | import type { Router } from "../router.ts"; |
| 23 | 24 | |
| 24 | 25 | function appPatch(input: Record<string, unknown>): AppSettingsInput { |
| @@ -336,7 +337,7 @@ export function appRoutes(router: Router<Handler>): void { | ||
| 336 | 337 | const preview = await previewBuild(ctx.app, ctx.auth.user, repo, sha); |
| 337 | 338 | if (!preview.ok) |
| 338 | 339 | return preview.error === "rate_limited" |
| 339 | ? json({ error: previewBuildRefusalText(preview.error) }, 429) | |
| 340 | ? tryAgain("rate_limited", 429, previewBuildRefusalText(preview.error), BUILD_ASKS_WINDOW_MS / 1000) | |
| 340 | 341 | : respondAccess(preview.error, 422); |
| 341 | 342 | return json({ env: preview.value.env, url: preview.value.url }, 201); |
| 342 | 343 | }); |
src/http/api/app_notes.ts +5 −2
| @@ -10,9 +10,12 @@ import { getAppByRepo } from "../../core/apps.ts"; | ||
| 10 | 10 | import { loadRepo } from "../api.ts"; |
| 11 | 11 | import type { Ctx, Handler } from "../context.ts"; |
| 12 | 12 | import { holds, weakEtag } from "../etag.ts"; |
| 13 | import { apiError, forbidden, json, notFound, requireScope } from "../respond.ts"; | |
| 13 | import { forbidden, json, notFound, requireScope, tryAgain } from "../respond.ts"; | |
| 14 | 14 | import type { Router } from "../router.ts"; |
| 15 | 15 | |
| 16 | /** Moderation needs the builders, who are away just now; no clock says when they are back, so a short stated guess. */ | |
| 17 | const NOTES_RETRY = 30; | |
| 18 | ||
| 16 | 19 | /** The repository and its app, or the refusal. */ |
| 17 | 20 | function target(ctx: Ctx) { |
| 18 | 21 | const denied = requireScope(ctx, "rw"); |
| @@ -55,7 +58,7 @@ function moderation(action: "hide" | "ban" | "unban"): Handler { | ||
| 55 | 58 | if (!done.ok) { |
| 56 | 59 | if (done.error === "forbidden") return forbidden(); |
| 57 | 60 | if (done.error === "not_found") return notFound(); |
| 58 | return apiError("unavailable", 503, "that could not be done just now"); | |
| 61 | return tryAgain("unavailable", 503, "that could not be done just now", NOTES_RETRY); | |
| 59 | 62 | } |
| 60 | 63 | const note = ownerNote(ctx.app, t.record, t.repo, ctx.auth.user, id); |
| 61 | 64 | return note === undefined ? notFound() : json(note); |
src/http/api/conversation_talk.ts +18 −4
| @@ -33,9 +33,23 @@ import { fileQueued, queuedOf, turnRunning } from "../../crew/conversation.ts"; | ||
| 33 | 33 | import { crossOrigin, type Handler } from "../context.ts"; |
| 34 | 34 | import { holds, weakEtag } from "../etag.ts"; |
| 35 | 35 | import { readBounded } from "../limits.ts"; |
| 36 | import { apiError, boundedBody, forbidden, json, unauthorized } from "../respond.ts"; | |
| 36 | import { apiError, boundedBody, forbidden, json, tryAgain, unauthorized, WINDOW } from "../respond.ts"; | |
| 37 | 37 | import type { Router } from "../router.ts"; |
| 38 | 38 | |
| 39 | /** | |
| 40 | * Seconds to wait after each talk refusal that says wait. The per-minute | |
| 41 | * limit and "busy" are a minute by their own words; the rest depend on a | |
| 42 | * turn ending or an operator, so they are stated conservative guesses. | |
| 43 | */ | |
| 44 | const TALK_WAIT: Readonly<Record<string, number>> = { | |
| 45 | rate_limited: WINDOW.minute, | |
| 46 | busy: WINDOW.minute, | |
| 47 | queue_full: 30, | |
| 48 | resting: 300, | |
| 49 | off: 300, | |
| 50 | }; | |
| 51 | const talkWait = (error: string): number => TALK_WAIT[error] ?? WINDOW.minute; | |
| 52 | ||
| 39 | 53 | /** Room for the words at their cap even when every character is escaped. */ |
| 40 | 54 | const BODY_MAX = 128_000; |
| 41 | 55 | |
| @@ -84,7 +98,7 @@ export function conversationTalkRoutes(router: Router<Handler>): void { | ||
| 84 | 98 | const said = talk(ctx.app, user, text, files.value, during === "queue" ? "queue" : during === "steer" ? "interrupt" : "inject", named, typeof input["about"] === "string" ? input["about"] : null); |
| 85 | 99 | if (!said.ok) { |
| 86 | 100 | const [status, words] = TALK_WORDS[said.error]; |
| 87 | return json({ error: said.error, message: words }, status, said.error === "rate_limited" || said.error === "busy" ? { "retry-after": "60" } : {}); | |
| 101 | return status === 429 || status === 503 ? tryAgain(said.error, status, words, talkWait(said.error)) : json({ error: said.error, message: words }, status); | |
| 88 | 102 | } |
| 89 | 103 | return json(said.value.queued === undefined ? { how: said.value.how } : { how: said.value.how, queued: said.value.queued }, 202); |
| 90 | 104 | }); |
| @@ -104,7 +118,7 @@ export function conversationTalkRoutes(router: Router<Handler>): void { | ||
| 104 | 118 | if (crossOrigin(ctx.request, ctx.url, ctx.app.log)) return forbidden(); |
| 105 | 119 | if (ctx.auth.via === "token" && !scopeAllows(ctx.auth.scope ?? "ro", "rw")) return forbidden(); |
| 106 | 120 | const stopped = stopTalk(ctx.app, user); |
| 107 | if (!stopped.ok) return json({ error: stopped.error, message: stopped.error === "idle" ? "no answer is running" : "the forge's agent is not running here" }, stopped.error === "idle" ? 409 : 503); | |
| 121 | if (!stopped.ok) return stopped.error === "idle" ? json({ error: stopped.error, message: "no answer is running" }, 409) : tryAgain(stopped.error, 503, "the forge's agent is not running here", talkWait("off")); | |
| 108 | 122 | return json({ stopped: true }, 202); |
| 109 | 123 | }); |
| 110 | 124 | |
| @@ -114,7 +128,7 @@ export function conversationTalkRoutes(router: Router<Handler>): void { | ||
| 114 | 128 | if (user === undefined) return unauthorized(); |
| 115 | 129 | if (crossOrigin(ctx.request, ctx.url, ctx.app.log)) return forbidden(); |
| 116 | 130 | if (ctx.auth.via === "token" && !scopeAllows(ctx.auth.scope ?? "ro", "rw")) return forbidden(); |
| 117 | if (!uploadLimiter(ctx.app).take(`upload:${user.id}`, "rw").allowed) return json({ error: "rate_limited", message: `at most ${UPLOADS_PER_MINUTE} files a minute` }, 429, { "retry-after": "60" }); | |
| 131 | if (!uploadLimiter(ctx.app).take(`upload:${user.id}`, "rw").allowed) return tryAgain("rate_limited", 429, `at most ${UPLOADS_PER_MINUTE} files a minute`, WINDOW.minute); | |
| 118 | 132 | const bytes = await readBounded(ctx.request, IMAGE_MAX); |
| 119 | 133 | const refuse = (why: keyof typeof FILE_WORDS) => apiError(why, FILE_WORDS[why][0], FILE_WORDS[why][1]); |
| 120 | 134 | if (bytes === "too_large") return refuse("too_large"); |
src/http/api/data.ts +6 −3
| @@ -17,7 +17,8 @@ import { | ||
| 17 | 17 | } from "../../data/repo_scope.ts"; |
| 18 | 18 | import { loadRepo } from "../api.ts"; |
| 19 | 19 | import type { Ctx, Handler } from "../context.ts"; |
| 20 | import { apiError, body, json, notFound, requireScope } from "../respond.ts"; | |
| 20 | import { QUERY_TIMEOUT_MS } from "../../data/query.ts"; | |
| 21 | import { apiError, body, json, notFound, requireScope, tryAgain } from "../respond.ts"; | |
| 21 | 22 | import type { Router } from "../router.ts"; |
| 22 | 23 | |
| 23 | 24 | const PRIVATE = { "cache-control": "private, no-store", vary: "authorization, cookie" }; |
| @@ -74,8 +75,10 @@ export function dataRoutes(router: Router<Handler>): void { | ||
| 74 | 75 | const result = await runSql(ctx.app, user, found.source, input["sql"]); |
| 75 | 76 | if (!result.ok) { |
| 76 | 77 | if (result.error === "rate") |
| 77 | return json({ error: "rate_limited", message: result.message, retryAfter: result.retryAfter }, 429, { ...PRIVATE, "retry-after": String(result.retryAfter) }); | |
| 78 | return apiError(result.error === "busy" ? "busy" : result.error === "timeout" ? "timeout" : "invalid_sql", result.error === "busy" ? 429 : 422, result.message); | |
| 78 | return tryAgain("rate_limited", 429, result.message, result.retryAfter, PRIVATE); | |
| 79 | // Every worker is taken; one frees within the query timeout at the latest. | |
| 80 | if (result.error === "busy") return tryAgain("busy", 429, result.message, QUERY_TIMEOUT_MS / 1000, PRIVATE); | |
| 81 | return apiError(result.error === "timeout" ? "timeout" : "invalid_sql", 422, result.message); | |
| 79 | 82 | } |
| 80 | 83 | return reply(result); |
| 81 | 84 | }); |
src/http/api/following.ts +7 −6
| @@ -19,7 +19,7 @@ import { upstreamPayload } from "../../spawn/upstream.ts"; | ||
| 19 | 19 | import { loadRepo, respondTake } from "../api.ts"; |
| 20 | 20 | import type { Handler } from "../context.ts"; |
| 21 | 21 | import { holds, weakEtag } from "../etag.ts"; |
| 22 | import { apiError, body, json, notFound, requireScope, respondAccess } from "../respond.ts"; | |
| 22 | import { apiError, body, json, notFound, requireScope, respondAccess, tryAgain, untilNextDay } from "../respond.ts"; | |
| 23 | 23 | import type { Router } from "../router.ts"; |
| 24 | 24 | |
| 25 | 25 | /** |
| @@ -242,8 +242,7 @@ export function followingRoutes(router: Router<Handler>): void { | ||
| 242 | 242 | : outcome.kind === "nobody" || outcome.kind === "forbidden" || outcome.kind === "not_member" |
| 243 | 243 | ? 403 |
| 244 | 244 | : 409; |
| 245 | return json( | |
| 246 | { | |
| 245 | const payload = { | |
| 247 | 246 | kind: outcome.kind, |
| 248 | 247 | issue: "issue" in outcome ? outcome.issue.number : null, |
| 249 | 248 | pull: "pull" in outcome ? outcome.pull.number : null, |
| @@ -251,9 +250,11 @@ export function followingRoutes(router: Router<Handler>): void { | ||
| 251 | 250 | "agent" in outcome && outcome.agent !== undefined |
| 252 | 251 | ? outcome.agent.user.username |
| 253 | 252 | : null, |
| 254 | }, | |
| 255 | status, | |
| 256 | ); | |
| 253 | }; | |
| 254 | // The crew's ceiling is a day's spend: it starts over at the next UTC day. | |
| 255 | if (outcome.kind === "ceiling") | |
| 256 | return tryAgain("ceiling", 429, "the crew is at its ceiling for today", untilNextDay(), {}, payload); | |
| 257 | return json(payload, status); | |
| 257 | 258 | }, |
| 258 | 259 | ); |
| 259 | 260 | } |
src/http/api/hooks.ts +2 −6
| @@ -14,7 +14,7 @@ import { getWebhook, listWebhooks, pingWebhook, recentDeliveries } from "../../c | ||
| 14 | 14 | import { loadRepo, pageParams, slicePage } from "../api.ts"; |
| 15 | 15 | import type { Ctx, Handler } from "../context.ts"; |
| 16 | 16 | import { holds, weakEtag } from "../etag.ts"; |
| 17 | import { body, json, notFound, requireScope, respondAccess, str } from "../respond.ts"; | |
| 17 | import { body, json, notFound, requireScope, respondAccess, str, tryAgain } from "../respond.ts"; | |
| 18 | 18 | import type { Router } from "../router.ts"; |
| 19 | 19 | |
| 20 | 20 | /** |
| @@ -64,11 +64,7 @@ export function hookRoutes(router: Router<Handler>): void { | ||
| 64 | 64 | if (!hook) return notFound(); |
| 65 | 65 | const pinged = pingWebhook(ctx.app.db, `${ctx.params.owner}/${ctx.params.name}`, hook); |
| 66 | 66 | if (!pinged.ok) |
| 67 | return json( | |
| 68 | { error: "rate_limited", message: `too many pings: try again in ${pinged.error.retryAfter} s` }, | |
| 69 | 429, | |
| 70 | { "retry-after": String(pinged.error.retryAfter) }, | |
| 71 | ); | |
| 67 | return tryAgain("rate_limited", 429, `too many pings: try again in ${pinged.error.retryAfter} s`, pinged.error.retryAfter); | |
| 72 | 68 | return json({ id: pinged.value.id }, 202); |
| 73 | 69 | }); |
| 74 | 70 |
src/http/api/print.ts +10 −3
| @@ -7,7 +7,7 @@ | ||
| 7 | 7 | import { PRINT_WORDS, PDF_MAX_BYTES, permitPrint, printFor, releasePermit, type PrintError } from "../../devices/print.ts"; |
| 8 | 8 | import type { Handler } from "../context.ts"; |
| 9 | 9 | import { readBounded } from "../limits.ts"; |
| 10 | import { apiError, json, unauthorized } from "../respond.ts"; | |
| 10 | import { apiError, json, tryAgain, unauthorized, untilNextDay, WINDOW } from "../respond.ts"; | |
| 11 | 11 | import type { Router } from "../router.ts"; |
| 12 | 12 | import { tokenSecret } from "../token_header.ts"; |
| 13 | 13 | |
| @@ -23,6 +23,13 @@ const STATUS: Record<PrintError, number> = { | ||
| 23 | 23 | printer_refused: 502, |
| 24 | 24 | }; |
| 25 | 25 | |
| 26 | /** A print refusal; the two 429s say when: the minute's window, or the next UTC day for the daily cap. */ | |
| 27 | function printRefusal(error: PrintError): Response { | |
| 28 | if (error === "rate_limited") return tryAgain(error, 429, PRINT_WORDS[error], WINDOW.minute); | |
| 29 | if (error === "daily_cap") return tryAgain(error, 429, PRINT_WORDS[error], untilNextDay()); | |
| 30 | return apiError(error, STATUS[error], PRINT_WORDS[error]); | |
| 31 | } | |
| 32 | ||
| 26 | 33 | export function printRoutes(router: Router<Handler>): void { |
| 27 | 34 | router.add("POST", "/api/v1/devices/:id/print", async (ctx) => { |
| 28 | 35 | const secret = tokenSecret(ctx.request.headers.get("authorization") ?? ""); |
| @@ -30,7 +37,7 @@ export function printRoutes(router: Router<Handler>): void { | ||
| 30 | 37 | const id = /^\d{1,9}$/.test(ctx.params["id"] ?? "") ? Number(ctx.params["id"]) : Number.NaN; |
| 31 | 38 | // Who and where first; the body (up to 10 MB) is read only for a granted app. |
| 32 | 39 | const permit = permitPrint(ctx.app, secret, id); |
| 33 | if (!permit.ok) return apiError(permit.error, STATUS[permit.error], PRINT_WORDS[permit.error]); | |
| 40 | if (!permit.ok) return printRefusal(permit.error); | |
| 34 | 41 | // The permit holds a place until printFor, or here when the body never gets that far. |
| 35 | 42 | let pdf: Awaited<ReturnType<typeof readBounded>>; |
| 36 | 43 | try { |
| @@ -44,7 +51,7 @@ export function printRoutes(router: Router<Handler>): void { | ||
| 44 | 51 | return apiError("too_large", 422, PRINT_WORDS.too_large); |
| 45 | 52 | } |
| 46 | 53 | const done = await printFor(ctx.app, permit.value, pdf); |
| 47 | if (!done.ok) return apiError(done.error, STATUS[done.error], PRINT_WORDS[done.error]); | |
| 54 | if (!done.ok) return printRefusal(done.error); | |
| 48 | 55 | return json({ job: { id: done.value.id, state: done.value.state, reasons: done.value.reasons } }); |
| 49 | 56 | }); |
| 50 | 57 | } |
src/http/api/reports.ts +6 −10
| @@ -31,6 +31,7 @@ import { | ||
| 31 | 31 | requireScope, |
| 32 | 32 | respondAccess, |
| 33 | 33 | str, |
| 34 | tryAgain, | |
| 34 | 35 | } from "../respond.ts"; |
| 35 | 36 | import type { Router } from "../router.ts"; |
| 36 | 37 | |
| @@ -41,6 +42,9 @@ const REPORT_HINT = | ||
| 41 | 42 | "page an http(s) address, agent a name of letters, digits, . _ - " + |
| 42 | 43 | "(a claim unless a token proves it)"; |
| 43 | 44 | |
| 45 | /** A full queue drains as the crew reads reports; no clock says when, so a minute. */ | |
| 46 | const REPORT_QUEUE_RETRY = 60; | |
| 47 | ||
| 44 | 48 | export function reportRoutes(router: Router<Handler>): void { |
| 45 | 49 | // ── reports from outside — no account, a person decides ────────────── |
| 46 | 50 | |
| @@ -66,17 +70,9 @@ export function reportRoutes(router: Router<Handler>): void { | ||
| 66 | 70 | const refusal = filed.error; |
| 67 | 71 | if (refusal.error === "not_found") return notFound(); |
| 68 | 72 | if (refusal.error === "rate_limited") |
| 69 | return json( | |
| 70 | { error: "rate_limited", message: "too many reports; wait and send again" }, | |
| 71 | 429, | |
| 72 | { "retry-after": String(refusal.retryAfter) }, | |
| 73 | ); | |
| 73 | return tryAgain("rate_limited", 429, "too many reports; wait and send again", refusal.retryAfter); | |
| 74 | 74 | if (refusal.error === "queue_full") |
| 75 | return apiError( | |
| 76 | "queue_full", | |
| 77 | 429, | |
| 78 | "this repository has many reports waiting; try again later", | |
| 79 | ); | |
| 75 | return tryAgain("queue_full", 429, "this repository has many reports waiting; try again later", REPORT_QUEUE_RETRY); | |
| 80 | 76 | return apiError(refusal.error, 422, REPORT_HINT); |
| 81 | 77 | } |
| 82 | 78 | return json( |
src/http/api/upstream.ts +11 −2
| @@ -6,6 +6,7 @@ | ||
| 6 | 6 | * `actions/repos.ts`. |
| 7 | 7 | */ |
| 8 | 8 | |
| 9 | import { forksSince } from "../../core/repos.ts"; | |
| 9 | 10 | import { forkRepo, FORKS_PER_DAY, repositoryLimitMessage } from "../../actions/repos.ts"; |
| 10 | 11 | import { |
| 11 | 12 | checkUpstreamNow, |
| @@ -22,9 +23,17 @@ import { getFollow } from "../../core/upstream_follow.ts"; | ||
| 22 | 23 | import { holds, weakEtag } from "../etag.ts"; |
| 23 | 24 | import { loadRepo, pageParams, slicePage } from "../api.ts"; |
| 24 | 25 | import type { Handler } from "../context.ts"; |
| 25 | import { apiError, body, json, notFound, requireScope, respondAccess, str } from "../respond.ts"; | |
| 26 | import { apiError, body, json, notFound, requireScope, respondAccess, str, tryAgain, WINDOW } from "../respond.ts"; | |
| 26 | 27 | import type { Router } from "../router.ts"; |
| 27 | 28 | |
| 29 | /** Seconds until the oldest fork in the last day ages out and frees a place. */ | |
| 30 | function forkWait(db: Parameters<typeof forksSince>[0], ownerId: string): number { | |
| 31 | const since = new Date(Date.now() - WINDOW.day * 1000).toISOString(); | |
| 32 | const made = forksSince(db, ownerId, since); | |
| 33 | const oldest = made[Math.max(0, made.length - FORKS_PER_DAY)]; | |
| 34 | return oldest === undefined ? WINDOW.day : (Date.parse(oldest) + WINDOW.day * 1000 - Date.now()) / 1000; | |
| 35 | } | |
| 36 | ||
| 28 | 37 | export function upstreamRoutes(router: Router<Handler>): void { |
| 29 | 38 | // Following an outside upstream (#3442): an admin reads, sets and checks. |
| 30 | 39 | router.add("GET", "/api/v1/repos/:owner/:name/upstream", (ctx) => { |
| @@ -138,7 +147,7 @@ export function upstreamRoutes(router: Router<Handler>): void { | ||
| 138 | 147 | : forked.error === "already_forked" |
| 139 | 148 | ? apiError("already_forked", 409, "you already have a fork of this repository") |
| 140 | 149 | : forked.error === "limit" |
| 141 | ? apiError("limit", 429, `at most ${FORKS_PER_DAY} forks a day: try again tomorrow`) | |
| 150 | ? tryAgain("limit", 429, `at most ${FORKS_PER_DAY} forks a day: try again tomorrow`, forkWait(ctx.app.db, ctx.auth.user!.id)) | |
| 142 | 151 | : forked.error === "fork_failed" |
| 143 | 152 | ? apiError("fork_failed", 422, "the fork could not be created") |
| 144 | 153 | : respondAccess(forked.error, 422); |
src/http/api/user.ts +20 −21
| @@ -8,8 +8,8 @@ | ||
| 8 | 8 | import { contribute, forkOrigin } from "../../actions/contribute.ts"; |
| 9 | 9 | import { CAPPED_WORDS } from "../../actions/mail.ts"; |
| 10 | 10 | import { ORIGIN_UNREACHABLE } from "../../actions/origin_app_words.ts"; |
| 11 | import { connectOrigin, connectRefusal, listOriginRepoGroup, deleteOriginApp, disconnectOrigin, listOriginApps, listPinned, listRemoteRepos, listUserOrigins, mirrorJobFor, pinRepo, pollOAuthConnect, registerOriginApp, startMirrorJob, startOAuthConnect, syncOriginNow, unpinRepo } from "../../actions/origins.ts"; | |
| 12 | import { addSshKey, changePassword, deleteSshKey, updateProfile } from "../../actions/users.ts"; | |
| 11 | import { CONNECT_LIMITS, connectOrigin, connectRefusal, listOriginRepoGroup, deleteOriginApp, disconnectOrigin, listOriginApps, listPinned, listRemoteRepos, listUserOrigins, mirrorJobFor, pinRepo, pollOAuthConnect, registerOriginApp, startMirrorJob, startOAuthConnect, syncOriginNow, unpinRepo } from "../../actions/origins.ts"; | |
| 12 | import { addSshKey, CHANGE_PASSWORD_WORDS, changePassword, deleteSshKey, updateProfile } from "../../actions/users.ts"; | |
| 13 | 13 | import { VERIFY_WORDS } from "../../actions/verify_words.ts"; |
| 14 | 14 | import { resendVerification } from "../../actions/verify.ts"; |
| 15 | 15 | import { NOTIFY_ERRORS, notifyHost, notifyUrlOf, setNotifyUrl } from "../../crew/notify.ts"; |
| @@ -20,7 +20,7 @@ import { listOrgsForUser } from "../../core/orgs.ts"; | ||
| 20 | 20 | import { canRead, getReposByIds } from "../../core/repos.ts"; |
| 21 | 21 | import { notePhase, takeSessionTime, timePhase, timePhaseAsync } from "../../core/route_timing.ts"; |
| 22 | 22 | import { mayView } from "../../core/sealed.ts"; |
| 23 | import { emailVerified } from "../../core/verify.ts"; | |
| 23 | import { emailVerified, RESEND_AFTER_MS } from "../../core/verify.ts"; | |
| 24 | 24 | import type { Result } from "../../kernel/result.ts"; |
| 25 | 25 | import { subscribe as subscribeJob } from "../../origins/jobs.ts"; |
| 26 | 26 | import { sshCloneUrl } from "../../ssh/command.ts"; |
| @@ -30,7 +30,7 @@ import { connectMessage, contributeFailure, contributionJson, editFilesFrom, job | ||
| 30 | 30 | import type { Ctx, Handler } from "../context.ts"; |
| 31 | 31 | import { holds, weakEtag } from "../etag.ts"; |
| 32 | 32 | import { clientAddress } from "../peer.ts"; |
| 33 | import { apiError, body, json, notFound, publicProfile, publicUser, requireAccount, requireScope, respondAccess, str, unauthorized } from "../respond.ts"; | |
| 33 | import { apiError, body, json, notFound, publicProfile, publicUser, requireAccount, requireScope, respondAccess, str, tryAgain, unauthorized, WINDOW } from "../respond.ts"; | |
| 34 | 34 | import type { Router } from "../router.ts"; |
| 35 | 35 | import { eventStream } from "../sse.ts"; |
| 36 | 36 | |
| @@ -73,11 +73,7 @@ function sourceFailure(error: SourceError): Response { | ||
| 73 | 73 | case "not_found": |
| 74 | 74 | return notFound(); |
| 75 | 75 | case "rate_limited": |
| 76 | return apiError( | |
| 77 | "rate_limited", | |
| 78 | 429, | |
| 79 | "the origin is rate-limiting us; try again shortly", | |
| 80 | ); | |
| 76 | return tryAgain("rate_limited", 429, "the origin is rate-limiting us; try again shortly", WINDOW.minute); | |
| 81 | 77 | case "unauthorized": |
| 82 | 78 | return apiError( |
| 83 | 79 | "origin_unauthorized", |
| @@ -95,6 +91,9 @@ function sourceFailure(error: SourceError): Response { | ||
| 95 | 91 | } |
| 96 | 92 | } |
| 97 | 93 | |
| 94 | /** No mailer is set: an operator must configure one, no clock ends it; ask again in five minutes. */ | |
| 95 | const NO_MAIL_RETRY = 300; | |
| 96 | ||
| 98 | 97 | export function userRoutes(router: Router<Handler>): void { |
| 99 | 98 | router.add("GET", "/api/v1/user", (ctx) => { |
| 100 | 99 | notePhase(ctx.request, "session", takeSessionTime(ctx.request)); |
| @@ -112,11 +111,11 @@ export function userRoutes(router: Router<Handler>): void { | ||
| 112 | 111 | return json({ outcome: sent.value }, sent.value === "sent" ? 202 : 200); |
| 113 | 112 | if (sent.error === "unauthorized") return unauthorized(); |
| 114 | 113 | if (sent.error === "too_soon") |
| 115 | return apiError("too_soon", 429, VERIFY_WORDS.too_soon); | |
| 114 | return tryAgain("too_soon", 429, VERIFY_WORDS.too_soon, RESEND_AFTER_MS / 1000); | |
| 116 | 115 | if (sent.error === "no_mail") |
| 117 | return apiError("no_mail", 503, VERIFY_WORDS.no_mail); | |
| 116 | return tryAgain("no_mail", 503, VERIFY_WORDS.no_mail, NO_MAIL_RETRY); | |
| 118 | 117 | if (sent.error === "capped") |
| 119 | return apiError("capped", 429, CAPPED_WORDS); | |
| 118 | return tryAgain("capped", 429, CAPPED_WORDS, WINDOW.day); | |
| 120 | 119 | return apiError("failed", 502, VERIFY_WORDS.failed); |
| 121 | 120 | }); |
| 122 | 121 | |
| @@ -152,14 +151,14 @@ export function userRoutes(router: Router<Handler>): void { | ||
| 152 | 151 | next: str(input.next), |
| 153 | 152 | source: clientAddress(ctx.request), |
| 154 | 153 | }); |
| 154 | if (!changed.ok && changed.error === "throttled") | |
| 155 | return tryAgain("throttled", 429, CHANGE_PASSWORD_WORDS.throttled, WINDOW.minute); | |
| 155 | 156 | if (!changed.ok) |
| 156 | 157 | return respondAccess( |
| 157 | 158 | changed.error, |
| 158 | 159 | changed.error === "bad_credentials" |
| 159 | 160 | ? 403 |
| 160 | : changed.error === "throttled" | |
| 161 | ? 429 | |
| 162 | : 422, | |
| 161 | : 422, | |
| 163 | 162 | ); |
| 164 | 163 | return json({}, 200); |
| 165 | 164 | }); |
| @@ -275,9 +274,11 @@ export function userRoutes(router: Router<Handler>): void { | ||
| 275 | 274 | // A refusal that names its own reason (a missing scope) keeps it; |
| 276 | 275 | // every other code has its sentence here. |
| 277 | 276 | const refused = connectRefusal(connected.error); |
| 277 | if (refused.code === "rate_limited") | |
| 278 | return tryAgain("rate_limited", 429, refused.message ?? connectMessage(refused.code), CONNECT_LIMITS.windowMs! / 1000); | |
| 278 | 279 | return apiError( |
| 279 | 280 | refused.code, |
| 280 | refused.code === "rate_limited" ? 429 : 422, | |
| 281 | 422, | |
| 281 | 282 | refused.message ?? connectMessage(refused.code), |
| 282 | 283 | ); |
| 283 | 284 | } |
| @@ -308,11 +309,9 @@ export function userRoutes(router: Router<Handler>): void { | ||
| 308 | 309 | redirectUri: callbackUri(ctx), |
| 309 | 310 | }); |
| 310 | 311 | if (!started.ok) |
| 311 | return apiError( | |
| 312 | started.error.error, | |
| 313 | started.error.error === "rate_limited" ? 429 : 422, | |
| 314 | started.error.message, | |
| 315 | ); | |
| 312 | return started.error.error === "rate_limited" | |
| 313 | ? tryAgain("rate_limited", 429, started.error.message, CONNECT_LIMITS.windowMs! / 1000) | |
| 314 | : apiError(started.error.error, 422, started.error.message); | |
| 316 | 315 | return json(started.value, 201); |
| 317 | 316 | }); |
| 318 | 317 |
src/http/health.ts +3 −3
| @@ -102,7 +102,7 @@ export function liveResponse(ping: () => unknown, log: Log, loopLagMs?: number): | ||
| 102 | 102 | error: error instanceof Error ? error.message : String(error), |
| 103 | 103 | }); |
| 104 | 104 | databaseDown = true; |
| 105 | return new Response("database", { status: 503 }); | |
| 105 | return new Response("database", { status: 503, headers: { "retry-after": "5" } }); | |
| 106 | 106 | } |
| 107 | 107 | if (databaseDown) log.info("forge alive again: the database answers"); |
| 108 | 108 | databaseDown = false; |
| @@ -142,9 +142,9 @@ export function drainLingerMs(raw: string | undefined): number | null { | ||
| 142 | 142 | * drains. |
| 143 | 143 | */ |
| 144 | 144 | export function readyResponse(now = Date.now()): Response { |
| 145 | if (health.draining) return new Response("draining", { status: 503 }); | |
| 145 | if (health.draining) return new Response("draining", { status: 503, headers: { "retry-after": "5" } }); | |
| 146 | 146 | const waiting = stillStarting(now); |
| 147 | 147 | return waiting > 0 |
| 148 | ? new Response(`starting ${waiting} app${waiting === 1 ? "" : "s"}`, { status: 503 }) | |
| 148 | ? new Response(`starting ${waiting} app${waiting === 1 ? "" : "s"}`, { status: 503, headers: { "retry-after": "5" } }) | |
| 149 | 149 | : new Response("ok", { status: 200 }); |
| 150 | 150 | } |
src/http/oauth.ts +6 −13
| @@ -22,7 +22,7 @@ import { RateLimiter } from "../core/ratelimit.ts"; | ||
| 22 | 22 | import type { Handler } from "./context.ts"; |
| 23 | 23 | import { clientBucket } from "./peer.ts"; |
| 24 | 24 | import { readBounded } from "./limits.ts"; |
| 25 | import { boundedBody, json } from "./respond.ts"; | |
| 25 | import { boundedBody, json, retryAfterOf, tryAgain } from "./respond.ts"; | |
| 26 | 26 | import type { Router } from "./router.ts"; |
| 27 | 27 | import { authorizePage, authorizeSubmit } from "../web/pages/oauth.ts"; |
| 28 | 28 | |
| @@ -148,10 +148,7 @@ export function oauthRoutes(router: Router<Handler>): void { | ||
| 148 | 148 | limiterIn(revokeAllLimiters, ctx.app, REVOKE_ALL_PER_MINUTE).take("all", "ro"), |
| 149 | 149 | ].find((v) => !v.allowed) ?? { allowed: true, retryAfter: 0 }; |
| 150 | 150 | if (!verdict.allowed) |
| 151 | return json({ error: "rate_limited" }, 429, { | |
| 152 | ...noStore, | |
| 153 | "retry-after": String(verdict.retryAfter), | |
| 154 | }); | |
| 151 | return tryAgain("rate_limited", 429, "too many revocations", verdict.retryAfter, noStore); | |
| 155 | 152 | const get = await tokenParams(ctx.request); |
| 156 | 153 | if (get === null) return grantError("invalid_request", 413); |
| 157 | 154 | const presented = get("token"); |
| @@ -243,14 +240,10 @@ export function oauthRoutes(router: Router<Handler>): void { | ||
| 243 | 240 | router.add("POST", "/oauth/register", async (ctx) => { |
| 244 | 241 | const verdict = limiterIn(limiters, ctx.app, REGISTER_PER_MINUTE).take(clientBucket(ctx.request), "ro"); |
| 245 | 242 | if (!verdict.allowed) |
| 246 | return json( | |
| 247 | { | |
| 248 | error: "rate_limited", | |
| 249 | message: `too many registrations — retry in ${verdict.retryAfter}s`, | |
| 250 | }, | |
| 251 | 429, | |
| 252 | { ...open, "retry-after": String(verdict.retryAfter), "cache-control": "no-store" }, | |
| 253 | ); | |
| 243 | return tryAgain("rate_limited", 429, `too many registrations — retry in ${retryAfterOf(verdict.retryAfter)}s`, verdict.retryAfter, { | |
| 244 | ...open, | |
| 245 | "cache-control": "no-store", | |
| 246 | }); | |
| 254 | 247 | const raw = await boundedBody(ctx.request, BODY_MAX); |
| 255 | 248 | if (raw === "too_large") |
| 256 | 249 | return json({ error: "invalid_client_metadata", error_description: `at most ${BODY_MAX} bytes` }, 413, { ...open, "cache-control": "no-store" }); |
src/http/proposal_gate.ts +3 −2
| @@ -18,7 +18,7 @@ import { getRepoById } from "../core/repos.ts"; | ||
| 18 | 18 | import { RateLimiter } from "../core/ratelimit.ts"; |
| 19 | 19 | import { PROPOSAL_BODY_MAX, pullProposalOf } from "../core/proposal_token.ts"; |
| 20 | 20 | import type { Auth } from "./context.ts"; |
| 21 | import { apiError } from "./respond.ts"; | |
| 21 | import { apiError, tryAgain } from "./respond.ts"; | |
| 22 | 22 | |
| 23 | 23 | const limiters = new WeakMap<App, RateLimiter>(); |
| 24 | 24 | |
| @@ -72,10 +72,11 @@ export function proposalGate( | ||
| 72 | 72 | } |
| 73 | 73 | const verdict = proposalLimiter(app).take(proposal.tokenId, "rw"); |
| 74 | 74 | if (!verdict.allowed) |
| 75 | return apiError( | |
| 75 | return tryAgain( | |
| 76 | 76 | "rate_limited", |
| 77 | 77 | 429, |
| 78 | 78 | `too many proposals: ${verdict.limit} per minute per token — retry in ${verdict.retryAfter}s`, |
| 79 | verdict.retryAfter, | |
| 79 | 80 | ); |
| 80 | 81 | |
| 81 | 82 | if (method === "GET" && path === `${at}/info/refs`) { |
src/http/read_only.ts +1 −1
| @@ -40,7 +40,7 @@ export function readOnlyRefusal(request: Request): Response | undefined { | ||
| 40 | 40 | const headers = { ...MAINTENANCE_HEADERS }; |
| 41 | 41 | if (wantsJson) { |
| 42 | 42 | headers["content-type"] = "application/json; charset=utf-8"; |
| 43 | return new Response(JSON.stringify({ error: "read_only", message: WORDS }), { status: 503, headers }); | |
| 43 | return new Response(JSON.stringify({ error: "read_only", message: WORDS, retryAfter: Number(MAINTENANCE_HEADERS["retry-after"]) }), { status: 503, headers }); | |
| 44 | 44 | } |
| 45 | 45 | headers["content-type"] = "text/plain; charset=utf-8"; |
| 46 | 46 | return new Response(`${WORDS}: this forge is serving reads while the main one is away; nothing can be changed here.\n`, { |
src/http/respond.ts +33 −8
| @@ -19,12 +19,39 @@ export function json( | ||
| 19 | 19 | status = 200, |
| 20 | 20 | headers: Record<string, string> = {}, |
| 21 | 21 | ): Response { |
| 22 | // The floor under every 429 and 503: a wait is always named, in the header | |
| 23 | // and in the body, even where a site did not say how long (docs/MCP.md). | |
| 24 | if (status === 429 || status === 503) { | |
| 25 | const named = Object.entries(headers).find(([k]) => k.toLowerCase() === "retry-after")?.[1]; | |
| 26 | const wait = retryAfterOf(named === undefined ? 60 : Number(named)); | |
| 27 | headers = { ...Object.fromEntries(Object.entries(headers).filter(([k]) => k.toLowerCase() !== "retry-after")), "retry-after": String(wait) }; | |
| 28 | if (typeof data === "object" && data !== null && !Array.isArray(data) && !("retryAfter" in data)) | |
| 29 | data = { ...data, retryAfter: wait }; | |
| 30 | } | |
| 22 | 31 | return new Response(JSON.stringify(data), { |
| 23 | 32 | status, |
| 24 | 33 | headers: { "content-type": "application/json", ...headers }, |
| 25 | 34 | }); |
| 26 | 35 | } |
| 27 | 36 | |
| 37 | /** | |
| 38 | * A 429 or 503 that says for how long: an integer `Retry-After` header and the | |
| 39 | * same seconds as `retryAfter` in the body, so an agent told to slow down | |
| 40 | * knows when to come back (docs/MCP.md, "Slowing down"). | |
| 41 | */ | |
| 42 | export const retryAfterOf = (seconds: number): number => Math.max(1, Math.ceil(Number.isFinite(seconds) ? seconds : 1)); | |
| 43 | export function tryAgain( | |
| 44 | code: string, | |
| 45 | status: 429 | 503, | |
| 46 | message: string, | |
| 47 | seconds: number, | |
| 48 | headers: Record<string, string> = {}, | |
| 49 | extra: Record<string, unknown> = {}, | |
| 50 | ): Response { | |
| 51 | const wait = retryAfterOf(seconds); | |
| 52 | return json({ error: code, message, ...extra, retryAfter: wait }, status, { ...headers, "retry-after": String(wait) }); | |
| 53 | } | |
| 54 | ||
| 28 | 55 | /** |
| 29 | 56 | * Every API error is `{ error, message }`. `error` is a stable machine code to |
| 30 | 57 | * branch on; `message` is a human hint for logs and automation. The codes are |
| @@ -58,14 +85,7 @@ export const unauthorized = () => | ||
| 58 | 85 | ); |
| 59 | 86 | // A password door past the sign-in windows (`core/signin-window.ts`). |
| 60 | 87 | export const tooManyAttempts = (retryAfter: number) => |
| 61 | json( | |
| 62 | { | |
| 63 | error: "too_many_attempts", | |
| 64 | message: `too many sign-in attempts: try again in ${retryAfter} s`, | |
| 65 | }, | |
| 66 | 429, | |
| 67 | { "retry-after": String(retryAfter) }, | |
| 68 | ); | |
| 88 | tryAgain("too_many_attempts", 429, `too many sign-in attempts: try again in ${retryAfter} s`, retryAfter); | |
| 69 | 89 | export const FORBIDDEN_MESSAGE = "your credentials lack the scope for this action"; |
| 70 | 90 | export const forbidden = () => apiError("forbidden", 403, FORBIDDEN_MESSAGE); |
| 71 | 91 | // A path matching no route at all — distinct from a missing resource, so a |
| @@ -391,3 +411,8 @@ export function paged<T>( | ||
| 391 | 411 | items.length >= limit && last !== undefined ? cursorOf(last) : null, |
| 392 | 412 | }; |
| 393 | 413 | } |
| 414 | ||
| 415 | /** Seconds a limiter's window lasts, named where a site has no sharper figure. */ | |
| 416 | export const WINDOW = { minute: 60, day: 86_400 } as const; | |
| 417 | /** Seconds until the next UTC midnight: when a per-day count starts over. */ | |
| 418 | export const untilNextDay = (now = Date.now()): number => (WINDOW.day * 1000 - (now % (WINDOW.day * 1000))) / 1000; |
src/http/rp.ts +3 −2
| @@ -25,6 +25,7 @@ import { | ||
| 25 | 25 | CALLBACK_PATH, |
| 26 | 26 | linkParent, |
| 27 | 27 | } from "../core/federation.ts"; |
| 28 | import { retryAfterOf } from "./respond.ts"; | |
| 28 | 29 | import { readCapped } from "../origins/providers/http.ts"; |
| 29 | 30 | import { clientAddress } from "./peer.ts"; |
| 30 | 31 | import { signinAllowed } from "../web/pages/site.ts"; |
| @@ -144,8 +145,8 @@ async function begin(ctx: Ctx, linkUser?: string): Promise<Response> { | ||
| 144 | 145 | // out the sign-ins of people in flight. By socket address, never a header. |
| 145 | 146 | const wait = signinAllowed(ctx, `parent:${clientAddress(ctx.request)}`); |
| 146 | 147 | if (wait !== undefined) { |
| 147 | const slow = failure(ctx, `Too many attempts. Try again in ${wait}s.`, 429); | |
| 148 | slow.headers.set("retry-after", String(wait)); | |
| 148 | const slow = failure(ctx, `Too many attempts. Try again in ${retryAfterOf(wait)}s.`, 429); | |
| 149 | slow.headers.set("retry-after", String(retryAfterOf(wait))); | |
| 149 | 150 | return slow; |
| 150 | 151 | } |
| 151 | 152 | const redirectUri = redirectUriOf(ctx); |
src/http/server.ts +9 −5
| @@ -539,6 +539,7 @@ import { | ||
| 539 | 539 | securityFloor, |
| 540 | 540 | privateWhenSignedIn, |
| 541 | 541 | tooManyAttempts, |
| 542 | tryAgain, | |
| 542 | 543 | } from "./respond.ts"; |
| 543 | 544 | import { compress } from "./compress.ts"; |
| 544 | 545 | import { RateLimiter, type Verdict } from "../core/ratelimit.ts"; |
| @@ -1775,10 +1776,11 @@ function tokenCredential(request: Request, auth: Auth): string | undefined { | ||
| 1775 | 1776 | } |
| 1776 | 1777 | |
| 1777 | 1778 | const tooManyRequests = (verdict: Verdict): Response => |
| 1778 | apiError( | |
| 1779 | tryAgain( | |
| 1779 | 1780 | "rate_limited", |
| 1780 | 1781 | 429, |
| 1781 | 1782 | `too many requests: ${verdict.limit} per minute per token — retry in ${verdict.retryAfter}s`, |
| 1783 | verdict.retryAfter, | |
| 1782 | 1784 | ); |
| 1783 | 1785 | |
| 1784 | 1786 | /** |
| @@ -1797,7 +1799,7 @@ function withRateHeaders( | ||
| 1797 | 1799 | response.headers.set("x-ratelimit-remaining", String(verdict.remaining)); |
| 1798 | 1800 | response.headers.set("x-ratelimit-reset", String(verdict.reset)); |
| 1799 | 1801 | if (!verdict.allowed) { |
| 1800 | response.headers.set("retry-after", String(verdict.retryAfter)); | |
| 1802 | response.headers.set("retry-after", String(Math.max(1, Math.ceil(verdict.retryAfter)))); | |
| 1801 | 1803 | } |
| 1802 | 1804 | } catch { |
| 1803 | 1805 | // An immutable header set is not worth failing a request over; the |
| @@ -2622,7 +2624,7 @@ export async function serve(app: App): Promise<Servers> { | ||
| 2622 | 2624 | if (!may.ok) { |
| 2623 | 2625 | const [status, text] = |
| 2624 | 2626 | may.refusal === "not_found" ? [404, "not found"] : may.refusal === "busy" ? [429, "too many shells"] : [403, "forbidden"]; |
| 2625 | return securityFloor(new Response(text, { status })); | |
| 2627 | return securityFloor(new Response(text, { status, ...(status === 429 ? { headers: { "retry-after": "5" } } : {}) })); | |
| 2626 | 2628 | } |
| 2627 | 2629 | // A machine's shell runs only where a cell holds it, asked at the |
| 2628 | 2630 | // first shell rather than at boot; a test that set its own keeps them. |
| @@ -2911,10 +2913,11 @@ async function answerGet( | ||
| 2911 | 2913 | if (verdict !== undefined && !verdict.allowed) { |
| 2912 | 2914 | const refusal = |
| 2913 | 2915 | credential === undefined |
| 2914 | ? apiError( | |
| 2916 | ? tryAgain( | |
| 2915 | 2917 | "rate_limited", |
| 2916 | 2918 | 429, |
| 2917 | 2919 | `too many requests without a token: ${verdict.limit} per minute — sign in, or retry in ${verdict.retryAfter}s`, |
| 2920 | verdict.retryAfter, | |
| 2918 | 2921 | ) |
| 2919 | 2922 | : tooManyRequests(verdict); |
| 2920 | 2923 | return withRateHeaders(refusal, verdict); |
| @@ -2925,10 +2928,11 @@ async function answerGet( | ||
| 2925 | 2928 | : undefined; |
| 2926 | 2929 | if (stranger !== undefined && !stranger.allowed) { |
| 2927 | 2930 | return withRateHeaders( |
| 2928 | apiError( | |
| 2931 | tryAgain( | |
| 2929 | 2932 | "rate_limited", |
| 2930 | 2933 | 429, |
| 2931 | 2934 | `too many requests: ${stranger.limit} per minute from one source without a token — retry in ${stranger.retryAfter}s`, |
| 2935 | stranger.retryAfter, | |
| 2932 | 2936 | ), |
| 2933 | 2937 | stranger, |
| 2934 | 2938 | ); |
test/retry-after.test.ts added +217
| @@ -0,0 +1,217 @@ | ||
| 1 | /** | |
| 2 | * Every 429 and 503 the HTTP layer answers says for how long (#10762). | |
| 3 | * Part one reads src/http per site: a status literal must sit in a `tryAgain` | |
| 4 | * call, or in a call that spells its own retry-after, or be named below with | |
| 5 | * the reason it is safe. Part two drives the real limits through a running | |
| 6 | * forge and asserts the integer header and an equal `retryAfter` in the body. | |
| 7 | */ | |
| 8 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 9 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 10 | import { readdirSync, readFileSync, statSync } from "node:fs"; | |
| 11 | import { tmpdir } from "node:os"; | |
| 12 | import { join } from "node:path"; | |
| 13 | import * as ts from "typescript-api"; | |
| 14 | import { createApp, type App } from "../src/app.ts"; | |
| 15 | import { createToken } from "../src/core/tokens.ts"; | |
| 16 | import { createUser } from "../src/core/users.ts"; | |
| 17 | import { readOnlyRefusal } from "../src/http/read_only.ts"; | |
| 18 | import { json, tryAgain } from "../src/http/respond.ts"; | |
| 19 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 20 | import { setStrangerLimits, strangerLimits } from "../src/http/strangers.ts"; | |
| 21 | ||
| 22 | const files = (dir: string): string[] => | |
| 23 | readdirSync(dir).flatMap((n) => { | |
| 24 | const p = join(dir, n); | |
| 25 | return statSync(p).isDirectory() ? files(p) : p.endsWith(".ts") ? [p] : []; | |
| 26 | }); | |
| 27 | ||
| 28 | const PERF_LINE = `src/http/server.ts: ${readFileSync("src/http/server.ts", "utf8").split("\n").find((l) => l.includes("refused: response.status === 429"))!.trim()}`; | |
outdated
This regex lets any site pass if |
||
| 29 | ||
| 30 | /** Sites that are not a response of their own, each with why it is safe. `file: trimmed line`. */ | |
| 31 | const EXEMPT: Record<string, string> = { | |
| 32 | [PERF_LINE]: "a comparison, not a response", | |
| 33 | "src/http/respond.ts: if (status === 429 || status === 503) {": "json()'s floor itself", | |
| 34 | "src/http/respond.ts: status: 429 | 503,": "tryAgain's own signature", | |
| 35 | "src/http/api/print.ts: rate_limited: 429,": "status table; printRefusal answers through tryAgain", | |
| 36 | "src/http/api/print.ts: daily_cap: 429,": "status table; printRefusal answers through tryAgain", | |
| 37 | "src/http/api/conversation_talk.ts: return status === 429 || status === 503 ? tryAgain(said.error, status, words, talkWait(said.error)) : json({ error: said.error, message: words }, status);": | |
| 38 | "the 429/503 branch is the tryAgain", | |
| 39 | "src/http/api/following.ts: ? 429": "the ceiling outcome is answered through tryAgain just below", | |
| 40 | "src/http/feed.ts: error.code === \"forbidden\" ? 403 : error.code === \"too_many\" ? 429 : error.code === \"already_undone\" || error.code === \"expired\" ? 409 : 422,": | |
| 41 | "a cap on one request's size, not a wait; json()'s 60 s floor answers it", | |
| 42 | "src/http/rp.ts: const slow = failure(ctx, `Too many attempts. Try again in ${retryAfterOf(wait)}s.`, 429);": "an HTML page; the header is set on the next line", | |
| 43 | "src/http/read_only.ts: status: 503,": "plain text; its headers are MAINTENANCE_HEADERS, which carry retry-after (the JSON answer above names it in the body too)", | |
| 44 | "src/http/server.ts: may.refusal === \"not_found\" ? [404, \"not found\"] : may.refusal === \"busy\" ? [429, \"too many shells\"] : [403, \"forbidden\"];": | |
| 45 | "plain text; the Response below carries retry-after", | |
| 46 | }; | |
| 47 | ||
| 48 | /** Every 429/503 numeric literal in src/http that no allowed shape covers, as `file: line`. */ | |
| 49 | function unnamedWaits(): string[] { | |
| 50 | const out: string[] = []; | |
| 51 | for (const f of files("src/http")) { | |
| 52 | const text = readFileSync(f, "utf8"); | |
| 53 | const sf = ts.createSourceFile(f, text, ts.ScriptTarget.Latest, true); | |
| 54 | const visit = (n: ts.Node) => { | |
| 55 | if (ts.isNumericLiteral(n) && (n.text === "429" || n.text === "503")) { | |
| 56 | let c: ts.Node | undefined = n.parent; | |
| 57 | while (c && !ts.isCallExpression(c) && !ts.isNewExpression(c) && !ts.isStatement(c)) c = c.parent; | |
| 58 | const call = c && (ts.isCallExpression(c) || ts.isNewExpression(c)) ? c : undefined; | |
| 59 | const callee = call?.expression.getText() ?? ""; | |
| 60 | const line = text.split("\n")[sf.getLineAndCharacterOfPosition(n.getStart()).line]!.trim(); | |
| 61 | const key = `${f}: ${line}`; | |
| 62 | const covered = callee === "tryAgain" || (call !== undefined && /retry-?after/i.test(call.getText())); | |
| 63 | if (!covered && EXEMPT[key] === undefined) out.push(key); | |
| 64 | } | |
| 65 | ts.forEachChild(n, visit); | |
| 66 | }; | |
| 67 | visit(sf); | |
| 68 | } | |
| 69 | return out; | |
| 70 | } | |
| 71 | ||
| 72 | describe("every 429 and 503 site in src/http", () => { | |
| 73 | test("is a tryAgain, spells its own retry-after, or is named with its reason", () => { | |
| 74 | expect(unnamedWaits()).toEqual([]); | |
| 75 | }); | |
| 76 | test("the scan catches a bare Response beside a helper call", () => { | |
| 77 | // The shape the old window scan let through: a helper nearby, a bare 429 of its own. | |
| 78 | const src = `json({}); return new Response("slow", { status: 429 });`; | |
| 79 | const sf = ts.createSourceFile("x.ts", src, ts.ScriptTarget.Latest, true); | |
| 80 | let bare = 0; | |
| 81 | const visit = (n: ts.Node) => { | |
| 82 | if (ts.isNumericLiteral(n) && n.text === "429") { | |
| 83 | let c: ts.Node = n.parent; | |
| 84 | while (!ts.isNewExpression(c)) c = c.parent; | |
| 85 | if (!/retry-?after/i.test(c.getText())) bare++; | |
| 86 | } | |
| 87 | ts.forEachChild(n, visit); | |
| 88 | }; | |
| 89 | visit(sf); | |
| 90 | expect(bare).toBe(1); | |
| 91 | }); | |
| 92 | }); | |
| 93 | ||
| 94 | /** The integer header and the body's `retryAfter` say the same seconds. */ | |
| 95 | async function told(res: Response, status: 429 | 503 = 429): Promise<number> { | |
| 96 | expect(res.status).toBe(status); | |
| 97 | const header = res.headers.get("retry-after"); | |
| 98 | expect(header).toMatch(/^[1-9]\d*$/); | |
| 99 | const body = (await res.json()) as { retryAfter?: unknown }; | |
| 100 | expect(body.retryAfter).toBe(Number(header)); | |
| 101 | return Number(header); | |
| 102 | } | |
| 103 | ||
| 104 | describe("the limits, driven through a running forge", () => { | |
| 105 | let dataDir: string; | |
| 106 | let app: App; | |
| 107 | let servers: Servers; | |
| 108 | let base: string; | |
| 109 | let n = 0; | |
| 110 | const tokens: string[] = []; | |
| 111 | ||
| 112 | beforeAll(async () => { | |
| 113 | dataDir = await mkdtemp(join(tmpdir(), "forge-retry-after-")); | |
| 114 | process.env["FORGE_RATE_LIMIT"] = "3"; | |
| 115 | app = createApp({ dataDir, port: 0 }); | |
| 116 | servers = await serve(app); | |
| 117 | delete process.env["FORGE_RATE_LIMIT"]; | |
| 118 | base = servers.url.replace("localhost", "127.0.0.1"); | |
| 119 | const made = await createUser(app.db, { origin: "seed", trusted: true, username: "rae", email: "[email protected]", password: "a-good-passphrase", isAdmin: true }); | |
| 120 | if (!made.ok) throw new Error(made.error); | |
| 121 | for (let i = 0; i < 4; i++) tokens.push(createToken(app.db, made.value.id, `t${i}`, "rw").token); | |
| 122 | }); | |
| 123 | afterAll(async () => { | |
| 124 | await servers.stop(); | |
| 125 | await rm(dataDir, { recursive: true, force: true }); | |
| 126 | }); | |
| 127 | ||
| 128 | /** Repeat `go` until it answers 429 (at most `max` tries); the refusal. */ | |
| 129 | async function until(go: () => Promise<Response>, max: number): Promise<Response> { | |
| 130 | for (let i = 0; i < max; i++) { | |
| 131 | const res = await go(); | |
| 132 | if (res.status === 429 || res.status === 503) return res; | |
| 133 | } | |
| 134 | throw new Error("never limited"); | |
| 135 | } | |
| 136 | ||
| 137 | test("the per-token minute limit on the API", async () => { | |
| 138 | const res = await until(() => fetch(`${base}/api/v1/user`, { headers: { authorization: `Bearer ${tokens[0]}` } }), 10); | |
| 139 | expect(await told(res)).toBeLessThanOrEqual(60); | |
| 140 | }); | |
| 141 | ||
| 142 | test("the same limit on /mcp", async () => { | |
| 143 | const res = await until( | |
| 144 | () => | |
| 145 | fetch(`${base}/mcp`, { | |
| 146 | method: "POST", | |
| 147 | headers: { authorization: `Bearer ${tokens[1]}`, "content-type": "application/json", accept: "application/json, text/event-stream" }, | |
| 148 | body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/list" }), | |
| 149 | }), | |
| 150 | 10, | |
| 151 | ); | |
| 152 | expect(await told(res)).toBeLessThanOrEqual(60); | |
| 153 | }); | |
| 154 | ||
| 155 | test("a stranger's per-address limit on the API", async () => { | |
| 156 | setStrangerLimits(app, { perAddress: { ...strangerLimits(app).perAddress, api: 2 } }); | |
| 157 | const res = await until(() => fetch(`${base}/api/v1/repos`), 10); | |
| 158 | expect(await told(res)).toBeLessThanOrEqual(60); | |
| 159 | }); | |
| 160 | ||
| 161 | test("a stranger's code search", async () => { | |
| 162 | setStrangerLimits(app, { perAddress: { ...strangerLimits(app).perAddress, code: 1 } }); | |
| 163 | const res = await until(() => fetch(`${base}/api/v1/search?kind=code&q=needle`), 10); | |
| 164 | expect(await told(res)).toBeLessThanOrEqual(60); | |
| 165 | }); | |
| 166 | ||
| 167 | test("sign-up", async () => { | |
| 168 | const res = await until( | |
| 169 | () => | |
| 170 | fetch(`${base}/api/v1/users`, { | |
| 171 | method: "POST", | |
| 172 | headers: { "content-type": "application/json" }, | |
| 173 | body: JSON.stringify({ username: `person${++n}`, email: `p${n}@outsider.test`, password: "a-good-passphrase" }), | |
| 174 | }), | |
| 175 | 12, | |
| 176 | ); | |
| 177 | expect(await told(res)).toBeLessThanOrEqual(60); | |
| 178 | }); | |
| 179 | ||
| 180 | test("oauth register", async () => { | |
| 181 | const res = await until( | |
| 182 | () => | |
| 183 | fetch(`${base}/oauth/register`, { | |
| 184 | method: "POST", | |
| 185 | headers: { "content-type": "application/json" }, | |
| 186 | body: JSON.stringify({ client_name: "x", redirect_uris: ["https://claude.ai/api/mcp/auth_callback"] }), | |
| 187 | }), | |
| 188 | 15, | |
| 189 | ); | |
| 190 | expect(await told(res)).toBeLessThanOrEqual(60); | |
| 191 | }); | |
| 192 | ||
| 193 | test("oauth revoke", async () => { | |
| 194 | const res = await until( | |
| 195 | () => | |
| 196 | fetch(`${base}/oauth/revoke`, { | |
| 197 | method: "POST", | |
| 198 | headers: { "content-type": "application/x-www-form-urlencoded" }, | |
| 199 | body: "token=forge_unknown", | |
| 200 | }), | |
| 201 | 70, | |
| 202 | ); | |
| 203 | expect(await told(res)).toBeLessThanOrEqual(60); | |
| 204 | }); | |
| 205 | ||
| 206 | test("standby: a write on a read-only forge", async () => { | |
| 207 | const post = new Request("http://x/api/v1/things", { method: "POST" }); | |
| 208 | expect(await told(readOnlyRefusal(post)!, 503)).toBe(300); | |
| 209 | }); | |
| 210 | ||
| 211 | test("the shared responses: a site's own wait wins, a silent one gets the floor", async () => { | |
| 212 | expect(await told(json({ error: "x" }, 429, { "Retry-After": "7" }))).toBe(7); | |
| 213 | expect(await told(json({ error: "x" }, 503), 503)).toBe(60); | |
| 214 | expect(await told(tryAgain("x", 429, "slow", 0.2))).toBe(1); | |
| 215 | expect(json({ error: "x" }, 404).headers.get("retry-after")).toBeNull(); | |
| 216 | }); | |
| 217 | }); |
Silent default of 60 makes sites that never state a wait look compliant. Keep as a backstop, but have each site name its real seconds (print, hooks, reports, following, forks, data, proposal gate, rp.ts).