Public: anyone can read this; the crew can change it.
Strangers cannot exhaust the forge: anonymous clones, code search and streams are rate-limited and capped #2948
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/2920-strangers-cannot-exhaust-the-forge-anony → main 4 commits fork 6 threads · 6 unresolved
20 files
changed +1103
−48
docs/ARCHITECTURE.md +14
| @@ -383,6 +383,20 @@ Paid-for lessons from open-platform-ts, designed in structurally: | ||
| 383 | 383 | answered. A body no route read, of no declared length, is treated the |
| 384 | 384 | same. A cancelled read is never used: it leaves the rest of the body on |
| 385 | 385 | the connection to be taken for the next request. |
| 386 | 7. No stranger can spend the forge (`src/http/strangers.ts`). A request with | |
| 387 | no session and no token is counted per client address (an IPv6 /64) and | |
| 388 | for every stranger together, at each door that costs: API reads (600 a | |
| 389 | minute per address, 6000 for all), clones and fetches (60 a minute, at | |
| 390 | most 4 in flight per address and half the transfer pool for all, killed | |
| 391 | after 300 s, and a negotiation that goes 30 s without a byte ends), code | |
| 392 | search (10 a minute, one stranger's at a time, so half the plumbing pool | |
| 393 | stays the forge's — the page, the API and the terminal's guest palette, | |
| 394 | which is counted under its SSH peer address, alike), event streams (8 open per address, 256 for all | |
| 395 | strangers, 4096 for anyone) and spawn plans (10 a minute). Past a limit | |
| 396 | the answer is a 429 with `Retry-After`. The audit card and the sitemap, | |
| 397 | which anyone may ask for, are made at most once a minute | |
| 398 | (`src/kernel/fresh.ts`). `/healthz` is never limited: refusing a probe | |
| 399 | turns load into a restart. | |
| 386 | 400 | |
| 387 | 401 | ## Two processes at once |
| 388 | 402 |
docs/THREATS.md +6
| @@ -106,6 +106,12 @@ Wants: accounts, spam, resources, or a way to exhaust the forge. | ||
| 106 | 106 | Test: `test/fuzz-parsers.test.ts` |
| 107 | 107 | - Anonymous spawn is rate limited per client address. |
| 108 | 108 | Test: `test/spawn-anon.test.ts` |
| 109 | - Every door a stranger can reach that costs (API reads, clones, code | |
| 110 | search on the page, the API and the terminal's guest palette, event | |
| 111 | streams, spawn plans) is limited per address and for every stranger | |
| 112 | together (`src/http/strangers.ts`, docs/ARCHITECTURE.md, concurrency | |
| 113 | invariant 7); a signed-in push keeps its share of the pool. | |
| 114 | Test: `test/strangers.test.ts` | |
| 109 | 115 | - The client address is the socket's peer unless that peer is a configured |
| 110 | 116 | trusted proxy; `X-Forwarded-For` is read from the right, past trusted hops. |
| 111 | 117 | Test: `test/client-address.test.ts` |
src/actions/search.ts +7 −1
| @@ -51,6 +51,12 @@ export interface SearchInput { | ||
| 51 | 51 | limit?: number; |
| 52 | 52 | /** Also ask each connected origin's own issue search. @default false */ |
| 53 | 53 | live?: boolean; |
| 54 | /** | |
| 55 | * Grep the code when `kind` asks for it. False when a stranger's code | |
| 56 | * search was refused (`http/strangers.ts`): the rest still answers. | |
| 57 | * @default true | |
| 58 | */ | |
| 59 | withCode?: boolean; | |
| 54 | 60 | } |
| 55 | 61 | |
| 56 | 62 | /** |
| @@ -113,7 +119,7 @@ export async function search( | ||
| 113 | 119 | issues.push(...(await liveIssues(app, actor, q, limit))); |
| 114 | 120 | } |
| 115 | 121 | |
| 116 | const code = wants("code") | |
| 122 | const code = wants("code") && input.withCode !== false | |
|
||
| 117 | 123 | ? await grepRepos( |
| 118 | 124 | app.reposRoot, |
| 119 | 125 | visible, |
src/git/smart-http.ts +36 −5
| @@ -35,8 +35,19 @@ export interface SmartHttpOptions { | ||
| 35 | 35 | env?: Record<string, string>; |
| 36 | 36 | /** Kill a wedged transfer after this long. @default 600_000 */ |
| 37 | 37 | timeoutMs?: number; |
| 38 | /** | |
| 39 | * End a fetch whose request body stops arriving for this long: a client | |
| 40 | * that opens a negotiation and trickles it holds a transfer permit the | |
| 41 | * whole time. Upload-pack only; a push's pack may pause. @default 30_000 | |
| 42 | */ | |
| 43 | idleMs?: number; | |
| 44 | /** Given back when the transfer ends, however it ends (a stranger's share). */ | |
| 45 | release?: () => void; | |
| 38 | 46 | } |
| 39 | 47 | |
| 48 | /** How long a fetch's negotiation may go without a byte before it ends. */ | |
| 49 | export const FETCH_IDLE_MS = 30_000; | |
| 50 | ||
| 40 | 51 | /** |
| 41 | 52 | * GET /:owner/:name.git/info/refs?service=… — the ref advertisement. |
| 42 | 53 | * The `# service=` preamble is required for smart clients on protocol v0/v1 |
| @@ -77,7 +88,9 @@ export async function serviceRpc( | ||
| 77 | 88 | options: SmartHttpOptions = {}, |
| 78 | 89 | ): Promise<Response> { |
| 79 | 90 | const proc = await spawnService(repoDir, service, request, options, []); |
| 80 | feedStdin(proc, service, request).catch(() => proc.kill("SIGKILL")); | |
| 91 | feedStdin(proc, service, request, options.idleMs ?? FETCH_IDLE_MS).catch(() => | |
| 92 | proc.kill("SIGKILL"), | |
| 93 | ); | |
| 81 | 94 | |
| 82 | 95 | return new Response(proc.stdout, { |
| 83 | 96 | headers: { |
| @@ -108,7 +121,11 @@ async function spawnService( | ||
| 108 | 121 | options: SmartHttpOptions, |
| 109 | 122 | extraArgs: readonly string[], |
| 110 | 123 | ) { |
| 111 | const release = await transferGate.acquire(); | |
| 124 | const permit = await transferGate.acquire(); | |
| 125 | const release = (): void => { | |
| 126 | permit(); | |
| 127 | options.release?.(); | |
| 128 | }; | |
| 112 | 129 | // "git-upload-pack" the service is "upload-pack" the subcommand. |
| 113 | 130 | const subcommand = service.slice(4); |
| 114 | 131 | |
| @@ -177,6 +194,7 @@ async function feedStdin( | ||
| 177 | 194 | proc: Bun.Subprocess<"pipe", "pipe", "ignore">, |
| 178 | 195 | service: GitService, |
| 179 | 196 | request: Request, |
| 197 | idleMs: number, | |
| 180 | 198 | ): Promise<void> { |
| 181 | 199 | const stdin = proc.stdin; |
| 182 | 200 | if (request.body === null) { |
| @@ -184,13 +202,23 @@ async function feedStdin( | ||
| 184 | 202 | return; |
| 185 | 203 | } |
| 186 | 204 | let gone = false; |
| 187 | void proc.exited.then(() => { | |
| 188 | gone = true; | |
| 189 | }); | |
| 190 | 205 | const stop = (): void => { |
| 191 | 206 | closeNow(request); |
| 192 | 207 | proc.kill("SIGKILL"); |
| 193 | 208 | }; |
| 209 | // A fetch's negotiation is a few kilobytes sent at once; one that stops | |
| 210 | // arriving is a client holding a permit open, and it ends. | |
| 211 | let idle: ReturnType<typeof setTimeout> | undefined; | |
| 212 | const poke = (): void => { | |
| 213 | if (service !== "git-upload-pack") return; | |
| 214 | clearTimeout(idle); | |
| 215 | idle = setTimeout(stop, idleMs); | |
| 216 | }; | |
| 217 | poke(); | |
| 218 | void proc.exited.then(() => { | |
| 219 | gone = true; | |
| 220 | clearTimeout(idle); | |
| 221 | }); | |
| 194 | 222 | const write = async (chunk: Uint8Array): Promise<void> => { |
| 195 | 223 | const wrote: unknown = stdin.write(chunk); |
| 196 | 224 | if (wrote instanceof Promise) await wrote; |
| @@ -211,11 +239,14 @@ async function feedStdin( | ||
| 211 | 239 | const max = service === "git-receive-pack" ? PUSH_MAX : FETCH_MAX; |
| 212 | 240 | for await (const chunk of counted(request.body, max, stop)) { |
| 213 | 241 | if (gone) break; |
| 242 | poke(); | |
| 214 | 243 | await write(chunk); |
| 215 | 244 | } |
| 216 | 245 | } |
| 246 | clearTimeout(idle); | |
| 217 | 247 | if (!gone) await stdin.end(); |
| 218 | 248 | } catch { |
| 249 | clearTimeout(idle); | |
| 219 | 250 | // Git went away mid-write: the body has nowhere to go, and its answer |
| 220 | 251 | // (or its absence) is the response's business. |
| 221 | 252 | proc.kill("SIGKILL"); |
src/http/api.ts +32 −11
| @@ -217,7 +217,8 @@ import { | ||
| 217 | 217 | canWrite, |
| 218 | 218 | getRepoById, |
| 219 | 219 | } from "../core/repos.ts"; |
| 220 | import { searchKind } from "../core/search.ts"; | |
| 220 | import { cleanQuery, searchKind } from "../core/search.ts"; | |
| 221 | import { strangerCodeSearch } from "./strangers.ts"; | |
| 221 | 222 | import { |
| 222 | 223 | isReviewState, |
| 223 | 224 | listReviewComments, |
| @@ -2146,15 +2147,35 @@ export function apiRoutes(router: Router<Handler>): void { | ||
| 2146 | 2147 | // caller has connected, conversations here, and the code itself. `live=1` |
| 2147 | 2148 | // additionally asks each origin's own issue search, inside one budget. |
| 2148 | 2149 | |
| 2150 | // A stranger's search that greps code (`kind` code or all) takes a slot | |
| 2151 | // of its own first (`strangers.ts`): `git grep` over fifty repositories | |
| 2152 | // is not a request to answer as fast as anyone can send it. | |
| 2149 | 2153 | router.add("GET", "/api/v1/search", async (ctx) => { |
| 2150 | 2154 | const q = ctx.url.searchParams; |
| 2151 | const found = await search(ctx.app, ctx.auth.user, { | |
| 2152 | q: q.get("q") ?? "", | |
| 2153 | kind: searchKind(q.get("kind")), | |
| 2154 | ...(q.has("limit") ? { limit: Number(q.get("limit")) } : {}), | |
| 2155 | live: q.get("live") === "1" || q.get("live") === "true", | |
| 2156 | }); | |
| 2157 | return json(found); | |
| 2155 | const kind = searchKind(q.get("kind")); | |
| 2156 | const query = q.get("q") ?? ""; | |
| 2157 | const greps = | |
| 2158 | ctx.auth.user === undefined && | |
| 2159 | (kind === "code" || kind === "all") && | |
| 2160 | cleanQuery(query) !== ""; | |
| 2161 | const held = greps ? strangerCodeSearch(ctx.app, ctx.request) : undefined; | |
| 2162 | if (held !== undefined && !held.allowed) { | |
| 2163 | const refused = apiError("rate_limited", 429, `${held.why}; sign in, or retry in ${held.retryAfter}s`); | |
| 2164 | refused.headers.set("retry-after", String(held.retryAfter)); | |
| 2165 | return refused; | |
| 2166 | } | |
| 2167 | try { | |
| 2168 | return json( | |
| 2169 | await search(ctx.app, ctx.auth.user, { | |
| 2170 | q: query, | |
| 2171 | kind, | |
| 2172 | ...(q.has("limit") ? { limit: Number(q.get("limit")) } : {}), | |
| 2173 | live: q.get("live") === "1" || q.get("live") === "true", | |
| 2174 | }), | |
| 2175 | ); | |
| 2176 | } finally { | |
| 2177 | held?.release(); | |
| 2178 | } | |
| 2158 | 2179 | }); |
| 2159 | 2180 | |
| 2160 | 2181 | // ── repositories ───────────────────────────────────────────────────── |
| @@ -4558,7 +4579,7 @@ export function apiRoutes(router: Router<Handler>): void { | ||
| 4558 | 4579 | ctx.params.job!, |
| 4559 | 4580 | ); |
| 4560 | 4581 | if (!found.ok) return respondAccess(found.error, 404); |
| 4561 | return eventStream(ctx.request, (emit, close) => | |
| 4582 | return eventStream(ctx.app, ctx.request, (emit, close) => | |
| 4562 | 4583 | subscribeJob(found.value.id, (job) => { |
| 4563 | 4584 | emit(`mirror.${job.stage}`, jobJson(ctx, job), job.id); |
| 4564 | 4585 | if (job.stage === "done" || job.stage === "failed") close(); |
| @@ -4984,7 +5005,7 @@ export function apiRoutes(router: Router<Handler>): void { | ||
| 4984 | 5005 | router.add("GET", "/api/v1/migrate/:job/stream", (ctx) => { |
| 4985 | 5006 | const found = migrationJobFor(ctx.auth.user, ctx.params.job!); |
| 4986 | 5007 | if (!found.ok) return respondAccess(found.error, 404); |
| 4987 | return eventStream(ctx.request, (emit, close) => | |
| 5008 | return eventStream(ctx.app, ctx.request, (emit, close) => | |
| 4988 | 5009 | subscribeJob(found.value.id, (job) => { |
| 4989 | 5010 | emit(`migrate.${job.stage}`, migrationJson(ctx, job), job.id); |
| 4990 | 5011 | if (job.stage === "done" || job.stage === "failed") close(); |
| @@ -5517,7 +5538,7 @@ export function apiRoutes(router: Router<Handler>): void { | ||
| 5517 | 5538 | (status.state === "running" || status.state === "deploying") |
| 5518 | 5539 | ); |
| 5519 | 5540 | }; |
| 5520 | return eventStream(ctx.request, (emit, close) => { | |
| 5541 | return eventStream(ctx.app, ctx.request, (emit, close) => { | |
| 5521 | 5542 | const controller = new AbortController(); |
| 5522 | 5543 | let index = 0; |
| 5523 | 5544 | const send = (line: string): void => { |
src/http/limits.ts +3
| @@ -48,6 +48,9 @@ export const noteSignedIn = (request: Request): void => { | ||
| 48 | 48 | signedIn.add(request); |
| 49 | 49 | }; |
| 50 | 50 | |
| 51 | /** Whether the request carries a signed-in person or a token. */ | |
| 52 | export const isSignedIn = (request: Request): boolean => signedIn.has(request); | |
| 53 | ||
| 51 | 54 | /** The most a shared reader keeps of this request's body. */ |
| 52 | 55 | export const bodyCapFor = (request: Request): number => |
| 53 | 56 | signedIn.has(request) ? BODY_MAX : STRANGER_MAX; |
src/http/server.ts +69 −9
| @@ -46,6 +46,7 @@ import { | ||
| 46 | 46 | armClose, |
| 47 | 47 | BODY_MAX, |
| 48 | 48 | carryNotes, |
| 49 | closeAfter, | |
| 49 | 50 | mustClose, |
| 50 | 51 | noteSignedIn, |
| 51 | 52 | readBounded, |
| @@ -82,7 +83,14 @@ import { | ||
| 82 | 83 | isGitService, |
| 83 | 84 | serviceRpc, |
| 84 | 85 | type GitService, |
| 86 | type SmartHttpOptions, | |
| 85 | 87 | } from "../git/smart-http.ts"; |
| 88 | import { | |
| 89 | strangerLimits, | |
| 90 | strangerTransfer, | |
| 91 | takeDoor, | |
| 92 | tooManyStrangers, | |
| 93 | } from "./strangers.ts"; | |
| 86 | 94 | import { timingSafeEqual } from "node:crypto"; |
| 87 | 95 | import { |
| 88 | 96 | forwardedGitEnv, |
| @@ -997,12 +1005,16 @@ export function buildRouter(): Router<Handler> { | ||
| 997 | 1005 | if (!isGitService(service)) |
| 998 | 1006 | return new Response("smart http only", { status: 400 }); |
| 999 | 1007 | return withGitAccess(ctx, service, (dir) => |
| 1000 | advertiseRefs(dir, service, ctx.request), | |
| 1008 | asStranger(ctx, (options) => | |
| 1009 | advertiseRefs(dir, service, ctx.request, options), | |
| 1010 | ), | |
| 1001 | 1011 | ); |
| 1002 | 1012 | }); |
| 1003 | 1013 | router.add("POST", "/:owner/:name.git/git-upload-pack", (ctx) => |
| 1004 | 1014 | withGitAccess(ctx, "git-upload-pack", (dir) => |
| 1005 | serviceRpc(dir, "git-upload-pack", ctx.request), | |
| 1015 | asStranger(ctx, (options) => | |
| 1016 | serviceRpc(dir, "git-upload-pack", ctx.request, options), | |
| 1017 | ), | |
| 1006 | 1018 | ), |
| 1007 | 1019 | ); |
| 1008 | 1020 | router.add("POST", "/:owner/:name.git/git-receive-pack", (ctx) => |
| @@ -1395,6 +1407,39 @@ export function knownPatterns(): string[] { | ||
| 1395 | 1407 | return (known ??= [...buildRouter().patterns(), ...buildApiRouter().patterns()]); |
| 1396 | 1408 | } |
| 1397 | 1409 | |
| 1410 | /** | |
| 1411 | * A transfer someone signed in makes runs as it always has; a stranger's | |
| 1412 | * takes a share of the transfer pool first (`strangers.ts`) — refused with | |
| 1413 | * a 429 when its address or every stranger together has none left — and | |
| 1414 | * runs on a shorter clock. Called once access is decided, so a private | |
| 1415 | * repository still answers a stranger with the credential challenge. | |
| 1416 | */ | |
| 1417 | async function asStranger( | |
| 1418 | ctx: Ctx, | |
| 1419 | run: (options: SmartHttpOptions) => Promise<Response>, | |
| 1420 | ): Promise<Response> { | |
| 1421 | if (ctx.auth.user !== undefined) return run({}); | |
| 1422 | const held = strangerTransfer(ctx.app, ctx.request); | |
| 1423 | if (!held.allowed) { | |
| 1424 | // Refused before a byte of the body is wanted: nothing more is read of | |
| 1425 | // it, and the connection ends once the 429 is out (`http/limits.ts`). | |
| 1426 | closeAfter(ctx.request); | |
| 1427 | void ctx.request.body?.cancel().catch(() => {}); | |
| 1428 | return tooManyStrangers(held); | |
| 1429 | } | |
| 1430 | const limits = strangerLimits(ctx.app); | |
| 1431 | try { | |
| 1432 | return await run({ | |
| 1433 | release: held.release, | |
| 1434 | timeoutMs: limits.transferTimeoutMs, | |
| 1435 | idleMs: limits.transferIdleMs, | |
| 1436 | }); | |
| 1437 | } catch (error) { | |
| 1438 | held.release(); | |
| 1439 | throw error; | |
| 1440 | } | |
| 1441 | } | |
| 1442 | ||
| 1398 | 1443 | async function withGitAccess( |
| 1399 | 1444 | ctx: Ctx, |
| 1400 | 1445 | service: GitService, |
| @@ -2307,9 +2352,11 @@ async function answer( | ||
| 2307 | 2352 | request: Request, |
| 2308 | 2353 | ): Promise<Response> { |
| 2309 | 2354 | const response = await answerGet(app, router, apiRouter, limiter, request); |
| 2310 | return request.method === "HEAD" | |
| 2311 | ? new Response(null, { status: response.status, headers: response.headers }) | |
| 2312 | : response; | |
| 2355 | if (request.method !== "HEAD") return response; | |
| 2356 | // The body nobody will read is cancelled, so a stream behind it (an | |
| 2357 | // event stream's slot, a git process) ends now rather than never. | |
| 2358 | void response.body?.cancel().catch(() => {}); | |
No test covers this. Add a stranger HEAD to |
||
| 2359 | return new Response(null, { status: response.status, headers: response.headers }); | |
| 2313 | 2360 | } |
| 2314 | 2361 | |
| 2315 | 2362 | async function answerGet( |
| @@ -2387,13 +2434,26 @@ async function answerGet( | ||
| 2387 | 2434 | // The two things standing between one runaway agent and everybody |
| 2388 | 2435 | // else's latency, applied before a handler runs and after auth is |
| 2389 | 2436 | // known — a limit needs an identity, and a replay needs a credential. |
| 2437 | // A stranger has no token to count against: its address does, and every | |
| 2438 | // stranger together (`strangers.ts`) — on the API, which is what a | |
| 2439 | // script reads; the doors that cost more keep tighter limits of their own. | |
| 2390 | 2440 | const credential = tokenCredential(request, auth); |
| 2391 | 2441 | const verdict = |
| 2392 | credential === undefined | |
| 2393 | ? undefined | |
| 2394 | : limiter.take(credential, auth.scope ?? "ro"); | |
| 2442 | credential !== undefined | |
| 2443 | ? limiter.take(credential, auth.scope ?? "ro") | |
| 2444 | : auth.user === undefined && isApiPath && limiter.enabled | |
| 2445 | ? takeDoor(app, "api", request) | |
| 2446 | : undefined; | |
| 2395 | 2447 | if (verdict !== undefined && !verdict.allowed) { |
| 2396 | return withRateHeaders(tooManyRequests(verdict), verdict); | |
| 2448 | const refusal = | |
| 2449 | credential === undefined | |
| 2450 | ? apiError( | |
| 2451 | "rate_limited", | |
| 2452 | 429, | |
| 2453 | `too many requests without a token: ${verdict.limit} per minute — sign in, or retry in ${verdict.retryAfter}s`, | |
| 2454 | ) | |
| 2455 | : tooManyRequests(verdict); | |
| 2456 | return withRateHeaders(refusal, verdict); | |
| 2397 | 2457 | } |
| 2398 | 2458 | |
| 2399 | 2459 | const themeCookie = cookieValue(request, "forge_theme") ?? ""; |
src/http/sse.ts +13 −2
| @@ -15,6 +15,8 @@ | ||
| 15 | 15 | */ |
| 16 | 16 | |
| 17 | 17 | import { holdForCaller } from "./caller.ts"; |
| 18 | import { isSignedIn } from "./limits.ts"; | |
| 19 | import { holdStream, tooManyStrangers } from "./strangers.ts"; | |
| 18 | 20 | import type { App } from "../app.ts"; |
| 19 | 21 | import type { User } from "../core/users.ts"; |
| 20 | 22 | import { listEvents, listEventsVisibleTo } from "../core/events.ts"; |
| @@ -82,11 +84,19 @@ export type Emit = (event: string, data: unknown, id?: string) => void; | ||
| 82 | 84 | * Open a stream. `register` is called once with the emitter and a `close`; |
| 83 | 85 | * it returns its own unsubscribe, which runs when the client goes away, when |
| 84 | 86 | * `close` is called, or when a write fails — whichever happens first. |
| 87 | * | |
| 88 | * Every stream is counted on `app` while it is open (`strangers.ts`): a | |
| 89 | * stranger's against its address and every stranger's together, anyone's | |
| 90 | * against the forge's whole budget. Past either, the answer is a 429 and | |
| 91 | * nothing is registered. | |
| 85 | 92 | */ |
| 86 | 93 | export function eventStream( |
| 94 | app: App, | |
| 87 | 95 | request: Request, |
| 88 | 96 | register: (emit: Emit, close: () => void, ready: () => boolean) => () => void, |
| 89 | 97 | ): Response { |
| 98 | const held = holdStream(app, request, !isSignedIn(request)); | |
| 99 | if (!held.allowed) return tooManyStrangers(held); | |
| 90 | 100 | const encoder = new TextEncoder(); |
| 91 | 101 | let cleanup = () => {}; |
| 92 | 102 | |
| @@ -132,6 +142,7 @@ export function eventStream( | ||
| 132 | 142 | cleanup = () => { |
| 133 | 143 | if (!open) return; |
| 134 | 144 | open = false; |
| 145 | held.release(); | |
| 135 | 146 | release(); |
| 136 | 147 | unsubscribe(); |
| 137 | 148 | clearInterval(heartbeat); |
| @@ -218,7 +229,7 @@ export function viewerStream( | ||
| 218 | 229 | // admins', a person's own choices are theirs (`core/event_audience.ts`). |
| 219 | 230 | const meant = audienceFilter(app.db, viewer); |
| 220 | 231 | const after = resumeFrom(request); |
| 221 | return eventStream(request, (emit, _close, ready) => { | |
| 232 | return eventStream(app, request, (emit, _close, ready) => { | |
| 222 | 233 | if (after !== undefined) { |
| 223 | 234 | const missed = listEventsVisibleTo(app.db, viewer, { |
| 224 | 235 | afterId: after, |
| @@ -269,7 +280,7 @@ export function sseStream( | ||
| 269 | 280 | ): Response { |
| 270 | 281 | const meant = audienceFilter(app.db, viewer); |
| 271 | 282 | const after = resumeFrom(request); |
| 272 | return eventStream(request, (emit) => { | |
| 283 | return eventStream(app, request, (emit) => { | |
| 273 | 284 | // The event id doubles as the SSE id, so a reconnecting client resumes |
| 274 | 285 | // from where it stopped: what it missed first, oldest first, then live. |
| 275 | 286 | if (after !== undefined) |
src/http/strangers.ts added +310
| @@ -0,0 +1,310 @@ | ||
| 1 | /** | |
| 2 | * What a stranger — a request with no session and no token — may spend. | |
| 3 | * | |
| 4 | * A signed-in caller is limited by its token (`core/ratelimit.ts`) and is | |
| 5 | * someone the forge can suspend. A stranger is neither, and the doors open | |
| 6 | * to everyone are the expensive ones: a clone holds a transfer permit for | |
| 7 | * minutes, a code search runs `git grep` over fifty repositories, a stream | |
| 8 | * holds a connection open for as long as the client likes, a spawn plan | |
| 9 | * walks what a catalog app carries. Each door here has two limits, one per | |
| 10 | * client address (`peer.ts`'s bucket, an IPv6 /64) and one for the whole | |
| 11 | * forge, so a thousand addresses cannot do together what one may not — | |
| 12 | * and the forge-wide one is what keeps a share of every pool for pushes, | |
| 13 | * page renders and agents. | |
| 14 | * | |
| 15 | * Rates are a sliding minute (`RateLimiter`); what is held — a transfer, a | |
| 16 | * search, a stream — is counted while it is held and given back on every | |
| 17 | * path out. One set per forge, so a test's forge never throttles another's. | |
| 18 | */ | |
| 19 | ||
| 20 | import type { App } from "../app.ts"; | |
| 21 | import { RateLimiter, type Verdict } from "../core/ratelimit.ts"; | |
| 22 | import { transferGate } from "../git/env.ts"; | |
| 23 | import { bucketOf, clientBucket } from "./peer.ts"; | |
| 24 | ||
| 25 | /** A door a stranger is counted at, each with its own minute. */ | |
| 26 | export type Door = "api" | "git" | "code" | "plan"; | |
| 27 | ||
| 28 | export interface StrangerLimits { | |
| 29 | /** Requests per minute through each door, per client address. */ | |
| 30 | readonly perAddress: Readonly<Record<Door, number>>; | |
| 31 | /** Requests per minute through each door, every stranger together. */ | |
| 32 | readonly forge: Readonly<Record<Door, number>>; | |
| 33 | /** Pack transfers one address may hold at once. */ | |
| 34 | readonly transfersPerAddress: number; | |
| 35 | /** | |
| 36 | * The share of the transfer pool strangers may hold together: the rest is | |
| 37 | * kept for pushes and signed-in clones. At least one permit. | |
| 38 | */ | |
| 39 | readonly transferShare: number; | |
| 40 | /** A stranger's transfer is killed after this long (a signed-in one: 600 s). */ | |
| 41 | readonly transferTimeoutMs: number; | |
| 42 | /** A fetch's request body that stops arriving for this long ends the fetch. */ | |
| 43 | readonly transferIdleMs: number; | |
| 44 | /** | |
| 45 | * Code searches strangers may run at once, every one together. One search | |
| 46 | * greps four repositories at a time (`core/search.ts`), so one is half the | |
| 47 | * eight-permit plumbing pool and the other half stays the forge's own. | |
| 48 | */ | |
| 49 | readonly codeAtOnce: number; | |
| 50 | /** Event streams one address may hold open. */ | |
| 51 | readonly streamsPerAddress: number; | |
| 52 | /** Event streams strangers may hold open, every one together. */ | |
| 53 | readonly strangerStreams: number; | |
| 54 | /** Event streams anyone may hold open on this forge. */ | |
| 55 | readonly allStreams: number; | |
| 56 | } | |
| 57 | ||
| 58 | export const STRANGER_LIMITS: StrangerLimits = { | |
| 59 | perAddress: { api: 600, git: 60, code: 10, plan: 10 }, | |
| 60 | forge: { api: 6000, git: 600, code: 60, plan: 60 }, | |
| 61 | transfersPerAddress: 4, | |
| 62 | transferShare: 0.5, | |
| 63 | transferTimeoutMs: 300_000, | |
| 64 | transferIdleMs: 30_000, | |
| 65 | codeAtOnce: 1, | |
| 66 | streamsPerAddress: 8, | |
| 67 | strangerStreams: 256, | |
| 68 | allStreams: 4096, | |
| 69 | }; | |
| 70 | ||
| 71 | /** A door refused: what to say, and when to come back (seconds). */ | |
| 72 | export interface Refused { | |
| 73 | readonly allowed: false; | |
| 74 | readonly retryAfter: number; | |
| 75 | readonly why: string; | |
| 76 | } | |
| 77 | ||
| 78 | /** Something held, given back by `release` (idempotent). */ | |
| 79 | export interface Held { | |
| 80 | readonly allowed: true; | |
| 81 | readonly release: () => void; | |
| 82 | } | |
| 83 | ||
| 84 | interface State { | |
| 85 | limits: StrangerLimits; | |
| 86 | readonly perAddress: Map<Door, RateLimiter>; | |
| 87 | readonly forge: Map<Door, RateLimiter>; | |
| 88 | /** What is held right now, by key: "transfer:<bucket>", "code", "stream:*". */ | |
| 89 | readonly held: Map<string, number>; | |
| 90 | } | |
| 91 | ||
| 92 | const states = new WeakMap<App, State>(); | |
| 93 | ||
| 94 | function stateOf(app: App): State { | |
| 95 | let state = states.get(app); | |
| 96 | if (state === undefined) { | |
| 97 | state = { | |
| 98 | limits: STRANGER_LIMITS, | |
| 99 | perAddress: new Map(), | |
| 100 | forge: new Map(), | |
| 101 | held: new Map(), | |
| 102 | }; | |
| 103 | states.set(app, state); | |
| 104 | } | |
| 105 | return state; | |
| 106 | } | |
| 107 | ||
| 108 | const minute = (perMinute: number): RateLimiter => | |
| 109 | new RateLimiter({ perMinute, adminPerMinute: perMinute, enabled: true }); | |
| 110 | ||
| 111 | function limiter( | |
| 112 | map: Map<Door, RateLimiter>, | |
| 113 | door: Door, | |
| 114 | perMinute: number, | |
| 115 | ): RateLimiter { | |
| 116 | let found = map.get(door); | |
| 117 | if (found === undefined) { | |
| 118 | found = minute(perMinute); | |
| 119 | map.set(door, found); | |
| 120 | } | |
| 121 | return found; | |
| 122 | } | |
| 123 | ||
| 124 | /** This forge's limits for strangers. */ | |
| 125 | export const strangerLimits = (app: App): StrangerLimits => stateOf(app).limits; | |
| 126 | ||
| 127 | /** | |
| 128 | * Test seam and operator lever: new limits for this forge's strangers. | |
| 129 | * Counted minutes start over. | |
| 130 | */ | |
| 131 | export function setStrangerLimits( | |
| 132 | app: App, | |
| 133 | change: Partial<StrangerLimits>, | |
| 134 | ): void { | |
| 135 | const state = stateOf(app); | |
| 136 | state.limits = { ...state.limits, ...change }; | |
| 137 | state.perAddress.clear(); | |
| 138 | state.forge.clear(); | |
| 139 | } | |
| 140 | ||
| 141 | /** | |
| 142 | * Count one stranger's request through `door`: refused when its address or | |
| 143 | * every stranger together has spent the minute. A refusal is not counted | |
| 144 | * (`RateLimiter.take`), and one refused forge-wide gives back the address's | |
| 145 | * count, so a closed door does not push anyone's window forward. | |
| 146 | */ | |
| 147 | export const takeDoor = (app: App, door: Door, request: Request): Verdict => | |
| 148 | takeDoorAt(app, door, clientBucket(request)); | |
| 149 | ||
| 150 | /** | |
| 151 | * `takeDoor` for a client that is not an HTTP request — a terminal guest — | |
| 152 | * keyed on its address's bucket (`peer.ts`'s `bucketOf`, an IPv6 /64). | |
| 153 | */ | |
| 154 | export function takeDoorAt(app: App, door: Door, bucket: string): Verdict { | |
| 155 | const state = stateOf(app); | |
| 156 | const mine = limiter(state.perAddress, door, state.limits.perAddress[door]); | |
| 157 | const own = mine.take(bucket, "ro"); | |
| 158 | if (!own.allowed) return own; | |
| 159 | const all = limiter(state.forge, door, state.limits.forge[door]).take("*", "ro"); | |
| 160 | if (!all.allowed) { | |
| 161 | mine.refund(bucket); | |
| 162 | return all; | |
| 163 | } | |
| 164 | return own; | |
| 165 | } | |
| 166 | ||
| 167 | /** Take one of `key` if fewer than `max` are held; the release gives it back. */ | |
| 168 | function hold(state: State, key: string, max: number): (() => void) | undefined { | |
| 169 | const now = state.held.get(key) ?? 0; | |
| 170 | if (now >= max) return undefined; | |
| 171 | state.held.set(key, now + 1); | |
| 172 | let released = false; | |
| 173 | return () => { | |
| 174 | if (released) return; | |
| 175 | released = true; | |
| 176 | const left = (state.held.get(key) ?? 1) - 1; | |
| 177 | if (left <= 0) state.held.delete(key); | |
| 178 | else state.held.set(key, left); | |
| 179 | }; | |
| 180 | } | |
| 181 | ||
| 182 | /** Take every hold in `wants`, or none of them. */ | |
| 183 | function holdAll( | |
| 184 | state: State, | |
| 185 | wants: readonly (readonly [string, number])[], | |
| 186 | ): (() => void) | undefined { | |
| 187 | const taken: (() => void)[] = []; | |
| 188 | for (const [key, max] of wants) { | |
| 189 | const release = hold(state, key, max); | |
| 190 | if (release === undefined) { | |
| 191 | for (const give of taken) give(); | |
| 192 | return undefined; | |
| 193 | } | |
| 194 | taken.push(release); | |
| 195 | } | |
| 196 | return () => { | |
| 197 | for (const give of taken) give(); | |
| 198 | }; | |
| 199 | } | |
| 200 | ||
| 201 | const refused = (why: string, retryAfter: number): Refused => ({ | |
| 202 | allowed: false, | |
| 203 | retryAfter, | |
| 204 | why, | |
| 205 | }); | |
| 206 | ||
| 207 | /** | |
| 208 | * A stranger's pack transfer (a clone's ref advertisement or its fetch): | |
| 209 | * counted against the minute, then held — at most `transfersPerAddress` per | |
| 210 | * address, and together at most `transferShare` of the transfer pool, so a | |
| 211 | * crowd of anonymous clones never stands between a push and its permit. | |
| 212 | */ | |
| 213 | export function strangerTransfer(app: App, request: Request): Held | Refused { | |
| 214 | const verdict = takeDoor(app, "git", request); | |
| 215 | if (!verdict.allowed) | |
| 216 | return refused( | |
| 217 | "too many anonymous git requests; sign in or retry later", | |
| 218 | verdict.retryAfter, | |
| 219 | ); | |
| 220 | const state = stateOf(app); | |
| 221 | const share = Math.max( | |
| 222 | 1, | |
| 223 | Math.floor(transferGate.limit * state.limits.transferShare), | |
| 224 | ); | |
| 225 | const release = holdAll(state, [ | |
| 226 | [`transfer:${clientBucket(request)}`, state.limits.transfersPerAddress], | |
| 227 | ["transfer", share], | |
| 228 | ]); | |
| 229 | if (release === undefined) | |
| 230 | return refused( | |
| 231 | "too many anonymous git transfers at once; sign in or retry shortly", | |
| 232 | 5, | |
| 233 | ); | |
| 234 | return { allowed: true, release }; | |
| 235 | } | |
| 236 | ||
| 237 | /** | |
| 238 | * A stranger's code search: counted against the minute, then held — at most | |
| 239 | * `codeAtOnce` searches by strangers at a time, the whole forge together. | |
| 240 | */ | |
| 241 | export const strangerCodeSearch = (app: App, request: Request): Held | Refused => | |
| 242 | codeSearchAt(app, clientBucket(request)); | |
| 243 | ||
| 244 | /** | |
| 245 | * `strangerCodeSearch` for a terminal guest (any SSH key the forge does not | |
| 246 | * know), counted under its peer address: the palette's search greps code too. | |
| 247 | */ | |
| 248 | export const guestCodeSearch = (app: App, remoteAddress: string): Held | Refused => | |
| 249 | codeSearchAt(app, bucketOf(remoteAddress)); | |
| 250 | ||
| 251 | function codeSearchAt(app: App, bucket: string): Held | Refused { | |
| 252 | const verdict = takeDoorAt(app, "code", bucket); | |
| 253 | if (!verdict.allowed) | |
| 254 | return refused( | |
| 255 | "too many anonymous code searches; sign in or retry later", | |
| 256 | verdict.retryAfter, | |
| 257 | ); | |
| 258 | const state = stateOf(app); | |
| 259 | const release = hold(state, "code", state.limits.codeAtOnce); | |
| 260 | if (release === undefined) | |
| 261 | return refused("code search is busy for visitors who are not signed in", 2); | |
| 262 | return { allowed: true, release }; | |
| 263 | } | |
| 264 | ||
| 265 | /** | |
| 266 | * An event stream: every one counts toward `allStreams`; a stranger's also | |
| 267 | * toward its address and toward every stranger's together. | |
| 268 | */ | |
| 269 | export function holdStream( | |
| 270 | app: App, | |
| 271 | request: Request, | |
| 272 | stranger: boolean, | |
| 273 | ): Held | Refused { | |
| 274 | const state = stateOf(app); | |
| 275 | const { limits } = state; | |
| 276 | const release = holdAll( | |
| 277 | state, | |
| 278 | stranger | |
| 279 | ? [ | |
| 280 | ["stream:*", limits.allStreams], | |
| 281 | ["stream:strangers", limits.strangerStreams], | |
| 282 | [`stream:${clientBucket(request)}`, limits.streamsPerAddress], | |
| 283 | ] | |
| 284 | : [["stream:*", limits.allStreams]], | |
| 285 | ); | |
| 286 | if (release === undefined) | |
| 287 | return refused( | |
| 288 | stranger | |
| 289 | ? "too many open streams for visitors who are not signed in" | |
| 290 | : "too many open streams", | |
| 291 | 30, | |
| 292 | ); | |
| 293 | return { allowed: true, release }; | |
| 294 | } | |
| 295 | ||
| 296 | /** How many of `key` are held now — for tests. */ | |
| 297 | export const heldNow = (app: App, key: string): number => | |
| 298 | stateOf(app).held.get(key) ?? 0; | |
| 299 | ||
| 300 | /** The plain 429 every refused door answers with. */ | |
| 301 | export function tooManyStrangers(refusal: { retryAfter: number; why: string }): Response { | |
| 302 | return new Response(`${refusal.why}\n`, { | |
| 303 | status: 429, | |
| 304 | headers: { | |
| 305 | "content-type": "text/plain; charset=utf-8", | |
| 306 | "retry-after": String(Math.max(1, refusal.retryAfter)), | |
| 307 | "cache-control": "no-store", | |
| 308 | }, | |
| 309 | }); | |
| 310 | } |
src/kernel/fresh.ts added +46
| @@ -0,0 +1,46 @@ | ||
| 1 | /** | |
| 2 | * A value kept for a short while: the first caller makes it, everyone | |
| 3 | * after it inside `ttlMs` is handed the same one — a promise included, so | |
| 4 | * a burst of callers while it is being made waits on one piece of work | |
| 5 | * instead of starting its own. A promise that rejects is forgotten at | |
| 6 | * once, so a failure is never what the next caller is handed. | |
| 7 | * | |
| 8 | * For an answer anyone may ask for and nobody needs to the second (a | |
| 9 | * public card, a sitemap): what a crowd of strangers can make the forge | |
| 10 | * do is then one piece of work per key per `ttlMs`, however many ask. | |
| 11 | * Keys are bounded (`maxKeys`, the oldest dropped first), so a caller who | |
| 12 | * varies the key cannot grow the map. | |
| 13 | */ | |
| 14 | export class Fresh<T> { | |
| 15 | private readonly kept = new Map<string, { value: T; until: number }>(); | |
| 16 | ||
| 17 | constructor( | |
| 18 | private readonly ttlMs: number, | |
| 19 | private readonly maxKeys = 16, | |
| 20 | private readonly now: () => number = Date.now, | |
| 21 | ) {} | |
| 22 | ||
| 23 | get(key: string, make: () => T): T { | |
| 24 | const at = this.now(); | |
| 25 | const hit = this.kept.get(key); | |
| 26 | if (hit !== undefined && hit.until > at) return hit.value; | |
| 27 | this.kept.delete(key); | |
| 28 | const value = make(); | |
| 29 | if (value instanceof Promise) | |
| 30 | value.catch(() => { | |
| 31 | if (this.kept.get(key)?.value === value) this.kept.delete(key); | |
| 32 | }); | |
| 33 | this.kept.set(key, { value, until: at + this.ttlMs }); | |
| 34 | while (this.kept.size > this.maxKeys) { | |
| 35 | const oldest = this.kept.keys().next().value; | |
| 36 | if (oldest === undefined) break; | |
| 37 | this.kept.delete(oldest); | |
| 38 | } | |
| 39 | return value; | |
| 40 | } | |
| 41 | ||
| 42 | /** Forget everything kept. */ | |
| 43 | clear(): void { | |
| 44 | this.kept.clear(); | |
| 45 | } | |
| 46 | } |
src/spawn/door.ts +14
| @@ -15,6 +15,7 @@ import { kindOfToken, parseApiBase } from "../core/subscriptions.ts"; | ||
| 15 | 15 | import type { User } from "../core/users.ts"; |
| 16 | 16 | import { repoDir } from "../git/repos.ts"; |
| 17 | 17 | import type { Ctx } from "../http/context.ts"; |
| 18 | import { takeDoor } from "../http/strangers.ts"; | |
| 18 | 19 | import { |
| 19 | 20 | childPublicUrl, |
| 20 | 21 | hostedPlacement, |
| @@ -154,6 +155,19 @@ export async function spawnDoor( | ||
| 154 | 155 | ): Promise<Door> { |
| 155 | 156 | const anon = user === undefined; |
| 156 | 157 | if (!anon && user.kind !== "user") return spawnWords({ kind: "forbidden" }); |
| 158 | // A plan is free of children but not of work — it reads every catalog | |
| 159 | // app it names — so a stranger's are counted per address and per forge. | |
| 160 | if (anon) { | |
| 161 | const verdict = takeDoor(ctx.app, "plan", ctx.request); | |
| 162 | if (!verdict.allowed) | |
| 163 | return { | |
| 164 | status: 429, | |
| 165 | body: { | |
| 166 | error: "rate_limited", | |
| 167 | message: `too many plans without signing in: ${verdict.limit} a minute; try again in ${verdict.retryAfter}s`, | |
| 168 | }, | |
| 169 | }; | |
| 170 | } | |
| 157 | 171 | if (raw["dryRun"] !== undefined && typeof raw["dryRun"] !== "boolean") |
| 158 | 172 | return invalid("dryRun", "must be true or false"); |
| 159 | 173 | const dry = raw["dryRun"] !== false; |
src/tui/forge.ts +30 −6
| @@ -10,6 +10,8 @@ | ||
| 10 | 10 | |
| 11 | 11 | import type { App as Forge } from "../app.ts"; |
| 12 | 12 | import { search } from "../actions/search.ts"; |
| 13 | import { cleanQuery } from "../core/search.ts"; | |
| 14 | import { guestCodeSearch } from "../http/strangers.ts"; | |
| 13 | 15 | import { sentinelSummary } from "../crew/sentinel.ts"; |
| 14 | 16 | import { canRead, getRepo, type Repo } from "../core/repos.ts"; |
| 15 | 17 | import { sourceFor } from "../view/source.ts"; |
| @@ -533,13 +535,26 @@ export function forgeCommands(session: Session): (ctx: Ctx) => Command[] { | ||
| 533 | 535 | * first, mirror if you mean it (`view/source.ts`). A repo somebody else's |
| 534 | 536 | * origin holds is `not_found` for you, and `sourceFor` is what says so. |
| 535 | 537 | */ |
| 536 | function forgeSearch(session: Session): (ctx: Ctx) => PaletteSearch { | |
| 538 | export function forgeSearch(session: Session): (ctx: Ctx) => PaletteSearch { | |
| 537 | 539 | return (ctx) => async (query, live, signal) => { |
| 538 | const found = await search(session.app, session.user, { | |
| 539 | q: query, | |
| 540 | limit: 8, | |
| 541 | live, | |
| 542 | }); | |
| 540 | // A guest is a stranger with a key: its search greps code under the same | |
| 541 | // limits as the page's and the API's (`http/strangers.ts`), counted | |
| 542 | // under its peer address. Refused, everything but the code is searched. | |
| 543 | const greps = session.user === undefined && cleanQuery(query) !== ""; | |
| 544 | const held = greps | |
| 545 | ? guestCodeSearch(session.app, session.identity.remoteAddress) | |
| 546 | : undefined; | |
| 547 | let found; | |
| 548 | try { | |
| 549 | found = await search(session.app, session.user, { | |
| 550 | q: query, | |
| 551 | limit: 8, | |
| 552 | live, | |
| 553 | withCode: held === undefined || held.allowed, | |
| 554 | }); | |
| 555 | } finally { | |
| 556 | if (held?.allowed) held.release(); | |
| 557 | } | |
| 543 | 558 | if (signal.aborted) return []; |
| 544 | 559 | const items: PaletteItem[] = []; |
| 545 | 560 | const set = ctx.icons; |
| @@ -655,6 +670,15 @@ function forgeSearch(session: Session): (ctx: Ctx) => PaletteSearch { | ||
| 655 | 670 | }, |
| 656 | 671 | }); |
| 657 | 672 | } |
| 673 | if (held !== undefined && !held.allowed) | |
| 674 | items.push({ | |
| 675 | id: "code:not-searched", | |
| 676 | section: "Code", | |
| 677 | mark: G.ring, | |
| 678 | title: "Code was not searched", | |
| 679 | hint: `${held.why}; retry in ${held.retryAfter}s`, | |
| 680 | run: () => ctx.toast("Sign in to search code without waiting.", "secondary"), | |
| 681 | }); | |
| 658 | 682 | return items; |
| 659 | 683 | }; |
| 660 | 684 | } |
src/web/pages/og.ts +17 −1
| @@ -18,6 +18,7 @@ | ||
| 18 | 18 | |
| 19 | 19 | import { createHash } from "node:crypto"; |
| 20 | 20 | import { join } from "node:path"; |
| 21 | import { Fresh } from "../../kernel/fresh.ts"; | |
| 21 | 22 | import type { App } from "../../app.ts"; |
| 22 | 23 | import type { Run } from "../../core/runs.ts"; |
| 23 | 24 | import { getUserById } from "../../core/users.ts"; |
| @@ -351,7 +352,7 @@ async function resolve(ctx: Ctx, kind: OgKind, ref: string): Promise<CardInput | | ||
| 351 | 352 | if (kind === "issue" || kind === "pull") return threadInput(ctx, kind, ref); |
| 352 | 353 | if (kind === "run") return runInput(ctx, ref); |
| 353 | 354 | if (kind === "map") return mapCardInput(ctx, ref); |
| 354 | if (kind === "audit") return ref === "" ? auditCardInput(ctx) : undefined; | |
| 355 | if (kind === "audit") return ref === "" ? auditCards(ctx.app).get("", () => auditCardInput(ctx)) : undefined; | |
The audit card cache has no test. Fetch |
||
| 355 | 356 | if (kind === "site") return ref === "" ? SITE_CARD : undefined; |
| 356 | 357 | if (kind === "repo") return repoCard(ctx, ref); |
| 357 | 358 | if (!isSlug(ref)) return undefined; |
| @@ -374,6 +375,21 @@ async function resolve(ctx: Ctx, kind: OgKind, ref: string): Promise<CardInput | | ||
| 374 | 375 | }; |
| 375 | 376 | } |
| 376 | 377 | |
| 378 | /** | |
| 379 | * The audit card reads every public app's repository with git; anyone may | |
| 380 | * ask for it, so it is made once per AUDIT_CARD_MS however many do. | |
| 381 | */ | |
| 382 | const AUDIT_CARD_MS = 45_000; | |
| 383 | const auditCache = new WeakMap<App, Fresh<Promise<CardInput>>>(); | |
| 384 | function auditCards(app: App): Fresh<Promise<CardInput>> { | |
| 385 | let kept = auditCache.get(app); | |
| 386 | if (kept === undefined) { | |
| 387 | kept = new Fresh(AUDIT_CARD_MS, 1); | |
| 388 | auditCache.set(app, kept); | |
| 389 | } | |
| 390 | return kept; | |
| 391 | } | |
| 392 | ||
| 377 | 393 | const caches = new WeakMap<App, PngCache>(); |
| 378 | 394 | |
| 379 | 395 | function cacheFor(app: App): PngCache { |
src/web/pages/search.ts +37 −3
| @@ -10,6 +10,7 @@ | ||
| 10 | 10 | * that was never meant to be confirmed. |
| 11 | 11 | */ |
| 12 | 12 | |
| 13 | import { strangerCodeSearch } from "../../http/strangers.ts"; | |
| 13 | 14 | import { search } from "../../actions/search.ts"; |
| 14 | 15 | import { appsRuntime, envsFor } from "../../apps/runtime.ts"; |
| 15 | 16 | import type { AppRecord } from "../../core/apps.ts"; |
| @@ -299,8 +300,32 @@ function section( | ||
| 299 | 300 | |
| 300 | 301 | /** The same `search()` the API and the palette call, so the sources can |
| 301 | 302 | * never disagree between surfaces. */ |
| 302 | async function find(ctx: Ctx, a: Asked): Promise<SearchResults> { | |
| 303 | return search(ctx.app, ctx.auth.user, { q: a.query, kind: a.kind, live: a.live }); | |
| 303 | async function find( | |
| 304 | ctx: Ctx, | |
| 305 | a: Asked, | |
| 306 | ): Promise<{ found: SearchResults; busy?: number }> { | |
| 307 | // A stranger's code search takes a slot of its own (`http/strangers.ts`); | |
| 308 | // refused, the page still answers with everything but the code, and says so. | |
| 309 | // The same test as the API's: a blank or stop-word query greps nothing. | |
outdated
The API also requires |
||
| 310 | const greps = | |
| 311 | ctx.auth.user === undefined && | |
| 312 | (a.kind === "code" || a.kind === "all") && | |
| 313 | cleanQuery(a.query) !== ""; | |
| 314 | const held = greps ? strangerCodeSearch(ctx.app, ctx.request) : undefined; | |
| 315 | const withCode = held === undefined || held.allowed; | |
| 316 | try { | |
| 317 | const found = await search(ctx.app, ctx.auth.user, { | |
| 318 | q: a.query, | |
| 319 | kind: a.kind, | |
| 320 | live: a.live, | |
| 321 | withCode, | |
| 322 | }); | |
| 323 | return held !== undefined && !held.allowed | |
| 324 | ? { found, busy: held.retryAfter } | |
| 325 | : { found }; | |
| 326 | } finally { | |
| 327 | if (held?.allowed) held.release(); | |
| 328 | } | |
| 304 | 329 | } |
| 305 | 330 | |
| 306 | 331 | /** |
| @@ -360,7 +385,7 @@ export async function searchPage(ctx: Ctx): Promise<Response> { | ||
| 360 | 385 | }); |
| 361 | 386 | } |
| 362 | 387 | |
| 363 | const found = await find(ctx, a); | |
| 388 | const { found, busy } = await find(ctx, a); | |
| 364 | 389 | const seen = appsOf(ctx, ctx.auth.user); |
| 365 | 390 | const apps: Apps = { |
| 366 | 391 | byId: seen.records, |
| @@ -425,6 +450,15 @@ export async function searchPage(ctx: Ctx): Promise<Response> { | ||
| 425 | 450 | const body = html`${pageHead(html`Results for “${a.query}”`)} ${form} |
| 426 | 451 | <div class="filters">${chips}</div> |
| 427 | 452 | ${liveToggle} |
| 453 | ${when( | |
| 454 | busy !== undefined, | |
| 455 | () => | |
| 456 | html`<p class="muted small" data-code-busy> | |
| 457 | Code was not searched: code search is busy for visitors who are not | |
| 458 | signed in. <a href="/signin">Sign in</a>, or try again in | |
| 459 | ${String(busy)} s. | |
| 460 | </p>`, | |
| 461 | )} | |
| 428 | 462 | ${WORDS |
| 429 | 463 | .filter( |
| 430 | 464 | ([kind]) => kind !== "all" && (a.kind === "all" || a.kind === kind), |
src/web/pages/system.ts +1 −1
| @@ -134,7 +134,7 @@ export function systemStream(ctx: Ctx): Response { | ||
| 134 | 134 | if (!ctx.auth.user) return signInFirst(ctx); |
| 135 | 135 | if (telemetryReader(ctx.app, ctx.auth.user)?.kind !== "owner") return notFoundPage(ctx); |
| 136 | 136 | const sampler = live(); |
| 137 | return eventStream(ctx.request, (emit, _close, ready) => { | |
| 137 | return eventStream(ctx.app, ctx.request, (emit, _close, ready) => { | |
| 138 | 138 | emit("sample", sampleFrame(sampler.read())); |
| 139 | 139 | return sampler.subscribe((view) => { |
| 140 | 140 | if (ready()) emit("sample", sampleFrame(view)); |
src/web/pages/teams.ts +1 −1
| @@ -100,7 +100,7 @@ export const teamsLiveJs = /* js */ ` | ||
| 100 | 100 | // GET /settings/teams/stream: the page's regions again, each time a team changes. |
| 101 | 101 | export function teamsStream(ctx: Ctx): Response { |
| 102 | 102 | if (!isAdminPerson(ctx)) return notFoundPage(ctx); |
| 103 | return eventStream(ctx.request, (emit, _close, ready) => { | |
| 103 | return eventStream(ctx.app, ctx.request, (emit, _close, ready) => { | |
| 104 | 104 | const send = () => { |
| 105 | 105 | const r = regions(ctx); |
| 106 | 106 | if (r && ready()) emit("teams", { regions: r }); |
src/web/standards.ts +30 −7
| @@ -24,6 +24,8 @@ import releases from "../core/releases.json" with { type: "json" }; | ||
| 24 | 24 | import { markPng, type MarkShape } from "../design/mark.ts"; |
| 25 | 25 | import { BODY_MAX, TITLE_MAX } from "../core/reports.ts"; |
| 26 | 26 | import { canRead, getRepo, listReposVisibleTo } from "../core/repos.ts"; |
| 27 | import type { App } from "../app.ts"; | |
| 28 | import { Fresh } from "../kernel/fresh.ts"; | |
| 27 | 29 | import type { Ctx } from "../http/context.ts"; |
| 28 | 30 | import { notFound } from "../http/respond.ts"; |
| 29 | 31 | |
| @@ -138,16 +140,30 @@ function xml(s: string): string { | ||
| 138 | 140 | ); |
| 139 | 141 | } |
| 140 | 142 | |
| 143 | /** | |
| 144 | * A sitemap is up to five thousand rows, asked for by anyone: its rows are | |
| 145 | * read once a minute, however many ask. They are kept without an origin, | |
| 146 | * under one key, and the origin is put on per answer — so a caller who | |
| 147 | * varies `Host` (where `FORGE_URL` does not fix the origin) gets their own | |
| 148 | * origin on the same rows and cannot make the forge read them again. | |
| 149 | */ | |
| 150 | const SITEMAP_MS = 60_000; | |
| 151 | const sitemaps = new WeakMap<App, Fresh<readonly string[]>>(); | |
| 152 | ||
| 141 | 153 | /** GET /sitemap.xml — the front page and every public repository. */ |
| 142 | 154 | export function sitemapXml(ctx: Ctx): Response { |
| 143 | const origin = ctx.url.origin; | |
| 144 | const repos = listReposVisibleTo(ctx.app.db, undefined, { limit: 5000 }); | |
| 155 | let kept = sitemaps.get(ctx.app); | |
| 156 | if (kept === undefined) { | |
| 157 | kept = new Fresh(SITEMAP_MS, 1); | |
| 158 | sitemaps.set(ctx.app, kept); | |
outdated
The key comes from the request's Host. With 4 keys evicted in insertion order, rotating Host values defeats the cache. Key on the configured public origin, or document why only known hosts get here. |
||
| 159 | } | |
| 160 | const paths = kept.get("", () => sitemapPaths(ctx.app)); | |
| 161 | // Escaping is per character, so the escaped origin and the escaped path | |
| 162 | // join into the escaped URL. | |
| 163 | const origin = xml(ctx.url.origin); | |
| 145 | 164 | const urls = [ |
| 146 | `<url><loc>${xml(origin)}/</loc></url>`, | |
| 147 | ...repos.map( | |
| 148 | (r) => | |
| 149 | `<url><loc>${xml(`${origin}/${r.ownerName}/${r.name}`)}</loc></url>`, | |
| 150 | ), | |
| 165 | `<url><loc>${origin}/</loc></url>`, | |
| 166 | ...paths.map((path) => `<url><loc>${origin}${path}</loc></url>`), | |
| 151 | 167 | ]; |
| 152 | 168 | return text( |
| 153 | 169 | `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urls.join("\n")}\n</urlset>\n`, |
| @@ -155,6 +171,13 @@ export function sitemapXml(ctx: Ctx): Response { | ||
| 155 | 171 | ); |
| 156 | 172 | } |
| 157 | 173 | |
| 174 | /** Every public repository's path, escaped for XML, newest first. */ | |
| 175 | function sitemapPaths(app: App): readonly string[] { | |
| 176 | return listReposVisibleTo(app.db, undefined, { limit: 5000 }).map((r) => | |
| 177 | xml(`/${r.ownerName}/${r.name}`), | |
| 178 | ); | |
| 179 | } | |
| 180 | ||
| 158 | 181 | /** GET /manifest.webmanifest — the forge as an app on a home screen. */ |
| 159 | 182 | export function webManifest(ctx: Ctx): Response { |
| 160 | 183 | const name = ctx.app.config.edgeName ? "forge" : "forge"; |
test/agents.test.ts +1
| @@ -673,6 +673,7 @@ describe("backpressure", () => { | ||
| 673 | 673 | let push: ((text: string) => void) | undefined; |
| 674 | 674 | const controller = new AbortController(); |
| 675 | 675 | const response = eventStream( |
| 676 | {} as App, | |
| 676 | 677 | new Request("http://forge/stream", { signal: controller.signal }), |
| 677 | 678 | (emit) => { |
| 678 | 679 | push = (text) => emit("noise", { text }); |
test/strangers.test.ts added +435
| @@ -0,0 +1,435 @@ | ||
| 1 | /** | |
| 2 | * Strangers cannot exhaust the forge (#2920): every door open to someone | |
| 3 | * with no session and no token — the API, a clone, a code search, an event | |
| 4 | * stream, a spawn plan — is limited per address and for every stranger | |
| 5 | * together, and while strangers flood each one a signed-in push still | |
| 6 | * lands. Addresses are told apart by `X-Forwarded-For`, which the forge | |
| 7 | * reads from this machine (`http/peer.ts`). | |
| 8 | */ | |
| 9 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; | |
| 10 | import { mkdtemp, rm, writeFile } from "node:fs/promises"; | |
| 11 | import { connect, type Socket } from "node:net"; | |
| 12 | import { tmpdir } from "node:os"; | |
| 13 | import { join } from "node:path"; | |
| 14 | import { createApp, type App } from "../src/app.ts"; | |
| 15 | import { createRepo } from "../src/actions/repos.ts"; | |
| 16 | import { createApp as createAppRecord } from "../src/core/apps.ts"; | |
| 17 | import { createToken } from "../src/core/tokens.ts"; | |
| 18 | import type { User } from "../src/core/users.ts"; | |
| 19 | import { transferGate } from "../src/git/env.ts"; | |
| 20 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 21 | import { | |
| 22 | heldNow, | |
| 23 | setStrangerLimits, | |
| 24 | STRANGER_LIMITS, | |
| 25 | strangerCodeSearch, | |
| 26 | } from "../src/http/strangers.ts"; | |
| 27 | import { Fresh } from "../src/kernel/fresh.ts"; | |
| 28 | import type { Ctx } from "../src/tui/app/app.ts"; | |
| 29 | import { forgeSearch } from "../src/tui/forge.ts"; | |
| 30 | import { Session } from "../src/tui/session.ts"; | |
| 31 | import { seedAdmin } from "./helpers/admin.ts"; | |
| 32 | import { gitClientEnv } from "./helpers/git-client.ts"; | |
| 33 | ||
| 34 | describe("Fresh: one piece of work per key per while", () => { | |
| 35 | test("kept for ttl, made again after, by the clock it is given", () => { | |
| 36 | let at = 1_000; | |
| 37 | let made = 0; | |
| 38 | const kept = new Fresh<number>(30_000, 16, () => at); | |
| 39 | expect(kept.get("k", () => ++made)).toBe(1); | |
| 40 | at += 29_999; | |
| 41 | expect(kept.get("k", () => ++made)).toBe(1); | |
| 42 | at += 1; // 30 000 ms after the first: stale | |
| 43 | expect(kept.get("k", () => ++made)).toBe(2); | |
| 44 | }); | |
| 45 | ||
| 46 | test("callers inside a making share one promise; a rejection is forgotten", async () => { | |
| 47 | const kept = new Fresh<Promise<string>>(60_000, 16, () => 0); | |
| 48 | let calls = 0; | |
| 49 | const a = kept.get("x", async () => { | |
| 50 | calls++; | |
| 51 | throw new Error("no"); | |
| 52 | }); | |
| 53 | const b = kept.get("x", async () => "never"); | |
| 54 | expect(b).toBe(a); | |
| 55 | await expect(a).rejects.toThrow("no"); | |
| 56 | expect(await kept.get("x", async () => "again")).toBe("again"); | |
| 57 | expect(calls).toBe(1); | |
| 58 | }); | |
| 59 | ||
| 60 | test("keys are bounded, the oldest dropped first", () => { | |
| 61 | const kept = new Fresh<string>(60_000, 2, () => 0); | |
| 62 | kept.get("a", () => "a1"); | |
| 63 | kept.get("b", () => "b1"); | |
| 64 | kept.get("c", () => "c1"); | |
| 65 | expect(kept.get("a", () => "a2")).toBe("a2"); | |
| 66 | expect(kept.get("c", () => "c2")).toBe("c1"); | |
| 67 | }); | |
| 68 | }); | |
| 69 | ||
| 70 | describe("strangers at every door", () => { | |
| 71 | let dataDir: string; | |
| 72 | let app: App; | |
| 73 | let servers: Servers; | |
| 74 | let base: string; | |
| 75 | let port: number; | |
| 76 | let token: string; | |
| 77 | let ada: User; | |
| 78 | let next = 1; | |
| 79 | /** A fresh address, so one test's minute is never another's. */ | |
| 80 | const address = (): string => `203.0.113.${next++}`; | |
| 81 | ||
| 82 | beforeAll(async () => { | |
| 83 | dataDir = await mkdtemp(join(tmpdir(), "forge-strangers-")); | |
| 84 | app = createApp({ dataDir, port: 0 }); | |
| 85 | servers = await serve(app); | |
| 86 | base = servers.url; | |
| 87 | port = servers.port; | |
| 88 | const owner = await seedAdmin(app, { | |
| 89 | username: "ada", | |
| 90 | email: "[email protected]", | |
| 91 | password: "owner-passphrase", | |
| 92 | }); | |
| 93 | if (!owner.ok) throw new Error(owner.error); | |
| 94 | token = createToken(app.db, owner.value.id, "t", "admin").token; | |
| 95 | ada = owner.value; | |
| 96 | for (const name of ["garden", "shed"]) { | |
| 97 | const repo = await createRepo(app, owner.value, { name, visibility: "public" }); | |
| 98 | if (!repo.ok) throw new Error(repo.error); | |
| 99 | } | |
| 100 | }); | |
| 101 | afterAll(async () => { | |
| 102 | servers.stop(); | |
| 103 | app.close(); | |
| 104 | await rm(dataDir, { recursive: true, force: true }); | |
| 105 | }); | |
| 106 | ||
| 107 | const from = (ip: string, extra: Record<string, string> = {}) => ({ | |
| 108 | "x-forwarded-for": ip, | |
| 109 | ...extra, | |
| 110 | }); | |
| 111 | const status = async (path: string, headers: Record<string, string>) => { | |
| 112 | const res = await fetch(`${base}${path}`, { headers }); | |
| 113 | await res.arrayBuffer(); | |
| 114 | return res.status; | |
| 115 | }; | |
| 116 | const until = async (done: () => boolean, ms = 10_000): Promise<boolean> => { | |
| 117 | const end = Date.now() + ms; | |
| 118 | while (!done()) { | |
| 119 | if (Date.now() > end) return false; | |
| 120 | await Bun.sleep(25); | |
| 121 | } | |
| 122 | return true; | |
| 123 | }; | |
| 124 | ||
| 125 | /** | |
| 126 | * A fetch's negotiation opened and never sent: the headers of a chunked | |
| 127 | * POST to upload-pack, then nothing. Resolves with the socket and, if the | |
| 128 | * forge answers before the body would, its status line. | |
| 129 | */ | |
| 130 | function stall(ip: string): { socket: Socket; answer: Promise<string> } { | |
| 131 | const socket = connect(port, "127.0.0.1"); | |
| 132 | let text = ""; | |
| 133 | const answer = new Promise<string>((resolve) => { | |
| 134 | socket.on("data", (chunk) => { | |
| 135 | text += chunk.toString("latin1"); | |
| 136 | if (text.includes("\r\n")) resolve(text.slice(0, text.indexOf("\r\n"))); | |
| 137 | }); | |
| 138 | socket.on("close", () => resolve(text.slice(0, text.indexOf("\r\n")))); | |
| 139 | socket.on("error", () => resolve("")); | |
| 140 | }); | |
| 141 | socket.write( | |
| 142 | `POST /ada/garden.git/git-upload-pack HTTP/1.1\r\nHost: x\r\nX-Forwarded-For: ${ip}\r\nContent-Type: application/x-git-upload-pack-request\r\nTransfer-Encoding: chunked\r\n\r\n`, | |
| 143 | ); | |
| 144 | return { socket, answer }; | |
| 145 | } | |
| 146 | ||
| 147 | test("a stranger's health checks are never throttled", async () => { | |
| 148 | const ip = address(); | |
| 149 | for (let i = 0; i < 20; i++) expect(await status("/healthz", from(ip))).toBe(200); | |
| 150 | }); | |
| 151 | ||
| 152 | test("anonymous API reads: per address, then every stranger together; a token is its own", async () => { | |
| 153 | setStrangerLimits(app, { | |
| 154 | perAddress: { ...STRANGER_LIMITS.perAddress, api: 5 }, | |
| 155 | forge: { ...STRANGER_LIMITS.forge, api: 8 }, | |
| 156 | }); | |
| 157 | try { | |
| 158 | const a = address(); | |
| 159 | for (let i = 0; i < 5; i++) expect(await status("/api/v1/repos", from(a))).toBe(200); | |
| 160 | const refused = await fetch(`${base}/api/v1/repos`, { headers: from(a) }); | |
| 161 | expect(refused.status).toBe(429); | |
| 162 | expect(Number(refused.headers.get("retry-after"))).toBeGreaterThan(0); | |
| 163 | expect(((await refused.json()) as { error: string }).error).toBe("rate_limited"); | |
| 164 | // Another address has its own five — until the forge's eight are spent. | |
| 165 | const b = address(); | |
| 166 | for (let i = 0; i < 3; i++) expect(await status("/api/v1/repos", from(b))).toBe(200); | |
| 167 | expect(await status("/api/v1/repos", from(b))).toBe(429); | |
| 168 | // A page is a person's, and a token is counted as a token. | |
| 169 | expect(await status("/explore", from(a))).toBe(200); | |
| 170 | expect( | |
| 171 | await status("/api/v1/repos", from(a, { authorization: `Bearer ${token}` })), | |
| 172 | ).toBe(200); | |
| 173 | } finally { | |
| 174 | setStrangerLimits(app, STRANGER_LIMITS); | |
| 175 | } | |
| 176 | }); | |
| 177 | ||
| 178 | test("anonymous ref advertisements are counted per address", async () => { | |
| 179 | setStrangerLimits(app, { perAddress: { ...STRANGER_LIMITS.perAddress, git: 3 } }); | |
| 180 | try { | |
| 181 | const ip = address(); | |
| 182 | const refs = "/ada/garden.git/info/refs?service=git-upload-pack"; | |
| 183 | for (let i = 0; i < 3; i++) expect(await status(refs, from(ip))).toBe(200); | |
| 184 | expect(await status(refs, from(ip))).toBe(429); | |
| 185 | expect(await status(refs, from(address()))).toBe(200); | |
| 186 | // Access is decided first: a repository nobody may read still asks | |
| 187 | // for a credential, not a retry. | |
| 188 | expect(await status("/ada/nothing.git/info/refs?service=git-upload-pack", from(ip))).toBe(401); | |
| 189 | } finally { | |
| 190 | setStrangerLimits(app, STRANGER_LIMITS); | |
| 191 | } | |
| 192 | }); | |
| 193 | ||
| 194 | test("a fetch that stops sending ends, and gives its transfer back", async () => { | |
| 195 | setStrangerLimits(app, { transferIdleMs: 300 }); | |
| 196 | const { socket } = stall(address()); | |
| 197 | try { | |
| 198 | expect(await until(() => heldNow(app, "transfer") === 1)).toBe(true); | |
| 199 | expect(await until(() => heldNow(app, "transfer") === 0)).toBe(true); | |
| 200 | } finally { | |
| 201 | socket.destroy(); | |
| 202 | setStrangerLimits(app, STRANGER_LIMITS); | |
| 203 | } | |
| 204 | }); | |
| 205 | ||
| 206 | test("a flood of stalled anonymous clones takes its share, and a signed-in push still lands", async () => { | |
| 207 | const share = Math.max(1, Math.floor(transferGate.limit * STRANGER_LIMITS.transferShare)); | |
| 208 | const stalls: ReturnType<typeof stall>[] = []; | |
| 209 | try { | |
| 210 | // One address holds its four, then is refused. | |
| 211 | const one = address(); | |
| 212 | for (let i = 0; i < STRANGER_LIMITS.transfersPerAddress; i++) stalls.push(stall(one)); | |
| 213 | expect(await until(() => heldNow(app, "transfer") === STRANGER_LIMITS.transfersPerAddress)).toBe(true); | |
| 214 | const own = stall(one); | |
| 215 | stalls.push(own); | |
| 216 | expect(await own.answer).toStartWith("HTTP/1.1 429"); | |
| 217 | // Many addresses together hold the strangers' share and no more. | |
| 218 | for (let i = 0; i < share; i++) stalls.push(stall(address())); | |
| 219 | expect(await until(() => heldNow(app, "transfer") === share)).toBe(true); | |
| 220 | const late = stall(address()); | |
| 221 | stalls.push(late); | |
| 222 | expect(await late.answer).toStartWith("HTTP/1.1 429"); | |
| 223 | expect(heldNow(app, "transfer")).toBe(share); | |
| 224 | ||
| 225 | // While they hold it, a signed-in push is made and lands. | |
| 226 | const work = await mkdtemp(join(tmpdir(), "forge-strangers-push-")); | |
| 227 | try { | |
| 228 | const git = async (args: string[]) => | |
| 229 | await Bun.spawn(["git", ...args], { | |
| 230 | cwd: work, | |
| 231 | env: gitClientEnv(), | |
| 232 | stdout: "ignore", | |
| 233 | stderr: "ignore", | |
| 234 | }).exited; | |
| 235 | expect(await git(["init", "-q", "-b", "main"])).toBe(0); | |
| 236 | await writeFile(join(work, "README.md"), "# shed\n"); | |
| 237 | expect(await git(["add", "."])).toBe(0); | |
| 238 | expect(await git(["commit", "-q", "-m", "first"])).toBe(0); | |
| 239 | const remote = `${base.replace("http://", `http://ada:${token}@`)}/ada/shed.git`; | |
| 240 | expect(await git(["push", "-q", remote, "main"])).toBe(0); | |
| 241 | } finally { | |
| 242 | await rm(work, { recursive: true, force: true }); | |
| 243 | } | |
| 244 | } finally { | |
| 245 | for (const s of stalls) s.socket.destroy(); | |
| 246 | } | |
| 247 | // Hung up, every stalled transfer gives its permit back. | |
| 248 | expect(await until(() => heldNow(app, "transfer") === 0)).toBe(true); | |
| 249 | }, 60_000); | |
| 250 | ||
| 251 | test("anonymous code search: per address, and one at a time for every stranger", async () => { | |
| 252 | setStrangerLimits(app, { perAddress: { ...STRANGER_LIMITS.perAddress, code: 2 } }); | |
| 253 | try { | |
| 254 | const ip = address(); | |
| 255 | const code = "/api/v1/search?q=shed&kind=code"; | |
| 256 | expect(await status(code, from(ip))).toBe(200); | |
| 257 | expect(await status(code, from(ip))).toBe(200); | |
| 258 | expect(await status(code, from(ip))).toBe(429); | |
| 259 | // A search that greps nothing is not a code search. | |
| 260 | expect(await status("/api/v1/search?q=shed&kind=repos", from(ip))).toBe(200); | |
| 261 | ||
| 262 | // Another stranger's search is running: a second is refused on the | |
| 263 | // API, and the page answers without the code and says why. | |
| 264 | const busy = strangerCodeSearch(app, new Request(`${base}/x`)); | |
| 265 | expect(busy.allowed).toBe(true); | |
| 266 | try { | |
| 267 | expect(await status(code, from(address()))).toBe(429); | |
| 268 | const page = await fetch(`${base}/search?q=shed`, { headers: from(address()) }); | |
| 269 | expect(page.status).toBe(200); | |
| 270 | expect(await page.text()).toContain("data-code-busy"); | |
| 271 | // Someone signed in searches code as ever. | |
| 272 | expect( | |
| 273 | await status(code, from(address(), { authorization: `Bearer ${token}` })), | |
| 274 | ).toBe(200); | |
| 275 | } finally { | |
| 276 | if (busy.allowed) busy.release(); | |
| 277 | } | |
| 278 | expect(heldNow(app, "code")).toBe(0); | |
| 279 | } finally { | |
| 280 | setStrangerLimits(app, STRANGER_LIMITS); | |
| 281 | } | |
| 282 | }); | |
| 283 | ||
| 284 | test("a terminal guest's palette search takes the same code slot, keyed on its peer", async () => { | |
| 285 | const wrote = await fetch(`${base}/api/v1/repos/ada/shed/contents`, { | |
| 286 | method: "POST", | |
| 287 | headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, | |
| 288 | body: JSON.stringify({ | |
| 289 | branch: "main", | |
| 290 | message: "first", | |
| 291 | files: [{ path: "rake.txt", content: "a trowel and a wheelbarrow\n" }], | |
| 292 | }), | |
| 293 | }); | |
| 294 | expect(wrote.status).toBeLessThan(300); | |
| 295 | await wrote.arrayBuffer(); | |
| 296 | ||
| 297 | // A guest: any SSH key the forge does not know. | |
| 298 | const guest = new Session(app, { | |
| 299 | sshUsername: "anyone", | |
| 300 | remoteAddress: address(), | |
| 301 | fingerprint: "SHA256:unknown", | |
| 302 | }); | |
| 303 | const toasts: string[] = []; | |
| 304 | const ctx = { icons: "plain", toast: (t: string) => toasts.push(t) } as unknown as Ctx; | |
| 305 | const ask = (q: string) => | |
| 306 | forgeSearch(guest)(ctx)(q, false, new AbortController().signal); | |
| 307 | const code = (items: { id: string }[]) => | |
| 308 | items.filter((i) => i.id.startsWith("code:") && i.id !== "code:not-searched"); | |
| 309 | ||
| 310 | expect(code(await ask("wheelbarrow")).length).toBeGreaterThan(0); | |
| 311 | expect(heldNow(app, "code")).toBe(0); | |
| 312 | ||
| 313 | const busy = strangerCodeSearch(app, new Request(`${base}/x`)); | |
| 314 | expect(busy.allowed).toBe(true); | |
| 315 | try { | |
| 316 | const items = await ask("wheelbarrow"); | |
| 317 | expect(code(items)).toEqual([]); | |
| 318 | const note = items.find((i) => i.id === "code:not-searched"); | |
| 319 | expect(note?.title).toBe("Code was not searched"); | |
| 320 | // The rest of the search still answers. | |
| 321 | expect((await ask("shed")).some((i) => i.id.startsWith("repo:"))).toBe(true); | |
| 322 | } finally { | |
| 323 | if (busy.allowed) busy.release(); | |
| 324 | } | |
| 325 | ||
| 326 | // Its own minute, counted under its address. | |
| 327 | setStrangerLimits(app, { perAddress: { ...STRANGER_LIMITS.perAddress, code: 1 } }); | |
| 328 | try { | |
| 329 | expect(code(await ask("wheelbarrow")).length).toBeGreaterThan(0); | |
| 330 | expect(code(await ask("wheelbarrow"))).toEqual([]); | |
| 331 | // A blank query greps nothing and spends nothing. | |
| 332 | expect((await ask(" ")).some((i) => i.id === "code:not-searched")).toBe(false); | |
| 333 | } finally { | |
| 334 | setStrangerLimits(app, STRANGER_LIMITS); | |
| 335 | } | |
| 336 | expect(heldNow(app, "code")).toBe(0); | |
| 337 | }); | |
| 338 | ||
| 339 | test("a blank query on the page does not take a stranger's code slot", async () => { | |
| 340 | const busy = strangerCodeSearch(app, new Request(`${base}/x`)); | |
| 341 | expect(busy.allowed).toBe(true); | |
| 342 | try { | |
| 343 | const page = await fetch(`${base}/search?q=`, { headers: from(address()) }); | |
| 344 | expect(page.status).toBe(200); | |
| 345 | expect(await page.text()).not.toContain("data-code-busy"); | |
| 346 | } finally { | |
| 347 | if (busy.allowed) busy.release(); | |
| 348 | } | |
| 349 | }); | |
| 350 | ||
| 351 | test("event streams: per stranger address and for every stranger, given back on hang-up", async () => { | |
| 352 | setStrangerLimits(app, { streamsPerAddress: 2, strangerStreams: 3 }); | |
| 353 | const open: AbortController[] = []; | |
| 354 | const stream = async (headers: Record<string, string>): Promise<number> => { | |
| 355 | const abort = new AbortController(); | |
| 356 | open.push(abort); | |
| 357 | const res = await fetch(`${base}/events`, { headers, signal: abort.signal }); | |
| 358 | if (res.status !== 200) await res.arrayBuffer(); | |
| 359 | return res.status; | |
| 360 | }; | |
| 361 | try { | |
| 362 | const a = address(); | |
| 363 | expect(await stream(from(a))).toBe(200); | |
| 364 | expect(await stream(from(a))).toBe(200); | |
| 365 | expect(await stream(from(a))).toBe(429); | |
| 366 | expect(await stream(from(address()))).toBe(200); | |
| 367 | expect(await stream(from(address()))).toBe(429); | |
| 368 | expect(heldNow(app, "stream:strangers")).toBe(3); | |
| 369 | // Someone signed in is not a stranger. | |
| 370 | expect(await stream(from(a, { authorization: `Bearer ${token}` }))).toBe(200); | |
| 371 | } finally { | |
| 372 | for (const abort of open) abort.abort(); | |
| 373 | setStrangerLimits(app, STRANGER_LIMITS); | |
| 374 | } | |
| 375 | expect(await until(() => heldNow(app, "stream:*") === 0)).toBe(true); | |
| 376 | }); | |
| 377 | ||
| 378 | test("a stranger's spawn plans are counted per address", async () => { | |
| 379 | setStrangerLimits(app, { perAddress: { ...STRANGER_LIMITS.perAddress, plan: 2 } }); | |
| 380 | try { | |
| 381 | const ip = address(); | |
| 382 | const plan = () => | |
| 383 | fetch(`${base}/api/v1/forges`, { | |
| 384 | method: "POST", | |
| 385 | headers: from(ip, { "content-type": "application/json", origin: base }), | |
| 386 | body: JSON.stringify({ name: "kiln", dryRun: true }), | |
| 387 | }).then(async (r) => { | |
| 388 | await r.arrayBuffer(); | |
| 389 | return r.status; | |
| 390 | }); | |
| 391 | expect(await plan()).not.toBe(429); | |
| 392 | expect(await plan()).not.toBe(429); | |
| 393 | expect(await plan()).toBe(429); | |
| 394 | } finally { | |
| 395 | setStrangerLimits(app, STRANGER_LIMITS); | |
| 396 | } | |
| 397 | }); | |
| 398 | ||
| 399 | test("a stranger's HEAD to /events gives its stream back at once", async () => { | |
| 400 | expect(heldNow(app, "stream:*")).toBe(0); | |
| 401 | const res = await fetch(`${base}/events`, { method: "HEAD", headers: from(address()) }); | |
| 402 | expect(res.status).toBe(200); | |
| 403 | await res.arrayBuffer(); | |
| 404 | // The GET's stream behind the HEAD is cancelled, not left holding a slot. | |
| 405 | expect(await until(() => heldNow(app, "stream:*") === 0, 2_000)).toBe(true); | |
| 406 | }); | |
| 407 | ||
| 408 | test("the audit card is made once in its while, however many ask", async () => { | |
| 409 | const first = await (await fetch(`${base}/og/audit.svg`)).text(); | |
| 410 | expect(first).toContain("<svg"); | |
| 411 | const repo = await createRepo(app, ada, { name: "greenhouse", visibility: "public" }); | |
| 412 | if (!repo.ok) throw new Error(String(repo.error)); | |
| 413 | const rec = createAppRecord(app.db, { repoId: repo.value.id, slug: "greenhouse", access: "public" }); | |
| 414 | expect(rec.ok).toBe(true); | |
| 415 | expect(await (await fetch(`${base}/og/audit.svg`)).text()).toBe(first); | |
| 416 | }); | |
| 417 | ||
| 418 | test("the sitemap is made once a minute, however many ask", async () => { | |
| 419 | const first = await (await fetch(`${base}/sitemap.xml`)).text(); | |
| 420 | expect(first).toContain("/ada/garden</loc>"); | |
| 421 | const made = await createRepo(app, ada, { | |
| 422 | name: "orchard", | |
| 423 | visibility: "public", | |
| 424 | }); | |
| 425 | expect(made.ok).toBe(true); | |
| 426 | expect(await (await fetch(`${base}/sitemap.xml`)).text()).toBe(first); | |
| 427 | // Another Host is answered under its own origin from the same rows: a | |
| 428 | // caller varying Host cannot make the forge read them again. | |
| 429 | const other = await ( | |
| 430 | await fetch(`${base}/sitemap.xml`, { headers: { host: "elsewhere.test" } }) | |
| 431 | ).text(); | |
| 432 | expect(other).toContain("<loc>http://elsewhere.test/ada/garden</loc>"); | |
| 433 | expect(other).not.toContain("orchard"); | |
| 434 | }); | |
| 435 | }); |
test/system-stream.test.ts +1 −1
| @@ -122,7 +122,7 @@ describe("/system over server-sent events", () => { | ||
| 122 | 122 | let emitter: ((e: string, d: unknown) => void) | undefined; |
| 123 | 123 | let ready: (() => boolean) | undefined; |
| 124 | 124 | let closed = false; |
| 125 | const res = eventStream(new Request("http://x/s"), (emit, _close, r) => { | |
| 125 | const res = eventStream({} as App, new Request("http://x/s"), (emit, _close, r) => { | |
| 126 | 126 | emitter = emit; |
| 127 | 127 | ready = r; |
| 128 | 128 | return () => { |
new above
offline
Nit: this line runs well past the ~76-column wrap the rest of the list uses (the terminal-palette clause was added without rewrapping). Rewrap it the next time this file is touched.