Public: anyone can read this; the crew can change it.
A miss under /.well-known or on a file-like path answers a few bytes of typed JSON or text, never the 10 KB not-found page #13896
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/13866-a-miss-under-well-known-or-on-a-file-lik → main 5 commits fork 3 threads · 3 unresolved
6 files
changed +114
−20
src/http/asset_miss.ts +37 −12
| @@ -1,21 +1,46 @@ | ||
| 1 | 1 | /** |
| 2 | * A path that names a file (`/feed.xml`, `/x.png`) and matches no route is a | |
| 3 | * crawler or a tool, not a person: it gets a 404 in a few dozen bytes instead | |
| 4 | * of the full not-found page. Extensionless and `.html` paths keep the page. | |
| 2 | * One rule decides the shape of a miss on a GET or HEAD that matched no route: | |
| 3 | * - a machine prefix (`/.well-known/…`, `/oauth/…`) answers `{error, message, hint}` JSON; | |
| 4 | * - a file-like path (`/feed.xml`, `/x.png`, `/docs.atom`) answers a few bytes of `no-store` plain | |
| 5 | * text; | |
| 6 | * - everything else keeps the full not-found page. | |
| 7 | * A crawler or tool is not a person, so it never costs the 10 KB page. `/api` | |
| 8 | * keeps its own `no_route` body (server.ts). The security floor is added by | |
| 9 | * the one place every response leaves through (`securityFloor`). | |
| 5 | 10 | */ |
| 6 | 11 | |
| 7 | const ASSET_EXTENSION = /\.(?:xml|json|txt|js|css|map|png|ico)$/; | |
| 12 | const FILE_EXTENSION = /\.(?:xml|atom|json|webmanifest|txt|md|js|css|map|png|svg|ico|wasm)$/; | |
| 8 | 13 | /** Machine surfaces that keep their own JSON miss; the one list server.ts shares. */ |
| 9 | 14 | export const MACHINE_PREFIX = /^\/(?:api|contract|mcp|oauth|\.well-known|metrics|health|healthz|readyz)(?:\/|$)/; |
| 10 | 15 | |
| 11 | export function isAssetMiss(request: Request, pathname: string): boolean { | |
| 12 | if (request.method !== "GET" && request.method !== "HEAD") return false; | |
| 13 | return ASSET_EXTENSION.test(pathname) && !MACHINE_PREFIX.test(pathname); | |
| 16 | export type MissShape = "machine" | "file" | "page"; | |
| 17 | ||
| 18 | export function missShape(request: Request, pathname: string): MissShape { | |
| 19 | if (request.method !== "GET" && request.method !== "HEAD") return "page"; | |
| 20 | if (MACHINE_PREFIX.test(pathname)) return "machine"; | |
| 21 | return FILE_EXTENSION.test(pathname) ? "file" : "page"; | |
| 14 | 22 | } |
| 15 | 23 | |
| 16 | export function assetNotFound(): Response { | |
| 17 | return new Response("not found\n", { | |
| 18 | status: 404, | |
| 19 | headers: { "content-type": "text/plain; charset=utf-8", "cache-control": "no-store" }, | |
| 20 | }); | |
| 24 | /** | |
| 25 | * The tiny answer for a miss that is not a page, or undefined when it is one. `/api` is left to its | |
| 26 | * own. | |
| 27 | */ | |
| 28 | export function typedMiss(request: Request, pathname: string): Response | undefined { | |
| 29 | if (pathname === "/api" || pathname.startsWith("/api/")) return undefined; | |
| 30 | const shape = missShape(request, pathname); | |
| 31 | if (shape === "file") | |
| 32 | return new Response("not found\n", { | |
| 33 | status: 404, | |
| 34 | headers: { "content-type": "text/plain; charset=utf-8", "cache-control": "no-store" }, | |
| 35 | }); | |
| 36 | if (shape === "machine") | |
| 37 | return Response.json( | |
| 38 | { | |
| 39 | error: "no_route", | |
| 40 | message: `no ${request.method} route for ${pathname}`, | |
| 41 | hint: "this path names nothing here; the API index is /api/v1 and the machine contract is /contract", | |
| 42 | }, | |
| 43 | { status: 404, headers: { "cache-control": "no-store" } }, | |
| 44 | ); | |
| 45 | return undefined; | |
| 21 | 46 | } |
src/http/atom_routes.ts +6 −3
| @@ -17,8 +17,12 @@ import { STARTED } from "../web/started.ts"; | ||
| 17 | 17 | import { ownerProfile } from "../web/pages/org.ts"; |
| 18 | 18 | import type { Ctx, Handler } from "./context.ts"; |
| 19 | 19 | import { notFound } from "./respond.ts"; |
| 20 | import { typedMiss } from "./asset_miss.ts"; | |
| 20 | 21 | import type { Router } from "./router.ts"; |
| 21 | 22 | |
| 23 | /** A feed that is not there: the few bytes a feed reader gets, the API's body otherwise. */ | |
| 24 | const feedMiss = (ctx: Ctx): Response => typedMiss(ctx.request, ctx.url.pathname) ?? notFound(); | |
| 25 | ||
| 22 | 26 | export const ATOM_ENTRIES = 30; |
| 23 | 27 | const HEADERS = { "content-type": "application/atom+xml; charset=utf-8", "cache-control": "public, max-age=300" }; |
| 24 | 28 | |
| @@ -95,7 +99,7 @@ ${entries.join("\n")} | ||
| 95 | 99 | export function shipsAtom(ctx: Ctx): Response { |
| 96 | 100 | notePhase(ctx.request, "session", takeSessionTime(ctx.request)); |
| 97 | 101 | const repo = getRepo(ctx.app.db, ctx.params["owner"]!, ctx.params["name"]!); |
| 98 | if (!repo || !canRead(ctx.app.db, repo, ctx.auth.user)) return notFound(); | |
| 102 | if (!repo || !canRead(ctx.app.db, repo, ctx.auth.user)) return feedMiss(ctx); | |
| 99 | 103 | const slug = `${repo.ownerName}/${repo.name}`; |
| 100 | 104 | const items = timePhase(ctx.request, "ships", () => shippedPulls(ctx.app.db, repo.id)).map(({ issue, at }) => ({ |
| 101 | 105 | id: `urn:forge:ship:${slug}#${issue.number}`, |
| @@ -114,8 +118,7 @@ export async function personAtom(ctx: Ctx): Promise<Response> { | ||
| 114 | 118 | const whole = getUserByName(ctx.app.db, `${ctx.params["owner"]}.atom`); |
| 115 | 119 | if (whole !== undefined) return ownerProfile({ ...ctx, params: { ...ctx.params, owner: whole.username } }); |
| 116 | 120 | const owner = getUserByName(ctx.app.db, ctx.params["owner"]!); |
| 117 | if (owner === undefined) return notFound(); | |
| 118 | if (hiddenFrom(owner, undefined)) return notFound(); | |
| 121 | if (owner === undefined || hiddenFrom(owner, undefined)) return feedMiss(ctx); | |
| 119 | 122 | const repos = new Map<string, Repo | undefined>(); |
| 120 | 123 | const readable = (id: string): Repo | undefined => { |
| 121 | 124 | if (!repos.has(id)) { |
src/http/server.ts +3 −2
| @@ -540,7 +540,7 @@ import { | ||
| 540 | 540 | } from "../web/pages/browse.ts"; |
| 541 | 541 | import { bringConnect, bringEnvSubmit, bringJobPage, bringPage, bringRun, bringSubmit } from "../web/pages/bring.ts"; |
| 542 | 542 | import { pushThrough } from "../origins/push-through.ts"; |
| 543 | import { assetNotFound, isAssetMiss, MACHINE_PREFIX } from "./asset_miss.ts"; | |
| 543 | import { MACHINE_PREFIX, typedMiss } from "./asset_miss.ts"; | |
| 544 | 544 | import { CAME_FROM_ELSEWHERE, forbiddenPage, notFoundPage, SENT_FROM_ELSEWHERE, signInHref } from "../web/chrome.ts"; |
| 545 | 545 | import { targetForShortCode } from "../origins/oauth.ts"; |
| 546 | 546 | import { apiRoutes, oauthCallbackUri } from "./api.ts"; |
| @@ -3273,7 +3273,8 @@ async function answerGet( | ||
| 3273 | 3273 | // even for someone who is. |
| 3274 | 3274 | // A plain GET (curl, a crawler, a link unfurler) names no type and gets the page too; |
| 3275 | 3275 | // machine paths and anyone who asks for JSON keep the JSON body. |
| 3276 | if (!isApiPath && isAssetMiss(request, url.pathname)) return assetNotFound(); | |
| 3276 | const typed = typedMiss(request, url.pathname); | |
| 3277 | if (!isApiPath && typed) return typed; | |
| 3277 | 3278 | const machinePath = MACHINE_PREFIX.test(url.pathname) || /\.(?:json|git)(?:\/|$)|\.git\//.test(url.pathname); |
| 3278 | 3279 | const plainGet = (request.method === "GET" || request.method === "HEAD") && !machinePath && !prefersJson(request); |
| 3279 | 3280 | if (!isApiPath && (prefersHtml(request, url.pathname) || plainGet)) { |
src/web/chrome.ts +5 −2
| @@ -9,7 +9,7 @@ | ||
| 9 | 9 | */ |
| 10 | 10 | |
| 11 | 11 | import { returnTo, withReturn } from "./return_to.ts"; |
| 12 | import { assetNotFound, isAssetMiss } from "../http/asset_miss.ts"; | |
| 12 | import { MACHINE_PREFIX, typedMiss } from "../http/asset_miss.ts"; | |
| 13 | 13 | import { audienceFacts, audienceOf } from "../core/audience.ts"; |
| 14 | 14 | import { activeEdgeToken, edgeLoginUrl, edgeStrangerLogin } from "../http/edge.ts"; |
| 15 | 15 | import { |
| @@ -74,7 +74,10 @@ export function crumbsFor(repo: Repo): { label: string; href: string }[] { | ||
| 74 | 74 | * this address; `plain` leaves that out for a miss that must reveal nothing |
| 75 | 75 | * (a dead `/go/:code`). */ |
| 76 | 76 | export function notFoundPage(ctx: Ctx, opts: { plain?: boolean } = {}): Response { |
| 77 | if (!opts.plain && isAssetMiss(ctx.request, ctx.url.pathname)) return assetNotFound(); | |
| 77 | // A machine path, or a miss at the owner level (no repository named), may be the few typed bytes; | |
outdated
notFoundPage serves every route handler's miss, so this turns a person's missing /owner/repo/blob/.../x.md or .svg into 10 bytes of plain text with no page. Keep the typed shape to the no-route branch and the feed routes. |
||
| 78 | // a person's missing repo file (`/o/r/blob/x.md`) keeps the page and its nearby links. | |
| 79 | const typed = opts.plain || (ctx.params.name !== undefined && !MACHINE_PREFIX.test(ctx.url.pathname)) ? undefined : typedMiss(ctx.request, ctx.url.pathname); | |
| 80 | if (typed) return typed; | |
| 78 | 81 | const get = !opts.plain && ctx.request.method === "GET"; |
| 79 | 82 | const near = get ? nearTo(ctx.app.db, ctx.auth.user, ctx.params.owner ?? "", ctx.params.name ?? "") : []; |
| 80 | 83 | const pages = get |
test/not-found-asset.test.ts +62 −1
| @@ -3,7 +3,9 @@ import { afterAll, beforeAll, expect, test } from "bun:test"; | ||
| 3 | 3 | import { mkdtemp, rm } from "node:fs/promises"; |
| 4 | 4 | import { tmpdir } from "node:os"; |
| 5 | 5 | import { join } from "node:path"; |
| 6 | import { createRepo } from "../src/actions/repos.ts"; | |
| 6 | 7 | import { createApp, type App } from "../src/app.ts"; |
| 8 | import { createUser } from "../src/core/users.ts"; | |
| 7 | 9 | import { serve, type Servers } from "../src/http/server.ts"; |
| 8 | 10 | |
| 9 | 11 | let dataDir = ""; |
| @@ -13,7 +15,11 @@ beforeAll(async () => { | ||
| 13 | 15 | dataDir = await mkdtemp(join(tmpdir(), "forge-asset404-")); |
| 14 | 16 | app = createApp({ dataDir, port: 0 }); |
| 15 | 17 | servers = await serve(app); |
| 16 | }); | |
| 18 | const u = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true }); | |
| 19 | if (!u.ok) throw new Error(u.error); | |
| 20 | const made = await createRepo(app, u.value, { name: "secret", visibility: "private", description: "d" }); | |
| 21 | if (!made.ok) throw new Error(String(made.error)); | |
| 22 | }, 30_000); | |
| 17 | 23 | |
| 18 | 24 | afterAll(async () => { |
| 19 | 25 | servers.stop(); |
| @@ -86,3 +92,58 @@ test("machine surfaces keep their own miss, not the plain text one", async () => | ||
| 86 | 92 | expect([path, await res.text()]).not.toEqual([path, "not found\n"]); |
| 87 | 93 | } |
| 88 | 94 | }); |
| 95 | ||
| 96 | // One rule, one table: a path of each kind, its status, type and size ceiling. | |
| 97 | const TABLE: [string, number, string, number][] = [ | |
| 98 | ["/.well-known/agent.json", 404, "application/json", 400], | |
| 99 | ["/oauth/nothing", 404, "application/json", 400], | |
| 100 | ["/docs.atom", 404, "text/plain", 20], | |
| 101 | ["/explore.json", 404, "text/plain", 20], | |
| 102 | ["/site.webmanifest", 404, "text/plain", 20], | |
| 103 | ["/readme.md", 404, "text/plain", 20], | |
| 104 | ["/logo.svg", 404, "text/plain", 20], | |
outdated
The page row's 1_000_000 ceiling can never fail; assert the not-found page marker. The table also lacks a feed miss, HEAD, and a repo blob .md path. |
||
| 105 | ["/m.wasm", 404, "text/plain", 20], | |
| 106 | ["/nobody.atom", 404, "text/plain", 20], | |
| 107 | ["/nobody/none/ships.atom", 404, "text/plain", 20], | |
| 108 | ["/alice/secret/ships.atom", 404, "text/plain", 20], | |
| 109 | ["/nothing-here", 404, "text/html", 1_000_000], | |
| 110 | ]; | |
| 111 | for (const [path, status, type, ceiling] of TABLE) { | |
| 112 | test(`miss shape: ${path} is ${status} ${type} under ${ceiling} bytes with the floor`, async () => { | |
| 113 | const res = await miss(path); | |
| 114 | const body = await res.text(); | |
| 115 | expect(res.status).toBe(status); | |
| 116 | expect(res.headers.get("content-type")).toContain(type); | |
| 117 | if (type === "text/html") expect(body.length).toBeGreaterThan(2000); | |
| 118 | else expect(body.length).toBeLessThanOrEqual(ceiling); | |
| 119 | expect(res.headers.get("x-content-type-options")).toBe("nosniff"); | |
| 120 | expect(res.headers.get("referrer-policy")).toBeTruthy(); | |
| 121 | if (type !== "text/html") { | |
| 122 | expect(res.headers.get("cache-control")).toBe("no-store"); | |
| 123 | expect(res.headers.get("content-security-policy")).toBeTruthy(); | |
| 124 | } | |
| 125 | if (type === "application/json") { | |
| 126 | const j = JSON.parse(body); | |
| 127 | expect(Object.keys(j).sort()).toEqual(["error", "hint", "message"]); | |
| 128 | } | |
| 129 | }); | |
| 130 | } | |
| 131 | ||
| 132 | test("a missing owner feed and a private repo's feed give the same bytes", async () => { | |
| 133 | const a = await miss("/alice/secret/ships.atom"); | |
| 134 | const b = await miss("/alice/nope/ships.atom"); | |
| 135 | expect([a.status, await a.text()]).toEqual([404, "not found\n"]); | |
| 136 | expect([b.status, await b.text()]).toEqual([404, "not found\n"]); | |
| 137 | const head = await miss("/nobody.atom", { method: "HEAD" }); | |
| 138 | expect([head.status, head.headers.get("content-type")?.includes("text/plain"), await head.text()]).toEqual([404, true, ""]); | |
| 139 | }); | |
| 140 | ||
| 141 | test("a person's missing repo file keeps the not-found page, not the typed bytes", async () => { | |
| 142 | for (const path of ["/alice/secret/blob/main/nope.md", "/alice/secret/blob/main/nope.svg"]) { | |
| 143 | const res = await miss(path); | |
| 144 | const body = await res.text(); | |
| 145 | expect([path, res.status, res.headers.get("content-type")?.includes("text/html")]).toEqual([path, 404, true]); | |
| 146 | expect(body).not.toBe("not found\n"); | |
| 147 | expect(body.length).toBeGreaterThan(2000); | |
| 148 | } | |
| 149 | }); |
test/web-ui.test.ts +1
| @@ -443,6 +443,7 @@ describe("web ui", () => { | ||
| 443 | 443 | t |
| 444 | 444 | .replaceAll(/nonce-?="?[\w-]+/g, "nonce") |
| 445 | 445 | .replaceAll(/here\?path=[^"&\s]*/g, "here?path=") |
| 446 | .replaceAll(/Quote <code>\w+<\/code>/g, "Quote <code>id</code>") | |
| 446 | 447 | .replaceAll(/next=[^"]*/g, "next=") |
| 447 | 448 | .replaceAll(/\/new\/issue\?from=[^"]*/g, "/new/issue?from="); |
| 448 | 449 | expect(scrub(await hidden.text())).toBe(scrub(await absent.text())); |
new above
offline
Const sits between imports (the next import follows it); move it below the last import.