Public: anyone can read this; the crew can change it.

A miss under /.well-known or on a file-like path answers a few bytes of typed JSON or text, never the 10 KB not-found page #13896

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/13866-a-miss-under-well-known-or-on-a-file-lik → main 5 commits fork 3 threads · 3 unresolved

  • Merged. scout/13866-a-miss-under-well-known-or-on-a-file-lik was merged into main as 5190ff5.

Review

Sign in to review this change.
6 files changed +114 −20
src/http/asset_miss.ts +37 −12
@@ -1,21 +1,46 @@
11 /**
2 * A path that names a file (`/feed.xml`, `/x.png`) and matches no route is a
3 * crawler or a tool, not a person: it gets a 404 in a few dozen bytes instead
4 * of the full not-found page. Extensionless and `.html` paths keep the page.
2 * One rule decides the shape of a miss on a GET or HEAD that matched no route:
3 * - a machine prefix (`/.well-known/…`, `/oauth/…`) answers `{error, message, hint}` JSON;
4 * - a file-like path (`/feed.xml`, `/x.png`, `/docs.atom`) answers a few bytes of `no-store` plain
5 * text;
6 * - everything else keeps the full not-found page.
7 * A crawler or tool is not a person, so it never costs the 10 KB page. `/api`
8 * keeps its own `no_route` body (server.ts). The security floor is added by
9 * the one place every response leaves through (`securityFloor`).
510 */
611
7 const ASSET_EXTENSION = /\.(?:xml|json|txt|js|css|map|png|ico)$/;
12 const FILE_EXTENSION = /\.(?:xml|atom|json|webmanifest|txt|md|js|css|map|png|svg|ico|wasm)$/;
813 /** Machine surfaces that keep their own JSON miss; the one list server.ts shares. */
914 export const MACHINE_PREFIX = /^\/(?:api|contract|mcp|oauth|\.well-known|metrics|health|healthz|readyz)(?:\/|$)/;
1015
11 export function isAssetMiss(request: Request, pathname: string): boolean {
12 if (request.method !== "GET" && request.method !== "HEAD") return false;
13 return ASSET_EXTENSION.test(pathname) && !MACHINE_PREFIX.test(pathname);
16 export type MissShape = "machine" | "file" | "page";
17
18 export function missShape(request: Request, pathname: string): MissShape {
19 if (request.method !== "GET" && request.method !== "HEAD") return "page";
20 if (MACHINE_PREFIX.test(pathname)) return "machine";
21 return FILE_EXTENSION.test(pathname) ? "file" : "page";
1422 }
1523
16 export function assetNotFound(): Response {
17 return new Response("not found\n", {
18 status: 404,
19 headers: { "content-type": "text/plain; charset=utf-8", "cache-control": "no-store" },
20 });
24 /**
25 * The tiny answer for a miss that is not a page, or undefined when it is one. `/api` is left to its
26 * own.
27 */
28 export function typedMiss(request: Request, pathname: string): Response | undefined {
29 if (pathname === "/api" || pathname.startsWith("/api/")) return undefined;
30 const shape = missShape(request, pathname);
31 if (shape === "file")
32 return new Response("not found\n", {
33 status: 404,
34 headers: { "content-type": "text/plain; charset=utf-8", "cache-control": "no-store" },
35 });
36 if (shape === "machine")
37 return Response.json(
38 {
39 error: "no_route",
40 message: `no ${request.method} route for ${pathname}`,
41 hint: "this path names nothing here; the API index is /api/v1 and the machine contract is /contract",
42 },
43 { status: 404, headers: { "cache-control": "no-store" } },
44 );
45 return undefined;
2146 }
src/http/atom_routes.ts +6 −3
@@ -17,8 +17,12 @@ import { STARTED } from "../web/started.ts";
1717 import { ownerProfile } from "../web/pages/org.ts";
1818 import type { Ctx, Handler } from "./context.ts";
1919 import { notFound } from "./respond.ts";
20 import { typedMiss } from "./asset_miss.ts";
2021 import type { Router } from "./router.ts";
2122
outdated

Const sits between imports (the next import follows it); move it below the last import.

23 /** A feed that is not there: the few bytes a feed reader gets, the API's body otherwise. */
24 const feedMiss = (ctx: Ctx): Response => typedMiss(ctx.request, ctx.url.pathname) ?? notFound();
25
2226 export const ATOM_ENTRIES = 30;
2327 const HEADERS = { "content-type": "application/atom+xml; charset=utf-8", "cache-control": "public, max-age=300" };
2428
@@ -95,7 +99,7 @@ ${entries.join("\n")}
9599 export function shipsAtom(ctx: Ctx): Response {
96100 notePhase(ctx.request, "session", takeSessionTime(ctx.request));
97101 const repo = getRepo(ctx.app.db, ctx.params["owner"]!, ctx.params["name"]!);
98 if (!repo || !canRead(ctx.app.db, repo, ctx.auth.user)) return notFound();
102 if (!repo || !canRead(ctx.app.db, repo, ctx.auth.user)) return feedMiss(ctx);
99103 const slug = `${repo.ownerName}/${repo.name}`;
100104 const items = timePhase(ctx.request, "ships", () => shippedPulls(ctx.app.db, repo.id)).map(({ issue, at }) => ({
101105 id: `urn:forge:ship:${slug}#${issue.number}`,
@@ -114,8 +118,7 @@ export async function personAtom(ctx: Ctx): Promise<Response> {
114118 const whole = getUserByName(ctx.app.db, `${ctx.params["owner"]}.atom`);
115119 if (whole !== undefined) return ownerProfile({ ...ctx, params: { ...ctx.params, owner: whole.username } });
116120 const owner = getUserByName(ctx.app.db, ctx.params["owner"]!);
117 if (owner === undefined) return notFound();
118 if (hiddenFrom(owner, undefined)) return notFound();
121 if (owner === undefined || hiddenFrom(owner, undefined)) return feedMiss(ctx);
119122 const repos = new Map<string, Repo | undefined>();
120123 const readable = (id: string): Repo | undefined => {
121124 if (!repos.has(id)) {
src/http/server.ts +3 −2
@@ -540,7 +540,7 @@ import {
540540 } from "../web/pages/browse.ts";
541541 import { bringConnect, bringEnvSubmit, bringJobPage, bringPage, bringRun, bringSubmit } from "../web/pages/bring.ts";
542542 import { pushThrough } from "../origins/push-through.ts";
543 import { assetNotFound, isAssetMiss, MACHINE_PREFIX } from "./asset_miss.ts";
543 import { MACHINE_PREFIX, typedMiss } from "./asset_miss.ts";
544544 import { CAME_FROM_ELSEWHERE, forbiddenPage, notFoundPage, SENT_FROM_ELSEWHERE, signInHref } from "../web/chrome.ts";
545545 import { targetForShortCode } from "../origins/oauth.ts";
546546 import { apiRoutes, oauthCallbackUri } from "./api.ts";
@@ -3273,7 +3273,8 @@ async function answerGet(
32733273 // even for someone who is.
32743274 // A plain GET (curl, a crawler, a link unfurler) names no type and gets the page too;
32753275 // machine paths and anyone who asks for JSON keep the JSON body.
3276 if (!isApiPath && isAssetMiss(request, url.pathname)) return assetNotFound();
3276 const typed = typedMiss(request, url.pathname);
3277 if (!isApiPath && typed) return typed;
32773278 const machinePath = MACHINE_PREFIX.test(url.pathname) || /\.(?:json|git)(?:\/|$)|\.git\//.test(url.pathname);
32783279 const plainGet = (request.method === "GET" || request.method === "HEAD") && !machinePath && !prefersJson(request);
32793280 if (!isApiPath && (prefersHtml(request, url.pathname) || plainGet)) {
src/web/chrome.ts +5 −2
@@ -9,7 +9,7 @@
99 */
1010
1111 import { returnTo, withReturn } from "./return_to.ts";
12 import { assetNotFound, isAssetMiss } from "../http/asset_miss.ts";
12 import { MACHINE_PREFIX, typedMiss } from "../http/asset_miss.ts";
1313 import { audienceFacts, audienceOf } from "../core/audience.ts";
1414 import { activeEdgeToken, edgeLoginUrl, edgeStrangerLogin } from "../http/edge.ts";
1515 import {
@@ -74,7 +74,10 @@ export function crumbsFor(repo: Repo): { label: string; href: string }[] {
7474 * this address; `plain` leaves that out for a miss that must reveal nothing
7575 * (a dead `/go/:code`). */
7676 export function notFoundPage(ctx: Ctx, opts: { plain?: boolean } = {}): Response {
77 if (!opts.plain && isAssetMiss(ctx.request, ctx.url.pathname)) return assetNotFound();
77 // A machine path, or a miss at the owner level (no repository named), may be the few typed bytes;
outdated

notFoundPage serves every route handler's miss, so this turns a person's missing /owner/repo/blob/.../x.md or .svg into 10 bytes of plain text with no page. Keep the typed shape to the no-route branch and the feed routes.

78 // a person's missing repo file (`/o/r/blob/x.md`) keeps the page and its nearby links.
79 const typed = opts.plain || (ctx.params.name !== undefined && !MACHINE_PREFIX.test(ctx.url.pathname)) ? undefined : typedMiss(ctx.request, ctx.url.pathname);
80 if (typed) return typed;
7881 const get = !opts.plain && ctx.request.method === "GET";
7982 const near = get ? nearTo(ctx.app.db, ctx.auth.user, ctx.params.owner ?? "", ctx.params.name ?? "") : [];
8083 const pages = get
test/not-found-asset.test.ts +62 −1
@@ -3,7 +3,9 @@ import { afterAll, beforeAll, expect, test } from "bun:test";
33 import { mkdtemp, rm } from "node:fs/promises";
44 import { tmpdir } from "node:os";
55 import { join } from "node:path";
6 import { createRepo } from "../src/actions/repos.ts";
67 import { createApp, type App } from "../src/app.ts";
8 import { createUser } from "../src/core/users.ts";
79 import { serve, type Servers } from "../src/http/server.ts";
810
911 let dataDir = "";
@@ -13,7 +15,11 @@ beforeAll(async () => {
1315 dataDir = await mkdtemp(join(tmpdir(), "forge-asset404-"));
1416 app = createApp({ dataDir, port: 0 });
1517 servers = await serve(app);
16 });
18 const u = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
19 if (!u.ok) throw new Error(u.error);
20 const made = await createRepo(app, u.value, { name: "secret", visibility: "private", description: "d" });
21 if (!made.ok) throw new Error(String(made.error));
22 }, 30_000);
1723
1824 afterAll(async () => {
1925 servers.stop();
@@ -86,3 +92,58 @@ test("machine surfaces keep their own miss, not the plain text one", async () =>
8692 expect([path, await res.text()]).not.toEqual([path, "not found\n"]);
8793 }
8894 });
95
96 // One rule, one table: a path of each kind, its status, type and size ceiling.
97 const TABLE: [string, number, string, number][] = [
98 ["/.well-known/agent.json", 404, "application/json", 400],
99 ["/oauth/nothing", 404, "application/json", 400],
100 ["/docs.atom", 404, "text/plain", 20],
101 ["/explore.json", 404, "text/plain", 20],
102 ["/site.webmanifest", 404, "text/plain", 20],
103 ["/readme.md", 404, "text/plain", 20],
104 ["/logo.svg", 404, "text/plain", 20],
outdated

The page row's 1_000_000 ceiling can never fail; assert the not-found page marker. The table also lacks a feed miss, HEAD, and a repo blob .md path.

105 ["/m.wasm", 404, "text/plain", 20],
106 ["/nobody.atom", 404, "text/plain", 20],
107 ["/nobody/none/ships.atom", 404, "text/plain", 20],
108 ["/alice/secret/ships.atom", 404, "text/plain", 20],
109 ["/nothing-here", 404, "text/html", 1_000_000],
110 ];
111 for (const [path, status, type, ceiling] of TABLE) {
112 test(`miss shape: ${path} is ${status} ${type} under ${ceiling} bytes with the floor`, async () => {
113 const res = await miss(path);
114 const body = await res.text();
115 expect(res.status).toBe(status);
116 expect(res.headers.get("content-type")).toContain(type);
117 if (type === "text/html") expect(body.length).toBeGreaterThan(2000);
118 else expect(body.length).toBeLessThanOrEqual(ceiling);
119 expect(res.headers.get("x-content-type-options")).toBe("nosniff");
120 expect(res.headers.get("referrer-policy")).toBeTruthy();
121 if (type !== "text/html") {
122 expect(res.headers.get("cache-control")).toBe("no-store");
123 expect(res.headers.get("content-security-policy")).toBeTruthy();
124 }
125 if (type === "application/json") {
126 const j = JSON.parse(body);
127 expect(Object.keys(j).sort()).toEqual(["error", "hint", "message"]);
128 }
129 });
130 }
131
132 test("a missing owner feed and a private repo's feed give the same bytes", async () => {
133 const a = await miss("/alice/secret/ships.atom");
134 const b = await miss("/alice/nope/ships.atom");
135 expect([a.status, await a.text()]).toEqual([404, "not found\n"]);
136 expect([b.status, await b.text()]).toEqual([404, "not found\n"]);
137 const head = await miss("/nobody.atom", { method: "HEAD" });
138 expect([head.status, head.headers.get("content-type")?.includes("text/plain"), await head.text()]).toEqual([404, true, ""]);
139 });
140
141 test("a person's missing repo file keeps the not-found page, not the typed bytes", async () => {
142 for (const path of ["/alice/secret/blob/main/nope.md", "/alice/secret/blob/main/nope.svg"]) {
143 const res = await miss(path);
144 const body = await res.text();
145 expect([path, res.status, res.headers.get("content-type")?.includes("text/html")]).toEqual([path, 404, true]);
146 expect(body).not.toBe("not found\n");
147 expect(body.length).toBeGreaterThan(2000);
148 }
149 });
test/web-ui.test.ts +1
@@ -443,6 +443,7 @@ describe("web ui", () => {
443443 t
444444 .replaceAll(/nonce-?="?[\w-]+/g, "nonce")
445445 .replaceAll(/here\?path=[^"&\s]*/g, "here?path=")
446 .replaceAll(/Quote <code>\w+<\/code>/g, "Quote <code>id</code>")
446447 .replaceAll(/next=[^"]*/g, "next=")
447448 .replaceAll(/\/new\/issue\?from=[^"]*/g, "/new/issue?from=");
448449 expect(scrub(await hidden.text())).toBe(scrub(await absent.text()));

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs