trevato/tempo

Public: anyone can read this; the crew can change it.

More
Forks5g fMapg aRunsg rEventsFlows

A board link does not end up in a search index: x-robots-tag, a robots.txt, and a sentence in the README #117

Merged masoncrew opened this

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

mason/tempo:mason/58-a-board-link-does-not-end-up-in-a-search → main 1 commit fork

  • Merged. mason/58-a-board-link-does-not-end-up-in-a-search was merged into main as f249144.

Review

Sign in to review this change.
7 files changed +166 −12
README.md +9
@@ -130,6 +130,15 @@ background, reverting if it fails.
130130 privacy. Anyone with the link can still see and edit any board,
131131 signed in or not — tempo never asks a visitor to sign in just to use
132132 a board they were given the link to.
133 - tempo asks crawlers to stay out of `/b/` (and `/mcp`): every response
134 under those paths carries `x-robots-tag: noindex, nofollow, noarchive`,
135 a board page repeats it in a `<meta name="robots">`, and `/robots.txt`
136 says the same — naming prefixes only, never a board id. That closes the
137 ordinary accident, a board link pasted somewhere public ending up in a
138 search index; it is a request, not a defence. A board link is still a
139 capability, and anyone who has it — crawler included — has the board.
140 `/` and `/static/*` are left indexable on purpose: the landing page is
141 the front door and names no board.
133142 - Because of that, `list_boards` over MCP is deliberately narrow: it
134143 answers only with boards the caller has kept, never every board on
135144 the server, so it can't be used to go discover a board by guessing.
src/pages/board.ts +8 −2
@@ -112,10 +112,15 @@ const shortcutsDialog = (extra: [string, string][]): string => {
112112 // that a plain form navigation just hit. It renders straight into the
113113 // toast, already shown — no JavaScript required to see why the action
114114 // didn't happen. static/board.js's own toast calls reuse the same element.
115 const shell = (title: string, body: string, shortcuts: [string, string][] = [], error?: string): string => `<!doctype html>
115 //
116 // `noindex` says the same thing as the x-robots-tag header src/server.ts
117 // puts on every `/b/…` response (src/robots.ts), for the crawler that reads
118 // the HTML and not the headers. The landing page doesn't ask for it: it is
119 // the front door and names no board.
120 const shell = (title: string, body: string, shortcuts: [string, string][] = [], error?: string, noindex = false): string => `<!doctype html>
116121 <meta charset="utf-8">
117122 <script src="/static/${THEME_JS}"></script>
118 <meta name="viewport" content="width=device-width, initial-scale=1">
123 ${noindex ? `<meta name="robots" content="noindex, nofollow">\n` : ""}<meta name="viewport" content="width=device-width, initial-scale=1">
119124 <title>${escape(title)}</title>
120125 <style>${styles}</style>
121126 <div class="tools">
@@ -308,6 +313,7 @@ export function boardPage({ board, columns, cards, query, who, error, presence =
308313 <script src="/static/${BOARDS_JS}" defer></script>`,
309314 boardShortcuts,
310315 error,
316 true, // a board is not for a search index
311317 );
312318 }
313319
src/robots.test.ts added +23
@@ -0,0 +1,23 @@
1 import { expect, test } from "bun:test";
2 import { DISALLOWED, isDisallowed, ROBOTS_TXT } from "./robots.ts";
3
4 test("boards and the JSON-RPC endpoint are asked to stay out of an index", () => {
5 expect(isDisallowed("/b/abcd1234")).toBe(true);
6 expect(isDisallowed("/b/abcd1234/events")).toBe(true);
7 expect(isDisallowed("/b/abcd1234/cards/abcdefabcdef/move")).toBe(true);
8 expect(isDisallowed("/mcp")).toBe(true);
9 });
10
11 test("the front door and its assets stay indexable", () => {
12 expect(isDisallowed("/")).toBe(false);
13 expect(isDisallowed("/static/board.232693062.js")).toBe(false);
14 expect(isDisallowed("/robots.txt")).toBe(false);
15 });
16
17 test("robots.txt says exactly what the header does, and names no board", () => {
18 expect(ROBOTS_TXT.split("\n")[0]).toBe("User-agent: *");
19 for (const prefix of DISALLOWED) expect(ROBOTS_TXT).toContain(`Disallow: ${prefix}`);
20 // Prefixes only: a board id in a public file would be the very leak this
21 // is here to prevent.
22 expect(ROBOTS_TXT).not.toMatch(/\/b\/[a-z0-9]/);
23 });
src/robots.ts added +32
@@ -0,0 +1,32 @@
1 /**
2 * What tempo asks crawlers to stay out of, in one place: the header put on
3 * a response (src/server.ts), the `<meta>` on the board page
4 * (src/pages/board.ts) and the /robots.txt that says the same thing
5 * (src/routes.ts) all read this list, so a new route under a path named
6 * here is covered the day it is written.
7 *
8 * A board's id is its access control, and indexing is the ordinary way a
9 * link-shared thing escapes: a board URL pasted into a public issue or an
10 * archived chat log is crawlable, and an indexed board is one anyone can
11 * find and edit without ever having been given the link. This closes that
12 * accident. It is not privacy — a crawler that ignores it, and anyone who
13 * has the link, still has the board.
14 */
15
16 // Paths a crawler is asked not to visit: every board (`/b/…` — its page,
17 // its event stream, its JSON) and the JSON-RPC endpoint. `/` and
18 // `/static/*` are deliberately left out of this list and stay indexable:
19 // the landing page is the app's front door and should be findable, and it
20 // names no board.
21 export const DISALLOWED = ["/b/", "/mcp"] as const;
22
23 /** The header on any response under a disallowed path. */
24 export const ROBOTS_TAG = "noindex, nofollow, noarchive";
25
26 export function isDisallowed(pathname: string): boolean {
27 return DISALLOWED.some((prefix) => pathname.startsWith(prefix));
28 }
29
30 // Prefixes only, never a board id: this file is public, so it must name
31 // nothing that isn't already public.
32 export const ROBOTS_TXT = ["User-agent: *", ...DISALLOWED.map((prefix) => `Disallow: ${prefix}`), ""].join("\n");
src/routes.ts +12
@@ -11,6 +11,7 @@ import { boardCreationBucket, callerKey, MAX_CARDS_PER_BOARD, MAX_COLUMNS_PER_BO
1111 import { rpc } from "./mcp.ts";
1212 import { boardPage, landingPage } from "./pages/board.ts";
1313 import * as presence from "./presence.ts";
14 import { ROBOTS_TXT } from "./robots.ts";
1415 import { cleanText } from "./text.ts";
1516 import { noteRevision } from "./watch.ts";
1617 import type { Who } from "./who.ts";
@@ -255,6 +256,17 @@ async function dispatch(request: Request, url: URL, env: string, who: Who, ip: s
255256 return Response.redirect(`/b/${board.id}`, 303);
256257 }
257258
259 // The etiquette half of src/robots.ts — the header on each response is
260 // the other half. Public, cacheable for a day, and it names prefixes
261 // only: no board id is ever in it, so it leaks nothing a crawler could
262 // then go and fetch.
263 if (parts.length === 1 && parts[0] === "robots.txt") {
264 if (method !== "GET") return methodNotAllowed("GET, HEAD");
265 return new Response(ROBOTS_TXT, {
266 headers: { "content-type": "text/plain; charset=utf-8", "cache-control": "public, max-age=86400" },
267 });
268 }
269
258270 if (method === "POST" && parts.length === 1 && parts[0] === "mcp") {
259271 const message = await request.json().catch(() => undefined);
260272 if (message === undefined) return json({ jsonrpc: "2.0", id: null, error: { code: -32700, message: "parse error" } });
src/server.test.ts +60 −1
@@ -12,6 +12,7 @@ const { sign } = await import("./who.ts");
1212 const { MAX_CARDS_PER_BOARD, MAX_COLUMNS_PER_BOARD, MAX_REQUEST_BODY_BYTES, MAX_SUBSCRIBERS_PER_BOARD, boardCreationBucket } =
1313 await import("./limits.ts");
1414 const db = await import("./db.ts");
15 const { THEME_JS } = await import("./pages/board.ts");
1516 // boardCreationBucket is one process-wide bucket; reset it so this file's
1617 // own board-creation tests aren't at the mercy of what other test files
1718 // already spent from it in this same `bun test` run.
@@ -27,8 +28,18 @@ const j = { accept: "application/json", "content-type": "application/x-www-form-
2728 const post = (path: string, body: Record<string, string> = {}, headers: Record<string, string> = j) =>
2829 fetch(base + path, { method: "POST", headers, body: new URLSearchParams(body) });
2930
31 // A test that only needs a board to act on gets one, whatever earlier
32 // tests already spent: this file creates enough boards to run the
33 // process-wide creation bucket (src/limits.ts, 30 a minute) dry, and every
34 // test here would then fail on a 429 for a reason it isn't about. The
35 // refusal itself is tested where it belongs — src/limits.ts's own tests.
3036 async function newBoard(name = "Test board"): Promise<string> {
31 const res = await fetch(base + "/boards", { method: "POST", body: new URLSearchParams({ name }) });
37 const create = () => fetch(base + "/boards", { method: "POST", body: new URLSearchParams({ name }) });
38 let res = await create();
39 if (res.status === 429) {
40 boardCreationBucket.reset();
41 res = await create();
42 }
3243 return new URL(res.url).pathname.split("/")[2];
3344 }
3445
@@ -44,6 +55,54 @@ test("stands on the security floor", async () => {
4455 expect(response.headers.get("x-content-type-options")).toBe("nosniff");
4556 });
4657
58 test("a board asks to stay out of a search index — page, stream and JSON alike", async () => {
59 const id = await newBoard("Findable");
60
61 const page = await fetch(base + `/b/${id}`);
62 expect(page.headers.get("x-robots-tag")).toBe("noindex, nofollow, noarchive");
63 // For the crawler that reads the HTML and not the headers.
64 expect(await page.text()).toContain('<meta name="robots" content="noindex, nofollow">');
65
66 // HEAD, so the endless body doesn't have to be read to see the headers.
67 const stream = await fetch(base + `/b/${id}/events`, { method: "HEAD" });
68 expect(stream.headers.get("content-type")).toBe("text/event-stream");
69 expect(stream.headers.get("x-robots-tag")).toBe("noindex, nofollow, noarchive");
70
71 const written = await post(`/b/${id}/undo`);
72 expect(written.headers.get("content-type")).toBe("application/json");
73 expect(written.headers.get("x-robots-tag")).toBe("noindex, nofollow, noarchive");
74
75 // A board that isn't there says it just as loudly: a 404 under /b/ is
76 // still a board URL someone could have pasted somewhere public.
77 const missing = await fetch(base + "/b/zzzzzzzz");
78 expect(missing.status).toBe(404);
79 expect(missing.headers.get("x-robots-tag")).toBe("noindex, nofollow, noarchive");
80 });
81
82 test("the front door stays indexable", async () => {
83 const landing = await fetch(base + "/");
84 expect(landing.headers.get("x-robots-tag")).toBeNull();
85 expect(await landing.text()).not.toContain('name="robots"');
86
87 const script = await fetch(base + `/static/${THEME_JS}`);
88 expect(script.status).toBe(200);
89 expect(script.headers.get("x-robots-tag")).toBeNull();
90 });
91
92 test("robots.txt asks crawlers off the boards and names none of them", async () => {
93 const id = await newBoard("Secret");
94 const response = await fetch(base + "/robots.txt");
95 expect(response.status).toBe(200);
96 expect(response.headers.get("content-type")).toBe("text/plain; charset=utf-8");
97 expect(response.headers.get("cache-control")).toBe("public, max-age=86400");
98 const body = await response.text();
99 expect(body).toContain("User-agent: *");
100 expect(body).toContain("Disallow: /b/");
101 expect(body).toContain("Disallow: /mcp");
102 expect(body).not.toContain(id);
103 expect(body).not.toMatch(/\/b\/[a-z0-9]{8}/);
104 });
105
47106 test("refuses a cross-site post", async () => {
48107 const posted = await fetch(base + "/boards", {
49108 method: "POST",
src/server.ts +22 −9
@@ -10,6 +10,7 @@ import { join } from "node:path";
1010 import "./db.ts";
1111 import { readBody } from "./body.ts";
1212 import { MAX_REQUEST_BODY_BYTES, MAX_REQUEST_DRAIN_BYTES } from "./limits.ts";
13 import { isDisallowed, ROBOTS_TAG } from "./robots.ts";
1314 import { route } from "./routes.ts";
1415 import { startWatching } from "./watch.ts";
1516 import { who } from "./who.ts";
@@ -32,8 +33,18 @@ const SECURITY = {
3233 "permissions-policy": "camera=(), microphone=(), geolocation=()",
3334 ...(publicOrigin.startsWith("https:") ? { "strict-transport-security": "max-age=31536000" } : {}),
3435 };
35 function withFloor(response: Response): Response {
36 // Beside the floor, and part of it: a response under a path a crawler is
37 // asked to stay out of (src/robots.ts — every `/b/…`, so the board page,
38 // its event stream and its JSON alike, and `/mcp`) says so in a header, so
39 // a new route under one of those paths carries it without being asked.
40 // `/` and `/static/*` are left indexable on purpose: the landing page is
41 // the app's front door and should be findable, and it names no board.
42 // `cache-control: private, no-store` already keeps a board out of a shared
43 // cache; it says nothing to a crawler, which is what this is for — and
44 // asking is all it is. A board link is still a capability.
45 function withFloor(response: Response, pathname: string): Response {
3646 for (const [key, value] of Object.entries(SECURITY)) response.headers.set(key, value);
47 if (isDisallowed(pathname)) response.headers.set("x-robots-tag", ROBOTS_TAG);
3748 return response;
3849 }
3950
@@ -67,7 +78,7 @@ export const server = Bun.serve({
6778 if (request.method === "POST") {
6879 // Refuse a cross-site write before spending anything on reading its
6980 // body at all.
70 if (!sameOrigin(request)) return withFloor(new Response("forbidden", { status: 403 }));
81 if (!sameOrigin(request)) return withFloor(new Response("forbidden", { status: 403 }), url.pathname);
7182 // A declared Content-Length over the cap is refused for free, before
7283 // a single byte is read — that's still worth doing even though it's
7384 // only a claim, because a truthful one is the common case. A chunked
@@ -76,9 +87,9 @@ export const server = Bun.serve({
7687 // truth of record either way. The cap is enforced here, once, and the
7788 // capped body replaces the original for every handler downstream.
7889 const declared = Number(request.headers.get("content-length") ?? "0");
79 if (declared > MAX_REQUEST_BODY_BYTES) return withFloor(new Response("request body too large", { status: 413 }));
90 if (declared > MAX_REQUEST_BODY_BYTES) return withFloor(new Response("request body too large", { status: 413 }), url.pathname);
8091 const body = await readBody(request, MAX_REQUEST_BODY_BYTES, MAX_REQUEST_DRAIN_BYTES);
81 if (body === undefined) return withFloor(new Response("request body too large", { status: 413 }));
92 if (body === undefined) return withFloor(new Response("request body too large", { status: 413 }), url.pathname);
8293 request = new Request(request, { body });
8394 }
8495
@@ -89,17 +100,17 @@ export const server = Bun.serve({
89100 const routed = await route(routingRequest, url, env, who(request, identityKey), ip);
90101 let response: Response;
91102 if (routed) {
92 response = withFloor(routed);
103 response = withFloor(routed, url.pathname);
93104 } else if (url.pathname.startsWith("/static/") && !url.pathname.includes("..")) {
94105 // Files under ./static are served as they are, cached for a year: name
95106 // them by content (board.<hash>.js) and a change is a new name, never a
96107 // stale cache. The path is confined to the directory.
97108 const file = Bun.file(join(import.meta.dir, "..", url.pathname.slice(1)));
98109 response = (await file.exists())
99 ? withFloor(new Response(file, { headers: { "cache-control": "public, max-age=31536000, immutable" } }))
100 : withFloor(new Response("not found", { status: 404 }));
110 ? withFloor(new Response(file, { headers: { "cache-control": "public, max-age=31536000, immutable" } }), url.pathname)
111 : withFloor(new Response("not found", { status: 404 }), url.pathname);
101112 } else {
102 response = withFloor(new Response("not found", { status: 404 }));
113 response = withFloor(new Response("not found", { status: 404 }), url.pathname);
103114 }
104115
105116 if (!isHead) return response;
@@ -121,7 +132,9 @@ export const server = Bun.serve({
121132 // here, answer with a plain 500 that still carries the security floor.
122133 error(err) {
123134 console.error(err);
124 return withFloor(new Response("internal error", { status: 500 }));
135 // No path reaches here — Bun hands the error, not the request — and a
136 // 500 carries nothing worth indexing anyway.
137 return withFloor(new Response("internal error", { status: 500 }), "");
125138 },
126139 });
127140

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs