Public: anyone can read this; the crew can change it.
GET /api/v1/feed answers 304 only when nothing the viewer would read has changed: events, shares, dismissals, mutes, chips and the minute #6252
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/6218-get-api-v1-feed-answers-304-only-when-no → main 2 commits fork 4 threads · 4 unresolved
src/feed/feed_tag.ts added +58
| @@ -0,0 +1,58 @@ | ||
| 1 | /** | |
| 2 | * The weak ETag of one viewer's feed page, and whether the viewer has shares | |
| 3 | * still waiting to be delivered. | |
| 4 | */ | |
| 5 | ||
| 6 | import type { Database } from "bun:sqlite"; | |
| 7 | import type { Chips } from "../core/feed_chips.ts"; | |
| 8 | import type { User } from "../core/users.ts"; | |
| 9 | import { writeStamp } from "../core/write_clock.ts"; | |
| 10 | ||
| 11 | /** One minute: the grain at which time itself changes a feed (ages, waits, decay). */ | |
| 12 | export const TAG_BUCKET_MS = 60_000; | |
| 13 | ||
| 14 | /** | |
| 15 | * The tag of the page `GET /api/v1/feed` would answer: a hash of the viewer, | |
| 16 | * the switched-off chips, the cursor, the limit, the forge's write clock | |
| 17 | * (`writeStamp`: any insert, update or delete, so events, shares, dismissals | |
| 18 | * and their undoing, mutes, watches, pins, settled issues and visibility all | |
| 19 | * move it) and the minute of `now`. It is built from those, never from the | |
| 20 | * body, so a 304 costs none of the ranking. A pull crossing the 24 h wait | |
| 21 | * line changes the minute. Inside a transaction the clock has no value and | |
| 22 | * there is no tag: the caller sends none and never answers 304. The caller | |
| 23 | * also drains the viewer's share queue first and answers 200 while | |
| 24 | * `hasQueuedShares` holds, since the queue's delivery depends on the clock. | |
| 25 | */ | |
| 26 | export function feedTag( | |
| 27 | db: Database, | |
| 28 | viewer: User, | |
| 29 | input: { | |
| 30 | readonly chips: Chips; | |
| 31 | readonly cursor: string | null; | |
| 32 | readonly limit: number | null; | |
| 33 | readonly now?: number; | |
| 34 | }, | |
| 35 | ): string | undefined { | |
| 36 | const stamp = writeStamp(db); | |
| 37 | if (stamp === undefined) return undefined; | |
| 38 | const now = input.now ?? Date.now(); | |
| 39 | const parts = [ | |
outdated
This reads up to 100 events and runs the audience filter on every request just to probe for change, and it still misses non-event sources: watches, pins, asks, decisions, settled issues and visibility changes. |
||
| 40 | viewer.id, | |
| 41 | [...input.chips.kinds].sort().join(","), | |
| 42 | [...input.chips.people].sort().join(","), | |
| 43 | input.cursor ?? "", | |
| 44 | input.limit ?? "", | |
| 45 | stamp, | |
| 46 | Math.floor(now / TAG_BUCKET_MS), | |
| 47 | ]; | |
| 48 | return `W/"${Bun.hash(JSON.stringify(parts)).toString(36)}"`; | |
outdated
|
||
| 49 | } | |
| 50 | ||
| 51 | /** Whether a share to the viewer is still queued behind their daily budget. */ | |
| 52 | export function hasQueuedShares(db: Database, viewerId: string): boolean { | |
| 53 | return ( | |
| 54 | db | |
| 55 | .query("SELECT 1 FROM share WHERE recipient_id = ?1 AND state = 'queued' LIMIT 1") | |
| 56 | .get(viewerId) !== null | |
| 57 | ); | |
| 58 | } |
src/http/feed.ts +26 −1
| @@ -21,6 +21,9 @@ import { | ||
| 21 | 21 | } from "../core/interactions.ts"; |
| 22 | 22 | import { chipsOf, isKindChip, setChip } from "../core/feed_chips.ts"; |
| 23 | 23 | import { getUserByName, getUsersByIds } from "../core/users.ts"; |
| 24 | import { drainQueue } from "../core/shares.ts"; | |
| 25 | import { feedTag, hasQueuedShares } from "../feed/feed_tag.ts"; | |
| 26 | import { holds } from "../web/feeds.ts"; | |
| 24 | 27 | import { share, RECIPIENTS_MAX } from "../feed/share.ts"; |
| 25 | 28 | import { canRead, getRepo, getRepoById } from "../core/repos.ts"; |
| 26 | 29 | import { |
| @@ -67,6 +70,28 @@ export function feedRoutes(router: Router<Handler>): void { | ||
| 67 | 70 | const cursor = rawCursor === null ? undefined : parseCursor(rawCursor); |
| 68 | 71 | if (rawCursor !== null && cursor === undefined) |
| 69 | 72 | return apiError("invalid_cursor", 422, "that cursor was not issued here"); |
| 73 | // A 304 only when nothing the viewer would read has changed (`feedTag`). | |
| 74 | // The queue drains first, as `readFeed` would; shares still waiting | |
| 75 | // depend on the clock, so they always get a page. | |
| 76 | const now = Date.now(); | |
| 77 | drainQueue(ctx.app.db, viewer.id, now); | |
Nit: |
||
| 78 | const tag = feedTag(ctx.app.db, viewer, { | |
| 79 | chips: chipsOf(ctx.app.db, viewer.id), | |
| 80 | cursor: rawCursor, | |
| 81 | limit: limit ?? null, | |
| 82 | now, | |
| 83 | }); | |
| 84 | const headers: Record<string, string> = { | |
| 85 | "cache-control": "private, no-store", | |
| 86 | vary: "authorization, cookie", | |
| 87 | ...(tag === undefined ? {} : { etag: tag }), | |
| 88 | }; | |
| 89 | if ( | |
| 90 | tag !== undefined && | |
| 91 | holds(ctx.request.headers.get("if-none-match"), tag) && | |
| 92 | !hasQueuedShares(ctx.app.db, viewer.id) | |
| 93 | ) | |
| 94 | return new Response(null, { status: 304, headers }); | |
| 70 | 95 | const page = timePhase(ctx.request, "feed", () => |
| 71 | 96 | readFeed(ctx.app.db, viewer, { |
| 72 | 97 | phase: (name, ms) => notePhase(ctx.request, name, ms), |
| @@ -75,7 +100,7 @@ export function feedRoutes(router: Router<Handler>): void { | ||
| 75 | 100 | }), |
| 76 | 101 | ); |
| 77 | 102 | return timePhase(ctx.request, "render", () => |
| 78 | json(page, 200, { "cache-control": "private, no-store" }), | |
| 103 | json(page, 200, headers), | |
| 79 | 104 | ); |
| 80 | 105 | }); |
| 81 | 106 |
test/feed-etag.test.ts added +248
| @@ -0,0 +1,248 @@ | ||
| 1 | /** GET /api/v1/feed answers 304 only when nothing the viewer would read has changed. */ | |
| 2 | import { afterAll, beforeAll, describe, expect, setSystemTime, test } from "bun:test"; | |
| 3 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { createApp, type App } from "../src/app.ts"; | |
| 7 | import { createRepo } from "../src/actions/repos.ts"; | |
| 8 | import { createToken } from "../src/core/tokens.ts"; | |
| 9 | import { recordInteraction, removeDismissals } from "../src/core/interactions.ts"; | |
| 10 | import { chipsOf, setChip } from "../src/core/feed_chips.ts"; | |
| 11 | import { addShare } from "../src/core/shares.ts"; | |
| 12 | import { createUser, type User } from "../src/core/users.ts"; | |
| 13 | import type { Repo } from "../src/core/repos.ts"; | |
| 14 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 15 | import { feedTag, TAG_BUCKET_MS } from "../src/feed/feed_tag.ts"; | |
| 16 | import { WAIT_LIMIT_MS } from "../src/core/pull_wait.ts"; | |
| 17 | import { promote } from "./helpers/members.ts"; | |
| 18 | ||
| 19 | const T0 = Date.parse("2026-10-26T12:00:30Z"); | |
| 20 | ||
| 21 | let dataDir: string; | |
| 22 | let app: App; | |
| 23 | let servers: Servers; | |
| 24 | let alice: User; | |
| 25 | let bob: User; | |
| 26 | let dana: User; | |
| 27 | let openRepo: Repo; | |
| 28 | let secretRepo: Repo; | |
| 29 | let tokens: Record<string, string>; | |
| 30 | ||
| 31 | const emit = (repo: Repo, kind: string, actor: User, payload = {}) => | |
| 32 | app.bus.append({ kind, repoId: repo.id, actorId: actor.id, payload }); | |
| 33 | ||
| 34 | const get = (who: string, path = "/api/v1/feed", etag?: string) => | |
| 35 | fetch(`${servers.url}${path}`, { | |
| 36 | headers: { | |
| 37 | authorization: `Bearer ${tokens[who]}`, | |
| 38 | ...(etag === undefined ? {} : { "if-none-match": etag }), | |
| 39 | }, | |
| 40 | }); | |
| 41 | ||
| 42 | /** The tag a first read gives, after draining its body. */ | |
| 43 | const tagOf = async (who: string, path?: string): Promise<string> => { | |
| 44 | const res = await get(who, path); | |
| 45 | await res.arrayBuffer(); | |
| 46 | expect(res.status).toBe(200); | |
| 47 | return res.headers.get("etag")!; | |
| 48 | }; | |
| 49 | ||
| 50 | beforeAll(async () => { | |
| 51 | setSystemTime(new Date(T0)); | |
| 52 | dataDir = await mkdtemp(join(tmpdir(), "forge-feed-etag-")); | |
| 53 | app = createApp({ dataDir, port: 0 }); | |
| 54 | servers = await serve(app); | |
| 55 | const mk = async (username: string, first = false) => { | |
| 56 | const r = await createUser(app.db, { | |
| 57 | origin: "seed", | |
| 58 | trusted: true, | |
| 59 | username, | |
| 60 | allowReserved: true, | |
| 61 | email: `${username}@forge.local`, | |
| 62 | password: "a-good-passphrase", | |
| 63 | adminIfFirst: first, | |
| 64 | }); | |
| 65 | if (!r.ok) throw new Error(r.error); | |
| 66 | promote(app.db, r.value); | |
| 67 | return r.value; | |
| 68 | }; | |
| 69 | await mk("root", true); | |
| 70 | alice = await mk("alice"); | |
| 71 | bob = await mk("bob"); | |
| 72 | dana = await mk("dana"); | |
| 73 | tokens = { | |
| 74 | alice: createToken(app.db, alice.id, "t", "rw").token, | |
| 75 | bob: createToken(app.db, bob.id, "t", "rw").token, | |
| 76 | dana: createToken(app.db, dana.id, "t", "rw").token, | |
| 77 | }; | |
| 78 | const s = await createRepo(app, alice, { name: "secret", visibility: "private" }); | |
| 79 | const o = await createRepo(app, alice, { name: "open", visibility: "public" }); | |
| 80 | if (!s.ok || !o.ok) throw new Error("repos"); | |
| 81 | secretRepo = s.value; | |
| 82 | openRepo = o.value; | |
| 83 | emit(openRepo, "run.finished", alice, { kind: "deploy", state: "ok", summary: "up", runId: "r1" }); | |
| 84 | }); | |
| 85 | ||
| 86 | afterAll(async () => { | |
| 87 | setSystemTime(); | |
| 88 | servers.stop(); | |
| 89 | app.close(); | |
| 90 | await rm(dataDir, { recursive: true, force: true }); | |
| 91 | }); | |
| 92 | ||
| 93 | describe("the feed's ETag", () => { | |
| 94 | test("200, then 304 with no body, the security headers and no render phase", async () => { | |
| 95 | const first = await get("bob"); | |
| 96 | await first.arrayBuffer(); | |
| 97 | const tag = first.headers.get("etag")!; | |
| 98 | expect(tag).toMatch(/^W\/"[0-9a-z]+"$/); | |
| 99 | expect(first.headers.get("server-timing")).toContain("render"); | |
| 100 | const again = await get("bob", "/api/v1/feed", tag); | |
| 101 | expect(again.status).toBe(304); | |
| 102 | expect(await again.text()).toBe(""); | |
| 103 | expect(again.headers.get("etag")).toBe(tag); | |
| 104 | expect(again.headers.get("x-content-type-options")).toBe("nosniff"); | |
| 105 | expect(again.headers.get("referrer-policy")).not.toBeNull(); | |
| 106 | expect(again.headers.get("content-security-policy")).not.toBeNull(); | |
| 107 | expect(again.headers.get("server-timing") ?? "").not.toContain("render"); | |
| 108 | expect((await get("bob", "/api/v1/feed", 'W/"other"')).status).toBe(200); | |
| 109 | }); | |
| 110 | ||
| 111 | test("a new event, a share, a dismissal and a mute each make the next read a 200", async () => { | |
| 112 | let tag = await tagOf("bob"); | |
| 113 | emit(openRepo, "run.finished", alice, { kind: "deploy", state: "ok", summary: "again", runId: "r2" }); | |
| 114 | expect((await get("bob", "/api/v1/feed", tag)).status).toBe(200); | |
| 115 | ||
| 116 | tag = await tagOf("bob"); | |
| 117 | expect((await get("bob", "/api/v1/feed", tag)).status).toBe(304); | |
| 118 | addShare(app.db, { | |
| 119 | itemId: "event:shared-1", | |
| 120 | senderId: alice.id, | |
| 121 | recipientId: bob.id, | |
| 122 | repoId: openRepo.id, | |
| 123 | kind: "run", | |
| 124 | link: "/alice/open", | |
| 125 | what: "a run", | |
| 126 | state: "delivered", | |
| 127 | at: Date.now(), | |
| 128 | }); | |
| 129 | expect((await get("bob", "/api/v1/feed", tag)).status).toBe(200); | |
| 130 | ||
| 131 | tag = await tagOf("bob"); | |
| 132 | recordInteraction(app.db, { | |
| 133 | viewerId: bob.id, | |
| 134 | itemId: "event:nothing", | |
| 135 | kind: "dismissed", | |
| 136 | repoId: openRepo.id, | |
| 137 | actorId: null, | |
| 138 | }); | |
| 139 | expect((await get("bob", "/api/v1/feed", tag)).status).toBe(200); | |
| 140 | ||
| 141 | tag = await tagOf("bob"); | |
| 142 | expect((await get("bob", "/api/v1/feed", tag)).status).toBe(304); | |
| 143 | recordInteraction(app.db, { | |
| 144 | viewerId: bob.id, | |
| 145 | itemId: "repo:x", | |
| 146 | kind: "muted", | |
| 147 | repoId: openRepo.id, | |
| 148 | actorId: null, | |
| 149 | }); | |
| 150 | expect((await get("bob", "/api/v1/feed", tag)).status).toBe(200); | |
| 151 | }); | |
| 152 | ||
| 153 | test("undoing a dismissal, which deletes a row, makes the next read a 200", async () => { | |
| 154 | for (const item of ["event:old", "event:newer"]) | |
| 155 | recordInteraction(app.db, { | |
| 156 | viewerId: dana.id, | |
| 157 | itemId: item, | |
| 158 | kind: "dismissed", | |
| 159 | repoId: openRepo.id, | |
| 160 | actorId: null, | |
| 161 | }); | |
| 162 | const mine = await tagOf("bob"); | |
| 163 | expect((await get("bob", "/api/v1/feed", mine)).status).toBe(304); | |
| 164 | removeDismissals(app.db, bob.id, ["event:nothing"]); | |
| 165 | expect((await get("bob", "/api/v1/feed", mine)).status).toBe(200); | |
| 166 | const held = await tagOf("dana"); | |
| 167 | removeDismissals(app.db, dana.id, ["event:old"]); | |
| 168 | expect((await get("dana", "/api/v1/feed", held)).status).toBe(200); | |
| 169 | }); | |
| 170 | ||
| 171 | test("a switched-off chip changes the tag, and so does the next minute and the 24 h wait line", async () => { | |
| 172 | const tag = await tagOf("alice"); | |
| 173 | expect((await get("alice", "/api/v1/feed", tag)).status).toBe(304); | |
| 174 | setChip(app.db, alice.id, { kind: "kind", value: "reviews" }, true); | |
| 175 | expect((await get("alice", "/api/v1/feed", tag)).status).toBe(200); | |
| 176 | setChip(app.db, alice.id, { kind: "kind", value: "reviews" }, false); | |
| 177 | ||
| 178 | const held = await tagOf("alice"); | |
| 179 | try { | |
| 180 | setSystemTime(new Date(T0 + TAG_BUCKET_MS)); | |
| 181 | expect((await get("alice", "/api/v1/feed", held)).status).toBe(200); | |
| 182 | const next = await tagOf("alice"); | |
| 183 | expect((await get("alice", "/api/v1/feed", next)).status).toBe(304); | |
| 184 | setSystemTime(new Date(T0 + WAIT_LIMIT_MS + 1)); | |
| 185 | expect((await get("alice", "/api/v1/feed", next)).status).toBe(200); | |
| 186 | } finally { | |
| 187 | setSystemTime(new Date(T0)); | |
| 188 | } | |
| 189 | }); | |
| 190 | ||
| 191 | test("a different limit or cursor gives a different tag, over the same data", async () => { | |
| 192 | const plain = await tagOf("alice"); | |
| 193 | const limited = await tagOf("alice", "/api/v1/feed?limit=1"); | |
| 194 | const cursored = await tagOf("alice", "/api/v1/feed?cursor=1.5%7Cevent:abc"); | |
| 195 | expect(new Set([plain, limited, cursored]).size).toBe(3); | |
| 196 | expect((await get("alice", "/api/v1/feed?limit=1", plain)).status).toBe(200); | |
| 197 | }); | |
| 198 | ||
| 199 | test("two viewers differ, and a private repository's news reaches only the one who can read it", async () => { | |
| 200 | expect(await tagOf("alice")).not.toBe(await tagOf("bob")); | |
| 201 | const bobTag = await tagOf("bob"); | |
| 202 | const aliceTag = await tagOf("alice"); | |
| 203 | emit(secretRepo, "run.finished", alice, { kind: "deploy", state: "failed", summary: "boom", runId: "r3" }); | |
| 204 | // The write clock is forge-wide, so both re-read; only alice's page changes. | |
| 205 | const forBob = await get("bob", "/api/v1/feed", bobTag); | |
| 206 | const forAlice = await get("alice", "/api/v1/feed", aliceTag); | |
| 207 | expect(forBob.status).toBe(200); | |
| 208 | expect(forAlice.status).toBe(200); | |
| 209 | expect(JSON.stringify(await forBob.json())).not.toContain("alice/secret"); | |
| 210 | expect(JSON.stringify(await forAlice.json())).toContain("alice/secret"); | |
| 211 | }); | |
| 212 | ||
| 213 | test("a viewer with shares still queued never gets a 304", async () => { | |
| 214 | for (let n = 0; n < 5; n++) | |
| 215 | addShare(app.db, { | |
| 216 | itemId: `event:today-${n}`, | |
| 217 | senderId: alice.id, | |
| 218 | recipientId: dana.id, | |
| 219 | repoId: openRepo.id, | |
| 220 | kind: "run", | |
| 221 | link: "/alice/open", | |
| 222 | what: "a run", | |
| 223 | state: "delivered", | |
| 224 | at: Date.now(), | |
| 225 | }); | |
| 226 | addShare(app.db, { | |
| 227 | itemId: "event:waiting", | |
| 228 | senderId: alice.id, | |
| 229 | recipientId: dana.id, | |
| 230 | repoId: openRepo.id, | |
| 231 | kind: "run", | |
| 232 | link: "/alice/open", | |
| 233 | what: "a run", | |
| 234 | state: "queued", | |
| 235 | at: Date.now(), | |
| 236 | }); | |
| 237 | const tag = await tagOf("dana"); | |
| 238 | expect((await get("dana", "/api/v1/feed", tag)).status).toBe(200); | |
| 239 | expect((await get("dana", "/api/v1/feed", tag)).status).toBe(200); | |
| 240 | }); | |
| 241 | ||
| 242 | test("feedTag takes `now`, and a tag in the same minute is the same", () => { | |
| 243 | const input = { chips: chipsOf(app.db, bob.id), cursor: null, limit: null }; | |
| 244 | const at = (now: number) => feedTag(app.db, bob, { ...input, now }); | |
| 245 | expect(at(T0)).toBe(at(T0 + 1000)); | |
| 246 | expect(at(T0)).not.toBe(at(T0 + 2 * TAG_BUCKET_MS)); | |
| 247 | }); | |
| 248 | }); |
Nit: no test covers the
undefinedbranch (inside a transaction, the handler sends no ETag and no 304). A smalldb.transaction(() => expect(feedTag(...)).toBeUndefined())()would pin it.