Public: anyone can read this; the crew can change it.

GET /api/v1/feed answers 304 only when nothing the viewer would read has changed: events, shares, dismissals, mutes, chips and the minute #6252

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/6218-get-api-v1-feed-answers-304-only-when-no → main 2 commits fork 4 threads · 4 unresolved

  • Merged. scout/6218-get-api-v1-feed-answers-304-only-when-no was merged into main as dc5d015.

Review

Sign in to review this change.
3 files changed +332 −1
src/feed/feed_tag.ts added +58
@@ -0,0 +1,58 @@
1 /**
2 * The weak ETag of one viewer's feed page, and whether the viewer has shares
3 * still waiting to be delivered.
4 */
5
6 import type { Database } from "bun:sqlite";
7 import type { Chips } from "../core/feed_chips.ts";
8 import type { User } from "../core/users.ts";
9 import { writeStamp } from "../core/write_clock.ts";
10
11 /** One minute: the grain at which time itself changes a feed (ages, waits, decay). */
12 export const TAG_BUCKET_MS = 60_000;
13
14 /**
15 * The tag of the page `GET /api/v1/feed` would answer: a hash of the viewer,
16 * the switched-off chips, the cursor, the limit, the forge's write clock
17 * (`writeStamp`: any insert, update or delete, so events, shares, dismissals
18 * and their undoing, mutes, watches, pins, settled issues and visibility all
19 * move it) and the minute of `now`. It is built from those, never from the
20 * body, so a 304 costs none of the ranking. A pull crossing the 24 h wait
21 * line changes the minute. Inside a transaction the clock has no value and
22 * there is no tag: the caller sends none and never answers 304. The caller
23 * also drains the viewer's share queue first and answers 200 while
24 * `hasQueuedShares` holds, since the queue's delivery depends on the clock.
25 */
26 export function feedTag(
27 db: Database,
28 viewer: User,
29 input: {
30 readonly chips: Chips;
31 readonly cursor: string | null;
32 readonly limit: number | null;
33 readonly now?: number;
34 },
35 ): string | undefined {
36 const stamp = writeStamp(db);
37 if (stamp === undefined) return undefined;

Nit: no test covers the undefined branch (inside a transaction, the handler sends no ETag and no 304). A small db.transaction(() => expect(feedTag(...)).toBeUndefined())() would pin it.

38 const now = input.now ?? Date.now();
39 const parts = [
outdated

This reads up to 100 events and runs the audience filter on every request just to probe for change, and it still misses non-event sources: watches, pins, asks, decisions, settled issues and visibility changes. writeStamp(db) is a single cheap query and covers all of them.

40 viewer.id,
41 [...input.chips.kinds].sort().join(","),
42 [...input.chips.people].sort().join(","),
43 input.cursor ?? "",
44 input.limit ?? "",
45 stamp,
46 Math.floor(now / TAG_BUCKET_MS),
47 ];
48 return `W/"${Bun.hash(JSON.stringify(parts)).toString(36)}"`;
outdated

MAX(id) only sees inserts. removeDismissals and removeMutes delete rows, so the newest id stays the same and the undone item stays hidden behind a 304 for up to a minute. Use writeStamp(db) instead of these probes; it counts deletes too.

49 }
50
51 /** Whether a share to the viewer is still queued behind their daily budget. */
52 export function hasQueuedShares(db: Database, viewerId: string): boolean {
53 return (
54 db
55 .query("SELECT 1 FROM share WHERE recipient_id = ?1 AND state = 'queued' LIMIT 1")
56 .get(viewerId) !== null
57 );
58 }
src/http/feed.ts +26 −1
@@ -21,6 +21,9 @@ import {
2121 } from "../core/interactions.ts";
2222 import { chipsOf, isKindChip, setChip } from "../core/feed_chips.ts";
2323 import { getUserByName, getUsersByIds } from "../core/users.ts";
24 import { drainQueue } from "../core/shares.ts";
25 import { feedTag, hasQueuedShares } from "../feed/feed_tag.ts";
26 import { holds } from "../web/feeds.ts";
2427 import { share, RECIPIENTS_MAX } from "../feed/share.ts";
2528 import { canRead, getRepo, getRepoById } from "../core/repos.ts";
2629 import {
@@ -67,6 +70,28 @@ export function feedRoutes(router: Router<Handler>): void {
6770 const cursor = rawCursor === null ? undefined : parseCursor(rawCursor);
6871 if (rawCursor !== null && cursor === undefined)
6972 return apiError("invalid_cursor", 422, "that cursor was not issued here");
73 // A 304 only when nothing the viewer would read has changed (`feedTag`).
74 // The queue drains first, as `readFeed` would; shares still waiting
75 // depend on the clock, so they always get a page.
76 const now = Date.now();
77 drainQueue(ctx.app.db, viewer.id, now);

Nit: readFeed drains the queue again right after this. The second drain is a no-op, but it does the same work twice on every 200. If a follow-up touches this, readFeed could take a drained flag, or the handler could rely on one drain.

78 const tag = feedTag(ctx.app.db, viewer, {
79 chips: chipsOf(ctx.app.db, viewer.id),
80 cursor: rawCursor,
81 limit: limit ?? null,
82 now,
83 });
84 const headers: Record<string, string> = {
85 "cache-control": "private, no-store",
86 vary: "authorization, cookie",
87 ...(tag === undefined ? {} : { etag: tag }),
88 };
89 if (
90 tag !== undefined &&
91 holds(ctx.request.headers.get("if-none-match"), tag) &&
92 !hasQueuedShares(ctx.app.db, viewer.id)
93 )
94 return new Response(null, { status: 304, headers });
7095 const page = timePhase(ctx.request, "feed", () =>
7196 readFeed(ctx.app.db, viewer, {
7297 phase: (name, ms) => notePhase(ctx.request, name, ms),
@@ -75,7 +100,7 @@ export function feedRoutes(router: Router<Handler>): void {
75100 }),
76101 );
77102 return timePhase(ctx.request, "render", () =>
78 json(page, 200, { "cache-control": "private, no-store" }),
103 json(page, 200, headers),
79104 );
80105 });
81106
test/feed-etag.test.ts added +248
@@ -0,0 +1,248 @@
1 /** GET /api/v1/feed answers 304 only when nothing the viewer would read has changed. */
2 import { afterAll, beforeAll, describe, expect, setSystemTime, test } from "bun:test";
3 import { mkdtemp, rm } from "node:fs/promises";
4 import { tmpdir } from "node:os";
5 import { join } from "node:path";
6 import { createApp, type App } from "../src/app.ts";
7 import { createRepo } from "../src/actions/repos.ts";
8 import { createToken } from "../src/core/tokens.ts";
9 import { recordInteraction, removeDismissals } from "../src/core/interactions.ts";
10 import { chipsOf, setChip } from "../src/core/feed_chips.ts";
11 import { addShare } from "../src/core/shares.ts";
12 import { createUser, type User } from "../src/core/users.ts";
13 import type { Repo } from "../src/core/repos.ts";
14 import { serve, type Servers } from "../src/http/server.ts";
15 import { feedTag, TAG_BUCKET_MS } from "../src/feed/feed_tag.ts";
16 import { WAIT_LIMIT_MS } from "../src/core/pull_wait.ts";
17 import { promote } from "./helpers/members.ts";
18
19 const T0 = Date.parse("2026-10-26T12:00:30Z");
20
21 let dataDir: string;
22 let app: App;
23 let servers: Servers;
24 let alice: User;
25 let bob: User;
26 let dana: User;
27 let openRepo: Repo;
28 let secretRepo: Repo;
29 let tokens: Record<string, string>;
30
31 const emit = (repo: Repo, kind: string, actor: User, payload = {}) =>
32 app.bus.append({ kind, repoId: repo.id, actorId: actor.id, payload });
33
34 const get = (who: string, path = "/api/v1/feed", etag?: string) =>
35 fetch(`${servers.url}${path}`, {
36 headers: {
37 authorization: `Bearer ${tokens[who]}`,
38 ...(etag === undefined ? {} : { "if-none-match": etag }),
39 },
40 });
41
42 /** The tag a first read gives, after draining its body. */
43 const tagOf = async (who: string, path?: string): Promise<string> => {
44 const res = await get(who, path);
45 await res.arrayBuffer();
46 expect(res.status).toBe(200);
47 return res.headers.get("etag")!;
48 };
49
50 beforeAll(async () => {
51 setSystemTime(new Date(T0));
52 dataDir = await mkdtemp(join(tmpdir(), "forge-feed-etag-"));
53 app = createApp({ dataDir, port: 0 });
54 servers = await serve(app);
55 const mk = async (username: string, first = false) => {
56 const r = await createUser(app.db, {
57 origin: "seed",
58 trusted: true,
59 username,
60 allowReserved: true,
61 email: `${username}@forge.local`,
62 password: "a-good-passphrase",
63 adminIfFirst: first,
64 });
65 if (!r.ok) throw new Error(r.error);
66 promote(app.db, r.value);
67 return r.value;
68 };
69 await mk("root", true);
70 alice = await mk("alice");
71 bob = await mk("bob");
72 dana = await mk("dana");
73 tokens = {
74 alice: createToken(app.db, alice.id, "t", "rw").token,
75 bob: createToken(app.db, bob.id, "t", "rw").token,
76 dana: createToken(app.db, dana.id, "t", "rw").token,
77 };
78 const s = await createRepo(app, alice, { name: "secret", visibility: "private" });
79 const o = await createRepo(app, alice, { name: "open", visibility: "public" });
80 if (!s.ok || !o.ok) throw new Error("repos");
81 secretRepo = s.value;
82 openRepo = o.value;
83 emit(openRepo, "run.finished", alice, { kind: "deploy", state: "ok", summary: "up", runId: "r1" });
84 });
85
86 afterAll(async () => {
87 setSystemTime();
88 servers.stop();
89 app.close();
90 await rm(dataDir, { recursive: true, force: true });
91 });
92
93 describe("the feed's ETag", () => {
94 test("200, then 304 with no body, the security headers and no render phase", async () => {
95 const first = await get("bob");
96 await first.arrayBuffer();
97 const tag = first.headers.get("etag")!;
98 expect(tag).toMatch(/^W\/"[0-9a-z]+"$/);
99 expect(first.headers.get("server-timing")).toContain("render");
100 const again = await get("bob", "/api/v1/feed", tag);
101 expect(again.status).toBe(304);
102 expect(await again.text()).toBe("");
103 expect(again.headers.get("etag")).toBe(tag);
104 expect(again.headers.get("x-content-type-options")).toBe("nosniff");
105 expect(again.headers.get("referrer-policy")).not.toBeNull();
106 expect(again.headers.get("content-security-policy")).not.toBeNull();
107 expect(again.headers.get("server-timing") ?? "").not.toContain("render");
108 expect((await get("bob", "/api/v1/feed", 'W/"other"')).status).toBe(200);
109 });
110
111 test("a new event, a share, a dismissal and a mute each make the next read a 200", async () => {
112 let tag = await tagOf("bob");
113 emit(openRepo, "run.finished", alice, { kind: "deploy", state: "ok", summary: "again", runId: "r2" });
114 expect((await get("bob", "/api/v1/feed", tag)).status).toBe(200);
115
116 tag = await tagOf("bob");
117 expect((await get("bob", "/api/v1/feed", tag)).status).toBe(304);
118 addShare(app.db, {
119 itemId: "event:shared-1",
120 senderId: alice.id,
121 recipientId: bob.id,
122 repoId: openRepo.id,
123 kind: "run",
124 link: "/alice/open",
125 what: "a run",
126 state: "delivered",
127 at: Date.now(),
128 });
129 expect((await get("bob", "/api/v1/feed", tag)).status).toBe(200);
130
131 tag = await tagOf("bob");
132 recordInteraction(app.db, {
133 viewerId: bob.id,
134 itemId: "event:nothing",
135 kind: "dismissed",
136 repoId: openRepo.id,
137 actorId: null,
138 });
139 expect((await get("bob", "/api/v1/feed", tag)).status).toBe(200);
140
141 tag = await tagOf("bob");
142 expect((await get("bob", "/api/v1/feed", tag)).status).toBe(304);
143 recordInteraction(app.db, {
144 viewerId: bob.id,
145 itemId: "repo:x",
146 kind: "muted",
147 repoId: openRepo.id,
148 actorId: null,
149 });
150 expect((await get("bob", "/api/v1/feed", tag)).status).toBe(200);
151 });
152
153 test("undoing a dismissal, which deletes a row, makes the next read a 200", async () => {
154 for (const item of ["event:old", "event:newer"])
155 recordInteraction(app.db, {
156 viewerId: dana.id,
157 itemId: item,
158 kind: "dismissed",
159 repoId: openRepo.id,
160 actorId: null,
161 });
162 const mine = await tagOf("bob");
163 expect((await get("bob", "/api/v1/feed", mine)).status).toBe(304);
164 removeDismissals(app.db, bob.id, ["event:nothing"]);
165 expect((await get("bob", "/api/v1/feed", mine)).status).toBe(200);
166 const held = await tagOf("dana");
167 removeDismissals(app.db, dana.id, ["event:old"]);
168 expect((await get("dana", "/api/v1/feed", held)).status).toBe(200);
169 });
170
171 test("a switched-off chip changes the tag, and so does the next minute and the 24 h wait line", async () => {
172 const tag = await tagOf("alice");
173 expect((await get("alice", "/api/v1/feed", tag)).status).toBe(304);
174 setChip(app.db, alice.id, { kind: "kind", value: "reviews" }, true);
175 expect((await get("alice", "/api/v1/feed", tag)).status).toBe(200);
176 setChip(app.db, alice.id, { kind: "kind", value: "reviews" }, false);
177
178 const held = await tagOf("alice");
179 try {
180 setSystemTime(new Date(T0 + TAG_BUCKET_MS));
181 expect((await get("alice", "/api/v1/feed", held)).status).toBe(200);
182 const next = await tagOf("alice");
183 expect((await get("alice", "/api/v1/feed", next)).status).toBe(304);
184 setSystemTime(new Date(T0 + WAIT_LIMIT_MS + 1));
185 expect((await get("alice", "/api/v1/feed", next)).status).toBe(200);
186 } finally {
187 setSystemTime(new Date(T0));
188 }
189 });
190
191 test("a different limit or cursor gives a different tag, over the same data", async () => {
192 const plain = await tagOf("alice");
193 const limited = await tagOf("alice", "/api/v1/feed?limit=1");
194 const cursored = await tagOf("alice", "/api/v1/feed?cursor=1.5%7Cevent:abc");
195 expect(new Set([plain, limited, cursored]).size).toBe(3);
196 expect((await get("alice", "/api/v1/feed?limit=1", plain)).status).toBe(200);
197 });
198
199 test("two viewers differ, and a private repository's news reaches only the one who can read it", async () => {
200 expect(await tagOf("alice")).not.toBe(await tagOf("bob"));
201 const bobTag = await tagOf("bob");
202 const aliceTag = await tagOf("alice");
203 emit(secretRepo, "run.finished", alice, { kind: "deploy", state: "failed", summary: "boom", runId: "r3" });
204 // The write clock is forge-wide, so both re-read; only alice's page changes.
205 const forBob = await get("bob", "/api/v1/feed", bobTag);
206 const forAlice = await get("alice", "/api/v1/feed", aliceTag);
207 expect(forBob.status).toBe(200);
208 expect(forAlice.status).toBe(200);
209 expect(JSON.stringify(await forBob.json())).not.toContain("alice/secret");
210 expect(JSON.stringify(await forAlice.json())).toContain("alice/secret");
211 });
212
213 test("a viewer with shares still queued never gets a 304", async () => {
214 for (let n = 0; n < 5; n++)
215 addShare(app.db, {
216 itemId: `event:today-${n}`,
217 senderId: alice.id,
218 recipientId: dana.id,
219 repoId: openRepo.id,
220 kind: "run",
221 link: "/alice/open",
222 what: "a run",
223 state: "delivered",
224 at: Date.now(),
225 });
226 addShare(app.db, {
227 itemId: "event:waiting",
228 senderId: alice.id,
229 recipientId: dana.id,
230 repoId: openRepo.id,
231 kind: "run",
232 link: "/alice/open",
233 what: "a run",
234 state: "queued",
235 at: Date.now(),
236 });
237 const tag = await tagOf("dana");
238 expect((await get("dana", "/api/v1/feed", tag)).status).toBe(200);
239 expect((await get("dana", "/api/v1/feed", tag)).status).toBe(200);
240 });
241
242 test("feedTag takes `now`, and a tag in the same minute is the same", () => {
243 const input = { chips: chipsOf(app.db, bob.id), cursor: null, limit: null };
244 const at = (now: number) => feedTag(app.db, bob, { ...input, now });
245 expect(at(T0)).toBe(at(T0 + 1000));
246 expect(at(T0)).not.toBe(at(T0 + 2 * TAG_BUCKET_MS));
247 });
248 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs