Public: anyone can read this; the crew can change it.

GET /api/v1/feed answers 304 to If-None-Match when the viewer's feed has not changed #5886

Closed opened this
Closed without merging

scout/forge:scout/5875-get-api-v1-feed-answers-304-to-if-none-m → main 3 commits fork 6 threads · 6 unresolved

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
3 files changed +253 −1
src/feed/feed_tag.ts added +59
@@ -0,0 +1,59 @@
1 /**
2 * The feed answer's ETag: who asked, which chips are off, where the page
3 * starts, how long it is, and the newest event this viewer can see. Never
4 * the body — a poll that holds the tag costs a header, not a render.
5 */
6
7 import type { Database } from "bun:sqlite";
8 import { chipsOf } from "../core/feed_chips.ts";
9 import { listEventsVisibleTo } from "../core/events.ts";
10 import type { User } from "../core/users.ts";
11
12 /**
13 * The tag, and whether the viewer has shares waiting: those are promoted by

This comment describes feedTag's return value but sits on TAG_BUCKET_MS. Move it to feedTag, and give the constant a one-liner of its own.

14 * `readFeed` (`drainQueue`), so while any wait the answer must be read.
15 */
16 export const TAG_BUCKET_MS = 60_000;
17
18 export function feedTag(
outdated

This is the only thing that can move the tag besides chips and paging. Shares delivered to the viewer (no event is emitted), dismissals and mutes all change readFeed's answer without moving it, so a held tag gets a stale 304. Fold the viewer's newest share, dismissal and mute rows into parts.

19 db: Database,
20 viewer: User,
21 page: { limit?: number; cursor?: string },
22 now = Date.now(),
23 ): { etag: string; queued: boolean } {
24 const chips = chipsOf(db, viewer.id);
25 const newest = listEventsVisibleTo(db, viewer, { limit: 1 })[0]?.id ?? "none";
26 // The viewer's own state, which moves the answer without an event: shares
27 // delivered to them, and what they dismissed or muted.
28 const shares = db
29 .query<{ n: number; last: string | null; queued: number }, [string]>(
30 `SELECT COUNT(*) FILTER (WHERE state = 'delivered') AS n,
31 MAX(id) FILTER (WHERE state = 'delivered') AS last,
32 COUNT(*) FILTER (WHERE state = 'queued') AS queued
33 FROM share WHERE recipient_id = ?1`,
34 )
35 .get(viewer.id)!;
36 const acts = db
37 .query<{ n: number; last: string | null }, [string]>(
38 `SELECT COUNT(*) AS n, MAX(id) AS last FROM interaction
39 WHERE viewer_id = ?1 AND kind IN ('dismissed', 'muted')`,
40 )
41 .get(viewer.id)!;
42 // Asks arise from age alone (a pull idle past its wait limit) and the rank
43 // reads the clock: a held tag lasts one bucket, never longer.
44 const parts = [
45 `t${Math.floor(now / TAG_BUCKET_MS)}`,
46 `s${shares.n}.${shares.last ?? ""}`,
47 `i${acts.n}.${acts.last ?? ""}`,
outdated

The tag covers events, shares, dismissals/mutes and chips, but not time. Asks in this answer appear by age alone (waiting: asks after WAIT_LIMIT_MS, src/web/studio/asks.ts:495), and rank/consequence scores take now. Add a time component (a minute bucket, or the next wait deadline) here, with now injectable for the test.

48 viewer.id,
49 [...chips.kinds].sort().join(","),
50 [...chips.people].sort().join(","),
51 page.cursor ?? "",
52 page.limit ?? "",
53 newest,
54 ];
55 return {
56 etag: `W/"${Bun.hash(parts.join("|")).toString(36)}"`,
57 queued: shares.queued > 0,
58 };
59 }
src/http/feed.ts +11 −1
@@ -38,6 +38,8 @@ import {
3838 } from "../actions/asks_bulk.ts";
3939 import type { FleetError } from "../actions/fleet.ts";
4040 import { notePhase, takeSessionTime, timePhase } from "../core/route_timing.ts";
41 import { feedTag } from "../feed/feed_tag.ts";
42 import { holds } from "./etag.ts";
4143 import type { Handler } from "./context.ts";
4244 import type { Router } from "./router.ts";
4345 import {
@@ -67,6 +69,14 @@ export function feedRoutes(router: Router<Handler>): void {
6769 const cursor = rawCursor === null ? undefined : parseCursor(rawCursor);
6870 if (rawCursor !== null && cursor === undefined)
6971 return apiError("invalid_cursor", 422, "that cursor was not issued here");
72 const { etag, queued } = timePhase(ctx.request, "tag", () =>
73 feedTag(ctx.app.db, viewer, {
74 ...(limit === undefined ? {} : { limit }),
75 ...(rawCursor === null ? {} : { cursor: rawCursor }),
76 }),
77 );
78 const headers = { "cache-control": "private, no-store", etag };
79 if (!queued && holds(ctx.request, etag)) return new Response(null, { status: 304, headers });
outdated

This 304 also skips drainQueue (called inside readFeed), so a queued share is never promoted for a client that polls with its tag. Either don't answer 304 while the viewer has queued shares, or drain before computing the tag.

With the tag missing time, this 304 can hold indefinitely for a viewer whose pull just crossed the 24h wait line: the new ask is never read. Once the tag includes time, add a test here that crosses the boundary and expects a 200.

7080 const page = timePhase(ctx.request, "feed", () =>
7181 readFeed(ctx.app.db, viewer, {
7282 ...(limit === undefined ? {} : { limit }),
@@ -74,7 +84,7 @@ export function feedRoutes(router: Router<Handler>): void {
7484 }),
7585 );
7686 return timePhase(ctx.request, "render", () =>
77 json(page, 200, { "cache-control": "private, no-store" }),
87 json(page, 200, headers),
7888 );
7989 });
8090
test/feed-etag.test.ts added +183
@@ -0,0 +1,183 @@
1 /** The feed answers 304 to a poll that has nothing new, and never shares a tag between viewers. */
2 import { afterAll, beforeAll, expect, test } from "bun:test";
3 import { mkdtemp, rm } from "node:fs/promises";
4 import { tmpdir } from "node:os";
5 import { join } from "node:path";
6 import { createApp, type App } from "../src/app.ts";
7 import { createRepo } from "../src/actions/repos.ts";
8 import { createToken } from "../src/core/tokens.ts";
9 import { createUser, getUserByName, type User } from "../src/core/users.ts";
10 import { feedTag, TAG_BUCKET_MS } from "../src/feed/feed_tag.ts";
11 import { share } from "../src/feed/share.ts";
12 import { addShare } from "../src/core/shares.ts";
13 import { serve, type Servers } from "../src/http/server.ts";
14 import { promote } from "./helpers/members.ts";
15 import { quietLogs } from "./helpers/globals.ts";
16
17 quietLogs();
18
19 let dir: string;
20 let app: App;
21 let servers: Servers;
22 let alice: User;
23 let bobToken: string;
24 let aliceToken: string;
25 let repoId: string;
26
27 const get = (token: string, path = "/api/v1/feed", inm?: string) =>
28 fetch(`${servers.url}${path}`, {
29 headers: { authorization: `Bearer ${token}`, ...(inm ? { "if-none-match": inm } : {}) },
30 });
31 const emit = () =>
32 app.bus.append({ kind: "look.taken", repoId, actorId: alice.id, payload: {} });
33
34 beforeAll(async () => {
35 dir = await mkdtemp(join(tmpdir(), "forge-feed-etag-"));
36 app = createApp({ dataDir: dir, port: 0 });
37 servers = await serve(app);
38 const mk = async (username: string, first = false) => {
39 const r = await createUser(app.db, {
40 origin: "seed", trusted: true, username, allowReserved: true,
41 email: `${username}@forge.local`, password: "a-good-passphrase", adminIfFirst: first,
42 });
43 if (!r.ok) throw new Error(r.error);
44 promote(app.db, r.value);
45 return r.value;
46 };
47 await mk("root", true);
48 alice = await mk("alice");
49 const bob = await mk("bob");
50 aliceToken = createToken(app.db, alice.id, "t", "rw").token;
51 bobToken = createToken(app.db, bob.id, "t", "rw").token;
52 const r = await createRepo(app, alice, { name: "secret", visibility: "private" });
53 if (!r.ok) throw new Error("repo");
54 repoId = r.value.id;
55 });
56 afterAll(async () => {
57 servers.stop();
58 app.close();
59 await rm(dir, { recursive: true, force: true });
60 });
61
62 test("200 with a weak tag, then 304 with no body, headers and no render", async () => {
63 const first = await get(aliceToken);
64 await first.arrayBuffer();
65 const tag = first.headers.get("etag")!;
66 expect(first.status).toBe(200);
67 expect(tag).toMatch(/^W\/"/);
68 const again = await get(aliceToken, "/api/v1/feed", tag);
69 expect(again.status).toBe(304);

Flaky at a minute boundary: if the first request lands at :59.999 and this one at :00.000, the bucket moves and the answer is 200. Pin the clock with setSystemTime from bun:test in beforeAll, and reset it in afterAll.

70 expect(await again.text()).toBe("");
71 expect(again.headers.get("etag")).toBe(tag);
72 expect(again.headers.get("x-content-type-options")).toBe("nosniff");
73 expect(again.headers.get("referrer-policy")).toBeTruthy();
74 expect(again.headers.get("content-security-policy")).toBeTruthy();
75 const phases = (again.headers.get("server-timing") ?? "").split(", ").map((p) => p.split(";")[0]);
76 expect(phases).not.toContain("render");
77 });
78
79 test("a new event the viewer can see gives a new tag and a 200", async () => {
80 const tag = (await get(aliceToken)).headers.get("etag")!;
81 emit();
82 const res = await get(aliceToken, "/api/v1/feed", tag);
83 await res.arrayBuffer();
84 expect(res.status).toBe(200);
85 expect(res.headers.get("etag")).not.toBe(tag);
86 });
87
88 test("a different chip, cursor or limit gives a different tag", async () => {
89 const plain = (await get(aliceToken)).headers.get("etag");
90 const limited = (await get(aliceToken, "/api/v1/feed?limit=5")).headers.get("etag");
91 expect(limited).not.toBe(plain);
92 await fetch(`${servers.url}/api/v1/feed/chips`, {
93 method: "POST",
94 headers: { authorization: `Bearer ${aliceToken}`, "content-type": "application/json" },
95 body: JSON.stringify({ kind: "kind", value: "crew", hidden: true }),
96 });
97 const chipped = (await get(aliceToken)).headers.get("etag");
98 expect(chipped).not.toBe(plain);
99 });
100
101 test("two viewers never share a tag for different answers", async () => {
102 const a = (await get(aliceToken)).headers.get("etag")!;
103 const b = await get(bobToken, "/api/v1/feed", a);
104 await b.arrayBuffer();
105 expect(b.status).toBe(200);
106 expect(b.headers.get("etag")).not.toBe(a);
107 // Bob cannot see the private repo's new event: his tag holds still, Alice's moves.
108 const bobTag = b.headers.get("etag")!;
109 emit();
110 expect((await get(bobToken, "/api/v1/feed", bobTag)).status).toBe(304);

Same wall-clock exposure as line 69: Bob's 304 depends on the minute not turning between his two requests. The pinned clock fixes both.

111 expect((await get(aliceToken, "/api/v1/feed", a)).status).toBe(200);
112 });
113
114 const openEvent = async () => {
115 const r = await createRepo(app, alice, { name: `open${Math.random().toString(36).slice(2, 8)}`, visibility: "public" });
116 if (!r.ok) throw new Error("repo");
117 return app.bus.append({ kind: "look.taken", repoId: r.value.id, actorId: alice.id, payload: {} });
118 };
119
120 test("a different cursor gives a different tag", async () => {
121 await fetch(`${servers.url}/api/v1/feed/chips`, {
122 method: "POST",
123 headers: { authorization: `Bearer ${aliceToken}`, "content-type": "application/json" },
124 body: JSON.stringify({ kind: "kind", value: "crew", hidden: false }),
125 });
126 await openEvent();
127 await openEvent();
128 const page = await get(aliceToken, "/api/v1/feed?limit=1");
129 const body = (await page.json()) as { next_cursor: string | null };
130 expect(body.next_cursor).not.toBeNull();
131 const next = await get(aliceToken, `/api/v1/feed?limit=1&cursor=${encodeURIComponent(body.next_cursor!)}`);
132 await next.arrayBuffer();
133 expect(next.status).toBe(200);
134 expect(next.headers.get("etag")).not.toBe(page.headers.get("etag"));
135 });
136
137 test("a share delivered to the viewer moves her tag", async () => {
138 const ev = await openEvent();
139 const tag = (await get(aliceToken)).headers.get("etag")!;
140 const made = await share(app, getUserByName(app.db, "bob")!, `event:${ev.id}`, ["alice"]);
141 expect(made.ok).toBe(true);
142 const res = await get(aliceToken, "/api/v1/feed", tag);
143 await res.arrayBuffer();
144 expect(res.status).toBe(200);
145 expect(res.headers.get("etag")).not.toBe(tag);
146 });
147
148 test("a dismissal moves the viewer's tag", async () => {
149 const ev = await openEvent();
150 const tag = (await get(aliceToken)).headers.get("etag")!;
151 const done = await fetch(`${servers.url}/api/v1/feed/interactions`, {
152 method: "POST",
153 headers: { authorization: `Bearer ${aliceToken}`, "content-type": "application/json" },
154 body: JSON.stringify({ item: `event:${ev.id}`, kind: "dismissed" }),
155 });
156 expect(done.status).toBe(201);
157 const res = await get(aliceToken, "/api/v1/feed", tag);
158 await res.arrayBuffer();
159 expect(res.status).toBe(200);
160 });
161
162 test("while a share waits in the queue the feed is read, so it can be promoted", async () => {
163 const ev = await openEvent();
164 const bob = getUserByName(app.db, "bob")!;
165 addShare(app.db, {
166 itemId: `event:${ev.id}`, senderId: bob.id, recipientId: alice.id, repoId: ev.repoId!,
167 kind: "look", link: "/", what: "x", state: "queued", at: Date.now(),
168 });
169 const first = await get(aliceToken);
170 await first.arrayBuffer();
171 const tag = first.headers.get("etag")!;
172 const again = await get(aliceToken, "/api/v1/feed", tag);
173 await again.arrayBuffer();
174 expect(again.status).toBe(200);
175 });
176
177 test("the tag moves with the clock: past the bucket, the same feed has a new tag", () => {
178 const page = {};
179 const t0 = Date.parse("2026-10-25T12:00:10Z");
180 const a = feedTag(app.db, alice, page, t0).etag;
181 expect(feedTag(app.db, alice, page, t0 + 20_000).etag).toBe(a);
182 expect(feedTag(app.db, alice, page, t0 + TAG_BUCKET_MS).etag).not.toBe(a);
183 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs