trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRuns11g rEventsFlows

Offline-first feed: a Mac with no wifi still shows the last feed and keeps your tells #371

Merged scoutcrew opened this

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

scout/forge:scout/363-offline-first-feed-a-mac-with-no-wifi-st → main 1 commit fork

  • Merged. scout/363-offline-first-feed-a-mac-with-no-wifi-st was merged into main as 37391dc.

Review

Sign in to review this change.
11 files changed +563 −19
docs/FEED.md +23
@@ -52,3 +52,26 @@ the app's `tell`), Dismiss and Mute app (the interactions API, same-origin).
5252 Keys: `j`/`k`/Enter are the shell's, `x` dismisses the focused card.
5353 Not yet: the `choose` deck card (nothing in the feed produces one), stop from
5454 a card, and paging past the first 20.
55
56 ## Offline (`src/web/sw.ts`, `outbox.ts`)
57
58 - `/sw.js` (root scope, `no-cache`) keeps `/` network-first and serves the
59 kept copy when the network fails, with `data-stale="1"` so the note above the
60 feed says "Offline: this feed is from 12 min ago". The stylesheet, `/assets/*`
61 and icons are cache-first. Nothing else is touched: the API, `/events`,
62 `/live/*` and every POST go to the network. Signing out clears the cache and
63 the outbox.
64 - Dismiss, mute, open and tell from a card go into an outbox in
65 `localStorage` (200 at most), each with an `Idempotency-Key` made when it is
66 queued, and are flushed on open, on `online` and after each add. No Background
67 Sync (Safari has none). Entries leave on any 2xx or a final 4xx; a network
68 error, 5xx, 408, 429 or 401 keeps them in order.
69 - Browser sessions now honour `Idempotency-Key` on `/api/v1/` mutations, scoped
70 to the person (as tokens' are to the token): a replay returns the first
71 answer. `POST /api/v1/feed/tells { app, text }` is the JSON tell the outbox
72 sends.
73 - The live client reopens `/events?since=<last id>` when the browser gives up
74 on the stream, and on `online`, and refreshes every region once.
75 - Not done: steering a run offline (a plain form post, needs the network),
76 updating the kept `/` from live swaps (it is the page as last navigated),
77 swipes (no swipe UI yet), push, and anything native.
forge.json +1 −1
@@ -6,7 +6,7 @@
66 "ready": 90
77 },
88 "check": {
9 "run": "bun x tsc --noEmit && bun scripts/test-shards.ts --timeout 60000 test/crew.test.ts test/crew-wrapup.test.ts test/crew-limits.test.ts test/crew-sandbox.test.ts test/steer.test.ts test/landing.test.ts test/decide.test.ts test/ledger.test.ts test/checks.test.ts test/dependencies.test.ts test/autonomy.test.ts test/studio.test.ts test/studio-words.test.ts test/forks.test.ts test/fork-flow.test.ts test/cross-repo-pull.test.ts test/pulls.test.ts test/apps.test.ts test/apps-adversarial.test.ts test/apps-busy.test.ts test/web.test.ts test/git-trust.test.ts test/helpers.test.ts test/shards.test.ts test/migrate.test.ts test/vitals.test.ts test/asks.test.ts test/feed.test.ts test/feed-page.test.ts",
9 "run": "bun x tsc --noEmit && bun scripts/test-shards.ts --timeout 60000 test/crew.test.ts test/crew-wrapup.test.ts test/crew-limits.test.ts test/crew-sandbox.test.ts test/steer.test.ts test/landing.test.ts test/decide.test.ts test/ledger.test.ts test/checks.test.ts test/dependencies.test.ts test/autonomy.test.ts test/studio.test.ts test/studio-words.test.ts test/forks.test.ts test/fork-flow.test.ts test/cross-repo-pull.test.ts test/pulls.test.ts test/apps.test.ts test/apps-adversarial.test.ts test/apps-busy.test.ts test/web.test.ts test/git-trust.test.ts test/helpers.test.ts test/shards.test.ts test/migrate.test.ts test/vitals.test.ts test/asks.test.ts test/feed.test.ts test/feed-page.test.ts test/offline.test.ts",
1010 "timeout": 1800
1111 }
1212 }
src/http/feed.ts +22
@@ -3,6 +3,7 @@
33 * are the signed-in viewer's own — there is no way to ask for another's.
44 */
55
6 import { tellCrew } from "../actions/crew.ts";
67 import { audienceFilter } from "../core/event_audience.ts";
78 import { getEvent } from "../core/events.ts";
89 import {
@@ -97,4 +98,25 @@ export function feedRoutes(router: Router<Handler>): void {
9798 });
9899 return json({ id: made.id }, 201);
99100 });
101
102 // A tell from a card, as JSON, so the outbox can replay it by key.
103 router.add("POST", "/api/v1/feed/tells", async (ctx) => {
104 const denied = requireScope(ctx, "rw");
105 if (denied) return denied;
106 const viewer = ctx.auth.user!;
107 const input = await body(ctx.request);
108 const text = str(input.text).trim();
109 if (text === "" || text.length > 2000)
110 return apiError("invalid_text", 422, "text is 1 to 2000 characters");
111 const [owner, name, extra] = str(input.app).split("/");
112 const repo =
113 owner && name && extra === undefined
114 ? getRepo(ctx.app.db, owner, name)
115 : undefined;
116 if (!repo || !canRead(ctx.app.db, repo, viewer)) return notFound();
117 const told = await tellCrew(ctx.app, viewer, repo, text);
118 if (!told.ok)
119 return apiError("refused", told.error === "unauthorized" ? 403 : 422, "that could not be told");
120 return json({ issue: told.value.issue.number }, 201);
121 });
100122 }
src/http/server.ts +11 −2
@@ -107,6 +107,7 @@ import {
107107 } from "../web/studio/actions.ts";
108108 import { asksPage } from "../web/studio/asks_page.ts";
109109 import { liveFeedFragment } from "../web/studio/feed.ts";
110 import { serviceWorker } from "../web/sw.ts";
110111 import { liveAppsFragment, liveOnYouFragment } from "../web/studio/home.ts";
111112 import { appNowFragment } from "../web/studio/now.ts";
112113 import { appStateFragment } from "../web/studio/previews.ts";
@@ -405,6 +406,9 @@ export function buildRouter(): Router<Handler> {
405406 if (asset === null || ctx.params.name !== asset.name) return notFound();
406407 return assetResponse(asset);
407408 });
409 // The service worker: root scope needs a root path, and it must never be
410 // cached by the browser's HTTP cache or an update would wait a day.
411 router.add("GET", "/sw.js", serviceWorker);
408412 router.add("GET", "/", home);
409413 router.add("GET", "/signin", signinPage);
410414 router.add("POST", "/signin", signinSubmit);
@@ -1416,10 +1420,15 @@ async function answerGet(
14161420 mode,
14171421 });
14181422
1423 // A signed-in browser's outbox replays by key too (web/outbox.ts): the
1424 // key is scoped to the person, like a token's is to the token.
1425 const replayAs =
1426 credential ??
1427 (auth.via === "session" && auth.user ? `user:${auth.user.id}` : undefined);
14191428 const response =
1420 credential === undefined
1429 replayAs === undefined
14211430 ? await invoke(request)
1422 : await withIdempotency(app, request, url, credential, invoke);
1431 : await withIdempotency(app, request, url, replayAs, invoke);
14231432 return withRateHeaders(
14241433 withModeCookie(response, modeOverride, url.protocol === "https:"),
14251434 verdict,
src/web/client.ts +28 −4
@@ -2119,12 +2119,28 @@ export const liveRegionsJs = /* js */ `
21192119 });
21202120 Object.keys(groups).forEach(function (src) { pull(src, groups[src]); });
21212121 }
2122 var source = new EventSource('/events');
2122 var source = null;
2123 var since = '';
2124 var retry = null;
21232125 // The stream's own state, for a page that shows whether it is connected.
21242126 function state(s) { document.dispatchEvent(new CustomEvent('forge:live', { detail: { state: s } })); }
2125 source.addEventListener('open', function () { state('open'); });
2126 source.addEventListener('error', function () { state('reconnecting'); });
2127 source.addEventListener('forge', function (event) {
2127 // The browser resumes a dropped stream by itself, from the last id it
2128 // saw. When it gives up (CLOSED: a long offline, a sleeping laptop) we
2129 // open a new one ourselves, from that same id: nothing is replayed twice
2130 // and nothing in between is lost.
2131 function connect() {
2132 if (retry) { clearTimeout(retry); retry = null; }
2133 if (source) source.close();
2134 source = since ? new EventSource('/events?since=' + encodeURIComponent(since)) : new EventSource('/events');
2135 source.addEventListener('open', function () { state('open'); });
2136 source.addEventListener('error', function () {
2137 state('reconnecting');
2138 if (source.readyState === 2 && retry === null) retry = setTimeout(connect, 3000);
2139 });
2140 source.addEventListener('forge', onForge);
2141 }
2142 function onForge(event) {
2143 if (event.lastEventId) since = event.lastEventId;
21282144 var data;
21292145 try { data = JSON.parse(event.data); } catch (err) { return; }
21302146 document.dispatchEvent(new CustomEvent('forge:event', { detail: data }));
@@ -2138,6 +2154,14 @@ export const liveRegionsJs = /* js */ `
21382154 // after the first, never more often than every MIN_GAP.
21392155 if (hit && timer === null)
21402156 timer = setTimeout(refresh, Math.max(600, last + MIN_GAP - Date.now()));
2157 }
2158 connect();
2159 // Back online: reconnect from the last id and look at every region once,
2160 // in case the replay window no longer reaches back that far.
2161 window.addEventListener('online', function () {
2162 connect();
2163 Array.prototype.forEach.call(document.querySelectorAll('[data-live]'), function (r) { if (r.id) due[r.id] = true; });
2164 if (timer === null) timer = setTimeout(refresh, 300);
21412165 });
21422166 document.addEventListener('visibilitychange', function () {
21432167 if (!document.hidden && Object.keys(due).length > 0 && timer === null) timer = setTimeout(refresh, 100);
src/web/layout.ts +1 −1
@@ -616,7 +616,7 @@ export function page(options: PageOptions, status = 200): Response {
616616 // The external stylesheet plus this page's own nonce'd <style>/<script>
617617 // are the only resources allowed; connect-src 'self' lets the live
618618 // event stream (EventSource) and fragment fetches reach same-origin.
619 "content-security-policy": `default-src 'none'; style-src 'self' 'nonce-${nonce}'; script-src 'nonce-${nonce}'${options.scriptAsset === undefined ? "" : " 'self'"}; connect-src 'self'; manifest-src 'self'; img-src 'self' data:; form-action 'self'; base-uri 'none'; frame-ancestors 'none'`,
619 "content-security-policy": `default-src 'none'; style-src 'self' 'nonce-${nonce}'; script-src 'nonce-${nonce}'${options.scriptAsset === undefined ? "" : " 'self'"}; connect-src 'self'; worker-src 'self'; manifest-src 'self'; img-src 'self' data:; form-action 'self'; base-uri 'none'; frame-ancestors 'none'`,
620620 "referrer-policy": "same-origin",
621621 "x-content-type-options": "nosniff",
622622 });
src/web/outbox.ts added +78
@@ -0,0 +1,78 @@
1 /**
2 * The outbox: what a person did while the network was away, kept in the
3 * browser and sent when it is back. Safari has no Background Sync, so
4 * nothing here waits for the browser to wake us: the page flushes when it
5 * opens, on `online`, and after every add.
6 *
7 * Every entry carries its own Idempotency-Key, made once when the act is
8 * queued. The server answers a repeated key with the first answer
9 * (`core/idempotency.ts`), so a request that landed but whose answer was
10 * lost is replayed without doing it twice. Entries leave the queue when the
11 * server has decided: any 2xx, or a 4xx that retrying cannot fix. A network
12 * failure, 5xx, 408, 429 or 401 stops the flush and keeps the rest, in order.
13 *
14 * Shipped to the page as source text (like the other client scripts) and
15 * evaluated by the test, so the code that runs is the code that is tested.
16 */
17
18 export const OUTBOX_KEY = "forge.outbox";
19 export const OUTBOX_MAX = 200;
20
21 export const outboxJs = /* js */ `
22 function forgeOutbox(storage, send, now) {
23 var KEY = ${JSON.stringify(OUTBOX_KEY)};
24 var MAX = ${OUTBOX_MAX};
25 var flushing = null;
26 function read() {
27 try {
28 var list = JSON.parse(storage.getItem(KEY) || '[]');
29 return Array.isArray(list) ? list : [];
30 } catch (e) { return []; }
31 }
32 function write(list) {
33 try { storage.setItem(KEY, JSON.stringify(list)); } catch (e) {}
34 }
35 function makeKey() {
36 if (typeof crypto !== 'undefined' && crypto.randomUUID) return crypto.randomUUID();
37 return 'k' + Math.random().toString(36).slice(2) + Date.now().toString(36);
38 }
39 function final(status) {
40 if (status >= 200 && status < 300) return true;
41 if (status >= 400 && status < 500) return status !== 401 && status !== 408 && status !== 429;
42 return false;
43 }
44 function add(url, payload) {
45 var list = read();
46 list.push({ key: makeKey(), url: url, body: JSON.stringify(payload), at: (now || Date.now)() });
47 write(list.slice(-MAX));
48 }
49 function run() {
50 var list = read();
51 if (list.length === 0) return Promise.resolve({ sent: 0, left: 0 });
52 var sent = 0;
53 function next() {
54 var entry = read()[0];
55 if (!entry) return Promise.resolve();
56 return send(entry.url, {
57 method: 'POST', credentials: 'same-origin', keepalive: true,
58 headers: { 'content-type': 'application/json', 'idempotency-key': entry.key },
59 body: entry.body
60 }).then(function (r) {
61 if (!final(r.status)) return;
62 write(read().filter(function (e) { return e.key !== entry.key; }));
63 sent++;
64 return next();
65 }, function () {});
66 }
67 return next().then(function () { return { sent: sent, left: read().length }; });
68 }
69 // One flush at a time: two tabs of one page share storage, two flushes in
70 // one page must not send an entry twice before the first has been dropped.
71 function flush() {
72 if (!flushing) flushing = run().then(function (r) { flushing = null; return r; }, function () { flushing = null; return { sent: 0, left: read().length }; });
73 return flushing;
74 }
75 function clear() { try { storage.removeItem(KEY); } catch (e) {} }
76 return { add: add, flush: flush, size: function () { return read().length; }, clear: clear };
77 }
78 `;
src/web/studio/feed.ts +84 −10
@@ -13,6 +13,7 @@ import { readFeed } from "../../feed/feed.ts";
1313 import type { FeedItem } from "../../feed/rank.ts";
1414 import { html, type Html } from "../html.ts";
1515 import { fragment } from "../layout.ts";
16 import { outboxJs } from "../outbox.ts";
1617 import { btn, empty } from "../ui.ts";
1718
1819 /** Events that can change what the feed says. */
@@ -62,6 +63,22 @@ export function feedCard(item: FeedItem): Html {
6263 </article>`;
6364 }
6465
66 /**
67 * One line above the feed, empty unless there is something to say: the feed
68 * on screen is old (the service worker served it with no network), or acts
69 * are waiting in the outbox. The script fills it; `hidden` until then.
70 */
71 export function feedNote(): Html {
72 return html`<p
73 id="feed-note"
74 class="feed-note small muted"
75 role="status"
76 data-rendered="${String(Date.now())}"
77 data-stale="0"
78 hidden
79 ></p>`;
80 }
81
6582 /** The region: cards, or one line saying there is nothing to do. */
6683 export function feedRegion(user: User, ctx: Ctx): Html {
6784 const { items } = readFeed(ctx.app.db, user, { limit: FEED_FIRST_PAGE });
@@ -95,15 +112,43 @@ export function liveFeedFragment(ctx: Ctx): Response {
95112 * Interactions go to the feed API, same-origin with the session cookie.
96113 */
97114 export const feedJs = /* js */ `
115 ${outboxJs}
98116 (function () {
99117 var feed = document.getElementById('live-feed');
100118 if (!feed || typeof fetch === 'undefined') return;
101 function send(card, kind, keepalive) {
102 var body = { item: card.getAttribute('data-item'), kind: kind, app: card.getAttribute('data-app') };
103 return fetch('/api/v1/feed/interactions', {
104 method: 'POST', credentials: 'same-origin', keepalive: !!keepalive,
105 headers: { 'content-type': 'application/json' }, body: JSON.stringify(body)
106 });
119 var note = document.getElementById('feed-note');
120 var outbox = null;
121 try { outbox = forgeOutbox(window.localStorage, function (u, o) { return fetch(u, o); }); } catch (e) {}
122 function ago(ms) {
123 var m = Math.max(0, Math.round((Date.now() - ms) / 60000));
124 if (m < 1) return 'just now';
125 if (m < 60) return m + ' min ago';
126 var h = Math.round(m / 60);
127 return h < 48 ? h + ' h ago' : Math.round(h / 24) + ' days ago';
128 }
129 function paint() {
130 if (!note) return;
131 var parts = [];
132 if (note && note.getAttribute('data-stale') === '1')
133 parts.push('Offline: this feed is from ' + ago(Number(note.getAttribute('data-rendered'))) + '.');
134 var n = outbox ? outbox.size() : 0;
135 if (n > 0) parts.push(n + (n === 1 ? ' act is' : ' acts are') + ' waiting to send.');
136 note.textContent = parts.join(' ');
137 note.hidden = parts.length === 0;
138 }
139 function flush() {
140 if (!outbox) return;
141 outbox.flush().then(paint);
142 }
143 function queue(url, payload) {
144 if (!outbox) return false;
145 outbox.add(url, payload);
146 paint();
147 flush();
148 return true;
149 }
150 function send(card, kind) {
151 return queue('/api/v1/feed/interactions', { item: card.getAttribute('data-item'), kind: kind, app: card.getAttribute('data-app') });
107152 }
108153 function gone(card) {
109154 var cards = Array.prototype.slice.call(feed.querySelectorAll('.fcard'));
@@ -111,16 +156,30 @@ export const feedJs = /* js */ `
111156 card.hidden = true;
112157 if (next) next.focus();
113158 }
114 function act(card, kind) {
115 send(card, kind).then(function (r) { if (r.ok) gone(card); }, function () {});
116 }
159 // Acting is instant: the card goes, the act is kept and sent.
160 function act(card, kind) { if (send(card, kind)) gone(card); }
117161 feed.addEventListener('click', function (e) {
118162 var t = e.target && e.target.closest ? e.target.closest('[data-feed],[data-feed-open]') : null;
119163 var card = t && t.closest('.fcard');
120164 if (!card) return;
121 if (t.hasAttribute('data-feed-open')) { send(card, 'opened', true).catch(function () {}); return; }
165 if (t.hasAttribute('data-feed-open')) { send(card, 'opened'); return; }
122166 act(card, t.getAttribute('data-feed'));
123167 });
168 // A tell is kept and sent the same way; a steer needs the run, so it
169 // stays an ordinary post and waits for the network.
170 feed.addEventListener('submit', function (e) {
171 var form = e.target;
172 var card = form.closest && form.closest('.fcard');
173 var box = form.elements && form.elements.text;
174 if (!card || !box || !/\\/tell$/.test(form.getAttribute('action') || '')) return;
175 var text = box.value.trim();
176 if (!text) return;
177 if (!queue('/api/v1/feed/tells', { app: card.getAttribute('data-app'), text: text })) return;
178 e.preventDefault();
179 box.value = '';
180 var d = form.closest('details');
181 if (d) d.open = false;
182 });
124183 document.addEventListener('keydown', function (e) {
125184 if (e.key !== 'x' || e.metaKey || e.ctrlKey || e.altKey) return;
126185 var a = document.activeElement;
@@ -128,5 +187,20 @@ export const feedJs = /* js */ `
128187 e.preventDefault();
129188 act(a, 'dismissed');
130189 });
190 window.addEventListener('online', function () {
191 flush();
192 // The page on screen came from the cache: the real one is back.
193 if (note && note.getAttribute('data-stale') === '1') location.reload();
194 });
195 // Signing out forgets the last feed and anything not yet sent.
196 document.addEventListener('submit', function (e) {
197 if (e.target && e.target.getAttribute && e.target.getAttribute('action') === '/logout') {
198 if (outbox) outbox.clear();
199 if (navigator.serviceWorker && navigator.serviceWorker.controller) navigator.serviceWorker.controller.postMessage('clear');
200 }
201 });
202 if ('serviceWorker' in navigator) navigator.serviceWorker.register('/sw.js', { scope: '/' }).catch(function () {});
203 paint();
204 flush();
131205 })();
132206 `;
src/web/studio/home.ts +2 −1
@@ -21,7 +21,7 @@ import { onYouCard } from "../pages/site.ts";
2121 import { writableAppsOf } from "./asks.ts";
2222 import { productionFactsForRepos, shelfRows } from "./shelf.ts";
2323 import { servingHistory } from "./words.ts";
24 import { feedJs, feedRegion } from "./feed.ts";
24 import { feedJs, feedNote, feedRegion } from "./feed.ts";
2525
2626 /**
2727 * The home in the studio: your apps as a shelf of
@@ -148,6 +148,7 @@ export async function studioHome(ctx: Ctx, user: User): Promise<Response> {
148148 ${costLine}
149149 <div class="dash">
150150 <div class="dash-main stack">
151 ${feedNote()}
151152 ${feedRegion(user, ctx)}
152153 ${appsRegion(apps)}
153154 </div>
src/web/sw.ts added +82
@@ -0,0 +1,82 @@
1 /**
2 * The service worker: the shell and the last feed, kept for when there is
3 * no network. Nothing else is its business.
4 *
5 * - `/` is network-first. Every good answer is kept; when the network
6 * fails the kept one is served, marked `data-stale="1"` so the page says
7 * it is old (the region carries `data-rendered`, the time it was made).
8 * - The stylesheet and content-addressed assets are cache-first (assets
9 * named by hash never change; the stylesheet is refreshed behind).
10 * - Everything else — the API, `/events`, `/live/*`, every POST — goes
11 * straight to the network untouched. Writes made offline wait in the
12 * page's outbox (`outbox.ts`), not here.
13 * - Signing out sends `clear`, which empties the cache: the last feed is
14 * one person's and must not outlive their session.
15 */
16
17 import { sha256 } from "../kernel/hash.ts";
18
19 export const SW_CACHE = "forge-shell-v1";
20
21 const source = /* js */ `
22 var CACHE = ${JSON.stringify(SW_CACHE)};
23 self.addEventListener('install', function () { self.skipWaiting(); });
24 self.addEventListener('activate', function (event) {
25 event.waitUntil(caches.keys().then(function (names) {
26 return Promise.all(names.filter(function (n) { return n !== CACHE; }).map(function (n) { return caches.delete(n); }));
27 }).then(function () { return self.clients.claim(); }));
28 });
29 self.addEventListener('message', function (event) {
30 if (event.data === 'clear') event.waitUntil(caches.delete(CACHE));
31 });
32 function keep(request, response) {
33 if (response.ok && response.type === 'basic' && !response.redirected) {
34 var copy = response.clone();
35 caches.open(CACHE).then(function (c) { return c.put(request, copy); });
36 }
37 return response;
38 }
39 function stale(response) {
40 return response.text().then(function (text) {
41 return new Response(text.replace('data-stale="0"', 'data-stale="1"'), {
42 status: 200, headers: { 'content-type': 'text/html; charset=utf-8' }
43 });
44 });
45 }
46 self.addEventListener('fetch', function (event) {
47 var request = event.request;
48 if (request.method !== 'GET') return;
49 var url = new URL(request.url);
50 if (url.origin !== location.origin) return;
51 if (url.pathname === '/' && request.mode === 'navigate') {
52 event.respondWith(fetch(request).then(function (r) {
53 return keep(new Request('/'), r);
54 }, function () {
55 return caches.match('/').then(function (hit) {
56 return hit ? stale(hit) : Response.error();
57 });
58 }));
59 return;
60 }
61 if (url.pathname.indexOf('/assets/') === 0 || url.pathname.indexOf('/icons/') === 0) {
62 event.respondWith(caches.match(request).then(function (hit) {
63 var fresh = fetch(request).then(function (r) { return keep(request, r); }, function () { return hit || Response.error(); });
64 return hit || fresh;
65 }));
66 }
67 });
68 `;
69
70 export const swSource = source;
71 const ETAG = `"${sha256(source).slice(0, 16)}"`;
72
73 export function serviceWorker(): Response {
74 return new Response(source, {
75 headers: {
76 "content-type": "text/javascript; charset=utf-8",
77 "cache-control": "no-cache",
78 etag: ETAG,
79 "service-worker-allowed": "/",
80 },
81 });
82 }
test/offline.test.ts added +231
@@ -0,0 +1,231 @@
1 /**
2 * Offline-first feed: the outbox keeps acts and replays them without
3 * doubling, the service worker is served and is valid, the page carries
4 * what it needs, a tell by key lands once, and /events resumes from an id.
5 */
6 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
7 import { mkdtemp, rm } from "node:fs/promises";
8 import { tmpdir } from "node:os";
9 import { join } from "node:path";
10 import { createApp, type App } from "../src/app.ts";
11 import { createRepo } from "../src/actions/repos.ts";
12 import { createSession } from "../src/core/sessions.ts";
13 import { createUser, type User } from "../src/core/users.ts";
14 import { listIssues } from "../src/core/issues.ts";
15 import { serve, type Servers } from "../src/http/server.ts";
16 import { outboxJs } from "../src/web/outbox.ts";
17 import { swSource } from "../src/web/sw.ts";
18
19 type Send = (url: string, init: any) => Promise<{ status: number }>;
20 const make = new Function(`${outboxJs}; return forgeOutbox;`)() as (
21 storage: Storage,
22 send: Send,
23 ) => {
24 add(url: string, body: unknown): void;
25 flush(): Promise<{ sent: number; left: number }>;
26 size(): number;
27 clear(): void;
28 };
29 const memory = (): Storage => {
30 const m = new Map<string, string>();
31 return {
32 getItem: (k: string) => m.get(k) ?? null,
33 setItem: (k: string, v: string) => void m.set(k, v),
34 removeItem: (k: string) => void m.delete(k),
35 } as Storage;
36 };
37
38 describe("the outbox", () => {
39 test("keeps acts while the network is down and sends them in order when it is back", async () => {
40 const store = memory();
41 let up = false;
42 const seen: string[] = [];
43 const box = make(store, async (_u, init) => {
44 if (!up) throw new TypeError("offline");
45 seen.push(init.body);
46 return { status: 201 };
47 });
48 box.add("/api/v1/feed/interactions", { n: 1 });
49 box.add("/api/v1/feed/tells", { n: 2 });
50 expect(await box.flush()).toEqual({ sent: 0, left: 2 });
51 // a reload: a new outbox over the same storage still holds them
52 up = true;
53 const reopened = make(store, async (_u, init) => {
54 seen.push(init.body);
55 return { status: 201 };
56 });
57 expect(await reopened.flush()).toEqual({ sent: 2, left: 0 });
58 expect(seen).toEqual(['{"n":1}', '{"n":2}']);
59 });
60
61 test("a replay carries the key it was queued with, so the server can refuse a double", async () => {
62 const store = memory();
63 const keys: string[] = [];
64 let lose = true;
65 const box = make(store, async (_u, init) => {
66 keys.push(init.headers["idempotency-key"]);
67 if (lose) throw new TypeError("answer lost");
68 return { status: 201 };
69 });
70 box.add("/x", { a: 1 });
71 await box.flush();
72 lose = false;
73 await box.flush();
74 expect(keys.length).toBe(2);
75 expect(keys[0]).toBe(keys[1]);
76 expect(box.size()).toBe(0);
77 });
78
79 test("a server error keeps the entry and stops; a refusal drops it", async () => {
80 const store = memory();
81 const statuses = [503, 404];
82 let calls = 0;
83 const box = make(store, async () => ({ status: statuses[calls++] ?? 201 }));
84 box.add("/a", {});
85 box.add("/b", {});
86 expect(await box.flush()).toEqual({ sent: 0, left: 2 });
87 expect(calls).toBe(1);
88 expect(await box.flush()).toEqual({ sent: 2, left: 0 });
89 });
90
91 test("two flushes at once send an entry once", async () => {
92 const store = memory();
93 let calls = 0;
94 const box = make(store, async () => {
95 calls++;
96 await Promise.resolve();
97 return { status: 201 };
98 });
99 box.add("/a", {});
100 await Promise.all([box.flush(), box.flush()]);
101 expect(calls).toBe(1);
102 });
103
104 test("clear forgets everything; junk in storage is ignored", async () => {
105 const store = memory();
106 store.setItem("forge.outbox", "{not json");
107 const box = make(store, async () => ({ status: 201 }));
108 expect(box.size()).toBe(0);
109 box.add("/a", {});
110 box.clear();
111 expect(box.size()).toBe(0);
112 });
113 });
114
115 describe("the service worker and the server", () => {
116 let dataDir: string;
117 let app: App;
118 let servers: Servers;
119 let alice: User;
120 let cookie: string;
121
122 beforeAll(async () => {
123 dataDir = await mkdtemp(join(tmpdir(), "forge-offline-"));
124 app = createApp({ dataDir, port: 0 });
125 servers = await serve(app);
126 const r = await createUser(app.db, {
127 username: "alice",
128 email: "[email protected]",
129 password: "a-good-passphrase",
130 adminIfFirst: true,
131 });
132 if (!r.ok) throw new Error(r.error);
133 alice = r.value;
134 cookie = `forge_session=${createSession(app.db, alice.id).token}`;
135 const made = await createRepo(app, alice, { name: "open", visibility: "public" });
136 if (!made.ok) throw new Error("repo");
137 });
138 afterAll(async () => {
139 servers.stop();
140 app.close();
141 await rm(dataDir, { recursive: true, force: true });
142 });
143
144 test("/sw.js is valid script, root-scoped, never cached by the browser", async () => {
145 const res = await fetch(`${servers.url}/sw.js`);
146 expect(res.status).toBe(200);
147 expect(res.headers.get("content-type")).toContain("javascript");
148 expect(res.headers.get("service-worker-allowed")).toBe("/");
149 expect(res.headers.get("cache-control")).toBe("no-cache");
150 expect(() => new Function(swSource)).not.toThrow();
151 expect(swSource).toContain("stale");
152 });
153
154 test("the home page registers the worker, has the note, and allows workers", async () => {
155 const res = await fetch(`${servers.url}/`, { headers: { cookie } });
156 const text = await res.text();
157 expect(text).toContain("serviceWorker");
158 expect(text).toContain('id="feed-note"');
159 expect(text).toContain("forgeOutbox");
160 expect(res.headers.get("content-security-policy")).toContain("worker-src 'self'");
161 });
162
163 test("the live client resumes /events from the last seen id", async () => {
164 const text = await (await fetch(`${servers.url}/`, { headers: { cookie } })).text();
165 expect(text).toContain("/events?since=");
166 expect(text).toContain("'online'");
167 });
168
169 test("a tell sent twice with one key makes one issue", async () => {
170 const send = (key: string, text = "make it faster") =>
171 fetch(`${servers.url}/api/v1/feed/tells`, {
172 method: "POST",
173 headers: {
174 cookie,
175 origin: servers.url,
176 "content-type": "application/json",
177 "idempotency-key": key,
178 },
179 body: JSON.stringify({ app: "alice/open", text }),
180 });
181 const before = listIssues(app.db, repoId()).length;
182 const a = await send("k1");
183 const b = await send("k1");
184 expect(a.status).toBe(201);
185 expect(b.status).toBe(201);
186 expect(b.headers.get("idempotent-replay")).toBe("true");
187 expect(await b.json()).toEqual(await a.json());
188 expect(listIssues(app.db, repoId()).length).toBe(before + 1);
189 expect((await send("k2")).status).toBe(201);
190 expect(listIssues(app.db, repoId()).length).toBe(before + 2);
191 });
192
193 test("a tell is validated and same-origin checked", async () => {
194 const post = (body: unknown, origin = servers.url) =>
195 fetch(`${servers.url}/api/v1/feed/tells`, {
196 method: "POST",
197 headers: { cookie, origin, "content-type": "application/json" },
198 body: JSON.stringify(body),
199 });
200 expect((await post({ app: "alice/open", text: " " })).status).toBe(422);
201 expect((await post({ app: "alice/nope", text: "hi" })).status).toBe(404);
202 expect((await post({ app: "alice/open", text: "hi" }, "http://evil.example")).status).toBe(403);
203 });
204
205 test("an interaction replayed by key is recorded once", async () => {
206 const post = () =>
207 fetch(`${servers.url}/api/v1/feed/interactions`, {
208 method: "POST",
209 headers: {
210 cookie,
211 origin: servers.url,
212 "content-type": "application/json",
213 "idempotency-key": "int-1",
214 },
215 body: JSON.stringify({ item: "ask:look:1", kind: "dismissed", app: "alice/open" }),
216 });
217 const a = await (await post()).json();
218 const b = await (await post()).json();
219 expect(b).toEqual(a);
220 const rows = app.db
221 .query<{ n: number }, []>("SELECT count(*) n FROM interaction WHERE item_id = 'ask:look:1'")
222 .get()!;
223 expect(rows.n).toBe(1);
224 });
225
226 function repoId(): string {
227 return app.db
228 .query<{ id: string }, []>("SELECT id FROM repo WHERE name = 'open'")
229 .get()!.id;
230 }
231 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs