Public: anyone can read this; the crew can change it.

Page and tool usage are counted per route group so unused paths can be dropped #15082

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/15073-page-and-tool-usage-are-counted-per-rout → main 7 commits fork 4 threads · 4 unresolved

  • Merged. scout/15073-page-and-tool-usage-are-counted-per-rout was merged into main as a02acd9.

Review

Sign in to review this change.
11 files changed +147 −10
docs/design/attention-budget.json +5 −5
@@ -175,16 +175,16 @@
175175 },
176176 "system": {
177177 "390": {
178 "words": 234,
179 "blocks": 77,
178 "words": 250,
179 "blocks": 79,
180180 "colours": 0
181181 },
182182 "1280": {
183 "words": 234,
184 "blocks": 77,
183 "words": 250,
184 "blocks": 79,
185185 "colours": 0
186186 },
187 "note": "Measured on the test forge's empty state with the machine reading replaced by an empty one and the two cell-cap sentences fixed (the live page's counts depend on the host's cores, disks, processes and cgroups); nothing seeded or waited for. Blocks 65 -> 67 (#11805): the off-site backup section's heading and one-sentence state stay on first view so a red lane is seen without a click; its rows sit behind the disclosure. Blocks 67 -> 69 (#12010): the database size section's heading and its one-sentence state (the file's size and growth, a badge when either is worth a look) stay on first view, as the off-site lane's do; its fact rows and the table of the ten largest tables and indexes sit behind the disclosure. Words +22, blocks +1 (#12092): the footprint section's heading and its one-sentence state (on an empty forge, not read yet and the memory budget) stay on first view; the measured facts and the optional parts sit behind the disclosure. Words +6, blocks +2 (#12420): the stale-pulls section's heading and \"None.\" on a forge with no quiet pull. Words +4, blocks +2 (#13052): the Restore section's heading and its one sentence (\"Restore never proven.\" on an empty forge) stay on first view, so the owner sees whether a restore works without a click."
187 "note": "Measured on the test forge's empty state with the machine reading replaced by an empty one and the two cell-cap sentences fixed (the live page's counts depend on the host's cores, disks, processes and cgroups); nothing seeded or waited for. Blocks 65 -> 67 (#11805): the off-site backup section's heading and one-sentence state stay on first view so a red lane is seen without a click; its rows sit behind the disclosure. Blocks 67 -> 69 (#12010): the database size section's heading and its one-sentence state (the file's size and growth, a badge when either is worth a look) stay on first view, as the off-site lane's do; its fact rows and the table of the ten largest tables and indexes sit behind the disclosure. Words +22, blocks +1 (#12092): the footprint section's heading and its one-sentence state (on an empty forge, not read yet and the memory budget) stay on first view; the measured facts and the optional parts sit behind the disclosure. Words +6, blocks +2 (#12420): the stale-pulls section's heading and \"None.\" on a forge with no quiet pull. Words +4, blocks +2 (#13052): the Restore section's heading and its one sentence (\"Restore never proven.\" on an empty forge) stay on first view, so the owner sees whether a restore works without a click. Words +16, blocks +2 (#15073): the least-used-routes section's heading and its one-sentence count stay on first view (how many route groups had no request in 30 days); the table sits behind the disclosure."
188188 },
189189 "explore": {
190190 "note": "Measured as an admin: the unfiltered count reads 'N apps, forges and people' (+3 words over 'N results'); a guest reads two fewer. An app row says forks by people (+1 word, #14323).",
docs/features/least-used-routes.md added +23
@@ -0,0 +1,23 @@
1 ---
2 topic: When something is wrong
3 row: Least used pages and tools on /system
4 ---
5 # Least used pages and tools (#15073)
6
7 `/system` has a section, **Least used pages and tools**, that lists route
8 groups by requests in the last 30 days, fewest first. It answers which pages
9 and tools could be dropped. A group is a route pattern, never a URL; no address
10 or text is kept.
11
12 ## How to read it
13
14 1. Open `/system` as a forge admin who is a person.
15 2. Find the section. Its first line says how many route groups had no request in 30 days.
16 3. Each row is a pattern, its request count, and the hour of its last use (UTC), or `never`.
17 4. **All of them** leads to `/settings/usage`.
18
19 ## In the terminal
20
21 Press the System command. Under "What the forge costs" a block named
22 **Least used pages and tools** names the three route patterns seen with the fewest requests.
23 It is empty for anyone `/system` shows no telemetry to.
src/core/usage_report.ts +5
@@ -114,3 +114,8 @@ export function forgeSpend(db: Database, repoIds: ReadonlySet<string>, now: Date
114114 last24h: spendSince(db, ids, last24hSince, until),
115115 };
116116 }
117
118 /** Least used first: fewest requests, then the longest unused. */
119 export function leastUsed(rows: readonly RouteUsage[]): RouteUsage[] {
120 return [...rows].sort((a, b) => a.total - b.total || (a.last ?? "").localeCompare(b.last ?? "") || a.pattern.localeCompare(b.pattern));
121 }
src/http/server.ts +1 −1
@@ -1151,7 +1151,7 @@ export function buildRouter(): Router<Handler> {
11511151 router.add("POST", "/flow/valve", valveSet);
11521152 router.add("POST", "/flow/backup/retry", backupRetry);
11531153 router.add("POST", "/flow/suggestion", suggestionTake);
1154 router.add("GET", "/system", systemPage);
1154 router.add("GET", "/system", (ctx) => systemPage(ctx, knownPatterns));
11551155 router.add("GET", "/system/stream", systemStream);
11561156 router.add("GET", "/system/bottleneck", bottleneckPage);
11571157 router.add("GET", "/offers/:id", offerReceiptPage);
src/tui/screens/system.ts +16
@@ -53,6 +53,8 @@ import { largestDirs, type LargestDirs } from "../../system/largest_dirs.ts";
5353 import { dbSize, dbSizeNeed, type DbSize } from "../../system/db_size.ts";
5454 import { footprintVerdict, lastFootprint, smallWords } from "../../system/footprint.ts";
5555 import { lastProof } from "../../release/prove.ts";
56 import { leastUsed, routeUsage, WINDOW_DAYS } from "../../core/usage_report.ts";
57 import { knownPatterns } from "../../http/server.ts";
5658 import type { PaletteEntry } from "./entries.ts";
outdated

A tui screen should not import from src/web/pages. Import leastUsed from core/usage_report.ts once moved.

5759 import { shapeWords } from "../../engine/mode.ts";
5860 import { readHealth } from "../../actions/health.ts";
@@ -209,6 +211,18 @@ export function footprintLines(session: Session, f = lastFootprint() ?? lastProo
209211 return [head, ...wrap(footprintVerdict(f), inner), ...small, ...(off.length === 0 ? [] : [truncate(`off: ${off.join(", ")}`, inner)])];
210212 }
211213
214 /**
215 * /system's "Least used pages and tools" (#15073), the three least used route patterns seen; the
216 * page and /settings/usage list more.
217 */
218 export function routeUseLines(session: Session, inner = WIDTH - 6): string[] {
219 if (!telemetryReader(session.app, session.user)) return [];
220 const db = session.app.db;
221 const rows = leastUsed(routeUsage(db, knownPatterns(), new Date())).slice(0, 3);
222 if (rows.length === 0) return [];
outdated

Only patterns already in usage_count are considered, so never-hit routes are missing from the terminal's 'least used' list, though the page lists them first. Use knownPatterns() as the page does, so both doors agree.

223 return ["Least used pages and tools", truncate(`${WINDOW_DAYS} days: ${rows.map((r) => `${r.pattern} ${r.total}`).join(", ")}`, inner)];
224 }
225
212226 /**
213227 * The database's size, as /system's "Database size" says it (#12010): the
214228 * file, its WAL, the room inside and its growth, then the largest tables
@@ -376,6 +390,7 @@ export function systemScreen(ctx: Ctx, lines: readonly string[], session: Sessio
376390 // and only for a reader.
377391 const db = reads ? dbSizeLines(session, dbSize.peek(session.app.db, session.app.config.dataDir, session.app.log), true) : [];
378392 const footprint = footprintLines(session);
393 const routes = routeUseLines(session);
379394 const buttons = dialog(s, area, ctx.theme.c, {
380395 title: "System",
381396 body: [
@@ -386,6 +401,7 @@ export function systemScreen(ctx: Ctx, lines: readonly string[], session: Sessio
386401 // Under the largest directories with no blank between: the dialog does not scroll.
387402 ...db,
388403 ...(footprint.length === 0 ? [] : ["", ...footprint]),
404 ...(routes.length === 0 ? [] : ["", ...routes]),
389405 ...(offered ? ["", ...readerLines(session)] : []),
390406 ...(features.length === 0 ? [] : ["", ...features]),
391407 ...(offered ? ["", grantLine(session), "g: grant or take back telemetry:read"] : []),
src/web/pages/system.ts +22 −1
@@ -66,6 +66,7 @@ import { LARGEST_DIRS, largestDirs, type LargestDirs } from "../../system/larges
6666 import { DB_FILE_CAP, DB_GROWTH_MAX, TOP, dbSize, dbSizeNeed, type DbSize } from "../../system/db_size.ts";
6767 import { resultWords, weekRows, weekWords, type WeekRow } from "../../maintain/week_line.ts";
6868 import { duties } from "../../maintain/registry.ts";
69 import { leastUsed, routeUsage, WINDOW_DAYS, type RouteUsage } from "../../core/usage_report.ts";
6970 import { childDirOf, listChildren } from "../../actions/spawn.ts";
7071 import { adoptionsOf } from "../../spawn/cell.ts";
7172 import type { AdoptionRow } from "../../spawn/adopted.ts";
@@ -392,6 +393,23 @@ export function footingSection(f: Footing): Html {
392393 ${(Object.keys(MACHINES) as Machine[]).map((m) => html`<p class="small muted" data-machine="${m}"><span class="mono">${m}</span>: ${MACHINES[m]}</p>`)}`, { id: "sys-footing-more" })}</section>`;
393394 }
394395
396 /** How many route groups /system lists; the rest are on the usage page. */
397 export const ROUTES_SHOWN = 15;
398
399 /**
400 * The route groups by requests in the last 30 days, least used first (#15073): the paths the owner
401 * could drop. A group is a route pattern, never a URL; a path that matched none is not one.
402 */
403 export function routeUseSection(rows: readonly RouteUsage[]): Html {
404 const unused = rows.filter((r) => r.total === 0).length;
405 const shown = rows.slice(0, ROUTES_SHOWN);
406 return html`<section class="settings-block" id="sys-routes" data-unused="${String(unused)}"><h2>Least used pages and tools</h2>
407 ${disclosure(`${String(unused)} of ${String(rows.length)} route groups had no request in ${String(WINDOW_DAYS)} days`, html`<p class="small muted">Counted by route pattern; no address or text is kept. <a href="/settings/usage">All of them</a>.</p><div class="table-scroll"><table class="table" data-dense aria-label="Route groups by requests in the last ${String(WINDOW_DAYS)} days, least used first">
408 <thead><tr><th scope="col">Route</th><th scope="col" class="num">${String(WINDOW_DAYS)} days</th><th scope="col">Last use (UTC)</th></tr></thead>
409 <tbody>${shown.map((r) => html`<tr data-route="${r.pattern}"><td class="mono">${r.pattern}</td><td class="num mono">${String(r.total)}</td><td class="mono">${r.last === null ? "never" : r.last.replace("T", " ") + ":00"}</td></tr>`)}</tbody>
410 </table></div>`, { id: "sys-routes-more" })}</section>`;
411 }
412
395413 /**
396414 * What the forge costs (#12092): the last reading at rest (this process's, else
397415 * the release proof's), the stated budget, and what is switched off and why.
outdated

leastUsed is plain data logic that the terminal also needs; put it in src/core/usage_report.ts next to routeUsage so tui/ does not import a web page module.

@@ -633,7 +651,7 @@ export function floorLine(held: FloorHold): Html {
633651 }
634652
635653 // GET /system
636 export async function systemPage(ctx: Ctx): Promise<Response> {
654 export async function systemPage(ctx: Ctx, patterns: () => readonly string[]): Promise<Response> {
637655 if (!ctx.auth.user) return signInFirst(ctx);
638656 notePhase(ctx.request, "session", takeSessionTime(ctx.request));
639657 const reader = timePhase(ctx.request, "reader", () => telemetryReader(ctx.app, ctx.auth.user!));
@@ -717,6 +735,9 @@ export async function systemPage(ctx: Ctx): Promise<Response> {
717735 <div class="js-only small muted">Sort by <button class="btn" data-size="sm" type="button" data-sort="cpu" aria-pressed="true"><kbd>c</kbd> CPU</button>
718736 <button class="btn" data-size="sm" type="button" data-sort="mem" aria-pressed="false"><kbd>m</kbd> memory</button></div>
719737 <div id="sys-procs" data-live-hold>${at("sys-procs")}</div></section>
738 ${routeUseSection(timePhase(ctx.request, "routes", () => {
739 return leastUsed(routeUsage(ctx.app.db, patterns(), new Date()));
740 }))}
720741 ${maintenanceSection(timePhase(ctx.request, "maintenance", () => weekRows(ctx.app.db, duties.map((d) => d.name))))}
721742 ${footprintSection(lastFootprint() ?? lastProof(ctx.app.config.dataDir)?.footprint ?? null)}
722743 ${footingSection(standsOn)}
test/engine-shape.test.ts +1 −1
@@ -54,7 +54,7 @@ const page = async (app: ReturnType<typeof fresh>, user: User | undefined) => {
5454 auth: { user, scope: undefined, via: user ? "session" : "none" },
5555 theme: undefined,
5656 };
57 const res = await systemPage(ctx);
57 const res = await systemPage(ctx, () => []);
5858 return { status: res.status, body: await res.text() };
5959 };
6060
test/forges-refused.test.ts +1 −1
@@ -62,7 +62,7 @@ describe("doors to /forges/refused", () => {
6262 const link = async (user: unknown) => {
6363 const { systemPage } = await import("../src/web/pages/system.ts");
6464 const app = fresh();
65 const res = await systemPage(ctxOf(app, user));
65 const res = await systemPage(ctxOf(app, user), () => []);
6666 return (await res.text()).includes('href="/forges/refused"');
6767 };
6868 test("/system links it for an admin who is a person, not for others", async () => {
test/route-use-system.test.ts added +59
@@ -0,0 +1,59 @@
1 /** /system lists route groups by requests in 30 days, least used first (#15073). */
2
3 import { afterAll, beforeAll, expect, test } from "bun:test";
4 import { mkdtemp, rm } from "node:fs/promises";
5 import { tmpdir } from "node:os";
6 import { join } from "node:path";
7 import { createApp, type App } from "../src/app.ts";
8 import { createToken } from "../src/core/tokens.ts";
9 import { createUser } from "../src/core/users.ts";
10 import { hourOf } from "../src/core/usage_count.ts";
11 import { leastUsed, routeUsage } from "../src/core/usage_report.ts";
12 import { knownPatterns, serve, type Servers } from "../src/http/server.ts";
13 import { routeUseSection } from "../src/web/pages/system.ts";
14
15 let dir: string;
16 let app: App;
17 let servers: Servers;
18 let token: string;
19
20 beforeAll(async () => {
21 dir = await mkdtemp(join(tmpdir(), "forge-route-use-"));
22 app = createApp({ dataDir: dir, port: 0 });
23 const r = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
24 if (!r.ok) throw new Error(r.error);
25 token = createToken(app.db, r.value.id, "t", "admin").token;
26 });
27
28 afterAll(async () => {
29 servers.stop();
30 app.close();
31 await rm(dir, { recursive: true, force: true });
32 });
33
34 test("two requests count as two, and a 404 is not a route group", async () => {
35 servers = await serve(app);
36 const pattern = "/api/v1/repos/:owner/:name/issues/:number";
37 for (const n of [1, 2]) await fetch(`${servers.url}/api/v1/repos/alice/none/issues/${n}`, { headers: { authorization: `Bearer ${token}` } });
38 await fetch(`${servers.url}/no/such/${crypto.randomUUID()}`);
39 servers.stop(); // stopping flushes the pending counts
40 const rows = routeUsage(app.db, knownPatterns(), new Date());
41 expect(rows.find((r) => r.pattern === pattern)?.total).toBe(2);
42 expect(rows.some((r) => r.pattern === "(unmatched)")).toBe(false);
43 expect(rows.some((r) => r.pattern.includes("such"))).toBe(false);
44 });
45
46 test("/system lists the least used first and keeps no URL", async () => {
47 servers = await serve(app);
48 const hour = hourOf(new Date());
49 app.db.run("INSERT INTO usage_count (hour, pattern, method, status, role, surface, n) VALUES (?, '/busy', 'GET', '2xx', 'user', 'web', 9)", [hour]);
50 const rows = leastUsed(routeUsage(app.db, ["/busy", "/quiet"], new Date()));
51 expect(rows.map((r) => r.pattern)).toEqual(["/quiet", "/busy"]);
52 const html = routeUseSection(rows).toString();
53 expect(html.indexOf("/quiet")).toBeLessThan(html.indexOf("/busy"));
54 const res = await fetch(`${servers.url}/system`, { headers: { authorization: `Bearer ${token}` } });
55 expect(res.status).toBe(200);
56 const text = await res.text();
57 expect(text).toContain('id="sys-routes"');
58 expect(text).not.toContain("(unmatched)");
59 });
test/system-parts.test.ts +1
@@ -54,6 +54,7 @@ test("one <h2> per part, each in a settings-block, and no card", async () => {
5454 "Largest under the forge's data",
5555 "Database size",
5656 "Processes",
57 "Least used pages and tools",
5758 "Maintenance, last 7 days",
5859 "What the forge costs",
5960 "What the forge stands on",
test/tui-system.test.ts +13 −1
@@ -16,7 +16,7 @@ import { createForgeApp, forgeCommands, Session } from "../src/tui/forge.ts";
1616 import { setTelemetryGrant } from "../src/actions/telemetry.ts";
1717 import { hasGrant } from "../src/core/telemetry.ts";
1818 import { countRead, resetTelemetryReads } from "../src/system/telemetry_reads.ts";
19 import { askingLines, dbSizeLines, dbSizeScreen, footprintLines, forgeLines, historyLines, largestLines, readerLines, memoryLines, memoryScreen, systemLines, systemScreen } from "../src/tui/screens/system.ts";
19 import { askingLines, dbSizeLines, dbSizeScreen, footprintLines, routeUseLines, forgeLines, historyLines, largestLines, readerLines, memoryLines, memoryScreen, systemLines, systemScreen } from "../src/tui/screens/system.ts";
2020 import type { DbSize } from "../src/system/db_size.ts";
2121 import type { SystemView } from "../src/system/view.ts";
2222 import { changeCapacity } from "../src/actions/capacity.ts";
@@ -494,3 +494,15 @@ describe("the machine line says when a cgroup holds the forge under the host (#1
494494 expect(systemLines(view).join("\n")).not.toContain("cgroup");
495495 });
496496 });
497
498 describe("tui system least used routes (#15073)", () => {
499 test("an admin reads the least used route patterns, never-hit ones first; a member reads nothing", async () => {
500 const { hourOf } = await import("../src/core/usage_count.ts");
501 app.db.run("INSERT INTO usage_count (hour, pattern, method, status, role, surface, n) VALUES (?, '/system', 'GET', '2xx', 'user', 'web', 9)", [hourOf(new Date())]);
502 const lines = routeUseLines(new Session(app, identity(carol)));
503 expect(lines[0]).toBe("Least used pages and tools");
504 expect(lines[1]!).toMatch(/ 0(,|$)/);
505 expect(lines[1]!).not.toContain("/system 9");
506 expect(routeUseLines(new Session(app, identity(bob)))).toEqual([]);
507 });
508 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs