Public: anyone can read this; the crew can change it.
How fast a change reaches its preview, on the app page: the direction's number, measured #109
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/83-how-fast-a-change-reaches-its-preview-on → main 1 commit fork
3 files
changed +269
−1
src/core/runs.ts +38
| @@ -590,6 +590,44 @@ export function latestAgentRunsForRepos( | ||
| 590 | 590 | return out; |
| 591 | 591 | } |
| 592 | 592 | |
| 593 | /** How fast this app's deploys have reached their preview: the median, and the count it is drawn from. */ | |
| 594 | export interface PreviewLatency { | |
| 595 | readonly medianSeconds: number; | |
| 596 | readonly n: number; | |
| 597 | } | |
| 598 | ||
| 599 | /** | |
| 600 | * The median seconds from push (`created_at`) to serving (`serving_at`) | |
| 601 | * over this repo's deploys since `since` — the same reference instant | |
| 602 | * `servedPhrase` (`web/ui.ts`) words a single deploy from. A deploy that | |
| 603 | * never served says nothing about speed and is left out, never a zero. | |
| 604 | * One query, bounded by `limit` and served by the `run_repo` index (it | |
| 605 | * walks newest-first and stops at the cap), never a scan of the table. | |
| 606 | */ | |
| 607 | export function previewLatency( | |
| 608 | db: Database, | |
| 609 | repoId: string, | |
| 610 | since: string, | |
| 611 | limit = 500, | |
| 612 | ): PreviewLatency | undefined { | |
| 613 | const secs = db | |
| 614 | .query<{ secs: number }, [string, string, number]>( | |
| 615 | `SELECT (julianday(serving_at) - julianday(created_at)) * 86400 AS secs | |
| 616 | FROM run | |
| 617 | WHERE repo_id = ?1 AND kind = 'deploy' AND serving_at IS NOT NULL | |
| 618 | AND created_at >= ?2 | |
| 619 | ORDER BY id DESC LIMIT ?3`, | |
| 620 | ) | |
| 621 | .all(repoId, since, limit) | |
| 622 | .map((r) => r.secs) | |
| 623 | .sort((a, b) => a - b); | |
| 624 | if (secs.length === 0) return undefined; | |
| 625 | const mid = Math.floor(secs.length / 2); | |
| 626 | const medianSeconds = | |
| 627 | secs.length % 2 === 0 ? (secs[mid - 1]! + secs[mid]!) / 2 : secs[mid]!; | |
| 628 | return { medianSeconds, n: secs.length }; | |
| 629 | } | |
| 630 | ||
| 593 | 631 | /** Agent runs queued or running across many repositories, newest first. */ |
| 594 | 632 | export function listActiveRuns( |
| 595 | 633 | db: Database, |
src/web/studio/app.ts +29 −1
| @@ -13,7 +13,12 @@ import { decisionStats, type DecisionStat } from "../../core/decisions.ts"; | ||
| 13 | 13 | import { getIssueById, listIssues, type Issue } from "../../core/issues.ts"; |
| 14 | 14 | import { listPulls, openPullsClosing } from "../../core/pulls.ts"; |
| 15 | 15 | import { canSeeOwnerFacts, type Repo } from "../../core/repos.ts"; |
| 16 | import { latestRunsForIssues, listRuns } from "../../core/runs.ts"; | |
| 16 | import { | |
| 17 | latestRunsForIssues, | |
| 18 | listRuns, | |
| 19 | previewLatency, | |
| 20 | type PreviewLatency, | |
| 21 | } from "../../core/runs.ts"; | |
| 17 | 22 | import { getUsersByIds } from "../../core/users.ts"; |
| 18 | 23 | import { autonomyStatus } from "../../crew/autonomy.ts"; |
| 19 | 24 | import { repoDir } from "../../git/repos.ts"; |
| @@ -65,6 +70,24 @@ function decisionWords(stats: readonly DecisionStat[]): Html { | ||
| 65 | 70 | >`; |
| 66 | 71 | } |
| 67 | 72 | |
| 73 | /** Below this many served deploys a median is one or two points, not a trend. */ | |
| 74 | const PREVIEW_LATENCY_MIN = 3; | |
| 75 | ||
| 76 | /** | |
| 77 | * How fast a change has reached its preview this week, worded like the | |
| 78 | * reference instant in `servedPhrase` (`web/ui.ts`): "reaches its preview | |
| 79 | * in 6.1s". Too few served deploys to trust a median, and the line says so | |
| 80 | * rather than printing one from a run or two. | |
| 81 | */ | |
| 82 | function previewLatencyWords(latency: PreviewLatency | undefined): Html { | |
| 83 | if (latency === undefined || latency.n < PREVIEW_LATENCY_MIN) | |
| 84 | return html`<span class="muted small" | |
| 85 | >not enough deploys this week to say</span | |
| 86 | >`; | |
| 87 | return html`reaches its preview in ${latency.medianSeconds.toFixed(1)}s | |
| 88 | <span class="muted">· ${latency.n} deploys</span>`; | |
| 89 | } | |
| 90 | ||
| 68 | 91 | export async function studioApp( |
| 69 | 92 | ctx: Ctx, |
| 70 | 93 | repo: Repo, |
| @@ -110,6 +133,9 @@ export async function studioApp( | ||
| 110 | 133 | : []; |
| 111 | 134 | const spent = weekRuns.reduce((s, r) => s + r.costUsd, 0); |
| 112 | 135 | const ledgerWeek = ownerFacts ? decisionStats(db, repo.id, week) : []; |
| 136 | const weekLatency = ownerFacts | |
| 137 | ? previewLatency(db, repo.id, week) | |
| 138 | : undefined; | |
| 113 | 139 | const autonomy = |
| 114 | 140 | !ownerFacts || repo.autonomy === "off" |
| 115 | 141 | ? undefined |
| @@ -305,6 +331,8 @@ export async function studioApp( | ||
| 305 | 331 | <dd>${shipped} change${shipped === 1 ? "" : "s"}</dd> |
| 306 | 332 | <dt>Runs</dt> |
| 307 | 333 | <dd>${weekRuns.length}</dd> |
| 334 | <dt>Preview</dt> | |
| 335 | <dd>${previewLatencyWords(weekLatency)}</dd> | |
| 308 | 336 | <dt>Spent</dt> |
| 309 | 337 | <dd> |
| 310 | 338 | $${spent.toFixed(2)}${shipped > 0 |
test/studio.test.ts +202
| @@ -38,6 +38,7 @@ import { | ||
| 38 | 38 | getRun, |
| 39 | 39 | markServing, |
| 40 | 40 | noteRunProgress, |
| 41 | previewLatency, | |
| 41 | 42 | type Run, |
| 42 | 43 | } from "../src/core/runs.ts"; |
| 43 | 44 | import { createUser, getUserByName, type User } from "../src/core/users.ts"; |
| @@ -2690,6 +2691,32 @@ describe("a stranger sees the app, not the owner's books", () => { | ||
| 2690 | 2691 | by: "rule", |
| 2691 | 2692 | ms: 3, |
| 2692 | 2693 | }); |
| 2694 | ||
| 2695 | // Three served deploys this week, median 4.0s, and a fourth that never | |
| 2696 | // answered — excluded, not a zero that would drag the median down. | |
| 2697 | const now = Date.now(); | |
| 2698 | for (const [offsetMs, durationMs] of [ | |
| 2699 | [0, 2_000], | |
| 2700 | [1_000, 4_000], | |
| 2701 | [2_000, 6_000], | |
| 2702 | [3_000, null], | |
| 2703 | ] as const) { | |
| 2704 | const deploy = createRun(app.db, { | |
| 2705 | repoId: repo.id, | |
| 2706 | kind: "deploy", | |
| 2707 | env: "production", | |
| 2708 | commitOid: "cafefeed", | |
| 2709 | reason: "push", | |
| 2710 | }); | |
| 2711 | app.db | |
| 2712 | .query("UPDATE run SET created_at = ?2 WHERE id = ?1") | |
| 2713 | .run(deploy.id, new Date(now + offsetMs).toISOString()); | |
| 2714 | if (durationMs !== null) { | |
| 2715 | app.db | |
| 2716 | .query("UPDATE run SET serving_at = ?2 WHERE id = ?1") | |
| 2717 | .run(deploy.id, new Date(now + offsetMs + durationMs).toISOString()); | |
| 2718 | } | |
| 2719 | } | |
| 2693 | 2720 | }); |
| 2694 | 2721 | |
| 2695 | 2722 | afterAll(async () => { |
| @@ -2715,12 +2742,20 @@ describe("a stranger sees the app, not the owner's books", () => { | ||
| 2715 | 2742 | expect(page).toContain(">Shipped<"); |
| 2716 | 2743 | expect(page).toContain("1 change"); |
| 2717 | 2744 | expect(page).toContain("Open the app"); |
| 2745 | // The direction's own number is the same owner's-books fact as spend | |
| 2746 | // and the ledger: a stranger and an anonymous visitor see neither. | |
| 2747 | expect(page).not.toContain(">Preview<"); | |
| 2748 | expect(page).not.toContain("reaches its preview"); | |
| 2718 | 2749 | } |
| 2719 | 2750 | |
| 2720 | 2751 | expect(owner).toContain("$42.50"); |
| 2721 | 2752 | expect(owner).toContain("keeper"); |
| 2722 | 2753 | expect(owner).toContain(">On its own<"); |
| 2723 | 2754 | expect(owner).toContain(">Decided<"); |
| 2755 | // Three served deploys this week — 2.0s, 4.0s and 6.0s — median 4.0s; | |
| 2756 | // the fourth, never served, does not drag it toward zero. | |
| 2757 | expect(owner).toContain("reaches its preview in 4.0s"); | |
| 2758 | expect(owner).toContain("· 3 deploys"); | |
| 2724 | 2759 | }); |
| 2725 | 2760 | |
| 2726 | 2761 | test("the app's live fragment carries no owner fact either", async () => { |
| @@ -2943,3 +2978,170 @@ describe("servedPhrase", () => { | ||
| 2943 | 2978 | ).toBeUndefined(); |
| 2944 | 2979 | }); |
| 2945 | 2980 | }); |
| 2981 | ||
| 2982 | describe("previewLatency: the median seconds from push to serving", () => { | |
| 2983 | let dataDir: string; | |
| 2984 | let app: App; | |
| 2985 | let alice: User; | |
| 2986 | let repo: Repo; | |
| 2987 | ||
| 2988 | beforeAll(async () => { | |
| 2989 | dataDir = await mkdtemp(join(tmpdir(), "forge-preview-latency-")); | |
| 2990 | app = createApp({ dataDir, port: 0 }); | |
| 2991 | const a = await createUser(app.db, { | |
| 2992 | username: "alice", | |
| 2993 | email: "[email protected]", | |
| 2994 | password: "a-good-passphrase", | |
| 2995 | adminIfFirst: true, | |
| 2996 | }); | |
| 2997 | if (!a.ok) throw new Error(a.error); | |
| 2998 | alice = a.value; | |
| 2999 | const made = await createRepo(app, alice, { | |
| 3000 | name: "latency", | |
| 3001 | visibility: "private", | |
| 3002 | description: "", | |
| 3003 | }); | |
| 3004 | if (!made.ok) throw new Error(String(made.error)); | |
| 3005 | repo = made.value; | |
| 3006 | }); | |
| 3007 | ||
| 3008 | afterAll(async () => { | |
| 3009 | app.close(); | |
| 3010 | await rm(dataDir, { recursive: true, force: true }); | |
| 3011 | }); | |
| 3012 | ||
| 3013 | /** A deploy run stamped with a known push instant and, unless null, a known serving instant. */ | |
| 3014 | const seedDeploy = (createdAt: string, servingAt: string | null): void => { | |
| 3015 | const deploy = createRun(app.db, { repoId: repo.id, kind: "deploy" }); | |
| 3016 | app.db | |
| 3017 | .query("UPDATE run SET created_at = ?2 WHERE id = ?1") | |
| 3018 | .run(deploy.id, createdAt); | |
| 3019 | if (servingAt !== null) { | |
| 3020 | app.db | |
| 3021 | .query("UPDATE run SET serving_at = ?2 WHERE id = ?1") | |
| 3022 | .run(deploy.id, servingAt); | |
| 3023 | } | |
| 3024 | }; | |
| 3025 | ||
| 3026 | test("no deploys, a never-served one, and the median of a growing set — all against one window", () => { | |
| 3027 | const now = Date.now(); | |
| 3028 | const since = new Date(now - 60_000).toISOString(); | |
| 3029 | ||
| 3030 | // Nothing seeded yet: undefined, never a zero. | |
| 3031 | expect(previewLatency(app.db, repo.id, since)).toBeUndefined(); | |
| 3032 | ||
| 3033 | // Before the window: a very slow deploy that must not pull the median up. | |
| 3034 | seedDeploy( | |
| 3035 | new Date(now - 10 * 24 * 60 * 60_000).toISOString(), | |
| 3036 | new Date(now - 10 * 24 * 60 * 60_000 + 600_000).toISOString(), | |
| 3037 | ); | |
| 3038 | expect(previewLatency(app.db, repo.id, since)).toBeUndefined(); | |
| 3039 | ||
| 3040 | // A deploy that never served is left out, not counted as a zero. | |
| 3041 | seedDeploy(new Date(now).toISOString(), null); | |
| 3042 | expect(previewLatency(app.db, repo.id, since)).toBeUndefined(); | |
| 3043 | ||
| 3044 | // Three served deploys, seeded out of order: an odd-count median. | |
| 3045 | const durations = [2, 6, 4]; // seconds | |
| 3046 | durations.forEach((d, i) => { | |
| 3047 | const createdAt = new Date(now + i * 1_000).toISOString(); | |
| 3048 | seedDeploy( | |
| 3049 | createdAt, | |
| 3050 | new Date(Date.parse(createdAt) + d * 1_000).toISOString(), | |
| 3051 | ); | |
| 3052 | }); | |
| 3053 | const odd = previewLatency(app.db, repo.id, since)!; | |
| 3054 | expect(odd.n).toBe(3); | |
| 3055 | expect(odd.medianSeconds).toBeCloseTo(4, 2); | |
| 3056 | ||
| 3057 | // A fourth deploy, 10s: the even-count median is the mean of the middle two. | |
| 3058 | const createdAt = new Date(now + 3_000).toISOString(); | |
| 3059 | seedDeploy( | |
| 3060 | createdAt, | |
| 3061 | new Date(Date.parse(createdAt) + 10_000).toISOString(), | |
| 3062 | ); | |
| 3063 | const even = previewLatency(app.db, repo.id, since)!; | |
| 3064 | expect(even.n).toBe(4); | |
| 3065 | expect(even.medianSeconds).toBeCloseTo(5, 2); | |
| 3066 | }); | |
| 3067 | }); | |
| 3068 | ||
| 3069 | describe("the studio's own words for how fast a change reaches its preview", () => { | |
| 3070 | let dataDir: string; | |
| 3071 | let app: App; | |
| 3072 | let servers: Servers; | |
| 3073 | let base: string; | |
| 3074 | let alice: User; | |
| 3075 | let repo: Repo; | |
| 3076 | let ownerCookie: string; | |
| 3077 | ||
| 3078 | beforeAll(async () => { | |
| 3079 | dataDir = await mkdtemp(join(tmpdir(), "forge-preview-latency-page-")); | |
| 3080 | app = createApp({ dataDir, port: 0 }); | |
| 3081 | servers = await serve(app); | |
| 3082 | base = servers.url; | |
| 3083 | const a = await createUser(app.db, { | |
| 3084 | username: "alice", | |
| 3085 | email: "[email protected]", | |
| 3086 | password: "a-good-passphrase", | |
| 3087 | adminIfFirst: true, | |
| 3088 | }); | |
| 3089 | if (!a.ok) throw new Error(a.error); | |
| 3090 | alice = a.value; | |
| 3091 | ownerCookie = ( | |
| 3092 | ( | |
| 3093 | await fetch(`${base}/signin`, { | |
| 3094 | method: "POST", | |
| 3095 | body: new URLSearchParams({ | |
| 3096 | username: "alice", | |
| 3097 | password: "a-good-passphrase", | |
| 3098 | }), | |
| 3099 | redirect: "manual", | |
| 3100 | }) | |
| 3101 | ).headers.get("set-cookie") ?? "" | |
| 3102 | ).split(";")[0]!; | |
| 3103 | const made = await createRepo(app, alice, { | |
| 3104 | name: "trickle", | |
| 3105 | visibility: "private", | |
| 3106 | description: "", | |
| 3107 | }); | |
| 3108 | if (!made.ok) throw new Error(String(made.error)); | |
| 3109 | repo = made.value; | |
| 3110 | const seeded = await commitToBranch(app, alice, repo, { | |
| 3111 | message: "Start", | |
| 3112 | files: [{ path: "README.md", content: "start\n" }], | |
| 3113 | }); | |
| 3114 | if (!seeded.ok) throw new Error(JSON.stringify(seeded.error)); | |
| 3115 | const enabled = await enableApp(app, alice, repo, { slug: "trickle" }); | |
| 3116 | if (!enabled.ok) throw new Error(String(enabled.error)); | |
| 3117 | }); | |
| 3118 | ||
| 3119 | afterAll(async () => { | |
| 3120 | servers.stop(); | |
| 3121 | app.close(); | |
| 3122 | await rm(dataDir, { recursive: true, force: true }); | |
| 3123 | }); | |
| 3124 | ||
| 3125 | const get = (path: string) => | |
| 3126 | fetch(`${base}${path}`, { | |
| 3127 | headers: { cookie: `${ownerCookie}; forge_mode=studio` }, | |
| 3128 | redirect: "manual", | |
| 3129 | }); | |
| 3130 | ||
| 3131 | test("fewer than three served deploys this week says so plainly rather than a median of one", async () => { | |
| 3132 | const before = await (await get("/alice/trickle")).text(); | |
| 3133 | expect(before).toContain("not enough deploys this week to say"); | |
| 3134 | expect(before).not.toContain("reaches its preview in"); | |
| 3135 | ||
| 3136 | const deploy = createRun(app.db, { | |
| 3137 | repoId: repo.id, | |
| 3138 | kind: "deploy", | |
| 3139 | env: "production", | |
| 3140 | reason: "push", | |
| 3141 | }); | |
| 3142 | markServing(app.db, deploy.id); | |
| 3143 | const one = await (await get("/alice/trickle")).text(); | |
| 3144 | expect(one).toContain("not enough deploys this week to say"); | |
| 3145 | expect(one).not.toContain("reaches its preview in"); | |
| 3146 | }); | |
| 3147 | }); |
new above
offline