Public: anyone can read this; the crew can change it.

How fast a change reaches its preview, on the app page: the direction's number, measured #109

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/83-how-fast-a-change-reaches-its-preview-on → main 1 commit fork

  • Merged. scout/83-how-fast-a-change-reaches-its-preview-on was merged into main as 28682a6.

Review

Sign in to review this change.
3 files changed +269 −1
src/core/runs.ts +38
@@ -590,6 +590,44 @@ export function latestAgentRunsForRepos(
590590 return out;
591591 }
592592
593 /** How fast this app's deploys have reached their preview: the median, and the count it is drawn from. */
594 export interface PreviewLatency {
595 readonly medianSeconds: number;
596 readonly n: number;
597 }
598
599 /**
600 * The median seconds from push (`created_at`) to serving (`serving_at`)
601 * over this repo's deploys since `since` — the same reference instant
602 * `servedPhrase` (`web/ui.ts`) words a single deploy from. A deploy that
603 * never served says nothing about speed and is left out, never a zero.
604 * One query, bounded by `limit` and served by the `run_repo` index (it
605 * walks newest-first and stops at the cap), never a scan of the table.
606 */
607 export function previewLatency(
608 db: Database,
609 repoId: string,
610 since: string,
611 limit = 500,
612 ): PreviewLatency | undefined {
613 const secs = db
614 .query<{ secs: number }, [string, string, number]>(
615 `SELECT (julianday(serving_at) - julianday(created_at)) * 86400 AS secs
616 FROM run
617 WHERE repo_id = ?1 AND kind = 'deploy' AND serving_at IS NOT NULL
618 AND created_at >= ?2
619 ORDER BY id DESC LIMIT ?3`,
620 )
621 .all(repoId, since, limit)
622 .map((r) => r.secs)
623 .sort((a, b) => a - b);
624 if (secs.length === 0) return undefined;
625 const mid = Math.floor(secs.length / 2);
626 const medianSeconds =
627 secs.length % 2 === 0 ? (secs[mid - 1]! + secs[mid]!) / 2 : secs[mid]!;
628 return { medianSeconds, n: secs.length };
629 }
630
593631 /** Agent runs queued or running across many repositories, newest first. */
594632 export function listActiveRuns(
595633 db: Database,
src/web/studio/app.ts +29 −1
@@ -13,7 +13,12 @@ import { decisionStats, type DecisionStat } from "../../core/decisions.ts";
1313 import { getIssueById, listIssues, type Issue } from "../../core/issues.ts";
1414 import { listPulls, openPullsClosing } from "../../core/pulls.ts";
1515 import { canSeeOwnerFacts, type Repo } from "../../core/repos.ts";
16 import { latestRunsForIssues, listRuns } from "../../core/runs.ts";
16 import {
17 latestRunsForIssues,
18 listRuns,
19 previewLatency,
20 type PreviewLatency,
21 } from "../../core/runs.ts";
1722 import { getUsersByIds } from "../../core/users.ts";
1823 import { autonomyStatus } from "../../crew/autonomy.ts";
1924 import { repoDir } from "../../git/repos.ts";
@@ -65,6 +70,24 @@ function decisionWords(stats: readonly DecisionStat[]): Html {
6570 >`;
6671 }
6772
73 /** Below this many served deploys a median is one or two points, not a trend. */
74 const PREVIEW_LATENCY_MIN = 3;
75
76 /**
77 * How fast a change has reached its preview this week, worded like the
78 * reference instant in `servedPhrase` (`web/ui.ts`): "reaches its preview
79 * in 6.1s". Too few served deploys to trust a median, and the line says so
80 * rather than printing one from a run or two.
81 */
82 function previewLatencyWords(latency: PreviewLatency | undefined): Html {
83 if (latency === undefined || latency.n < PREVIEW_LATENCY_MIN)
84 return html`<span class="muted small"
85 >not enough deploys this week to say</span
86 >`;
87 return html`reaches its preview in ${latency.medianSeconds.toFixed(1)}s
88 <span class="muted">· ${latency.n} deploys</span>`;
89 }
90
6891 export async function studioApp(
6992 ctx: Ctx,
7093 repo: Repo,
@@ -110,6 +133,9 @@ export async function studioApp(
110133 : [];
111134 const spent = weekRuns.reduce((s, r) => s + r.costUsd, 0);
112135 const ledgerWeek = ownerFacts ? decisionStats(db, repo.id, week) : [];
136 const weekLatency = ownerFacts
137 ? previewLatency(db, repo.id, week)
138 : undefined;
113139 const autonomy =
114140 !ownerFacts || repo.autonomy === "off"
115141 ? undefined
@@ -305,6 +331,8 @@ export async function studioApp(
305331 <dd>${shipped} change${shipped === 1 ? "" : "s"}</dd>
306332 <dt>Runs</dt>
307333 <dd>${weekRuns.length}</dd>
334 <dt>Preview</dt>
335 <dd>${previewLatencyWords(weekLatency)}</dd>
308336 <dt>Spent</dt>
309337 <dd>
310338 $${spent.toFixed(2)}${shipped > 0
test/studio.test.ts +202
@@ -38,6 +38,7 @@ import {
3838 getRun,
3939 markServing,
4040 noteRunProgress,
41 previewLatency,
4142 type Run,
4243 } from "../src/core/runs.ts";
4344 import { createUser, getUserByName, type User } from "../src/core/users.ts";
@@ -2690,6 +2691,32 @@ describe("a stranger sees the app, not the owner's books", () => {
26902691 by: "rule",
26912692 ms: 3,
26922693 });
2694
2695 // Three served deploys this week, median 4.0s, and a fourth that never
2696 // answered — excluded, not a zero that would drag the median down.
2697 const now = Date.now();
2698 for (const [offsetMs, durationMs] of [
2699 [0, 2_000],
2700 [1_000, 4_000],
2701 [2_000, 6_000],
2702 [3_000, null],
2703 ] as const) {
2704 const deploy = createRun(app.db, {
2705 repoId: repo.id,
2706 kind: "deploy",
2707 env: "production",
2708 commitOid: "cafefeed",
2709 reason: "push",
2710 });
2711 app.db
2712 .query("UPDATE run SET created_at = ?2 WHERE id = ?1")
2713 .run(deploy.id, new Date(now + offsetMs).toISOString());
2714 if (durationMs !== null) {
2715 app.db
2716 .query("UPDATE run SET serving_at = ?2 WHERE id = ?1")
2717 .run(deploy.id, new Date(now + offsetMs + durationMs).toISOString());
2718 }
2719 }
26932720 });
26942721
26952722 afterAll(async () => {
@@ -2715,12 +2742,20 @@ describe("a stranger sees the app, not the owner's books", () => {
27152742 expect(page).toContain(">Shipped<");
27162743 expect(page).toContain("1 change");
27172744 expect(page).toContain("Open the app");
2745 // The direction's own number is the same owner's-books fact as spend
2746 // and the ledger: a stranger and an anonymous visitor see neither.
2747 expect(page).not.toContain(">Preview<");
2748 expect(page).not.toContain("reaches its preview");
27182749 }
27192750
27202751 expect(owner).toContain("$42.50");
27212752 expect(owner).toContain("keeper");
27222753 expect(owner).toContain(">On its own<");
27232754 expect(owner).toContain(">Decided<");
2755 // Three served deploys this week — 2.0s, 4.0s and 6.0s — median 4.0s;
2756 // the fourth, never served, does not drag it toward zero.
2757 expect(owner).toContain("reaches its preview in 4.0s");
2758 expect(owner).toContain("· 3 deploys");
27242759 });
27252760
27262761 test("the app's live fragment carries no owner fact either", async () => {
@@ -2943,3 +2978,170 @@ describe("servedPhrase", () => {
29432978 ).toBeUndefined();
29442979 });
29452980 });
2981
2982 describe("previewLatency: the median seconds from push to serving", () => {
2983 let dataDir: string;
2984 let app: App;
2985 let alice: User;
2986 let repo: Repo;
2987
2988 beforeAll(async () => {
2989 dataDir = await mkdtemp(join(tmpdir(), "forge-preview-latency-"));
2990 app = createApp({ dataDir, port: 0 });
2991 const a = await createUser(app.db, {
2992 username: "alice",
2993 email: "[email protected]",
2994 password: "a-good-passphrase",
2995 adminIfFirst: true,
2996 });
2997 if (!a.ok) throw new Error(a.error);
2998 alice = a.value;
2999 const made = await createRepo(app, alice, {
3000 name: "latency",
3001 visibility: "private",
3002 description: "",
3003 });
3004 if (!made.ok) throw new Error(String(made.error));
3005 repo = made.value;
3006 });
3007
3008 afterAll(async () => {
3009 app.close();
3010 await rm(dataDir, { recursive: true, force: true });
3011 });
3012
3013 /** A deploy run stamped with a known push instant and, unless null, a known serving instant. */
3014 const seedDeploy = (createdAt: string, servingAt: string | null): void => {
3015 const deploy = createRun(app.db, { repoId: repo.id, kind: "deploy" });
3016 app.db
3017 .query("UPDATE run SET created_at = ?2 WHERE id = ?1")
3018 .run(deploy.id, createdAt);
3019 if (servingAt !== null) {
3020 app.db
3021 .query("UPDATE run SET serving_at = ?2 WHERE id = ?1")
3022 .run(deploy.id, servingAt);
3023 }
3024 };
3025
3026 test("no deploys, a never-served one, and the median of a growing set — all against one window", () => {
3027 const now = Date.now();
3028 const since = new Date(now - 60_000).toISOString();
3029
3030 // Nothing seeded yet: undefined, never a zero.
3031 expect(previewLatency(app.db, repo.id, since)).toBeUndefined();
3032
3033 // Before the window: a very slow deploy that must not pull the median up.
3034 seedDeploy(
3035 new Date(now - 10 * 24 * 60 * 60_000).toISOString(),
3036 new Date(now - 10 * 24 * 60 * 60_000 + 600_000).toISOString(),
3037 );
3038 expect(previewLatency(app.db, repo.id, since)).toBeUndefined();
3039
3040 // A deploy that never served is left out, not counted as a zero.
3041 seedDeploy(new Date(now).toISOString(), null);
3042 expect(previewLatency(app.db, repo.id, since)).toBeUndefined();
3043
3044 // Three served deploys, seeded out of order: an odd-count median.
3045 const durations = [2, 6, 4]; // seconds
3046 durations.forEach((d, i) => {
3047 const createdAt = new Date(now + i * 1_000).toISOString();
3048 seedDeploy(
3049 createdAt,
3050 new Date(Date.parse(createdAt) + d * 1_000).toISOString(),
3051 );
3052 });
3053 const odd = previewLatency(app.db, repo.id, since)!;
3054 expect(odd.n).toBe(3);
3055 expect(odd.medianSeconds).toBeCloseTo(4, 2);
3056
3057 // A fourth deploy, 10s: the even-count median is the mean of the middle two.
3058 const createdAt = new Date(now + 3_000).toISOString();
3059 seedDeploy(
3060 createdAt,
3061 new Date(Date.parse(createdAt) + 10_000).toISOString(),
3062 );
3063 const even = previewLatency(app.db, repo.id, since)!;
3064 expect(even.n).toBe(4);
3065 expect(even.medianSeconds).toBeCloseTo(5, 2);
3066 });
3067 });
3068
3069 describe("the studio's own words for how fast a change reaches its preview", () => {
3070 let dataDir: string;
3071 let app: App;
3072 let servers: Servers;
3073 let base: string;
3074 let alice: User;
3075 let repo: Repo;
3076 let ownerCookie: string;
3077
3078 beforeAll(async () => {
3079 dataDir = await mkdtemp(join(tmpdir(), "forge-preview-latency-page-"));
3080 app = createApp({ dataDir, port: 0 });
3081 servers = await serve(app);
3082 base = servers.url;
3083 const a = await createUser(app.db, {
3084 username: "alice",
3085 email: "[email protected]",
3086 password: "a-good-passphrase",
3087 adminIfFirst: true,
3088 });
3089 if (!a.ok) throw new Error(a.error);
3090 alice = a.value;
3091 ownerCookie = (
3092 (
3093 await fetch(`${base}/signin`, {
3094 method: "POST",
3095 body: new URLSearchParams({
3096 username: "alice",
3097 password: "a-good-passphrase",
3098 }),
3099 redirect: "manual",
3100 })
3101 ).headers.get("set-cookie") ?? ""
3102 ).split(";")[0]!;
3103 const made = await createRepo(app, alice, {
3104 name: "trickle",
3105 visibility: "private",
3106 description: "",
3107 });
3108 if (!made.ok) throw new Error(String(made.error));
3109 repo = made.value;
3110 const seeded = await commitToBranch(app, alice, repo, {
3111 message: "Start",
3112 files: [{ path: "README.md", content: "start\n" }],
3113 });
3114 if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
3115 const enabled = await enableApp(app, alice, repo, { slug: "trickle" });
3116 if (!enabled.ok) throw new Error(String(enabled.error));
3117 });
3118
3119 afterAll(async () => {
3120 servers.stop();
3121 app.close();
3122 await rm(dataDir, { recursive: true, force: true });
3123 });
3124
3125 const get = (path: string) =>
3126 fetch(`${base}${path}`, {
3127 headers: { cookie: `${ownerCookie}; forge_mode=studio` },
3128 redirect: "manual",
3129 });
3130
3131 test("fewer than three served deploys this week says so plainly rather than a median of one", async () => {
3132 const before = await (await get("/alice/trickle")).text();
3133 expect(before).toContain("not enough deploys this week to say");
3134 expect(before).not.toContain("reaches its preview in");
3135
3136 const deploy = createRun(app.db, {
3137 repoId: repo.id,
3138 kind: "deploy",
3139 env: "production",
3140 reason: "push",
3141 });
3142 markServing(app.db, deploy.id);
3143 const one = await (await get("/alice/trickle")).text();
3144 expect(one).toContain("not enough deploys this week to say");
3145 expect(one).not.toContain("reaches its preview in");
3146 });
3147 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs