Public: anyone can read this; the crew can change it.

Sleep windows follow the slider: the runtime reads its idle windows from the position, live #10181

Parked opened this
Closed without merging

scout/forge:scout/506-sleep-windows-follow-the-slider-the-runt → main 3 commits fork

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
8 files changed +318 −19
docs/APPS.md +14 −6
@@ -534,6 +534,10 @@ that before the deploy fails; `tty` is the app's terminal interface (below);
534534 file is read from the commit being deployed, never from the database — a
535535 pull that changes how the app runs is previewed the way it will run.
536536
537 `app.always_on: true` keeps production awake whatever the profit slider
538 says — the one per-app exception to it, shown on the app page. Previews
539 still sleep as the forge says.
540
537541 `toolbar`, at the top level, is `true` unless you set it to `false`. The
538542 forge adds one `<script>` tag to the HTML pages your app serves (before
539543 `</body>`, or at the end): a small pill in the bottom-right corner, collapsed
@@ -1431,17 +1435,21 @@ works the crew page says so, in words, before anyone assigns anything.
14311435
14321436 ### Asleep
14331437
1434 A preview whose pull has had no push and no comment for three days is
1435 stopped and shown as _asleep_ with when and why — its row, data and pictures
1438 A preview whose pull has had no push and no comment for as long as the
1439 profit slider says (three days at its default of 50, one at 0, never at
1440 100; `docs/PROFIT.md`) is stopped and shown as _asleep_ with when and why — its row, data and pictures
14361441 stay. It is not failed, and a restart leaves it stopped. Asking for it (a
14371442 request to its host, or **Wake it** on the change's page) deploys it again
14381443 on the ordinary deploy path. Production, and a pull with a run queued or
14391444 running on it, are never put to sleep this way. `FORGE_PREVIEW_IDLE_DAYS`
1440 sets the age; `0` keeps every preview warm.
1445 pins the age instead of the slider; `0` keeps every preview warm.
14411446
14421447 An environment nobody asks for does not keep a process. A preview that has
1443 served no request for thirty minutes, or production for a day, is stopped
1444 and marked _asleep_, its checkout kept whole; the next request through the
1448 served no request for thirty minutes, or production for a day — the profit
1449 slider's 50; the sweep reads the slider's position on every pass, so moving
1450 it changes when environments sleep within a minute, with no restart — is
1451 stopped and marked _asleep_ (its log names the window and the position:
1452 "asleep: nothing asked for it in 10 min (economy 0)"), its checkout kept whole; the next request through the
14451453 proxy starts it again — warm, without install or test, in about the time a
14461454 page takes — and waits for it rather than showing a closed door. The forge
14471455 coming back does the same: production starts warm from the checkout it kept,
@@ -1604,7 +1612,7 @@ a link to the milestone (`docs/PLANNING.md`); its progress never does.
16041612 | `FORGE_APPS` | `1` | `0` runs no apps and no crew |
16051613 | `FORGE_APP_DOMAIN` | the host in `FORGE_URL`/`_HOST` | what app hostnames attach to (`example` or `--x`) |
16061614 | `FORGE_APPS_CONCURRENCY` | `2` | deploys running their heavy steps at once |
1607 | `FORGE_PREVIEW_IDLE_DAYS` | `3` | days a pull may sit untouched before its preview sleeps; `0` = never |
1615 | `FORGE_PREVIEW_IDLE_DAYS` | the profit slider | days a pull may sit untouched before its preview sleeps; `0` = never; unset follows the slider (3 at 50) |
16081616 | `FORGE_CREW_CONCURRENCY` | 1 per 3 GiB above 2 GiB, 2–16 | agent runs at once (live: Settings → Capacity) |
16091617 | `FORGE_CHECKS_CONCURRENCY` | `1` | checks at once (live: Settings → Capacity) |
16101618 | `FORGE_CHECKS_PLACEMENT` | `process` | `job` runs a check's commands as a cluster Job (see Placement above) |
src/apps/manifest.ts +10
@@ -52,6 +52,12 @@ export interface AppManifest {
5252 readonly pages?: readonly string[] | undefined;
5353 /** `app.launch`: how this repository's demos open unless a pin says otherwise. */
5454 readonly launch?: Launch | undefined;
55 /**
56 * `app.always_on: true`: production never sleeps, whatever the profit
57 * slider says (docs/PROFIT.md) — the one per-app exception. Previews
58 * still follow the forge. Unset or false follows the slider.
59 */
60 readonly alwaysOn?: true | undefined;
5561 /**
5662 * Set when `app.kind` is `"service"`: declared TCP ports, always on,
5763 * a placement preference and synced volumes. `null` for an ordinary
@@ -167,6 +173,9 @@ export function parseManifest(
167173 const launch = app["launch"];
168174 if (launch !== undefined && !isLaunch(launch))
169175 return invalid(`forge.json: app.launch must be one of ${LAUNCHES.map((l) => `"${l}"`).join(", ")}`);
176 const alwaysOn = app["always_on"];
177 if (alwaysOn !== undefined && typeof alwaysOn !== "boolean")
178 return invalid("forge.json: app.always_on must be true or false");
170179 const egress = egressField(parsed.value);
171180 if (!egress.ok) return egress;
172181 const limits = limitsField(app, "app");
@@ -211,6 +220,7 @@ export function parseManifest(
211220 looksAsOwner: looksAsOwner.value,
212221 pages: pages.value,
213222 ...(launch === undefined ? {} : { launch }),
223 ...(alwaysOn === true ? { alwaysOn: true as const } : {}),
214224 service,
215225 egress: egress.value,
216226 limits: limits.value,
src/apps/runtime.ts +51 −11
@@ -101,6 +101,9 @@ import { parseManifest, type AppManifest } from "./manifest.ts";
101101 import { identityKey } from "./proxy.ts";
102102 import { noteReached } from "./egress_report.ts";
103103 import { egressSetting } from "../core/capacity.ts";
104 import { policyAt } from "../core/slider.ts";
105 import { getEconomics } from "../settings/economics.ts";
106 import { asleepWords, sleepWindow, type WindowSource } from "./sleep_window.ts";
104107 import { noteProtection } from "../kernel/protection.ts";
105108 import { cellsAvailable, resolveLimits, type CellSpec } from "./contain.ts";
106109 import {
@@ -126,18 +129,26 @@ export interface RuntimeOptions {
126129 readonly concurrency?: number | undefined;
127130 /**
128131 * How long an environment serves nobody before its process is stopped
129 * and it sleeps, to be woken by the next request. A preview is looked at
130 * and forgotten; production is left alone for a day before it rests.
132 * and it sleeps, to be woken by the next request. Unset (the forge's
133 * own runtime) reads the profit slider's position on every sweep
134 * (`apps/sleep_window.ts`); set, it is fixed — tests and nothing else.
131135 */
132136 readonly sleepAfterMs?:
133137 | { readonly production: number; readonly preview: number }
134138 | undefined;
139 /**
140 * The sweep's clock; a test hands in one that runs ahead to prove a
141 * window of minutes without waiting minutes. @default Date.now
142 */
143 readonly clock?: (() => number) | undefined;
135144 /** How often the runtime looks for environments to put to sleep. */
136145 readonly sweepEveryMs?: number | undefined;
137146 /**
138147 * How long a pull may go without a push or a comment before its preview
139148 * is stopped (not removed: its row, data and pictures stay) and reads as
140 * asleep until someone asks for it. Unset or 0 keeps every preview warm.
149 * asleep until someone asks for it. Unset follows the profit slider
150 * (`policyAt(position).idlePullMs`, read each sweep; `null` is never);
151 * 0 keeps every preview warm.
141152 */
142153 readonly idlePullMs?: number | undefined;
143154 /**
@@ -246,6 +257,8 @@ export interface EnvStatus {
246257 * person's words (`BUSY_WORDS`) — set only while the sweep is sparing it.
247258 */
248259 readonly busy?: string | undefined;
260 /** Production whose `forge.json` sets `app.always_on: true`: it never sleeps. */
261 readonly alwaysOn?: true | undefined;
249262 /**
250263 * Set while a preview is stopped because its pull went quiet: when it was
251264 * stopped, and the last touch (push, comment, wake) it was measured from.
@@ -593,6 +606,9 @@ export class AppsRuntime {
593606 looks: state.live.manifest.looks,
594607 looksAsOwner: state.live.manifest.looksAsOwner,
595608 toolbar: state.live.manifest.toolbar,
609 ...(state.env === "production" && state.live.manifest.alwaysOn === true
610 ? { alwaysOn: true as const }
611 : {}),
596612 ...(state.live.deployStartedAt === null
597613 ? {}
598614 : { deployStartedAt: state.live.deployStartedAt }),
@@ -1916,8 +1932,8 @@ export class AppsRuntime {
19161932 * or someone is deciding it right now (a queued or running run on it).
19171933 */
19181934 private idleSince(repo: Repo, pull: Pull, env: EnvName): string | undefined {
1919 const age = this.options.idlePullMs ?? 0;
1920 if (age <= 0 || pull.state !== "open") return undefined;
1935 const age = this.idlePullAge();
1936 if (age === null || pull.state !== "open") return undefined;
19211937 const now = Date.now();
19221938 let last = Date.parse(pull.updatedAt);
19231939 if (now - last < age) return undefined;
@@ -1968,7 +1984,7 @@ export class AppsRuntime {
19681984
19691985 /** Stop the previews whose pulls have gone quiet; keep everything else. */
19701986 private async hibernate(): Promise<void> {
1971 if ((this.options.idlePullMs ?? 0) <= 0) return;
1987 if (this.idlePullAge() === null) return;
19721988 for (const state of [...this.envs.values()]) {
19731989 const number = envPull(state.env);
19741990 if (
@@ -2007,13 +2023,32 @@ export class AppsRuntime {
20072023 }
20082024 }
20092025
2026 /**
2027 * How long a pull may sit untouched before its preview stops; `null` is
2028 * never. A fixed option wins (0 is off); unset follows the slider, read
2029 * now — so moving it changes the next sweep, with no restart.
2030 */
2031 private idlePullAge(): number | null {
2032 const fixed = this.options.idlePullMs;
2033 if (fixed !== undefined) return fixed > 0 ? fixed : null;
2034 return policyAt(getEconomics(this.app.db).position).idlePullMs;
2035 }
2036
2037 /** Where this sweep's idle windows come from: a fixed option, else the slider as it stands. */
2038 private windowSource(): WindowSource {
2039 const fixed = this.options.sleepAfterMs;
2040 return fixed !== undefined
2041 ? { kind: "fixed", sleepAfterMs: fixed }
2042 : { kind: "slider", position: getEconomics(this.app.db).position };
2043 }
2044
20102045 /** Stop what nobody has asked for lately, and keep it ready to wake. */
20112046 private async sweep(): Promise<void> {
20122047 if (this.stopped) return;
20132048 await this.hibernate();
2014 const after = this.options.sleepAfterMs ?? SLEEP_AFTER_MS;
2049 const source = this.windowSource();
20152050 const grace = this.options.busyGraceMs ?? BUSY_GRACE_MS;
2016 const now = Date.now();
2051 const now = this.options.clock?.() ?? Date.now();
20172052 this.metered(() => beatAwake(this.app.db, new Date(now).toISOString()));
20182053 for (const state of this.envs.values()) {
20192054 const live = state.live;
@@ -2029,8 +2064,13 @@ export class AppsRuntime {
20292064 state.waking !== undefined
20302065 )
20312066 continue;
2032 const limit =
2033 state.env === "production" ? after.production : after.preview;
2067 const window = sleepWindow(
2068 source,
2069 state.env === "production",
2070 live.manifest.alwaysOn === true,
2071 );
2072 const limit = window.ms;
2073 if (limit === null) continue;
20342074 const last = state.lastRequestAt ?? Date.parse(live.since);
20352075 if (now - last < limit) continue;
20362076 // Idle by the clock; asked whether it is mid-work, and spared while
@@ -2065,7 +2105,7 @@ export class AppsRuntime {
20652105 this.app.config.dataDir,
20662106 live.runId,
20672107 busy === undefined
2068 ? `asleep: nothing asked for it in ${Math.round(limit / 60_000)} minutes; the next request wakes it`
2108 ? asleepWords({ ms: limit, why: window.why })
20692109 : `asleep: still busy ${Math.round(grace / 60_000)} minutes past its idle window, which is as long as busy keeps an app awake; the next request wakes it`,
20702110 );
20712111 log.info("asleep", { app: state.app.slug, env: state.env });
src/apps/sleep_window.ts added +43
@@ -0,0 +1,43 @@
1 /**
2 * How long an environment may serve nobody before the sweep puts it to
3 * sleep, and the words its log says when it does. The window comes from
4 * the profit slider's position (`core/slider.ts`), read on every sweep so
5 * moving the slider takes effect within one pass; an app's
6 * `app.always_on: true` keeps its production awake whatever the position.
7 *
8 * Plain data in, plain data out.
9 */
10
11 import { duration, policyAt } from "../core/slider.ts";
12
13 /** Where a window came from: the slider at a position, or a fixed setting (tests, a child forge). */
14 export type WindowSource =
15 | { readonly kind: "slider"; readonly position: number }
16 | { readonly kind: "fixed"; readonly sleepAfterMs: { readonly production: number; readonly preview: number } };
17
18 export interface SleepWindow {
19 /** Idle milliseconds before it sleeps; `null` is never. */
20 readonly ms: number | null;
21 /** Why it is this long, for the log: "economy 20", "always_on". */
22 readonly why: string;
23 }
24
25 /** The window for one environment: production or a preview. */
26 export function sleepWindow(
27 source: WindowSource,
28 production: boolean,
29 alwaysOn: boolean,
30 ): SleepWindow {
31 if (production && alwaysOn) return { ms: null, why: "always_on" };
32 if (source.kind === "fixed") {
33 const { sleepAfterMs } = source;
34 return { ms: production ? sleepAfterMs.production : sleepAfterMs.preview, why: "set for this forge" };
35 }
36 const after = policyAt(source.position).sleepAfterMs;
37 return { ms: production ? after.production : after.preview, why: `economy ${source.position}` };
38 }
39
40 /** "asleep: nothing asked for it in 10 min (economy 20); the next request wakes it" */
41 export function asleepWords(window: SleepWindow & { readonly ms: number }): string {
42 return `asleep: nothing asked for it in ${duration(window.ms)} (${window.why}); the next request wakes it`;
43 }
src/main.ts +5 −2
@@ -410,8 +410,11 @@ const apps = appDomain?.ok
410410 concurrency: positive(process.env.FORGE_APPS_CONCURRENCY, 2),
411411 busyGraceMs: crewRunMs,
412412 // Days a pull may sit untouched before its preview sleeps; 0 keeps
413 // every preview warm.
414 idlePullMs: nonNegative(process.env.FORGE_PREVIEW_IDLE_DAYS, IDLE_PULL_DAYS) * 86_400_000,
413 // every preview warm. Unset follows the profit slider, live.
414 idlePullMs:
415 process.env.FORGE_PREVIEW_IDLE_DAYS === undefined
416 ? undefined
417 : nonNegative(process.env.FORGE_PREVIEW_IDLE_DAYS, IDLE_PULL_DAYS) * 86_400_000,
415418 })
416419 : undefined;
417420 // Which other forge serves this data's apps, read before this one starts its own.
src/web/ui.ts +8
@@ -516,6 +516,14 @@ export function envFacts(
516516 )}
517517 ${when(env.note, (note) => html`<p class="muted small">${note}</p>`)}
518518 ${when(dormantNote(env), (note) => html`<p class="muted small" data-dormant>${note}</p>`)}
519 ${when(
520 env.alwaysOn === true,
521 () =>
522 html`<p class="muted small" data-always-on>
523 always on: its forge.json sets app.always_on, so it never sleeps
524 whatever the profit slider says
525 </p>`,
526 )}
519527 ${when(
520528 env.busy,
521529 (busy) =>
test/apps-sleep-slider.test.ts added +128
@@ -0,0 +1,128 @@
1 /**
2 * The sweep reads its idle windows from the profit slider's position on
3 * every pass: moving the slider changes when production sleeps with no
4 * restart, and `app.always_on` keeps production awake even at 0. The
5 * sweep's clock runs ahead so a window of minutes takes milliseconds.
6 */
7 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
8 import { mkdtemp, rm } from "node:fs/promises";
9 import { tmpdir } from "node:os";
10 import { join } from "node:path";
11 import { createApp, type App } from "../src/app.ts";
12 import { AppsRuntime, type EnvStatus } from "../src/apps/runtime.ts";
13 import { newApp } from "../src/actions/apps.ts";
14 import { commitToBranch } from "../src/actions/repos.ts";
15 import type { AppRecord } from "../src/core/apps.ts";
16 import type { Repo } from "../src/core/repos.ts";
17 import { listRuns, type Run } from "../src/core/runs.ts";
18 import { createUser, type User } from "../src/core/users.ts";
19 import { serve, type Servers } from "../src/http/server.ts";
20 import { readRunLog } from "../src/runs/log.ts";
21 import { setEconomics } from "../src/settings/economics.ts";
22
23 describe("sleep windows follow the slider", () => {
24 let dataDir: string;
25 let app: App;
26 let servers: Servers;
27 let runtime: AppsRuntime;
28 let alice: User;
29 let repo: Repo;
30 let record: AppRecord;
31 const DOMAIN = "apps.test";
32 const ELEVEN_MINUTES = 11 * 60_000;
33 let ahead = 0;
34
35 const viaHost = (host: string) =>
36 fetch(`${servers.url}/`, { headers: { host }, redirect: "manual" });
37 const production = (): EnvStatus | undefined =>
38 runtime.status(record).find((e) => e.env === "production");
39 const position = (n: number) => setEconomics(app.db, { position: n, marginPct: 0 }, alice.id);
40
41 async function until<T>(read: () => T | undefined, ms = 60_000): Promise<T> {
42 const deadline = Date.now() + ms;
43 while (Date.now() < deadline) {
44 const value = read();
45 if (value !== undefined) return value;
46 await Bun.sleep(50);
47 }
48 throw new Error("did not happen in time");
49 }
50 const settled = (where: (run: Run) => boolean) =>
51 until(() =>
52 listRuns(app.db, repo.id, { kind: "deploy", limit: 50 }).find(
53 (r) => where(r) && (r.state === "ok" || r.state === "failed"),
54 ),
55 );
56 const log = (runId: string): string =>
57 readRunLog(dataDir, runId, 64 * 1024).lines.join("\n");
58
59 beforeAll(async () => {
60 dataDir = await mkdtemp(join(tmpdir(), "forge-sleep-slider-"));
61 app = createApp({ dataDir, port: 0 });
62 servers = await serve(app);
63 runtime = new AppsRuntime(app, {
64 domain: DOMAIN,
65 scheme: "http",
66 publicPort: servers.port,
67 forgeUrl: servers.url,
68 sweepEveryMs: 100,
69 clock: () => Date.now() + ahead,
70 });
71 await runtime.start();
72 const created = await createUser(app.db, { origin: "seed", trusted: true,
73 username: "alice",
74 email: "[email protected]",
75 password: "a-good-passphrase",
76 adminIfFirst: true,
77 });
78 if (!created.ok) throw new Error(created.error);
79 alice = created.value;
80 const made = await newApp(app, alice, { name: "hello", description: "Notes", visibility: "public" });
81 if (!made.ok) throw new Error(String(made.error));
82 repo = made.value.repo;
83 record = made.value.app;
84 expect((await settled((r) => r.env === "production")).state).toBe("ok");
85 }, 120_000);
86
87 afterAll(async () => {
88 await runtime.stop();
89 servers.stop();
90 app.close();
91 await rm(dataDir, { recursive: true, force: true });
92 });
93
94 test("at 100 production idle past the 0-anchor window stays up; moved to 0, the next sweep puts it to sleep", async () => {
95 position(100);
96 ahead = ELEVEN_MINUTES;
97 await Bun.sleep(600);
98 expect(production()?.state).toBe("running");
99 position(0);
100 await until(() => (production()?.state === "asleep" ? true : undefined), 5_000);
101 expect(log(production()!.runId!)).toContain(
102 "asleep: nothing asked for it in 10 min (economy 0); the next request wakes it",
103 );
104 }, 30_000);
105
106 test("always_on wins over 0, and the status says so", async () => {
107 ahead = 0;
108 expect((await viaHost(`hello.${DOMAIN}`)).status).toBe(200);
109 const before = new Set(listRuns(app.db, repo.id, { kind: "deploy", limit: 50 }).map((r) => r.id));
110 const pinned = await commitToBranch(app, alice, repo, {
111 branch: "main",
112 message: "Always on",
113 files: [{
114 path: "forge.json",
115 content: JSON.stringify({ app: { run: "bun run src/server.ts", health: "/healthz", always_on: true } }),
116 }],
117 });
118 if (!pinned.ok) throw new Error(JSON.stringify(pinned.error));
119 const run = await settled((r) => r.env === "production" && !before.has(r.id));
120 expect(run.state).toBe("ok");
121 await until(() => (production()?.alwaysOn === true ? true : undefined), 10_000);
122 position(0);
123 ahead = ELEVEN_MINUTES;
124 await Bun.sleep(600);
125 expect(production()?.state).toBe("running");
126 expect(production()?.alwaysOn).toBe(true);
127 }, 120_000);
128 });
test/sleep-window.test.ts added +59
@@ -0,0 +1,59 @@
1 /**
2 * The idle window an environment sleeps after comes from the slider's
3 * position, and `app.always_on` is the one per-app exception.
4 */
5 import { describe, expect, test } from "bun:test";
6 import { asleepWords, sleepWindow } from "../src/apps/sleep_window.ts";
7 import { parseManifest } from "../src/apps/manifest.ts";
8 import { envFacts } from "../src/web/ui.ts";
9
10 const at = (position: number) => ({ kind: "slider" as const, position });
11
12 describe("sleep windows", () => {
13 test("production follows the slider: 10 min at 0, never at 100", () => {
14 expect(sleepWindow(at(0), true, false)).toEqual({ ms: 600_000, why: "economy 0" });
15 expect(sleepWindow(at(100), true, false).ms).toBeNull();
16 expect(sleepWindow(at(50), true, false).ms).toBe(86_400_000);
17 expect(sleepWindow(at(0), false, false).ms).toBe(300_000);
18 });
19
20 test("always_on wins over 0 for production, and previews still follow the forge", () => {
21 expect(sleepWindow(at(0), true, true)).toEqual({ ms: null, why: "always_on" });
22 expect(sleepWindow(at(0), false, true).ms).toBe(300_000);
23 });
24
25 test("a fixed setting is used as given", () => {
26 const fixed = { kind: "fixed" as const, sleepAfterMs: { production: 700, preview: 900 } };
27 expect(sleepWindow(fixed, true, false).ms).toBe(700);
28 expect(sleepWindow(fixed, false, false).ms).toBe(900);
29 });
30
31 test("the asleep line names the window and the slider", () => {
32 expect(asleepWords({ ms: 600_000, why: "economy 20" })).toBe(
33 "asleep: nothing asked for it in 10 min (economy 20); the next request wakes it",
34 );
35 });
36 });
37
38 describe("app.always_on in forge.json", () => {
39 const manifest = (app: Record<string, unknown>) =>
40 parseManifest(JSON.stringify({ app: { run: "bun run s.ts", ...app } }), undefined);
41
42 test("true is kept, false and unset follow the forge, anything else is refused", () => {
43 const on = manifest({ always_on: true });
44 expect(on.ok && on.value.alwaysOn).toBe(true);
45 const off = manifest({ always_on: false });
46 expect(off.ok && off.value.alwaysOn).toBeUndefined();
47 const unset = manifest({});
48 expect(unset.ok && unset.value.alwaysOn).toBeUndefined();
49 const bad = manifest({ always_on: "yes" });
50 expect(bad.ok).toBe(false);
51 if (!bad.ok) expect(JSON.stringify(bad.error)).toContain("app.always_on must be true or false");
52 });
53
54 test("the app page says production is always on, and says nothing otherwise", () => {
55 const env = { env: "production" as const, state: "running" as const, commitOid: "abc1234", url: "http://hello.apps.test" };
56 expect(String(envFacts("/alice/hello", { ...env, alwaysOn: true }))).toContain("data-always-on");
57 expect(String(envFacts("/alice/hello", env))).not.toContain("data-always-on");
58 });
59 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs