Public: anyone can read this; the crew can change it.

src/apps/runtime.ts and src/checks/service.ts are each split into small modules named for one concept, with no behaviour change #14016

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

torque/forge:torque/14010-src-apps-runtime-ts-and-src-checks-servi → main 2 commits fork

  • Merged. torque/14010-src-apps-runtime-ts-and-src-checks-servi was merged into main as ff22503.

Review

Sign in to review this change.
11 files changed +2737 −2363
src/apps/runtime.ts +173 −2363

Diff not shown — 103,610 bytes of diff (limit 100,000). +173 −2363

View file
src/apps/runtime_bringup.ts added +204
@@ -0,0 +1,204 @@
1 /**
2 * Bringing one checkout up as a process (`AppsRuntime.bringUp`): export the
3 * tree, install, build, start it in a cell or a plain process, wait for its
4 * health path to answer.
5 */
6
7 import { runDir } from "../core/pulls.ts";
8 import { existsSync, rmSync, writeFileSync } from "node:fs";
9 import { join } from "node:path";
10 import { mintAppToken, revokeAppToken } from "./token.ts";
11 import { checkInstallCache } from "../checks/install.ts";
12 import { getPull } from "../core/pulls.ts";
13 import { guestWaits } from "../origins/guest.ts";
14 import type { Repo } from "../core/repos.ts";
15 import type { Run } from "../core/runs.ts";
16 import type { Result } from "../kernel/result.ts";
17 import { buildCacheRoot } from "./build_cache.ts";
18 import { exportTree, readFileAt } from "./checkout.ts";
19 import { Stopwatch } from "./timing.ts";
20 import { ensureEnvData, envDataDir, envDir, envPull, reseedEmpty, waitedEmptyNote } from "./data.ts";
21 import { parseManifest } from "./manifest.ts";
22 import { identityKey } from "./proxy.ts";
23 import { startProcess, type StartInput } from "./process.ts";
24 import type { AppsRuntime } from "./runtime.ts";
25 import { type Live, diedBeforeAnswering, rootFault, type Environment } from "./runtime_env.ts";
26
27 export async function bringUp(
28 rt: AppsRuntime,
29 state: Environment,
30 repo: Repo,
31 run: Run,
32 write: (line: string) => void,
33 signal: AbortSignal,
34 clock: Stopwatch,
35 ): Promise<
36 | { ok: true; value: Live; tested: Promise<Result<void, string>> }
37 | { ok: false; error: string }
38 > {
39 // A sealed pull's preview is built from where its work is (`pullDir`).
40 const bare = await runDir(rt.app.db, rt.app.reposRoot, repo, run.issueId);
41 const [forgeJson, packageJson] = await Promise.all([
42 readFileAt(bare, run.commitOid, "forge.json"),
43 readFileAt(bare, run.commitOid, "package.json"),
44 ]);
45 const parsed = parseManifest(forgeJson, packageJson);
46 const manifest = parsed.ok ? { ...parsed, value: await rt.heldFor(repo, state.env, parsed.value, write) } : parsed;
47 clock.lap("manifest");
48 if (!manifest.ok) {
49 return {
50 ok: false,
51 error:
52 manifest.error.kind === "not_an_app"
53 ? "this commit is not an app: no forge.json with an app, and no package.json start script"
54 : manifest.error.message,
55 };
56 }
57 const dir = join(
58 envDir(rt.app.config.dataDir, state.app.id, state.env),
59 run.commitOid.slice(0, 7),
60 );
61 // A checkout that is already here was installed and tested when it was
62 // made; the forge coming back starts it, and only starts it.
63 const warm = existsSync(join(dir, "package.json"));
64 if (warm) {
65 write(`warm start: ${run.commitOid.slice(0, 7)} was built here before`);
66 } else {
67 write(`checking out ${run.commitOid.slice(0, 7)}`);
68 const exported = await exportTree(bare, run.commitOid, dir);
69 if (!exported.ok) return { ok: false, error: exported.error };
70 }
71 clock.lap(warm ? "checkout (warm)" : "checkout");
72
73 // A guest pull nobody has approved runs a stranger's code: it starts on
74 // empty data, never a copy of production's, and says so. Once a writer
75 // approves the pull, this deploy (the approval's own) seeds the data the
76 // way the app's previews are configured.
77 const number = envPull(state.env);
78 const pull = number === undefined ? undefined : getPull(rt.app.db, repo.id, number);
79 const waits = pull !== undefined && guestWaits(rt.app.db, pull);
80 const note = waitedEmptyNote(rt.app.config.dataDir, state.app.id, state.env);
81 const approvedSince = !waits && existsSync(note);
82 if (approvedSince) {
83 rmSync(envDataDir(rt.app.config.dataDir, state.app.id, state.env), { recursive: true, force: true });
84 rmSync(note, { force: true });
85 }
86 const seeded = ensureEnvData(
87 rt.app.config.dataDir,
88 state.app.id,
89 state.env,
90 waits ? "empty" : state.app.previewData,
91 );
92 const dataDir = envDataDir(
93 rt.app.config.dataDir,
94 state.app.id,
95 state.env,
96 );
97 // Only a preview that would have had a copy has anything to switch to on approval.
98 if (waits && seeded === "empty" && state.app.previewData === "copy") writeFileSync(note, "");
99 if (waits && seeded !== "kept")
100 write("data: starting empty until a writer approves the pull");
101 else if (seeded === "cloned" || seeded === "copied")
102 write(`data: ${seeded} from production${approvedSince ? ", now the pull is approved (it ran on empty data while it waited)" : ""}`);
103 else if (seeded === "empty" && state.env !== "production")
104 write("data: starting empty");
105 clock.lap("data");
106
107 // The process's own credential; every way out of here without a
108 // process to hold it revokes it.
109 const token = mintAppToken(
110 rt.app.db,
111 repo.ownerId,
112 state.app.slug,
113 state.env,
114 );
115 const startInput: Omit<StartInput, "signal"> = {
116 dir,
117 dataDir,
118 manifest: manifest.value,
119 env: state.env,
120 url: rt.urlFor(state.app, state.env),
121 commitOid: run.commitOid,
122 forgeUrl: rt.options.forgeUrl,
123 identityKey: identityKey(rt.app.config.dataDir, state.app.id),
124 ssh: rt.sshFor(state.app, state.env),
125 // Read at start, never cached: a change is a redeploy, and the
126 // process that comes up is the one that sees it.
127 ...rt.secretsFor(repo, state.app.id, state.env, token.token, dir),
128 feature: rt.featureFor(repo, state.app.id, state.env, dir),
129 log: write,
130 warm,
131 cell: rt.cellFor(state.app, state.env),
132 // Every environment, production too, installs from the cache the
133 // repository's checks fill, so a new preview fetches nothing it has.
134 installCache: checkInstallCache(rt.app.config.dataDir, repo.id),
135 // Its own, per environment: a preview never touches production's.
136 buildCache: buildCacheRoot(rt.app.config.dataDir, state.app.id, state.env),
137 };
138 rt.noteCell(run.id, write);
139 clock.lap("setup");
140 write(clock.line("starting the process"));
141 const start = async (): ReturnType<typeof startProcess> => {
142 try {
143 return await startProcess({ ...startInput, signal });
144 } catch (error) {
145 revokeAppToken(rt.app.db, token.id);
146 throw error;
147 }
148 };
149 let started = await start();
150
151 // The one retry this deploy gets: a preview seeded from production whose
152 // process died before it ever answered — schema ahead of what this build
153 // knows, most often — is tried again on empty data, once. A production
154 // deploy is never seeded from a copy, so it never takes this path.
155 if (
156 !started.ok &&
157 (seeded === "cloned" || seeded === "copied") &&
158 diedBeforeAnswering(started.error)
159 ) {
160 const first = started.error;
161 write(`failed: ${first.stage}: ${first.message}`);
162 write(
163 "the copied data was not usable by this build; the preview is starting on empty data",
164 );
165 reseedEmpty(rt.app.config.dataDir, state.app.id, state.env);
166 started = await start();
167 if (!started.ok) {
168 revokeAppToken(rt.app.db, token.id);
169 rmSync(dir, { recursive: true, force: true });
170 return {
171 ok: false,
172 error: `${started.error.stage}: ${started.error.message} (retried on empty data after the copy failed with ${first.stage}: ${first.message})`,
173 };
174 }
175 } else if (!started.ok) {
176 revokeAppToken(rt.app.db, token.id);
177 rmSync(dir, { recursive: true, force: true });
178 return {
179 ok: false,
180 error: `${started.error.stage}: ${started.error.message}`,
181 };
182 }
183 // The health path answered; the page people open must too.
184 const fault = await rootFault(started.value.handle.port);
185 if (fault !== undefined) {
186 await started.value.handle.stop();
187 revokeAppToken(rt.app.db, token.id);
188 rmSync(dir, { recursive: true, force: true });
189 return { ok: false, error: `ready: ${fault}` };
190 }
191 return {
192 ok: true,
193 value: {
194 commitOid: run.commitOid,
195 handle: started.value.handle,
196 dir,
197 runId: run.id,
198 since: new Date().toISOString(),
199 manifest: manifest.value,
200 tokenId: token.id,
201 },
202 tested: started.value.tested,
203 };
204 }
src/apps/runtime_deploy.ts added +450
@@ -0,0 +1,450 @@
1 /**
2 * One deploy run (`AppsRuntime.execute`): the serial lane per environment,
3 * the run row and its log, the gate, the swap-in, the tests after; and the
4 * build previews and variants that ride the same lane.
5 */
6
7 import { recordCarried } from "../core/carried.ts";
8 import { noteRelease, noteReleaseTests, recordsReleases } from "./release_record.ts";
9 import { gateBudgetMs, gatesPush, gateSummary, withinBudget } from "./push_gate.ts";
10 import { tellGated } from "./gate_notice.ts";
11 import { releaseOfCommit } from "../core/release_subject.ts";
12 import { runDir } from "../core/pulls.ts";
13 import { checkPreview } from "./preview_spec.ts";
14 import { rm } from "node:fs/promises";
15 import type { AppRecord } from "../core/apps.ts";
16 import { getVariant } from "../core/variants.ts";
17 import { revokeAppToken } from "./token.ts";
18 import { announceRun } from "../actions/runs.ts";
19 import { getPull } from "../core/pulls.ts";
20 import { getRepoById, type Repo } from "../core/repos.ts";
21 import { runningLimitSentence } from "../settings/quotas.ts";
22 import { cancelRun, claimRun, finishRun, markServing, STOPPED_SUMMARY, type Run } from "../core/runs.ts";
23 import { resolveOid } from "../git/merge.ts";
24 import { repoDir } from "../git/repos.ts";
25 import type { Result } from "../kernel/result.ts";
26 import { holdLease } from "../runs/lease.ts";
27 import { appendRunLog } from "../runs/log.ts";
28 import { readFileAt } from "./checkout.ts";
29 import { seconds, Stopwatch } from "./timing.ts";
30 import { DRAIN_BUDGET_MS, drainPort } from "./handoff.ts";
31 import { buildEnv, variantEnv, envPull, type EnvName } from "./data.ts";
32 import type { AppManifest } from "./manifest.ts";
33 import { heldManifest } from "../core/starts.ts";
34 import { ASKS_NEXT, asksLine, heldPull } from "../checks/held.ts";
35 import type { AppsRuntime } from "./runtime.ts";
36 import { log, TESTS_PASSED_TAIL, NO_TESTS_TAIL, type Live, PUSH_REASONS, elapsedSeconds, type Environment, BUILD_PREVIEWS_MAX, key } from "./runtime_env.ts";
37
38 /**
39 * Wait for an environment a request asked for to answer: until its deploy serves or ends, or
40 * `signal` aborts. Every caller waits on the one deploy; none starts another.
41 */
42 export function served(rt: AppsRuntime, record: AppRecord, env: EnvName, signal: AbortSignal): Promise<void> {
43 const state = rt.envs.get(key(record.id, env));
44 const coming = (): boolean =>
45 state !== undefined &&
46 state.live === undefined &&
47 (state.queued !== undefined || state.deploying !== undefined);
48 if (!coming() || signal.aborted) return Promise.resolve();
49 return new Promise((resolve) => {
50 const done = (): void => {
51 off();
52 signal.removeEventListener("abort", done);
53 resolve();
54 };
55 const off = rt.app.bus.subscribe((event) => {
56 if (event.repoId === record.repoId && (event.kind === "app.serving" || event.kind === "run.finished") && !coming())
57 done();
58 });
59 signal.addEventListener("abort", done, { once: true });
60 });
61 }
62
63 /**
64 * Serve one commit now, at `build-<sha7>`. Asked again for a build already
65 * here, it redeploys nothing and hands back the same environment. A new
66 * one beyond `BUILD_PREVIEWS_MAX` takes the place of the oldest, which is
67 * torn down with its data. Nothing of it outlives this process: a forge
68 * coming back removes what a build preview left on disk.
69 */
70 export async function previewBuild(
71 rt: AppsRuntime,
72 record: AppRecord,
73 repo: Repo,
74 commitOid: string,
75 actorId?: string,
76 ): Promise<{ env: EnvName; run: Run | undefined }> {
77 const env = buildEnv(commitOid);
78 const k = key(record.id, env);
79 if (!rt.envs.has(k)) {
80 // Claimed before anything waits, so two asks at once count each other.
81 rt.envFor(record, env).buildOrder = ++rt.buildCount;
82 const builds = [...rt.envs.values()]
83 .filter((s) => s.app.id === record.id && s.buildOrder !== undefined)
84 .sort((a, b) => b.buildOrder! - a.buildOrder!);
85 for (const old of builds.slice(BUILD_PREVIEWS_MAX)) {
86 log.info("build preview replaced", { app: record.slug, env: old.env, by: env });
87 await rt.teardown(record, old.env);
88 }
89 }
90 // Torn down by a newer ask while the old ones went: nothing to deploy.
91 if (!rt.envs.has(k)) return { env, run: undefined };
92 const run = rt.requestDeploy(record, repo, env, commitOid, "build", actorId);
93 return { env, run };
94 }
95
96 /**
97 * Start (or move) a variant's environment at its branch's tip, now: the
98 * ask that registers a variant, or saves its values. Nothing when the
99 * branch has no tip.
100 */
101 export async function deployVariant(rt: AppsRuntime, record: AppRecord, repo: Repo, label: string, actorId?: string): Promise<Run | undefined> {
102 const variant = getVariant(rt.app.db, record.id, label);
103 if (variant === undefined) return undefined;
104 const dir = repoDir(rt.app.reposRoot, repo.ownerName, repo.name);
105 const tip = await resolveOid(dir, `refs/heads/${variant.branch}`);
106 if (tip === undefined) return undefined;
107 return rt.requestDeploy(record, repo, variantEnv(label), tip, "variant", actorId);
108 }
109
110 export function pullIssueId(rt: AppsRuntime, repo: Repo, env: EnvName): string | undefined {
111 const pull = getPull(rt.app.db, repo.id, Number(env.slice(5)));
112 return pull?.id;
113 }
114
115 export function envFor(rt: AppsRuntime, record: AppRecord, env: EnvName): Environment {
116 const k = key(record.id, env);
117 let state = rt.envs.get(k);
118 if (state === undefined) {
119 state = {
120 app: record,
121 env,
122 chain: Promise.resolve(),
123 restarts: [],
124 openStreams: 0,
125 openRequests: 0,
126 };
127 rt.envs.set(k, state);
128 }
129 return state;
130 }
131
132 /**
133 * Previews of the account's apps that are up, `except` aside, and those
134 * coming up whose run is older than `before`: of two deploying together
135 * the older counts against the younger, never both against each other.
136 * With no `before`, every one coming up counts (a wake has no run).
137 */
138 export function livePreviews(
139 rt: AppsRuntime,
140 ownerId: string,
141 except: Environment,
142 before?: string,
143 ): number {
144 let n = 0;
145 for (const other of rt.envs.values()) {
146 if (other === except || other.env === "production") continue;
147 const up =
148 other.live !== undefined ||
149 (other.deploying !== undefined &&
150 (before === undefined || other.deploying.run.id < before));
151 if (!up) continue;
152 if (getRepoById(rt.app.db, other.app.repoId)?.ownerId === ownerId) n++;
153 }
154 return n;
155 }
156
157 export async function execute(
158 rt: AppsRuntime,
159 state: Environment,
160 queued: Repo,
161 run: Run,
162 ): Promise<void> {
163 // The row as it is now, not as it was when the deploy was asked for: a
164 // move (#570) between the two gives the repository a new owner and a
165 // new place on disk, and a queued run follows it there.
166 let repo = getRepoById(rt.app.db, queued.id) ?? queued;
167 const claimed = claimRun(rt.app.db, run.id);
168 if (!claimed) return; // cancelled meanwhile
169 const lease = holdLease(rt.app.db, run.id);
170 const abort = new AbortController();
171 state.deploying = { run: claimed, abort };
172 state.lastRunId = run.id;
173 const write = (line: string): void =>
174 appendRunLog(rt.app.config.dataDir, run.id, line);
175 announceRun(rt.app, claimed);
176 // An account's live previews are bounded (`settings/quotas.ts`): a
177 // preview past the limit does not start, and its run — the check the
178 // pull shows — fails saying why, so the pull never waits for it.
179 const refusal =
180 state.env === "production" || state.live !== undefined
181 ? undefined
182 : runningLimitSentence(
183 rt.app.db,
184 "previews",
185 "live previews",
186 repo.ownerId,
187 () => rt.livePreviews(repo.ownerId, state, run.id),
188 );
189 if (refusal !== undefined) {
190 state.deploying = undefined;
191 lease();
192 const message = `not started: ${refusal}. Close a pull or stop a preview to make room.`;
193 write(message);
194 const done = finishRun(rt.app.db, run.id, { state: "failed", summary: message });
195 if (done) announceRun(rt.app, done);
196 return;
197 }
198 write(
199 `deploy ${state.env} of ${repo.ownerName}/${repo.name} at ${run.commitOid.slice(0, 7)} (${run.reason})`,
200 );
201
202 // The clock starts at the claim, the instant the log calls "the deploy
203 // started"; the first lap is the wait for a permit. Production goes
204 // ahead of previews waiting for one: it is the environment a person
205 // actually visits.
206 const clock = new Stopwatch();
207 // A deploy that throws is a failed run, never a fallen forge: the chain
208 // this runs on would otherwise reject unhandled and take the process.
209 const outcome = await rt.gate
210 .run(
211 () => {
212 clock.lap("turn");
213 // Read again after the wait for a permit: a move may land in it.
214 repo = getRepoById(rt.app.db, repo.id) ?? repo;
215 return rt.bringUp(state, repo, run, write, abort.signal, clock);
216 },
217 { first: state.env === "production" },
218 )
219 .catch((error: unknown) => ({
220 ok: false as const,
221 error: `deploy failed: ${error instanceof Error ? error.message : String(error)}`,
222 }));
223
224 if (!outcome.ok) {
225 state.deploying = undefined;
226 lease();
227 // The runtime itself stopping is what aborted the step in flight —
228 // the forge's own business, never this commit's fault, and the same
229 // honest reason the still-queued case already carries. A real
230 // failure (a non-zero exit, a genuine timeout) still fails the run
231 // below even if the runtime happens to be stopping at the same time:
232 // only an abort this runtime raised earns the quieter outcome.
233 if (rt.stopped && abort.signal.aborted) {
234 const done = cancelRun(rt.app.db, run.id, STOPPED_SUMMARY);
235 write(`cancelled: ${STOPPED_SUMMARY}`);
236 if (done) announceRun(rt.app, done);
237 return;
238 }
239 state.lastError = outcome.error;
240 state.lastErrorCommitOid = run.commitOid;
241 const done = finishRun(rt.app.db, run.id, {
242 state: "failed",
243 summary: outcome.error,
244 });
245 write(`failed: ${outcome.error}`);
246 if (state.live !== undefined)
247 write(
248 `the previous deployment (${state.live.commitOid.slice(0, 7)}) keeps serving`,
249 );
250 if (done) announceRun(rt.app, done);
251 return;
252 }
253
254 // A direct push to production is gated (#13709): the declared tests run on the new tree
255 // before the switch, and while they fail the last green commit keeps serving. A merged
256 // pull already passed its check, and a first deploy has nothing to keep, so both swap in
257 // at once as before.
258 let gated: Result<void, string> | undefined;
259 if (gatesPush(state.env, run.reason, state.live !== undefined, outcome.value.manifest.test !== null)) {
260 write(`gate: running the tests on ${run.commitOid.slice(0, 7)} before the switch`);
261 gated = await withinBudget(outcome.tested, gateBudgetMs());
262 if (!gated.ok) {
263 const kept = state.live!;
264 await outcome.value.handle.stop();
265 revokeAppToken(rt.app.db, outcome.value.tokenId);
266 if (outcome.value.dir !== kept.dir) await rm(outcome.value.dir, { recursive: true, force: true });
267 state.deploying = undefined;
268 lease();
269 const summary = gateSummary(gated.error, kept.commitOid);
270 state.lastError = summary;
271 state.lastErrorCommitOid = run.commitOid;
272 const done = finishRun(rt.app.db, run.id, { state: "failed", summary });
273 write(`failed: ${summary}`);
274 if (done) announceRun(rt.app, done);
275 rt.app.bus.append({
276 kind: "app.gated",
277 repoId: repo.id,
278 payload: { app: state.app.slug, env: state.env, commitOid: run.commitOid, keptOid: kept.commitOid, why: gated.error.slice(0, 300) },
279 });
280 await tellGated(rt.app.db, repo.ownerId, `${state.app.slug}: ${summary}\n${rt.urlFor(state.app, state.env)}`);
281 return;
282 }
283 write("gate: the tests passed; switching");
284 }
285
286 // It answers: swap it in now, before its tests are done. `state.deploying`
287 // stays set — the run is still open, and a stop or a teardown must still
288 // be able to reach in and kill the test step — until they finish below.
289 // `servingAt` is stamped here, at the swap-in, the same instant the log
290 // says so — never the moment the tests beside it finish.
291 const previous = state.live;
292 const rested = state.asleep;
293 const served = markServing(rt.app.db, run.id);
294 const servingAt = served?.servingAt ?? new Date().toISOString();
295 const deployStartedAt = claimed.startedAt;
296 // The run was created the instant the push asked for it; it started when
297 // the gate handed it a permit. Both travel with the deployment: the log
298 // below is measured from the start, the studio's phrase from the push.
299 const createdAt = PUSH_REASONS.includes(run.reason)
300 ? run.createdAt
301 : undefined;
302 const live: Live = {
303 ...outcome.value,
304 servingAt,
305 createdAt,
306 deployStartedAt,
307 };
308 state.live = live;
309 rt.awake(state, servingAt);
310 state.asleep = undefined;
311 state.lastRequestAt = rt.now();
312 state.lastError = undefined;
313 state.lastErrorCommitOid = undefined;
314 // A checkout and its node_modules: tens of thousands of files, removed off this thread.
315 if (rested !== undefined && rested.dir !== live.dir)
316 await rm(rested.dir, { recursive: true, force: true });
317 if (previous !== undefined) {
318 write(
319 `stopping the previous deployment (${previous.commitOid.slice(0, 7)})`,
320 );
321 revokeAppToken(rt.app.db, previous.tokenId);
322 // Traffic already goes to the new one: the old one's sockets are told
323 // to reconnect (1012), and what it was answering is answered (#4496).
324 const unanswered = await drainPort(previous.handle.port);
325 if (unanswered > 0)
326 write(`${unanswered} request(s) to the previous deployment were still waiting after ${seconds(DRAIN_BUDGET_MS)}`);
327 await previous.handle.stop();
328 if (previous.dir !== live.dir)
329 await rm(previous.dir, { recursive: true, force: true });
330 }
331 rt.watch(state, repo, live);
332 const url = rt.urlFor(state.app, state.env);
333 write(
334 deployStartedAt
335 ? `serving ${url} — ${elapsedSeconds(deployStartedAt, servingAt)} after the deploy started`
336 : `serving ${url}`,
337 );
338 await rt.noteChange(repo, state.app.id, state.env, live.commitOid);
339 // The same statement the log line above just made, on the bus: a
340 // subscriber (the looks service) must never be able to take the
341 // deploy path down with it.
342 try {
343 const release = await releaseOfCommit(
344 repoDir(rt.app.reposRoot, repo.ownerName, repo.name),
345 state.env,
346 live.commitOid,
347 );
348 rt.app.bus.append({
349 kind: "app.serving",
350 repoId: repo.id,
351 payload: {
352 app: state.app.slug,
353 env: state.env,
354 commitOid: live.commitOid,
355 servingAt,
356 ...(deployStartedAt === null ? {} : { deployStartedAt }),
357 testsRunning: live.manifest.test !== null,
358 ...(release === undefined ? {} : { version: release.version, pulls: release.pulls }),
359 },
360 });
361 } catch (error) {
362 log.error("could not publish app.serving", { error: String(error) });
363 }
364
365 // A new version served by production is a release (#13707); never fails the deploy.
366 if (recordsReleases(state.env, live.manifest))
367 await noteRelease(rt.app.db, {
368 repoId: repo.id,
369 repoDir: repoDir(rt.app.reposRoot, repo.ownerName, repo.name),
370 source: live.manifest.version,
371 port: live.handle.port,
372 oid: live.commitOid,
373 servingAt,
374 pushedAt: createdAt,
375 startedAt: deployStartedAt ?? undefined,
376 tested: live.manifest.test !== null,
377 }).catch((error: unknown) => log.error("could not record a release", { error: String(error) }));
378
379 // A preview is held to the website specification once it is live, beside
380 // its tests and after the swap-in (#2955). Report-only and unawaited: the
381 // run neither waits for it nor can it fail.
382 if (state.env !== "production")
383 void checkPreview(
384 rt.app.db,
385 run.id,
386 `http://127.0.0.1:${live.handle.port}`,
387 live.manifest.looks,
388 );
389
390 const tested = gated ?? (await outcome.tested);
391 state.deploying = undefined;
392 lease();
393 noteReleaseTests(rt.app.db, state.env, live.manifest, repo.id, live.commitOid, tested.ok);
394 if (tested.ok) {
395 const done = finishRun(rt.app.db, run.id, {
396 state: "ok",
397 summary: `running at ${url}${live.manifest.test === null ? NO_TESTS_TAIL : TESTS_PASSED_TAIL}`,
398 });
399 write(`done: ${url}`);
400 if (done && state.env === "production") await rt.noteCarried(repo, done);
401 if (done) announceRun(rt.app, done);
402 return;
403 }
404 // The tests failed, but the deployment that answered is what was asked
405 // for: it keeps serving, and the run — the check a pull shows — fails.
406 if (state.live === live)
407 state.live = { ...live, testsFailed: tested.error };
408 const done = finishRun(rt.app.db, run.id, {
409 state: "failed",
410 summary: `test: ${tested.error}`,
411 });
412 write(`failed: test: ${tested.error}`);
413 write(`serving ${url} anyway — its tests failed`);
414 if (done) announceRun(rt.app, done);
415 }
416
417 /** Note the merged pulls a finished production deploy carries; never fails the deploy. */
418 export async function noteCarried(rt: AppsRuntime, repo: Repo, run: Run): Promise<void> {
419 try {
420 const dir = await runDir(rt.app.db, rt.app.reposRoot, repo, null);
421 await recordCarried(rt.app.db, {
422 runId: run.id,
423 repoId: repo.id,
424 commitOid: run.commitOid,
425 dir,
426 });
427 } catch (error) {
428 log.error("could not record what a deploy carries", { error: String(error) });
429 }
430 }
431
432 /**
433 * The manifest a start of `env` runs under. A pull's preview takes the hosts it may reach and
434 * its limits from the pull's base branch until the pull lands (`checks/held.ts`), and its log
435 * says what it asked for beyond them; production, a build preview and a variant keep their
436 * own. Then an agent's unclaimed account runs small, with no host but the registries (#10912).
437 */
438 export async function heldFor(rt: AppsRuntime, repo: Repo, env: EnvName, own: AppManifest, write?: (line: string) => void): Promise<AppManifest> {
439 const number = envPull(env);
440 const pull = number === undefined ? undefined : getPull(rt.app.db, repo.id, number);
441 let manifest = own;
442 if (pull !== undefined) {
443 const dir = repoDir(rt.app.reposRoot, repo.ownerName, repo.name);
444 const tip = await resolveOid(dir, `refs/heads/${pull.baseBranch}`);
445 const held = heldPull(own, tip === undefined ? undefined : await readFileAt(dir, tip, "forge.json"), "app");
446 manifest = held.manifest;
447 if (held.asks !== undefined) write?.(`${asksLine(held.asks, pull.baseBranch)} ${ASKS_NEXT}`);
448 }
449 return heldManifest(rt.app.db, repo.ownerId, manifest);
450 }
src/apps/runtime_env.ts added +389
@@ -0,0 +1,389 @@
1 /**
2 * The shapes and constants of `AppsRuntime` (`apps/runtime.ts`): its options, an
3 * environment's record and the status read from it, and the small helpers its
4 * modules share. Plain data; nothing here holds state.
5 */
6
7 import type { Database } from "bun:sqlite";
8 import type { CertificateProbe } from "./certificate.ts";
9 import { listEgressGrants } from "../core/egress_grants.ts";
10 import type { AppRecord } from "../core/apps.ts";
11 import type { Run } from "../core/runs.ts";
12 import { Gate } from "../git/env.ts";
13 import { createLog } from "../kernel/log.ts";
14 import type { BusyReason } from "./busy.ts";
15 import { seconds } from "./timing.ts";
16 import type { DomainResolver } from "./domains.ts";
17 import { envVariant, envPull, type EnvName } from "./data.ts";
18 import type { AppManifest } from "./manifest.ts";
19 import type { CellSpec } from "./contain.ts";
20 import type { ProcessHandle, StartError } from "./process.ts";
21
22 export const log = createLog("apps");
23
24 export interface RuntimeOptions {
25 /** The domain app hostnames hang off: `<slug>.<domain>`. */
26 readonly domain: string;
27 /** `http` or `https` — what the public URL of an app starts with. */
28 readonly scheme: "http" | "https";
29 /** The port in public app URLs, or undefined for the scheme's default. */
30 readonly publicPort?: number | undefined;
31 /** The forge's own public URL, for the app's `FORGE_URL` and links. */
32 readonly forgeUrl: string;
33 /** How many deploys may run their heavy steps at once. @default 2 */
34 readonly concurrency?: number | undefined;
35 /**
36 * How long an environment serves nobody before its process is stopped
37 * and it sleeps, to be woken by the next request. Unset (the forge's
38 * own runtime) reads the profit slider's position on every sweep
39 * (`apps/sleep_window.ts`); set, it is fixed — tests and nothing else.
40 */
41 readonly sleepAfterMs?:
42 | { readonly production: number; readonly preview: number }
43 | undefined;
44 /**
45 * The sweep's clock; a test hands in one that runs ahead to prove a
46 * window of minutes without waiting minutes. @default Date.now
47 */
48 readonly clock?: (() => number) | undefined;
49 /** How often the runtime looks for environments to put to sleep. */
50 readonly sweepEveryMs?: number | undefined;
51 /**
52 * Whose timers these are when this runtime serves a forge hosted in its
53 * parent's process: `child:<name>`, so a stall they ran across names that
54 * child and their slices are counted against it (`perf/child_time.ts`).
55 */
56 readonly owner?: string | undefined;
57 /**
58 * How long a pull may go without a push or a comment before its preview
59 * is stopped (not removed: its row, data and pictures stay) and reads as
60 * asleep until someone asks for it. Unset follows the profit slider
61 * (`policyAt(position).idlePullMs`, read each sweep; `null` is never);
62 * 0 keeps every preview warm.
63 */
64 readonly idlePullMs?: number | undefined;
65 /**
66 * How long past its idle window a busy environment (`apps/busy.ts`) is
67 * left awake before it sleeps anyway — one crew run's length, so an app
68 * that only claims to be busy cannot stay up forever.
69 */
70 readonly busyGraceMs?: number | undefined;
71 /**
72 * What a domain check asks: the system resolver unless a test hands in
73 * its own, so proving `app.example.com` needs no network in a test.
74 */
75 readonly dns?: DomainResolver | undefined;
76 /**
77 * What the registrar adapter calls (`apps/dns_provider.ts`); the global `fetch` unless a test
78 * hands in its own.
79 */
80 readonly dnsFetch?: typeof fetch | undefined;
81 /**
82 * What a check asks of a verified domain to read its certificate's expiry:
83 * a TLS handshake unless a test hands in its own.
84 */
85 readonly certificate?: CertificateProbe | undefined;
86 /**
87 * What the cut-over's preview check asks of each side (`apps/parity_probe.ts`): a status per
88 * URL, or null. The vetted outbound fetch unless a test hands in its own.
89 */
90 readonly probe?: ((url: string) => Promise<number | null>) | undefined;
91 /**
92 * Whether a cloud body can take a burst (`spawn/placement.ts`). Without
93 * one every environment is placed on owned metal and a burst queues.
94 * @default CLOUD_BODY_CONFIGURED (none yet)
95 */
96 readonly cloudBody?: boolean | undefined;
97 /**
98 * How many environments owned metal holds, read at each placement.
99 * @default unbounded — no limit is known, so owned is never full
100 */
101 readonly ownedCapacity?: (() => number) | undefined;
102 /** How often storage is sampled for the meter. @default one hour */
103 readonly storageEveryMs?: number | undefined;
104 /**
105 * The gate deploys take their turn at. A hosted child's runtime is handed
106 * its parent's, so a child's deploys count against the parent's capacity
107 * instead of beside it. @default a gate of `concurrency`
108 */
109 readonly gate?: Gate | undefined;
110 }
111
112 export type EnvState =
113 | "stopped"
114 | "deploying"
115 | "running"
116 | "asleep"
117 | "failed";
118
119 /** How long an always-on app may go unopened while it keeps calling the forge. */
120 export const UNOPENED_MS = 24 * 60 * 60_000;
121
122 /** Idle windows before an environment sleeps: the profit slider's 50 (`core/slider.ts`). */
123 export const SLEEP_AFTER_MS = { production: 24 * 3_600_000, preview: 30 * 60_000 };
124 /** Days a quiet pull keeps its preview by default: the slider's 50 too. */
125 export const IDLE_PULL_DAYS = 3;
126 export const SWEEP_EVERY_MS = 60_000;
127 export const BUSY_GRACE_MS = 45 * 60_000;
128
129 /** What a finished deploy's summary ends with: the studio words its green
130 * from these, so a pull says whether its tests ran (`words.ts`). */
131 export const TESTS_PASSED_TAIL = " · its tests passed";
132 export const NO_TESTS_TAIL = " · no tests declared, none run";
133
134 export interface EnvStatus {
135 readonly env: EnvName;
136 readonly state: EnvState;
137 /** The commit serving, or being brought up. */
138 readonly commitOid: string;
139 readonly url: string;
140 readonly port?: number | undefined;
141 /** The run that produced this state — a link to its log. */
142 readonly runId?: string | undefined;
143 readonly since?: string | undefined;
144 /** The instant this deployment started serving, set once at the swap-in. */
145 readonly servingAt?: string | undefined;
146 /**
147 * The instant the push created this deploy run — what a person means by
148 * "after the push", and the reference `servedPhrase` measures from. Unset
149 * for a deploy no push asked for (a redeploy someone pressed, a boot
150 * reconcile), so the words for those never claim a push that never was.
151 */
152 readonly createdAt?: string | undefined;
153 /** The deploy run's own start — after however long it waited for a permit. */
154 readonly deployStartedAt?: string | undefined;
155 readonly error?: string | undefined;
156 /**
157 * A running environment whose deploy is still checking itself: its tests
158 * are still running, or ran and failed. Never set while `state` is
159 * anything but `running` — a failed test never stops a deployment that
160 * already answers.
161 */
162 readonly note?: string | undefined;
163 /**
164 * The app's own `tty` command, once known live — `null` when its
165 * `forge.json` declares none. Unset while never live, since a checkout
166 * that has not started has not said either way.
167 */
168 readonly tty?: string | null | undefined;
169 /**
170 * The pages `app.looks` names worth a picture, once known live — `["/"]`
171 * when its `forge.json` says nothing. Unset while never live, the same
172 * as `tty`.
173 */
174 readonly looks?: readonly string[] | undefined;
175 /** The pages of `looks` pictured as the owner (`app.looksAsOwner`), once known live. */
176 readonly looksAsOwner?: readonly string[] | undefined;
177 /** `false` when its `forge.json` turns the forge pill off; unset is on. */
178 readonly toolbar?: boolean | undefined;
179 /**
180 * A deploy newer than the one serving that never came up — set only
181 * while an older deployment keeps answering in its place. A later
182 * successful deploy of this environment clears it (it is the newest
183 * attempt's own outcome, never a history), and it is never set once
184 * nothing serves at all: `state: "failed"` and `error` already say that.
185 */
186 readonly failedDeploy?:
187 | {
188 readonly commitOid: string;
189 readonly runId: string;
190 readonly error: string;
191 }
192 | undefined;
193 /**
194 * Why a running environment is still awake past its idle window, in a
195 * person's words (`BUSY_WORDS`) — set only while the sweep is sparing it.
196 */
197 readonly busy?: string | undefined;
198 /** Production whose `forge.json` sets `app.always_on: true`: it never sleeps. */
199 readonly alwaysOn?: true | undefined;
200 /**
201 * Set while a preview is stopped because its pull went quiet: when it was
202 * stopped, and the last touch (push, comment, wake) it was measured from.
203 * A stopped preview with this set is asleep, never failed.
204 */
205 readonly dormant?:
206 | {
207 readonly since: string;
208 readonly idleSince: string;
209 /** Never started at this head: recorded asleep at the push (`apps/prepared.ts`). */
210 readonly prepared?: true | undefined;
211 }
212 | undefined;
213 /**
214 * Set while it comes up because a request asked for a preview that was stopped or not started.
215 */
216 readonly waking?: true | undefined;
217 }
218
219 export interface Live {
220 readonly commitOid: string;
221 readonly handle: ProcessHandle;
222 readonly dir: string;
223 readonly runId: string;
224 readonly since: string;
225 /** How this checkout runs — its `tty` is what an ssh session opens. */
226 readonly manifest: AppManifest;
227 /** Unset while tests are still running or there are none; the message once they fail. */
228 readonly testsFailed?: string | undefined;
229 /**
230 * The instant this deployment started serving, set once at the swap-in —
231 * unset on the value `bringUp` returns, which does not yet know it.
232 */
233 readonly servingAt?: string | undefined;
234 /** The instant the push created this deploy run, when a push did. */
235 readonly createdAt?: string | undefined;
236 /** The deploy run's own start — the reference the deploy log measures from. */
237 readonly deployStartedAt?: string | null | undefined;
238 /** This process's forge token (`apps/token.ts`), revoked when it stops. */
239 readonly tokenId?: string | undefined;
240 }
241
242 /**
243 * The deploy reasons a ref moving created: for these the run's `createdAt`
244 * is the instant of the push that asked for the deploy, so the wait for a
245 * permit sits inside the number the studio prints. A redeploy someone
246 * pressed, a crash restart, and a boot or enable reconcile are not pushes.
247 */
248 export const PUSH_REASONS: readonly string[] = ["push", "opened", "merge"];
249
250 /**
251 * The deploy reasons that start the same commit again, even while it serves
252 * or is coming up: someone pressed redeploy, or the repository moved to
253 * another owner (#570) and what runs must start again as the new owner's.
254 */
255 export const AGAIN: ReadonlySet<string> = new Set(["redeploy", "moved"]);
256
257 /** How long between two instants, in seconds to one decimal place: `3.4s`. */
258 export function elapsedSeconds(fromIso: string, toIso: string): string {
259 return seconds(Date.parse(toIso) - Date.parse(fromIso));
260 }
261
262 /**
263 * Whether a start failure is the process itself exiting before it ever
264 * answered its health path — as opposed to install failing, or the process
265 * staying up but never answering. Only this shape of failure is worth
266 * retrying on empty data: it is what a build too old for the data it was
267 * given looks like.
268 */
269 export function diedBeforeAnswering(error: StartError): boolean {
270 return (
271 error.stage === "start" &&
272 /^exited with code \d+ before answering/.test(error.message)
273 );
274 }
275
276 /**
277 * What is wrong with an app's `/`, if anything: the line naming the status
278 * when it answers 4xx or 5xx, `undefined` when it answers 200 or redirects.
279 * It asks the process on its own port, so a private app's access door never
280 * answers in its place: whatever status comes back is the app's own.
281 */
282 export async function rootFault(port: number): Promise<string | undefined> {
283 try {
284 const response = await fetch(`http://127.0.0.1:${port}/`, {
285 signal: AbortSignal.timeout(5_000),
286 redirect: "manual",
287 });
288 await response.body?.cancel();
289 if (response.status >= 400)
290 return `/ answered ${response.status}; the app is up but has no page at its root`;
291 } catch {
292 // No answer is the health path's business, not this check's.
293 }
294 return undefined;
295 }
296
297 /** A running environment, as a terminal session needs it. */
298 /**
299 * A checkout that served and was stopped for want of visitors, kept whole:
300 * the next request starts it again, warm, in about the time a page takes.
301 */
302 export interface Asleep {
303 readonly commitOid: string;
304 readonly dir: string;
305 readonly manifest: AppManifest;
306 readonly runId: string;
307 readonly since: string;
308 }
309
310 export interface LiveEnv {
311 readonly env: EnvName;
312 readonly dir: string;
313 readonly dataDir: string;
314 readonly port: number;
315 readonly commitOid: string;
316 readonly url: string;
317 readonly tty: string | null;
318 /** The cell a terminal session runs in, the app's own; unset where cells cannot work. */
319 readonly cell?: CellSpec | undefined;
320 }
321
322 export interface Environment {
323 readonly app: AppRecord;
324 readonly env: EnvName;
325 live?: Live | undefined;
326 asleep?: Asleep | undefined;
327 /** Stopped for want of a push or comment; asked for, it redeploys. */
328 dormant?:
329 | { since: string; idleSince: string; commitOid: string; prepared?: true }
330 | undefined;
331 /** Why the last wake did not happen (the owner's preview limit), for the page a visitor sees. */
332 wakeRefused?: string | undefined;
333 /** One wake at a time; every request that arrives meanwhile waits on it. */
334 waking?: Promise<void> | undefined;
335 /** When something last came through the proxy for this environment. */
336 lastRequestAt?: number | undefined;
337 /** Streams the proxy holds open to it: websockets, event streams. */
338 openStreams: number;
339 /** Requests passed to it whose answer has not begun. */
340 openRequests: number;
341 /** Why the last sweep spared it, while it still is. */
342 keptAwake?: BusyReason | undefined;
343 /** The deploy in flight, if one is. */
344 deploying?: { run: Run; abort: AbortController } | undefined;
345 /** The next deploy, waiting its turn; replaced by a newer request. */
346 queued?: Run | undefined;
347 chain: Promise<void>;
348 lastError?: string | undefined;
349 lastRunId?: string | undefined;
350 /** The commit `lastError` belongs to — set together, cleared together. */
351 lastErrorCommitOid?: string | undefined;
352 /** Crash restarts in the current window, for backing off. */
353 restarts: number[];
354 /** A build preview's place in the order build previews were started, oldest lowest. */
355 buildOrder?: number | undefined;
356 }
357
358 /**
359 * How many build previews (`build-<sha7>`) one app runs at once; a newer one takes the oldest's
360 * place.
361 */
362 export const BUILD_PREVIEWS_MAX = 3;
363
364 /** The deploy reasons that mean a person touched the change. */
365 export const TOUCH_REASONS: readonly string[] = [
366 "push",
367 "opened",
368 "redeploy",
369 "wake",
370 ];
371
372 export const RESTART_WINDOW_MS = 10 * 60_000;
373 export const RESTART_MAX = 3;
374 export const RESTART_BACKOFF_MS = [2_000, 10_000, 30_000];
375
376 /**
377 * Production first, then build previews, then previews by
378 * pull number — a number, not a string, so `pull-10` follows `pull-9`.
379 */
380 export const envRank = (env: EnvName): number =>
381 env === "production" ? -1 : envVariant(env) !== undefined ? -0.5 : (envPull(env) ?? 0);
382 export const byEnv = (a: EnvStatus, b: EnvStatus): number => envRank(a.env) - envRank(b.env);
383
384 export const key = (appId: string, env: EnvName): string => `${appId}/${env}`;
385
386 /** The `host:port` destinations a site admin granted this repository's cells. */
387 export function grantedFor(db: Database, repoId: string): string[] {
388 return listEgressGrants(db, repoId).map((g) => g.destination);
389 }↵
src/apps/runtime_events.ts added +402
@@ -0,0 +1,402 @@
1 /**
2 * What the outbox asks of the runtime (`AppsRuntime.onEvent`): a push, a pull
3 * opening or closing, a repository leaving. Each turns into a deploy request
4 * (`requestDeploy`), a prepared asleep preview (`prepare`) or a teardown.
5 */
6
7 import { ASKER_WORDS, declaredAsker, type PushAsker } from "./prepared.ts";
8 import { lookPagesAt } from "../checks/look_pages.ts";
9 import { dropCheckouts, STRAY_CHECKOUT_AGE_MS } from "./dormant_checkout.ts";
10 import { commitLog } from "../git/read.ts";
11 import { noteLanded } from "./landed.ts";
12 import { pullDir } from "../core/pulls.ts";
13 import { accountActive } from "../core/accounts.ts";
14 import { existsSync, readdirSync, rmSync } from "node:fs";
15 import { join } from "node:path";
16 import { getAppByRepo, type AppRecord } from "../core/apps.ts";
17 import { variantOfBranch } from "../core/variants.ts";
18 import type { ForgeEvent } from "../core/events.ts";
19 import { announceRun } from "../actions/runs.ts";
20 import { checksFor } from "../checks/service.ts";
21 import { getPull, listPulls, type Pull } from "../core/pulls.ts";
22 import { canWrite, getRepoById, type Repo } from "../core/repos.ts";
23 import { getUserById } from "../core/users.ts";
24 import { cancelRun, createRun, type Run } from "../core/runs.ts";
25 import { resolveOid } from "../git/merge.ts";
26 import { repoDir } from "../git/repos.ts";
27 import { previewCommit } from "../git/combined.ts";
28 import { readFileAt } from "./checkout.ts";
29 import { appsRoot, variantEnv, envPull, isEnvName, previewEnv, type EnvName } from "./data.ts";
30 import type { AppsRuntime } from "./runtime.ts";
31 import { log, AGAIN, key } from "./runtime_env.ts";
32
33 /**
34 * At boot, remove checkouts no deployment serves, rests on or is making: what a failed or
35 * interrupted deploy left (#13772). Ones touched within `STRAY_CHECKOUT_AGE_MS` stay.
36 */
37 export async function dropStrayCheckouts(rt: AppsRuntime, record: AppRecord): Promise<void> {
38 const root = join(appsRoot(rt.app.config.dataDir), record.id);
39 let names: string[];
40 try {
41 names = readdirSync(root);
42 } catch {
43 return;
44 }
45 for (const name of names) {
46 if (!isEnvName(name)) continue;
47 const state = rt.envs.get(key(record.id, name));
48 const keep = [state?.live, state?.asleep, state?.deploying?.run, state?.queued]
49 .map((x) => x?.commitOid)
50 .filter((oid): oid is string => oid !== undefined && oid !== "");
51 try {
52 await dropCheckouts(rt.app.config.dataDir, record.id, name, keep, STRAY_CHECKOUT_AGE_MS);
53 } catch (error) {
54 log.warn("could not drop stray checkouts", { app: record.slug, env: name, error: String(error) });
55 }
56 }
57 }
58
59 /**
60 * Note the subject of the commit an environment serves, for the toolbar's "what changed"
61 * (`apps/landed.ts`): at the swap-in, and at boot for one adopted asleep or about to come back,
62 * so a restart does not leave every app saying "no change yet". A nicety: git that cannot be
63 * read is logged, never thrown into the deploy or the boot.
64 */
65 export async function noteChange(rt: AppsRuntime, repo: Repo, appId: string, env: EnvName, oid: string): Promise<void> {
66 try {
67 const read = await commitLog(repoDir(rt.app.reposRoot, repo.ownerName, repo.name), oid, { limit: 1 });
68 noteLanded(rt.app, appId, env, oid, read.ok ? (read.value[0]?.subject ?? "") : "");
69 } catch (error) {
70 log.warn("could not read the landed change", { repo: repo.name, env, commit: oid.slice(0, 7), error: String(error) });
71 }
72 }
73
74 /** Data on disk for apps that no longer exist. */
75 export function pruneDirs(rt: AppsRuntime, known: Set<string>): void {
76 const root = appsRoot(rt.app.config.dataDir);
77 if (!existsSync(root)) return;
78 for (const name of readdirSync(root)) {
79 if (!known.has(name))
80 rmSync(join(root, name), { recursive: true, force: true });
81 }
82 }
83
84 export async function pullHead(rt: AppsRuntime, repo: Repo, pull: Pull): Promise<string | undefined> {
85 const dir = await pullDir(rt.app.db, rt.app.reposRoot, repo, pull);
86 // A fork's pull combined on our side previews the combined build (#9012).
87 return previewCommit(dir, pull);
88 }
89
90 // ---- events ----
91
92 export function pusherMayRunVariant(rt: AppsRuntime, repo: Repo, actorId: string | null): boolean {
93 const who = actorId === null ? undefined : getUserById(rt.app.db, actorId);
94 if (who === undefined || who.kind === "agent" || !canWrite(rt.app.db, repo, who)) {
95 log.info("variant waits for a person who can write", { repo: repo.name });
96 return false;
97 }
98 return true;
99 }
100
101 export async function onEvent(rt: AppsRuntime, event: ForgeEvent): Promise<void> {
102 if (rt.stopped || event.repoId === null) return;
103 if (event.kind === "repo.deleted") {
104 await rt.teardownAll(event.repoId);
105 return;
106 }
107 const record = getAppByRepo(rt.app.db, event.repoId);
108 if (!record) return;
109 const repo = getRepoById(rt.app.db, event.repoId);
110 if (!repo) return;
111 const dir = repoDir(rt.app.reposRoot, repo.ownerName, repo.name);
112
113 switch (event.kind) {
114 case "app.created":
115 await rt.reconcileApp(record, repo, "enabled");
116 return;
117 case "app.deleted":
118 await rt.teardownAll(event.repoId);
119 return;
120 case "push": {
121 const updates = event.payload["updates"];
122 if (!Array.isArray(updates)) return;
123 for (const update of updates as { ref?: unknown; newOid?: unknown }[]) {
124 if (
125 typeof update.ref !== "string" ||
126 typeof update.newOid !== "string"
127 )
128 continue;
129 if (!update.ref.startsWith("refs/heads/")) continue;
130 const branch = update.ref.slice("refs/heads/".length);
131 const deleted = /^0+$/.test(update.newOid);
132 if (branch === repo.defaultBranch && !deleted) {
133 rt.requestDeploy(
134 record,
135 repo,
136 "production",
137 update.newOid,
138 "push",
139 event.actorId ?? undefined,
140 );
141 }
142 const variant = deleted ? undefined : variantOfBranch(rt.app.db, record.id, branch);
143 // A variant runs with the app's sealed values and a copy of its data:
144 // a push to its branch redeploys only when a person who can write made it.
145 // An agent proposes; the writer's own "set values" deploys directly.
146 if (variant !== undefined && rt.pusherMayRunVariant(repo, event.actorId))
147 rt.requestDeploy(
148 record,
149 repo,
150 variantEnv(variant.label),
151 update.newOid,
152 "push",
153 event.actorId ?? undefined,
154 );
155 // Every open pull whose head is this branch previews the new tip.
156 for (const pull of listPulls(rt.app.db, repo.id, { sealed: true,
157 state: "open",
158 limit: 200,
159 })) {
160 if (pull.crossRepo || pull.headBranch !== branch || deleted)
161 continue;
162 void rt.prepare(record, repo, pull, update.newOid, "push", event.actorId ?? undefined);
163 }
164 }
165 return;
166 }
167 case "pull.pushed": {
168 // A fork's branch moved: the base's snapshot followed, and so
169 // does the preview.
170 const number = event.payload["number"];
171 const headOid = event.payload["headOid"];
172 if (typeof number !== "number" || typeof headOid !== "string") return;
173 const pull = getPull(rt.app.db, repo.id, number);
174 if (pull) void rt.prepare(record, repo, pull, headOid, "push", event.actorId ?? undefined);
175 return;
176 }
177 case "pull.opened":
178 case "pull.reopened": {
179 const pull = rt.pullFromEvent(repo, event);
180 if (!pull) return;
181 const head = await rt.pullHead(repo, pull);
182 if (head !== undefined)
183 void rt.prepare(record, repo, pull, head, "opened", event.actorId ?? undefined);
184 return;
185 }
186 // An approval is what lets a reader's pull run: its head may be asked for from now.
187 // A guest's approval restarts a preview that is up with the app's variables.
188 case "pull.run_granted":
189 case "pull.guest_approved":
190 case "pull.reviewed": {
191 // A writer asking for changes may withdraw what let the preview run:
192 // if the code it serves may no longer run, it stops now, not at its
193 // next sleep or restart.
194 if (
195 event.kind === "pull.reviewed" &&
196 event.payload["state"] === "changes_requested"
197 ) {
198 const pull = rt.pullFromEvent(repo, event);
199 if (!pull) return;
200 const env = previewEnv(pull.number);
201 const state = rt.envs.get(key(record.id, env));
202 const serving =
203 state?.live?.commitOid ??
204 state?.deploying?.run.commitOid ??
205 state?.asleep?.commitOid;
206 if (serving !== undefined && !rt.mayRun(record, repo, env, serving))
207 await rt.teardown(record, env);
208 return;
209 }
210 if (
211 event.kind === "pull.reviewed" &&
212 event.payload["state"] !== "approved"
213 )
214 return;
215 const pull = rt.pullFromEvent(repo, event);
216 if (!pull || pull.state !== "open") return;
217 const head = await rt.pullHead(repo, pull);
218 if (head !== undefined)
219 void rt.prepare(
220 record,
221 repo,
222 pull,
223 head,
224 // The same head may be serving already: start it again so it
225 // comes up holding the variables it ran without.
226 event.kind === "pull.guest_approved" ? "redeploy" : "approved",
227 event.actorId ?? undefined,
228 );
229 return;
230 }
231 // A move (#570): a process started before it holds the old owner's
232 // token and limits, and a deploy in flight may have read the old
233 // place on disk. Each starts again at the commit it serves, under the
234 // new owner; an asleep one wakes as the new owner's anyway, and a
235 // queued one reads the repository when its turn comes.
236 case "repo.transferred": {
237 for (const state of rt.envs.values()) {
238 if (state.app.id !== record.id) continue;
239 const commit = state.deploying?.run.commitOid ?? state.live?.commitOid;
240 if (commit === undefined || state.queued !== undefined) continue;
241 rt.requestDeploy(record, repo, state.env, commit, "moved", event.actorId ?? undefined);
242 }
243 return;
244 }
245 case "pull.closed":
246 case "pull.merged": {
247 const number = event.payload["number"];
248 if (typeof number === "number")
249 await rt.teardown(record, previewEnv(number));
250 // A merge moves the default branch with `update-ref`, which runs no
251 // hooks and so announces no push: production follows it from here.
252 if (event.kind === "pull.merged") {
253 const tip = await resolveOid(dir, `refs/heads/${repo.defaultBranch}`);
254 if (tip !== undefined)
255 rt.requestDeploy(
256 record,
257 repo,
258 "production",
259 tip,
260 "merge",
261 event.actorId ?? undefined,
262 );
263 }
264 return;
265 }
266 default:
267 return;
268 }
269 }
270
271 export function pullFromEvent(rt: AppsRuntime, repo: Repo, event: ForgeEvent): Pull | undefined {
272 const number = event.payload["number"];
273 return typeof number === "number"
274 ? getPull(rt.app.db, repo.id, number)
275 : undefined;
276 }
277
278 // ---- deploying ----
279
280 /**
281 * Ask for `env` to run `commitOid`. Returns the run that will do it — or
282 * the one already serving that commit, when nothing needs to happen.
283 */
284 export function requestDeploy(
285 rt: AppsRuntime,
286 record: AppRecord,
287 repo: Repo,
288 env: EnvName,
289 commitOid: string,
290 reason: string,
291 actorId?: string,
292 ): Run | undefined {
293 // A suspended or removed owner's app does not deploy, nor does an
294 // archived repository's: it is put away, its data kept.
295 if (
296 rt.stopped ||
297 repo.archivedAt !== null ||
298 !accountActive(rt.app.db, repo.ownerId)
299 )
300 return undefined;
301 if (!rt.mayRun(record, repo, env, commitOid)) return undefined;
302 const state = rt.envFor(record, env);
303 state.dormant = undefined;
304 if (
305 (state.live?.commitOid === commitOid ||
306 state.asleep?.commitOid === commitOid) &&
307 state.deploying === undefined &&
308 state.queued === undefined &&
309 reason !== "restart" &&
310 reason !== "oom" &&
311 !AGAIN.has(reason)
312 ) {
313 return undefined;
314 }
315 if (state.deploying?.run.commitOid === commitOid && !AGAIN.has(reason)) {
316 return state.deploying.run;
317 }
318 const issue =
319 envPull(env) === undefined ? undefined : rt.pullIssueId(repo, env);
320 const run = createRun(rt.app.db, {
321 repoId: repo.id,
322 kind: "deploy",
323 env,
324 commitOid,
325 reason,
326 ...(issue === undefined ? {} : { issueId: issue }),
327 ...(actorId === undefined ? {} : { actorId }),
328 });
329 announceRun(rt.app, run, actorId);
330 if (state.queued !== undefined) {
331 cancelRun(
332 rt.app.db,
333 state.queued.id,
334 `superseded by ${commitOid.slice(0, 7)}`,
335 );
336 }
337 state.queued = run;
338 if (reason === "boot" && env === "production") rt.bootRuns.set(state, run.id);
339 state.chain = state.chain.then(async () => {
340 if (state.queued?.id !== run.id) return; // superseded
341 state.queued = undefined;
342 await rt.execute(state, repo, run);
343 });
344 return run;
345 }
346
347 /**
348 * A pull's preview at `head`, after a push, an opening or an approval. One that something asks
349 * for at the push (`apps/prepared.ts`), or that is up or coming up already, deploys the
350 * ordinary way. Any other costs nothing until it is asked for: it is recorded asleep at `head`
351 * with no checkout and no process, and the first request builds and starts it (`rouse`).
352 * Decided on the environment's chain, so a burst of pushes ends at the last head.
353 */
354 export function prepare(rt: AppsRuntime, record: AppRecord, repo: Repo, pull: Pull, head: string, reason: string, actorId?: string): Promise<void> {
355 const env = previewEnv(pull.number);
356 if (rt.stopped || repo.archivedAt !== null || !accountActive(rt.app.db, repo.ownerId)) return Promise.resolve();
357 if (!rt.mayRun(record, repo, env, head)) return Promise.resolve();
358 const state = rt.envFor(record, env);
359 const decide = async (): Promise<void> => {
360 if (rt.stopped || rt.envs.get(key(record.id, env)) !== state) return;
361 const up =
362 state.live !== undefined ||
363 state.deploying !== undefined ||
364 state.queued !== undefined ||
365 state.waking !== undefined;
366 // A repository that cannot be read is deployed, as it always was: the deploy says why.
367 const asker = up ? "up" : await rt.pushAsker(repo, pull, head, reason !== "boot").catch(() => "check" as const);
368 if (asker !== undefined) {
369 rt.requestDeploy(record, repo, env, head, reason, actorId);
370 return;
371 }
372 // Asleep at this very head already (a boot, an approval): it stays as it is.
373 if (state.asleep?.commitOid === head || state.dormant?.commitOid === head) return;
374 const now = new Date().toISOString();
375 state.asleep = undefined;
376 state.dormant = { since: now, idleSince: now, commitOid: head, prepared: true };
377 state.lastError = undefined;
378 state.lastErrorCommitOid = undefined;
379 log.info("preview asleep until asked for", { app: record.slug, env, commit: head.slice(0, 7), reason });
380 // An older head's checkout is dead weight now: the first request checks this one out.
381 await dropCheckouts(rt.app.config.dataDir, record.id, env);
382 };
383 const decided = state.chain.then(decide).catch((error: unknown) =>
384 log.error("could not prepare a preview", { app: record.slug, env, error: String(error) }),
385 );
386 state.chain = decided;
387 return decided;
388 }
389
390 /** What asks for this pull's preview at the push, if anything does (`apps/prepared.ts`). */
391 export async function pushAsker(rt: AppsRuntime, repo: Repo, pull: Pull, head: string, pictured: boolean): Promise<PushAsker | undefined> {
392 // A forge that runs no checks (`FORGE_CHECKS=0`) judges every pull by its deploy.
393 if (checksFor(rt.app) === undefined) return "check";
394 const dir = await pullDir(rt.app.db, rt.app.reposRoot, repo, pull);
395 const asker =
396 declaredAsker(await readFileAt(dir, head, "forge.json")) ??
397 // A boot is no push: a head pictured before it has its pictures, and nothing new is owed one.
398 (pictured && (await lookPagesAt(dir, pull.baseOid, head)).length > 0 ? ("look" as const) : undefined);
399 if (asker !== undefined)
400 log.info("preview deploys at the push", { repo: repo.name, pull: pull.number, why: ASKER_WORDS[asker] });
401 return asker;
402 }
src/apps/runtime_meter.ts added +142
@@ -0,0 +1,142 @@
1 /**
2 * The awake meter (`AppsRuntime.awake`, `resting`): each interval an
3 * environment serves, written as it opens and closes, and the storage sample
4 * beside it.
5 */
6
7 import { existsSync, readdirSync } from "node:fs";
8 import { join } from "node:path";
9 import { listApps } from "../core/apps.ts";
10 import { getRepoById } from "../core/repos.ts";
11 import { placeFor } from "../core/placement_policy.ts";
12 import { recordPlacement } from "../core/decisions.ts";
13 import { bodyTarget, CLOUD_BODY_CONFIGURED } from "../spawn/placement.ts";
14 import { repoDir } from "../git/repos.ts";
15 import { closeAllAwake, closeAwake, lastStorageAt, openAwake, recordStorage, SAMPLE_SPAN_MS } from "../core/meter.ts";
16 import { dirBytes } from "./bytes.ts";
17 import { appCacheSize } from "./build_cache.ts";
18 import { appsRoot, envDataDir, isEnvName } from "./data.ts";
19 import { policyAt } from "../core/slider.ts";
20 import { getEconomics, getOwnedNodeLabel } from "../settings/economics.ts";
21 import type { AppsRuntime } from "./runtime.ts";
22 import { log, type Environment } from "./runtime_env.ts";
23
24 // ---- the meter ----
25
26 /**
27 * Run one write to the meter. What an app used is worth recording, never
28 * worth taking a deploy, a sleep or a shutdown down with it.
29 */
30 export function metered<T>(rt: AppsRuntime, write: () => T): T | undefined {
31 if (rt.app.closing.signal.aborted) return undefined;
32 try {
33 return write();
34 } catch (error) {
35 log.error("meter write failed", { error: String(error) });
36 return undefined;
37 }
38 }
39
40 /** The forge is closing: stop the sweeper, end open intervals, then the meter goes quiet. */
41 export function closeMeter(rt: AppsRuntime): void {
42 rt.stopped = true;
43 if (rt.sweeper !== undefined) clearInterval(rt.sweeper);
44 try {
45 closeAllAwake(rt.app.db, new Date().toISOString());
46 } catch (error) {
47 log.error("meter write failed", { error: String(error) });
48 }
49 }
50
51 /**
52 * The environment began serving: its serving interval opens, if not open,
53 * placed by the policy (`core/placement_policy.ts`) and with that decision
54 * kept in the ledger, so cost knows which price applies and why.
55 */
56 export function awake(rt: AppsRuntime, state: Environment, at: string): void {
57 rt.metered(() => {
58 const db = rt.app.db;
59 const service = state.live?.manifest.service ?? null;
60 const seen = {
61 ownedUsed: rt.ownedUsed(state),
62 ownedCapacity: rt.options.ownedCapacity?.() ?? Infinity,
63 spillAtOwnedUse: policyAt(getEconomics(db).position).spillAtOwnedUse,
64 alwaysOn: state.env === "production" || service?.alwaysOn === true,
65 preference: service?.placement ?? null,
66 cloudBody: rt.options.cloudBody ?? CLOUD_BODY_CONFIGURED,
67 };
68 const placed = placeFor(seen);
69 const opened = openAwake(db, { repoId: state.app.repoId, env: state.env, placement: placed.where, at });
70 if (!opened) return;
71 recordPlacement(db, {
72 repoId: state.app.repoId,
73 env: state.env,
74 where: placed.where,
75 reason: placed.reason,
76 // JSON has no Infinity: an unbounded capacity is kept as null.
77 seen: { ...seen, ownedCapacity: Number.isFinite(seen.ownedCapacity) ? seen.ownedCapacity : null },
78 target: bodyTarget(placed.where, getOwnedNodeLabel(db)),
79 });
80 });
81 }
82
83 /** Environments up on this forge now, `except` aside: what owned metal holds. */
84 export function ownedUsed(rt: AppsRuntime, except: Environment): number {
85 let n = 0;
86 for (const other of rt.envs.values()) if (other !== except && other.live !== undefined) n++;
87 return n;
88 }
89
90 /** The environment stopped serving: its serving interval closes. */
91 export function resting(rt: AppsRuntime, state: Environment, at: number = Date.now()): void {
92 rt.metered(() =>
93 closeAwake(rt.app.db, {
94 repoId: state.app.repoId,
95 env: state.env,
96 at: new Date(at).toISOString(),
97 }),
98 );
99 }
100
101 /**
102 * Once per `storageEveryMs`: the bytes of each app's `FORGE_APP_DATA`,
103 * every environment summed, and of its bare repository.
104 */
105 export async function sampleStorage(rt: AppsRuntime): Promise<void> {
106 if (rt.stopped || rt.sampling) return;
107 const every = rt.options.storageEveryMs ?? SAMPLE_SPAN_MS;
108 const last = rt.metered(() => lastStorageAt(rt.app.db));
109 if (last !== undefined && Date.now() - Date.parse(last) < every) return;
110 rt.sampling = true;
111 try {
112 const at = new Date().toISOString();
113 for (const record of listApps(rt.app.db)) {
114 const repo = getRepoById(rt.app.db, record.repoId);
115 if (!repo) continue;
116 let dataBytes = 0;
117 const root = join(appsRoot(rt.app.config.dataDir), record.id);
118 if (existsSync(root))
119 for (const name of readdirSync(root))
120 if (isEnvName(name))
121 dataBytes += await dirBytes(
122 envDataDir(rt.app.config.dataDir, record.id, name),
123 );
124 // The build cache is the app's disk too (#10526).
125 dataBytes += appCacheSize(rt.app.config.dataDir, record.id);
126 const repoBytes = await dirBytes(
127 repoDir(rt.app.reposRoot, repo.ownerName, repo.name),
128 );
129 if (rt.stopped) return;
130 rt.metered(() =>
131 recordStorage(rt.app.db, {
132 repoId: record.repoId,
133 dataBytes,
134 repoBytes,
135 at,
136 }),
137 );
138 }
139 } finally {
140 rt.sampling = false;
141 }
142 }
src/apps/runtime_sleep.ts added +505
@@ -0,0 +1,505 @@
1 /**
2 * Sleeping and waking (`AppsRuntime.wake`, `hibernate`, `sweep`): what is
3 * wanted is touched, what is idle past its window sleeps, what is asleep
4 * wakes on its first request.
5 */
6
7 import { dropCheckouts } from "./dormant_checkout.ts";
8 import { accountActive } from "../core/accounts.ts";
9 import { existsSync, readFileSync } from "node:fs";
10 import { join } from "node:path";
11 import type { AppRecord } from "../core/apps.ts";
12 import { appTokenName, mintAppToken, renewAppToken, revokeAppToken } from "./token.ts";
13 import { askingOf } from "../perf/asking.ts";
14 import { getPull, type Pull } from "../core/pulls.ts";
15 import { isContributionFork } from "../core/contribute.ts";
16 import { previewMayRun } from "./preview_gate.ts";
17 import { getRepoById, type Repo } from "../core/repos.ts";
18 import { runningLimitSentence } from "../settings/quotas.ts";
19 import { listRuns } from "../core/runs.ts";
20 import { repoDir } from "../git/repos.ts";
21 import { headUnder } from "../git/combined.ts";
22 import { appendRunLog } from "../runs/log.ts";
23 import { beatAwake } from "../core/meter.ts";
24 import { dropPreviewBuildCache } from "./build_cache.ts";
25 import { BUSY_WORDS, busyReason } from "./busy.ts";
26 import { envBuild, envVariant, envDataDir, envDir, envPull, type EnvName } from "./data.ts";
27 import { parseManifest } from "./manifest.ts";
28 import { heldManifest } from "../core/starts.ts";
29 import { identityKey } from "./proxy.ts";
30 import { policyAt } from "../core/slider.ts";
31 import { getEconomics } from "../settings/economics.ts";
32 import { asleepWords, sleepWindow, type WindowSource } from "./sleep_window.ts";
33 import { startProcess } from "./process.ts";
34 import type { AppsRuntime } from "./runtime.ts";
35 import { log, UNOPENED_MS, BUSY_GRACE_MS, type Live, type LiveEnv, type Environment, TOUCH_REASONS, key } from "./runtime_env.ts";
36
37 // ---- tearing down ----
38
39 /** Stop an environment and remove everything it owned, data included. */
40 /** Something came through the proxy: the environment is wanted. */
41 export function touch(rt: AppsRuntime, record: AppRecord, env: EnvName): void {
42 const state = rt.envs.get(key(record.id, env));
43 if (state === undefined) return;
44 state.lastRequestAt = rt.now();
45 state.keptAwake = undefined;
46 }
47
48 /**
49 * A stream through the proxy opened: the environment is busy until the
50 * returned release is called — once; a second call does nothing.
51 */
52 export function holdStream(rt: AppsRuntime, record: AppRecord, env: EnvName): () => void {
53 const state = rt.envs.get(key(record.id, env));
54 if (state === undefined) return () => {};
55 state.openStreams++;
56 let held = true;
57 return () => {
58 if (!held) return;
59 held = false;
60 state.openStreams--;
61 state.lastRequestAt = rt.now();
62 };
63 }
64
65 /**
66 * A request passed to the app is waiting on its answer: the environment
67 * is busy until the returned release is called — once; a second call
68 * does nothing. Still inside the busy grace, so a hung app sleeps anyway.
69 */
70 export function holdRequest(rt: AppsRuntime, record: AppRecord, env: EnvName): () => void {
71 const state = rt.envs.get(key(record.id, env));
72 if (state === undefined) return () => {};
73 state.openRequests++;
74 let held = true;
75 return () => {
76 if (!held) return;
77 held = false;
78 state.openRequests--;
79 state.lastRequestAt = rt.now();
80 };
81 }
82
83 /**
84 * An environment asleep is started again, warm, from the checkout it
85 * kept; one start serves every request that arrives while it happens.
86 * Returns once it is live, or once it is clear it will not be.
87 */
88 /**
89 * A preview runs a pull's code: from someone who cannot write, only once
90 * a writer has approved this exact head (`core/pull_trust.ts`). Asked by
91 * every door that starts one — a deploy and a waking.
92 */
93 export function mayRun(
94 rt: AppsRuntime,
95 record: AppRecord,
96 repo: Repo,
97 env: EnvName,
98 commitOid: string,
99 ): boolean {
100 // A guest's fork has no app; were one ever to reach here, it starts nothing.
101 if (isContributionFork(rt.app.db, repo.id)) return false;
102 // Production is the default branch and a build preview a commit a writer asked to see
103 // (`actions/build_preview.ts`): each is a writer's choice already.
104 if (env === "production" || envBuild(env) !== undefined || envVariant(env) !== undefined)
105 return true;
106 const number = envPull(env);
107 const pull =
108 number === undefined ? undefined : getPull(rt.app.db, repo.id, number);
109 // A combined preview runs under the head it carries, read from git (#9012).
110 const head =
111 pull === undefined
112 ? commitOid
113 : headUnder(repoDir(rt.app.reposRoot, repo.ownerName, repo.name), pull, commitOid);
114 // An outside agent's branch (`contrib/…`) is previewed only once a person who can write has
115 // approved the pull (`origins/guest.ts`); its check, in a cell, is what runs before that.
116 if (pull !== undefined && previewMayRun(rt.app.db, repo, pull, head)) return true;
117 log.info("preview waits for a writer's approval", {
118 app: record.slug,
119 env,
120 commit: commitOid.slice(0, 7),
121 });
122 return false;
123 }
124
125 export async function wake(rt: AppsRuntime, record: AppRecord, env: EnvName): Promise<LiveEnv | undefined> {
126 const state = rt.envs.get(key(record.id, env));
127 if (state === undefined) return undefined;
128 if (state.waking !== undefined) await state.waking;
129 if (state.live !== undefined) return rt.liveFor(record, env);
130 const asleep = state.asleep;
131 if (asleep === undefined || rt.stopped) return undefined;
132 const repo = getRepoById(rt.app.db, record.repoId);
133 // Waking runs code without a run row: the owner's standing is asked here.
134 if (!repo || !accountActive(rt.app.db, repo.ownerId)) return undefined;
135 // So is whose code it is: a preview asleep at a head nobody who can
136 // write approved — built before the gate, or approved by someone who
137 // can no longer write — is taken down, not woken. Approving the head
138 // deploys it again.
139 if (!rt.mayRun(record, repo, env, asleep.commitOid)) {
140 await rt.teardown(record, env);
141 return undefined;
142 }
143 // A wake starts a process too: past the account's preview limit it stays asleep.
144 const full = runningLimitSentence(
145 rt.app.db,
146 "previews",
147 "live previews",
148 repo.ownerId,
149 () => rt.livePreviews(repo.ownerId, state),
150 );
151 if (full !== undefined) {
152 log.info("preview stays asleep", { app: record.slug, env, limit: full });
153 state.wakeRefused = `This preview stays asleep: its owner is at ${full}. Close a pull or stop a preview to make room.`;
154 return undefined;
155 }
156 state.wakeRefused = undefined;
157 state.waking = (async () => {
158 const write = (line: string): void => {
159 if (asleep.runId !== "")
160 appendRunLog(rt.app.config.dataDir, asleep.runId, line);
161 };
162 write("waking: a request came");
163 // Asked again at every start: a preview that slept wakes under what its base reaches now.
164 const manifest = await rt.heldFor(repo, env, asleep.manifest);
165 const token = mintAppToken(rt.app.db, repo.ownerId, record.slug, env);
166 const started = await startProcess({
167 dir: asleep.dir,
168 dataDir: envDataDir(rt.app.config.dataDir, record.id, env),
169 manifest,
170 env,
171 url: rt.urlFor(record, env),
172 commitOid: asleep.commitOid,
173 forgeUrl: rt.options.forgeUrl,
174 identityKey: identityKey(rt.app.config.dataDir, record.id),
175 ssh: rt.sshFor(record, env),
176 ...rt.secretsFor(repo, record.id, env, token.token, asleep.dir),
177 feature: rt.featureFor(repo, record.id, env, asleep.dir),
178 log: write,
179 signal: new AbortController().signal,
180 warm: true,
181 cell: rt.cellFor(record, env),
182 }).catch((error: unknown) => {
183 revokeAppToken(rt.app.db, token.id);
184 throw error;
185 });
186 if (state.asleep !== asleep || !started.ok)
187 revokeAppToken(rt.app.db, token.id);
188 if (state.asleep !== asleep) {
189 // Replaced while waking — a deploy landed. What we started is spare.
190 if (started.ok) await started.value.handle.stop();
191 return;
192 }
193 if (!started.ok) {
194 state.asleep = undefined;
195 state.lastError = `did not wake — ${started.error.stage}: ${started.error.message}`;
196 write(`failed: ${state.lastError}`);
197 return;
198 }
199 const now = new Date().toISOString();
200 const live: Live = {
201 commitOid: asleep.commitOid,
202 handle: started.value.handle,
203 dir: asleep.dir,
204 runId: asleep.runId,
205 since: now,
206 manifest,
207 servingAt: now,
208 tokenId: token.id,
209 };
210 state.asleep = undefined;
211 state.live = live;
212 rt.awake(state, now);
213 state.lastRequestAt = rt.now();
214 rt.watch(state, repo, live);
215 write("awake");
216 })().finally(() => {
217 state.waking = undefined;
218 });
219 await state.waking;
220 return rt.liveFor(record, env);
221 }
222
223 /**
224 * At boot, an environment whose checkout for this commit is still here
225 * is registered asleep rather than deployed. True when it was.
226 */
227 export function settleAsleep(
228 rt: AppsRuntime,
229 record: AppRecord,
230 env: EnvName,
231 commitOid: string,
232 ): boolean {
233 const dir = join(
234 envDir(rt.app.config.dataDir, record.id, env),
235 commitOid.slice(0, 7),
236 );
237 if (!existsSync(join(dir, "package.json"))) return false;
238 const read = (name: string): string | undefined => {
239 try {
240 return readFileSync(join(dir, name), "utf8");
241 } catch {
242 return undefined;
243 }
244 };
245 const manifest = parseManifest(read("forge.json"), read("package.json"));
246 if (!manifest.ok) return false;
247 const owner = getRepoById(rt.app.db, record.repoId)?.ownerId;
248 const state = rt.envFor(record, env);
249 if (state.live !== undefined || state.deploying !== undefined) return true;
250 // The deploy that built this checkout, in an earlier life: its log is
251 // where the environment's sleeping and waking keep being written.
252 const built = listRuns(rt.app.db, record.repoId, {
253 kind: "deploy",
254 limit: 100,
255 }).find(
256 (r) => r.env === env && r.state === "ok" && r.commitOid === commitOid,
257 );
258 state.asleep = {
259 commitOid,
260 dir,
261 manifest: owner === undefined ? manifest.value : heldManifest(rt.app.db, owner, manifest.value),
262 runId: built?.id ?? "",
263 since: new Date().toISOString(),
264 };
265 state.lastRunId ??= built?.id;
266 log.info("asleep from boot", {
267 app: record.slug,
268 env,
269 commit: commitOid.slice(0, 7),
270 });
271 return true;
272 }
273
274 /**
275 * When a pull was last touched, if that is longer ago than the idle age:
276 * the later of its `updatedAt` (comments, reviews) and the newest deploy
277 * run a person caused — a push, an opening, a press of Redeploy, a waking.
278 * Undefined when the feature is off, the pull was touched within the age,
279 * or someone is deciding it at this moment (a queued or running run on it).
280 */
281 export function idleSince(rt: AppsRuntime, repo: Repo, pull: Pull, env: EnvName): string | undefined {
282 const age = rt.idlePullAge();
283 if (age === null || pull.state !== "open") return undefined;
284 const now = Date.now();
285 let last = Date.parse(pull.updatedAt);
286 if (now - last < age) return undefined;
287 const runs = listRuns(rt.app.db, repo.id, {
288 issueId: pull.id,
289 limit: 50,
290 });
291 if (runs.some((r) => r.state === "queued" || r.state === "running"))
292 return undefined;
293 for (const r of runs) {
294 if (r.kind === "deploy" && r.env === env && TOUCH_REASONS.includes(r.reason))
295 last = Math.max(last, Date.parse(r.createdAt));
296 }
297 return now - last < age ? undefined : new Date(last).toISOString();
298 }
299
300 /** At boot: register a preview idle past the age as stopped, no process. */
301 export function hibernateAtBoot(
302 rt: AppsRuntime,
303 record: AppRecord,
304 env: EnvName,
305 commitOid: string,
306 idleSince: string,
307 ): void {
308 const state = rt.envFor(record, env);
309 if (state.live !== undefined || state.deploying !== undefined) return;
310 state.dormant = {
311 since: new Date().toISOString(),
312 idleSince,
313 commitOid,
314 };
315 const built = listRuns(rt.app.db, record.repoId, {
316 kind: "deploy",
317 env,
318 limit: 50,
319 }).find((r) => r.state === "ok");
320 state.lastRunId ??= built?.id;
321 void rt.dropDormantCheckouts(state);
322 log.info("dormant from boot", { app: record.slug, env });
323 }
324
325 /**
326 * Give back a dormant preview's checkouts on the environment's deploy chain, so a waking
327 * deploy starts only after the removal ends. Skipped when it was roused first; the commits
328 * serving or coming up are spared. A failure is logged, never thrown.
329 */
330 export function dropDormantCheckouts(rt: AppsRuntime, state: Environment): Promise<void> {
331 const run = async (): Promise<void> => {
332 if (state.dormant === undefined) return;
333 const keep = [state.live, state.deploying?.run, state.queued]
334 .map((x) => x?.commitOid)
335 .filter((oid): oid is string => oid !== undefined && oid !== "");
336 try {
337 await dropCheckouts(rt.app.config.dataDir, state.app.id, state.env, keep);
338 if (state.dormant !== undefined && state.deploying === undefined && state.queued === undefined)
339 dropPreviewBuildCache(rt.app.config.dataDir, state.app.id, state.env);
340 } catch (error) {
341 log.warn("could not drop dormant checkouts", { app: state.app.slug, env: state.env, error: String(error) });
342 }
343 };
344 const done = state.chain.then(run);
345 state.chain = done;
346 return done;
347 }
348
349 /** A dormant preview someone asked for deploys again, the ordinary way. */
350 export function rouse(rt: AppsRuntime, record: AppRecord, env: EnvName): void {
351 const state = rt.envs.get(key(record.id, env));
352 const dormant = state?.dormant;
353 if (dormant === undefined) return;
354 const repo = getRepoById(rt.app.db, record.repoId);
355 if (repo) rt.requestDeploy(record, repo, env, dormant.commitOid, "wake");
356 }
357
358 /** Stop the previews whose pulls have gone quiet; keep everything else. */
359 export async function hibernate(rt: AppsRuntime): Promise<void> {
360 if (rt.idlePullAge() === null) return;
361 for (const state of [...rt.envs.values()]) {
362 const number = envPull(state.env);
363 if (
364 number === undefined ||
365 state.dormant !== undefined ||
366 state.deploying !== undefined ||
367 state.queued !== undefined ||
368 state.waking !== undefined ||
369 (state.live === undefined && state.asleep === undefined)
370 )
371 continue;
372 const repo = getRepoById(rt.app.db, state.app.repoId);
373 const pull = repo && getPull(rt.app.db, repo.id, number);
374 if (!repo || !pull) continue;
375 const idleSince = rt.idleSince(repo, pull, state.env);
376 if (idleSince === undefined) continue;
377 const commitOid = state.live?.commitOid ?? state.asleep!.commitOid;
378 const runId = state.live?.runId ?? state.asleep!.runId;
379 await rt.stopEnv(state.app, state.env);
380 if (rt.stopped || state.live !== undefined || state.queued !== undefined)
381 continue;
382 state.dormant = {
383 since: new Date().toISOString(),
384 idleSince,
385 commitOid,
386 };
387 state.lastError = undefined;
388 state.lastErrorCommitOid = undefined;
389 if (runId !== "")
390 appendRunLog(
391 rt.app.config.dataDir,
392 runId,
393 `asleep: nobody pushed to or commented on pull #${number} since ${idleSince}; asking for it, or Redeploy, brings it back`,
394 );
395 await rt.dropDormantCheckouts(state);
396 log.info("dormant", { app: state.app.slug, env: state.env });
397 }
398 }
399
400 /**
401 * How long a pull may sit untouched before its preview stops; `null` is
402 * never. A fixed option wins (0 is off); unset follows the slider, read
403 * at this moment — so moving it changes the next idle pass, with no restart.
404 */
405 export function idlePullAge(rt: AppsRuntime): number | null {
406 const fixed = rt.options.idlePullMs;
407 if (fixed !== undefined) return fixed > 0 ? fixed : null;
408 return policyAt(getEconomics(rt.app.db).position).idlePullMs;
409 }
410
411 /** Where the idle windows of this pass come from: a fixed option, else the slider as it stands. */
412 export function windowSource(rt: AppsRuntime): WindowSource {
413 const fixed = rt.options.sleepAfterMs;
414 return fixed !== undefined
415 ? { kind: "fixed", sleepAfterMs: fixed }
416 : { kind: "slider", position: getEconomics(rt.app.db).position };
417 }
418
419 /** Stop what nobody has asked for lately, and keep it ready to start again on the next request. */
420 export async function sweep(rt: AppsRuntime): Promise<void> {
421 if (rt.stopped) return;
422 await rt.hibernate();
423 const source = rt.windowSource();
424 const grace = rt.options.busyGraceMs ?? BUSY_GRACE_MS;
425 const now = rt.now();
426 rt.metered(() => beatAwake(rt.app.db, new Date(now).toISOString()));
427 for (const state of rt.envs.values()) {
428 const live = state.live;
429 // Still running: its credential is still wanted for another window.
430 if (live?.tokenId !== undefined)
431 renewAppToken(rt.app.db, live.tokenId, now);
432 const spared = state.keptAwake;
433 state.keptAwake = undefined;
434 if (
435 live === undefined ||
436 state.deploying !== undefined ||
437 state.queued !== undefined ||
438 state.waking !== undefined
439 )
440 continue;
441 const window = sleepWindow(
442 source,
443 state.env === "production" || envVariant(state.env) !== undefined,
444 live.manifest.alwaysOn === true,
445 );
446 const last = state.lastRequestAt ?? Date.parse(live.since);
447 // An always-on app nobody has opened for a day, still calling the forge, is idle like any
448 // other: its own calls are not a visit (`touch` is the proxy's), and they do not keep it
449 // awake (#12265).
450 const limit =
451 window.ms ??
452 (now - last >= UNOPENED_MS && askingOf(rt.app).calling(appTokenName(state.app.slug, state.env), 60 * 60_000, now)
453 ? UNOPENED_MS
454 : null);
455 if (limit === null) continue;
456 if (now - last < limit) continue;
457 // Idle by the clock; asked whether it is mid-work, and spared while
458 // it is — but only for one grace past the window, never forever.
459 const busy = busyReason(
460 live.handle.pid,
461 state.openStreams,
462 state.openRequests,
463 );
464 const overdue = now - last - limit >= grace;
465 if (busy !== undefined && !overdue) {
466 if (spared !== busy && live.runId !== "")
467 appendRunLog(
468 rt.app.config.dataDir,
469 live.runId,
470 `kept awake: ${BUSY_WORDS[busy]}`,
471 );
472 state.keptAwake = busy;
473 continue;
474 }
475 state.live = undefined;
476 rt.resting(state, now);
477 state.asleep = {
478 commitOid: live.commitOid,
479 dir: live.dir,
480 manifest: live.manifest,
481 runId: live.runId,
482 since: new Date(now).toISOString(),
483 };
484 if (live.runId !== "")
485 appendRunLog(
486 rt.app.config.dataDir,
487 live.runId,
488 busy === undefined
489 ? asleepWords({ ms: limit, why: window.ms === null ? "always_on, but nobody opened it and it kept calling the forge" : window.why })
490 : `asleep: still busy ${Math.round(grace / 60_000)} minutes past its idle window, which is as long as busy keeps an app awake; the next request wakes it`,
491 );
492 log.info("asleep", { app: state.app.slug, env: state.env });
493 revokeAppToken(rt.app.db, live.tokenId);
494 await live.handle.stop();
495 }
496 await rt.sampleStorage();
497 }
498
499 /**
500 * The sweep's clock. Requests stamp `lastRequestAt` with it too, so a
501 * window measured by the sweep starts again at a request on the same clock.
502 */
503 export function now(rt: AppsRuntime): number {
504 return rt.options.clock?.() ?? Date.now();
505 }
src/apps/runtime_status.ts added +253
@@ -0,0 +1,253 @@
1 /**
2 * How an environment reads from outside (`AppsRuntime.describe`, `liveFor`,
3 * `cellFor`, `startingCount`): its status, its live process and its cell,
4 * computed from the in-memory record without starting anything.
5 */
6
7 import { cellHides, grantsHere } from "../spawn/hosted-mark.ts";
8 import { VERIFYING_NOTE } from "./env_phrase.ts";
9 import { join } from "node:path";
10 import type { AppRecord } from "../core/apps.ts";
11 import { getRun, noteRunProgress } from "../core/runs.ts";
12 import { BUSY_WORDS } from "./busy.ts";
13 import { envDataDir, envDir, type EnvName } from "./data.ts";
14 import { noteReached } from "./egress_report.ts";
15 import { cellHooks } from "./egress_refused.ts";
16 import { egressSetting } from "../core/capacity.ts";
17 import { resolveLimits } from "./contain.ts";
18 import type { CellBase } from "./process.ts";
19 import type { AppsRuntime } from "./runtime.ts";
20 import { type EnvStatus, type LiveEnv, type Environment, key, grantedFor } from "./runtime_env.ts";
21
22 export function describe(
23 rt: AppsRuntime,
24 state: Environment,
25 hosts?: ReadonlyMap<string, string>,
26 ): EnvStatus {
27 const url = rt.urlFor(state.app, state.env, hosts);
28 // Once a deploy's own commit has answered and swapped in, it is
29 // "running" from here — even though its run stays open, checking its
30 // tests, until they finish. Only a build still short of that point (no
31 // live process yet, or the live one is an older commit) is "deploying".
32 const swapped =
33 state.deploying !== undefined &&
34 state.live?.commitOid === state.deploying.run.commitOid;
35 if (state.deploying !== undefined && !swapped) {
36 return {
37 env: state.env,
38 state: "deploying",
39 commitOid: state.deploying.run.commitOid,
40 url,
41 ...(state.live === undefined && state.deploying.run.reason === "wake" ? { waking: true as const } : {}),
42 ...(state.live === undefined
43 ? {}
44 : {
45 port: state.live.handle.port,
46 since: state.live.since,
47 servingAt: state.live.servingAt,
48 createdAt: state.live.createdAt,
49 tty: state.live.manifest.tty,
50 looks: state.live.manifest.looks,
51 looksAsOwner: state.live.manifest.looksAsOwner,
52 toolbar: state.live.manifest.toolbar,
53 ...(state.live.deployStartedAt === null
54 ? {}
55 : { deployStartedAt: state.live.deployStartedAt }),
56 }),
57 runId: state.deploying.run.id,
58 };
59 }
60 if (state.live !== undefined) {
61 // The most recent deploy attempt failed, and it was not this one — a
62 // newer commit that never came up, while this older deployment kept
63 // serving. `swapped` already covers the alternative (the newest
64 // attempt is this one and it succeeded, however its tests read).
65 const failedDeploy =
66 !swapped &&
67 state.lastError !== undefined &&
68 state.lastErrorCommitOid !== undefined &&
69 state.lastErrorCommitOid !== state.live.commitOid &&
70 state.lastRunId !== undefined
71 ? {
72 commitOid: state.lastErrorCommitOid,
73 runId: state.lastRunId,
74 error: state.lastError,
75 }
76 : undefined;
77 return {
78 env: state.env,
79 state: "running",
80 commitOid: state.live.commitOid,
81 url,
82 port: state.live.handle.port,
83 runId: swapped ? state.deploying!.run.id : state.live.runId,
84 since: state.live.since,
85 servingAt: state.live.servingAt,
86 createdAt: state.live.createdAt,
87 tty: state.live.manifest.tty,
88 looks: state.live.manifest.looks,
89 looksAsOwner: state.live.manifest.looksAsOwner,
90 toolbar: state.live.manifest.toolbar,
91 ...(state.env === "production" && state.live.manifest.alwaysOn === true
92 ? { alwaysOn: true as const }
93 : {}),
94 ...(state.live.deployStartedAt === null
95 ? {}
96 : { deployStartedAt: state.live.deployStartedAt }),
97 ...(swapped ? { note: VERIFYING_NOTE } : {}),
98 ...(state.live.testsFailed === undefined
99 ? {}
100 : { error: `serving · tests failed: ${state.live.testsFailed}` }),
101 ...(failedDeploy === undefined ? {} : { failedDeploy }),
102 ...(state.keptAwake === undefined
103 ? {}
104 : { busy: BUSY_WORDS[state.keptAwake] }),
105 };
106 }
107 // Nothing is live and nothing is in flight yet, but a deploy already
108 // waits its turn on the chain — a restart's own reconcile queues one
109 // the instant it boots. That is "starting", not the stale error (if
110 // any) a cancelled attempt before it left behind.
111 if (state.queued !== undefined) {
112 return {
113 env: state.env,
114 state: "deploying",
115 commitOid: state.queued.commitOid,
116 url,
117 runId: state.queued.id,
118 ...(state.queued.reason === "wake" ? { waking: true as const } : {}),
119 };
120 }
121 if (state.dormant !== undefined) {
122 return {
123 env: state.env,
124 state: "stopped",
125 commitOid: state.dormant.commitOid,
126 url,
127 runId: state.lastRunId,
128 dormant: {
129 since: state.dormant.since,
130 idleSince: state.dormant.idleSince,
131 ...(state.dormant.prepared === true ? { prepared: true as const } : {}),
132 },
133 };
134 }
135 if (state.asleep !== undefined) {
136 return {
137 env: state.env,
138 state: "asleep",
139 commitOid: state.asleep.commitOid,
140 url,
141 runId: state.asleep.runId,
142 since: state.asleep.since,
143 error: state.wakeRefused,
144 };
145 }
146 return {
147 env: state.env,
148 state: state.lastError === undefined ? "stopped" : "failed",
149 commitOid: "",
150 url,
151 runId: state.lastRunId,
152 error: state.lastError,
153 };
154 }
155
156 /**
157 * How many of the productions this boot asked for are neither serving
158 * nor settled yet. Until it reads 0 the forge is not ready (`/readyz`),
159 * so in a rollout the forge it replaces keeps proxying to its own copy
160 * of each app and no request finds an app that is still starting (#4496).
161 */
162 export function startingCount(rt: AppsRuntime): number {
163 let count = 0;
164 for (const [state, runId] of rt.bootRuns) {
165 const run = state.live?.runId === runId ? undefined : getRun(rt.app.db, runId);
166 const pending = run?.state === "queued" || run?.state === "running";
167 if (pending) count++;
168 else rt.bootRuns.delete(state);
169 }
170 return count;
171 }
172
173 /**
174 * What an environment's processes are held to: the forge's data and home
175 * hidden, the environment's own home. `undefined` where cells cannot work.
176 */
177 export function cellFor(rt: AppsRuntime, record: AppRecord, env: EnvName): CellBase | undefined {
178 if (!rt.cells.ok) return undefined;
179 const dataDir = rt.app.config.dataDir;
180 return {
181 hide: cellHides(rt.app),
182 home: join(envDir(dataDir, record.id, env), "home"),
183 onUndeclared: (host, port) =>
184 noteReached(dataDir, record.repoId, host, port),
185 // Asked on every connection: an owner's Allow reaches the running cell.
186 // Said once per host: the first refusal is an event, the rest a count.
187 ...cellHooks(dataDir, record.repoId, env, (host, port) =>
188 rt.app.bus.append({ kind: "app.egress.refused", repoId: record.repoId, payload: { host, port } }),
189 (line) => rt.app.log.warn(line),
190 record.slug,
191 ),
192 // Asked at every start: an admin's grant reaches the next cell.
193 granted: grantsHere(rt.app, () => grantedFor(rt.app.db, record.repoId)),
194 // Likewise an admin's flip of the egress mode.
195 egressMode: egressSetting(rt.app.db),
196 label: `app:${record.slug}/${env}`,
197 repoId: record.repoId,
198 };
199 }
200
201 /** Say on the run, once, whether its processes were held. */
202 export function noteCell(rt: AppsRuntime, runId: string, write: (line: string) => void): void {
203 noteRunProgress(rt.app.db, runId, {
204 sandbox: {
205 enforced: rt.cells.ok,
206 note: rt.cells.ok ? "" : rt.cells.reason,
207 },
208 });
209 if (!rt.cells.ok)
210 write(`uncontained here: ${rt.cells.reason}; the app's processes run unrestricted`);
211 }
212
213 /** The running environment a terminal session would attach to, if any. */
214 export function liveFor(rt: AppsRuntime, record: AppRecord, env: EnvName): LiveEnv | undefined {
215 const state = rt.envs.get(key(record.id, env));
216 const live = state?.live;
217 if (state === undefined || live === undefined) return undefined;
218 const dataDir = envDataDir(rt.app.config.dataDir, record.id, env);
219 const base = rt.cellFor(record, env);
220 return {
221 env,
222 dir: live.dir,
223 dataDir,
224 port: live.handle.port,
225 commitOid: live.commitOid,
226 url: rt.urlFor(record, env),
227 tty: live.manifest.tty,
228 cell:
229 base === undefined
230 ? undefined
231 : {
232 dir: live.dir,
233 writable: [dataDir],
234 home: base.home,
235 hide: base.hide,
236 egress: live.manifest.egress ?? [],
237 granted: base.granted,
238 // A terminal is the app's own client: it reaches the app on
239 // its port, as it would beside it.
240 reach: [
241 rt.options.forgeUrl,
242 `http://127.0.0.1:${live.handle.port}`,
243 ],
244 limits: resolveLimits("app", live.manifest.limits),
245 terminal: true,
246 onUndeclared: base.onUndeclared,
247 allowed: base.allowed,
248 onRefused: base.onRefused,
249 egressMode: base.egressMode,
250 label: base.label,
251 },
252 };
253 }
src/apps/runtime_teardown.ts added +69
@@ -0,0 +1,69 @@
1 /**
2 * Tearing down (`AppsRuntime.teardown`, `stopEnv`): stop an environment and
3 * remove what it owned, or only stop it.
4 */
5
6 import { forgetLanded } from "./landed.ts";
7 import type { AppRecord } from "../core/apps.ts";
8 import { revokeAppToken } from "./token.ts";
9 import { cancelRun } from "../core/runs.ts";
10 import { removeApp, removeEnv, type EnvName } from "./data.ts";
11 import type { AppsRuntime } from "./runtime.ts";
12 import { key } from "./runtime_env.ts";
13
14 export async function teardown(rt: AppsRuntime, record: AppRecord, env: EnvName): Promise<void> {
15 const k = key(record.id, env);
16 const state = rt.envs.get(k);
17 if (state !== undefined) {
18 rt.envs.delete(k);
19 forgetLanded(rt.app, record.id, env);
20 if (state.queued)
21 cancelRun(rt.app.db, state.queued.id, "the environment was removed");
22 state.deploying?.abort.abort();
23 await state.chain.catch(() => {});
24 // Null the reference before stopping it: `watch()`'s exit handler
25 // only no-ops on a deliberate stop when `state.live` no longer points
26 // at the process it is watching. Leaving it set here (as `stopEnv`
27 // knows not to) had the exit handler read this as a crash and
28 // schedule a restart — resurrecting a torn-down preview seconds
29 // after its pull closed, checkout, cloned data and all.
30 const live = state.live;
31 state.live = undefined;
32 rt.resting(state);
33 revokeAppToken(rt.app.db, live?.tokenId);
34 if (live) await live.handle.stop();
35 }
36 removeEnv(rt.app.config.dataDir, record.id, env);
37 }
38
39 export async function teardownAll(rt: AppsRuntime, repoId: string): Promise<void> {
40 const states = [...rt.envs.values()].filter(
41 (s) => s.app.repoId === repoId,
42 );
43 for (const state of states) await rt.teardown(state.app, state.env);
44 for (const state of states)
45 removeApp(rt.app.config.dataDir, state.app.id);
46 }
47
48 /** Stop every environment of a repository's app; the data stays. */
49 export async function stopRepo(rt: AppsRuntime, repoId: string): Promise<void> {
50 const states = [...rt.envs.values()].filter(
51 (s) => s.app.repoId === repoId,
52 );
53 for (const state of states) await rt.stopEnv(state.app, state.env);
54 }
55
56 /** Stop an environment's process without removing its data. */
57 export async function stopEnv(rt: AppsRuntime, record: AppRecord, env: EnvName): Promise<void> {
58 const state = rt.envs.get(key(record.id, env));
59 if (state === undefined) return;
60 if (state.queued) cancelRun(rt.app.db, state.queued.id, "stopped");
61 state.deploying?.abort.abort();
62 await state.chain.catch(() => {});
63 const live = state.live;
64 state.live = undefined;
65 rt.resting(state);
66 state.asleep = undefined;
67 revokeAppToken(rt.app.db, live?.tokenId);
68 if (live) await live.handle.stop();
69 }
src/apps/runtime_vars.ts added +88
@@ -0,0 +1,88 @@
1 /**
2 * The variables an app's process starts with (`AppsRuntime.featureFor`,
3 * `sealedFor`, `secretsFor`): its feature flags, the forge's own sealed
4 * values and the owner's secrets.
5 */
6
7 import { readFileSync } from "node:fs";
8 import { join } from "node:path";
9 import { resolveAppEnv } from "../core/app_env.ts";
10 import { neededNames, resolveAppSecrets } from "../core/app_secrets.ts";
11 import { featureVarsAt } from "./feature_env.ts";
12 import { readVariantValues } from "./variant_file.ts";
13 import { getPull } from "../core/pulls.ts";
14 import { guestWaits } from "../origins/guest.ts";
15 import type { Repo } from "../core/repos.ts";
16 import { envVariant, envPull, type EnvName } from "./data.ts";
17 import type { AppsRuntime } from "./runtime.ts";
18
19 /**
20 * An absorbed app's settings for a process about to start in `dir`, read
21 * from that checkout's forge.json; nothing for an app not absorbed. A
22 * guest's preview nobody has approved holds none, as it holds no secrets,
23 * and only production renames the menu entry: a pull's forge.json cannot
24 * relabel the forge's menu for every reader.
25 */
26 export function featureFor(rt: AppsRuntime, repo: Repo, appId: string, env: EnvName, dir: string): Record<string, string> {
27 const number = envPull(env);
28 const pull = number === undefined ? undefined : getPull(rt.app.db, repo.id, number);
29 if (pull !== undefined && guestWaits(rt.app.db, pull)) return {};
30 let forgeJson: string | undefined;
31 try {
32 forgeJson = readFileSync(join(dir, "forge.json"), "utf8");
33 } catch {
34 forgeJson = undefined;
35 }
36 return featureVarsAt(rt.app.db, rt.app.config.dataDir, appId, forgeJson);
37 }
38
39 /**
40 * The app's sealed secrets for a start: its own, and each granted name its forge.json asks
41 * for.
42 */
43 export function sealedFor(rt: AppsRuntime, repo: Repo, appId: string, env: EnvName, dir?: string, start = true): Record<string, string> {
44 let forgeJson: string | undefined;
45 try {
46 forgeJson = dir === undefined ? undefined : readFileSync(join(dir, "forge.json"), "utf8");
47 } catch {
48 forgeJson = undefined;
49 }
50 return resolveAppSecrets(rt.app.db, rt.app.config.dataDir, rt.app.bus, {
51 appId,
52 repoId: repo.id,
53 ownerId: repo.ownerId,
54 env,
55 needs: neededNames(forgeJson),
56 start,
57 });
58 }
59
60 /**
61 * What a process for `env` starts holding: the operator's variables, read
62 * at start, and the forge token. A preview of a guest pull nobody has
63 * approved yet (`origins/guest.ts`) holds neither. The one reader of an
64 * app's variables for a process: its terminal (`apps/tty.ts`) asks here
65 * too, with no token.
66 */
67 export function secretsFor(
68 rt: AppsRuntime,
69 repo: Repo,
70 appId: string,
71 env: EnvName,
72 forgeToken: string | undefined,
73 dir?: string,
74 start = true,
75 ): { vars: Record<string, string>; forgeToken: string | undefined } {
76 const number = envPull(env);
77 const pull =
78 number === undefined ? undefined : getPull(rt.app.db, repo.id, number);
79 if (pull !== undefined && guestWaits(rt.app.db, pull))
80 return { vars: {}, forgeToken: undefined };
81 // A variant's own values (`variant.json` on its branch) go over the
82 // app's defaults; they are plain settings, never secrets.
83 const own = dir !== undefined && envVariant(env) !== undefined ? readVariantValues(dir) : {};
84 return {
85 vars: { ...rt.sealedFor(repo, appId, env, dir, start), ...resolveAppEnv(rt.app.db, rt.app.config.dataDir, appId, env), ...own },
86 forgeToken,
87 };
88 }
test/apps-runtime-split.test.ts added +62
@@ -0,0 +1,62 @@
1 /**
2 * #14010: `apps/runtime.ts` is cut into modules named for one concept, and
3 * the one registry of runtimes (`runtimes`, the WeakMap behind `appsRuntime`
4 * and `envsFor`) stays in exactly one of them. Were it made twice, the runtime
5 * a constructor registered would be one map's and the reader another's.
6 */
7
8 import { describe, expect, test } from "bun:test";
9 import { mkdtemp, rm } from "node:fs/promises";
10 import { readdirSync, readFileSync } from "node:fs";
11 import { tmpdir } from "node:os";
12 import { join } from "node:path";
13 import { createApp } from "../src/app.ts";
14 import { AppsRuntime, appsRuntime, envsFor } from "../src/apps/runtime.ts";
15 import * as status from "../src/apps/runtime_status.ts";
16
17 const root = new URL("..", import.meta.url).pathname;
18 const parts = readdirSync(join(root, "src/apps")).filter((f) => /^runtime(_[a-z]+)?\.ts$/.test(f));
19
20 describe("the runtime's modules", () => {
21 test("the runtime a constructor registers is the one appsRuntime and envsFor read", async () => {
22 const dataDir = await mkdtemp(join(tmpdir(), "forge-runtime-split-"));
23 const app = createApp({ dataDir, port: 0, host: "forge.test" });
24 try {
25 expect(appsRuntime(app)).toBeUndefined();
26 const runtime = new AppsRuntime(app, { domain: "apps.test", scheme: "http", forgeUrl: "http://forge.test" });
27 expect(appsRuntime(app)).toBe(runtime);
28 const record = { id: "a1", repoId: "r1", slug: "notes", access: "public" } as Parameters<typeof envsFor>[2];
29 // Nothing deployed: the registered runtime answers with its (empty) status, not the "no
30 // runtime" [].
31 let asked = 0;
32 const own = runtime.status.bind(runtime);
33 runtime.status = (r) => (asked++, own(r));
34 expect(envsFor(app, undefined, record)).toEqual([]);
35 expect(asked).toBe(1);
36 } finally {
37 app.close();
38 await rm(dataDir, { recursive: true, force: true });
39 }
40 });
41
42 test("a method handed to its module acts on the instance it is handed, not a copy", () => {
43 const bootRuns = new Map<unknown, string>([[{ live: { runId: "run-1" } }, "run-1"]]);
44 const fake = { bootRuns, app: {} } as unknown as AppsRuntime;
45 // Its boot run is the one serving: not starting, and dropped from the instance's own map.
46 expect(status.startingCount(fake)).toBe(0);
47 expect(bootRuns.size).toBe(0);
48 });
49
50 test("each module is under 600 lines, and only runtime.ts holds module-level state", () => {
51 expect(parts.length).toBeGreaterThan(5);
52 const held: string[] = [];
53 for (const file of parts) {
54 const text = readFileSync(join(root, "src/apps", file), "utf8");
55 expect(`${file}: ${text.split("\n").length}`).toBe(`${file}: ${Math.min(text.split("\n").length, 599)}`);
56 for (const line of text.split("\n"))
57 if (/^(?:export\s+)?(?:const|let)\s+\w+\s*=\s*new\s+(?:Weak)?(?:Map|Set)\b|^(?:export\s+)?let\s/.test(line)) held.push(`${file}: ${line}`);
58 }
59 expect(held).toEqual(["runtime.ts: const runtimes = new WeakMap<App, AppsRuntime>();"]);
60 });
61 });
62

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs