Public: anyone can read this; the crew can change it.
src/apps/runtime.ts and src/checks/service.ts are each split into small modules named for one concept, with no behaviour change #14016
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/14010-src-apps-runtime-ts-and-src-checks-servi → main 2 commits fork
11 files
changed +2737
−2363
src/apps/runtime.ts +173 −2363
Diff not shown — 103,610 bytes of diff (limit 100,000). +173 −2363
View filesrc/apps/runtime_bringup.ts added +204
| @@ -0,0 +1,204 @@ | ||
| 1 | /** | |
| 2 | * Bringing one checkout up as a process (`AppsRuntime.bringUp`): export the | |
| 3 | * tree, install, build, start it in a cell or a plain process, wait for its | |
| 4 | * health path to answer. | |
| 5 | */ | |
| 6 | ||
| 7 | import { runDir } from "../core/pulls.ts"; | |
| 8 | import { existsSync, rmSync, writeFileSync } from "node:fs"; | |
| 9 | import { join } from "node:path"; | |
| 10 | import { mintAppToken, revokeAppToken } from "./token.ts"; | |
| 11 | import { checkInstallCache } from "../checks/install.ts"; | |
| 12 | import { getPull } from "../core/pulls.ts"; | |
| 13 | import { guestWaits } from "../origins/guest.ts"; | |
| 14 | import type { Repo } from "../core/repos.ts"; | |
| 15 | import type { Run } from "../core/runs.ts"; | |
| 16 | import type { Result } from "../kernel/result.ts"; | |
| 17 | import { buildCacheRoot } from "./build_cache.ts"; | |
| 18 | import { exportTree, readFileAt } from "./checkout.ts"; | |
| 19 | import { Stopwatch } from "./timing.ts"; | |
| 20 | import { ensureEnvData, envDataDir, envDir, envPull, reseedEmpty, waitedEmptyNote } from "./data.ts"; | |
| 21 | import { parseManifest } from "./manifest.ts"; | |
| 22 | import { identityKey } from "./proxy.ts"; | |
| 23 | import { startProcess, type StartInput } from "./process.ts"; | |
| 24 | import type { AppsRuntime } from "./runtime.ts"; | |
| 25 | import { type Live, diedBeforeAnswering, rootFault, type Environment } from "./runtime_env.ts"; | |
| 26 | ||
| 27 | export async function bringUp( | |
| 28 | rt: AppsRuntime, | |
| 29 | state: Environment, | |
| 30 | repo: Repo, | |
| 31 | run: Run, | |
| 32 | write: (line: string) => void, | |
| 33 | signal: AbortSignal, | |
| 34 | clock: Stopwatch, | |
| 35 | ): Promise< | |
| 36 | | { ok: true; value: Live; tested: Promise<Result<void, string>> } | |
| 37 | | { ok: false; error: string } | |
| 38 | > { | |
| 39 | // A sealed pull's preview is built from where its work is (`pullDir`). | |
| 40 | const bare = await runDir(rt.app.db, rt.app.reposRoot, repo, run.issueId); | |
| 41 | const [forgeJson, packageJson] = await Promise.all([ | |
| 42 | readFileAt(bare, run.commitOid, "forge.json"), | |
| 43 | readFileAt(bare, run.commitOid, "package.json"), | |
| 44 | ]); | |
| 45 | const parsed = parseManifest(forgeJson, packageJson); | |
| 46 | const manifest = parsed.ok ? { ...parsed, value: await rt.heldFor(repo, state.env, parsed.value, write) } : parsed; | |
| 47 | clock.lap("manifest"); | |
| 48 | if (!manifest.ok) { | |
| 49 | return { | |
| 50 | ok: false, | |
| 51 | error: | |
| 52 | manifest.error.kind === "not_an_app" | |
| 53 | ? "this commit is not an app: no forge.json with an app, and no package.json start script" | |
| 54 | : manifest.error.message, | |
| 55 | }; | |
| 56 | } | |
| 57 | const dir = join( | |
| 58 | envDir(rt.app.config.dataDir, state.app.id, state.env), | |
| 59 | run.commitOid.slice(0, 7), | |
| 60 | ); | |
| 61 | // A checkout that is already here was installed and tested when it was | |
| 62 | // made; the forge coming back starts it, and only starts it. | |
| 63 | const warm = existsSync(join(dir, "package.json")); | |
| 64 | if (warm) { | |
| 65 | write(`warm start: ${run.commitOid.slice(0, 7)} was built here before`); | |
| 66 | } else { | |
| 67 | write(`checking out ${run.commitOid.slice(0, 7)}`); | |
| 68 | const exported = await exportTree(bare, run.commitOid, dir); | |
| 69 | if (!exported.ok) return { ok: false, error: exported.error }; | |
| 70 | } | |
| 71 | clock.lap(warm ? "checkout (warm)" : "checkout"); | |
| 72 | ||
| 73 | // A guest pull nobody has approved runs a stranger's code: it starts on | |
| 74 | // empty data, never a copy of production's, and says so. Once a writer | |
| 75 | // approves the pull, this deploy (the approval's own) seeds the data the | |
| 76 | // way the app's previews are configured. | |
| 77 | const number = envPull(state.env); | |
| 78 | const pull = number === undefined ? undefined : getPull(rt.app.db, repo.id, number); | |
| 79 | const waits = pull !== undefined && guestWaits(rt.app.db, pull); | |
| 80 | const note = waitedEmptyNote(rt.app.config.dataDir, state.app.id, state.env); | |
| 81 | const approvedSince = !waits && existsSync(note); | |
| 82 | if (approvedSince) { | |
| 83 | rmSync(envDataDir(rt.app.config.dataDir, state.app.id, state.env), { recursive: true, force: true }); | |
| 84 | rmSync(note, { force: true }); | |
| 85 | } | |
| 86 | const seeded = ensureEnvData( | |
| 87 | rt.app.config.dataDir, | |
| 88 | state.app.id, | |
| 89 | state.env, | |
| 90 | waits ? "empty" : state.app.previewData, | |
| 91 | ); | |
| 92 | const dataDir = envDataDir( | |
| 93 | rt.app.config.dataDir, | |
| 94 | state.app.id, | |
| 95 | state.env, | |
| 96 | ); | |
| 97 | // Only a preview that would have had a copy has anything to switch to on approval. | |
| 98 | if (waits && seeded === "empty" && state.app.previewData === "copy") writeFileSync(note, ""); | |
| 99 | if (waits && seeded !== "kept") | |
| 100 | write("data: starting empty until a writer approves the pull"); | |
| 101 | else if (seeded === "cloned" || seeded === "copied") | |
| 102 | write(`data: ${seeded} from production${approvedSince ? ", now the pull is approved (it ran on empty data while it waited)" : ""}`); | |
| 103 | else if (seeded === "empty" && state.env !== "production") | |
| 104 | write("data: starting empty"); | |
| 105 | clock.lap("data"); | |
| 106 | ||
| 107 | // The process's own credential; every way out of here without a | |
| 108 | // process to hold it revokes it. | |
| 109 | const token = mintAppToken( | |
| 110 | rt.app.db, | |
| 111 | repo.ownerId, | |
| 112 | state.app.slug, | |
| 113 | state.env, | |
| 114 | ); | |
| 115 | const startInput: Omit<StartInput, "signal"> = { | |
| 116 | dir, | |
| 117 | dataDir, | |
| 118 | manifest: manifest.value, | |
| 119 | env: state.env, | |
| 120 | url: rt.urlFor(state.app, state.env), | |
| 121 | commitOid: run.commitOid, | |
| 122 | forgeUrl: rt.options.forgeUrl, | |
| 123 | identityKey: identityKey(rt.app.config.dataDir, state.app.id), | |
| 124 | ssh: rt.sshFor(state.app, state.env), | |
| 125 | // Read at start, never cached: a change is a redeploy, and the | |
| 126 | // process that comes up is the one that sees it. | |
| 127 | ...rt.secretsFor(repo, state.app.id, state.env, token.token, dir), | |
| 128 | feature: rt.featureFor(repo, state.app.id, state.env, dir), | |
| 129 | log: write, | |
| 130 | warm, | |
| 131 | cell: rt.cellFor(state.app, state.env), | |
| 132 | // Every environment, production too, installs from the cache the | |
| 133 | // repository's checks fill, so a new preview fetches nothing it has. | |
| 134 | installCache: checkInstallCache(rt.app.config.dataDir, repo.id), | |
| 135 | // Its own, per environment: a preview never touches production's. | |
| 136 | buildCache: buildCacheRoot(rt.app.config.dataDir, state.app.id, state.env), | |
| 137 | }; | |
| 138 | rt.noteCell(run.id, write); | |
| 139 | clock.lap("setup"); | |
| 140 | write(clock.line("starting the process")); | |
| 141 | const start = async (): ReturnType<typeof startProcess> => { | |
| 142 | try { | |
| 143 | return await startProcess({ ...startInput, signal }); | |
| 144 | } catch (error) { | |
| 145 | revokeAppToken(rt.app.db, token.id); | |
| 146 | throw error; | |
| 147 | } | |
| 148 | }; | |
| 149 | let started = await start(); | |
| 150 | ||
| 151 | // The one retry this deploy gets: a preview seeded from production whose | |
| 152 | // process died before it ever answered — schema ahead of what this build | |
| 153 | // knows, most often — is tried again on empty data, once. A production | |
| 154 | // deploy is never seeded from a copy, so it never takes this path. | |
| 155 | if ( | |
| 156 | !started.ok && | |
| 157 | (seeded === "cloned" || seeded === "copied") && | |
| 158 | diedBeforeAnswering(started.error) | |
| 159 | ) { | |
| 160 | const first = started.error; | |
| 161 | write(`failed: ${first.stage}: ${first.message}`); | |
| 162 | write( | |
| 163 | "the copied data was not usable by this build; the preview is starting on empty data", | |
| 164 | ); | |
| 165 | reseedEmpty(rt.app.config.dataDir, state.app.id, state.env); | |
| 166 | started = await start(); | |
| 167 | if (!started.ok) { | |
| 168 | revokeAppToken(rt.app.db, token.id); | |
| 169 | rmSync(dir, { recursive: true, force: true }); | |
| 170 | return { | |
| 171 | ok: false, | |
| 172 | error: `${started.error.stage}: ${started.error.message} (retried on empty data after the copy failed with ${first.stage}: ${first.message})`, | |
| 173 | }; | |
| 174 | } | |
| 175 | } else if (!started.ok) { | |
| 176 | revokeAppToken(rt.app.db, token.id); | |
| 177 | rmSync(dir, { recursive: true, force: true }); | |
| 178 | return { | |
| 179 | ok: false, | |
| 180 | error: `${started.error.stage}: ${started.error.message}`, | |
| 181 | }; | |
| 182 | } | |
| 183 | // The health path answered; the page people open must too. | |
| 184 | const fault = await rootFault(started.value.handle.port); | |
| 185 | if (fault !== undefined) { | |
| 186 | await started.value.handle.stop(); | |
| 187 | revokeAppToken(rt.app.db, token.id); | |
| 188 | rmSync(dir, { recursive: true, force: true }); | |
| 189 | return { ok: false, error: `ready: ${fault}` }; | |
| 190 | } | |
| 191 | return { | |
| 192 | ok: true, | |
| 193 | value: { | |
| 194 | commitOid: run.commitOid, | |
| 195 | handle: started.value.handle, | |
| 196 | dir, | |
| 197 | runId: run.id, | |
| 198 | since: new Date().toISOString(), | |
| 199 | manifest: manifest.value, | |
| 200 | tokenId: token.id, | |
| 201 | }, | |
| 202 | tested: started.value.tested, | |
| 203 | }; | |
| 204 | } |
src/apps/runtime_deploy.ts added +450
| @@ -0,0 +1,450 @@ | ||
| 1 | /** | |
| 2 | * One deploy run (`AppsRuntime.execute`): the serial lane per environment, | |
| 3 | * the run row and its log, the gate, the swap-in, the tests after; and the | |
| 4 | * build previews and variants that ride the same lane. | |
| 5 | */ | |
| 6 | ||
| 7 | import { recordCarried } from "../core/carried.ts"; | |
| 8 | import { noteRelease, noteReleaseTests, recordsReleases } from "./release_record.ts"; | |
| 9 | import { gateBudgetMs, gatesPush, gateSummary, withinBudget } from "./push_gate.ts"; | |
| 10 | import { tellGated } from "./gate_notice.ts"; | |
| 11 | import { releaseOfCommit } from "../core/release_subject.ts"; | |
| 12 | import { runDir } from "../core/pulls.ts"; | |
| 13 | import { checkPreview } from "./preview_spec.ts"; | |
| 14 | import { rm } from "node:fs/promises"; | |
| 15 | import type { AppRecord } from "../core/apps.ts"; | |
| 16 | import { getVariant } from "../core/variants.ts"; | |
| 17 | import { revokeAppToken } from "./token.ts"; | |
| 18 | import { announceRun } from "../actions/runs.ts"; | |
| 19 | import { getPull } from "../core/pulls.ts"; | |
| 20 | import { getRepoById, type Repo } from "../core/repos.ts"; | |
| 21 | import { runningLimitSentence } from "../settings/quotas.ts"; | |
| 22 | import { cancelRun, claimRun, finishRun, markServing, STOPPED_SUMMARY, type Run } from "../core/runs.ts"; | |
| 23 | import { resolveOid } from "../git/merge.ts"; | |
| 24 | import { repoDir } from "../git/repos.ts"; | |
| 25 | import type { Result } from "../kernel/result.ts"; | |
| 26 | import { holdLease } from "../runs/lease.ts"; | |
| 27 | import { appendRunLog } from "../runs/log.ts"; | |
| 28 | import { readFileAt } from "./checkout.ts"; | |
| 29 | import { seconds, Stopwatch } from "./timing.ts"; | |
| 30 | import { DRAIN_BUDGET_MS, drainPort } from "./handoff.ts"; | |
| 31 | import { buildEnv, variantEnv, envPull, type EnvName } from "./data.ts"; | |
| 32 | import type { AppManifest } from "./manifest.ts"; | |
| 33 | import { heldManifest } from "../core/starts.ts"; | |
| 34 | import { ASKS_NEXT, asksLine, heldPull } from "../checks/held.ts"; | |
| 35 | import type { AppsRuntime } from "./runtime.ts"; | |
| 36 | import { log, TESTS_PASSED_TAIL, NO_TESTS_TAIL, type Live, PUSH_REASONS, elapsedSeconds, type Environment, BUILD_PREVIEWS_MAX, key } from "./runtime_env.ts"; | |
| 37 | ||
| 38 | /** | |
| 39 | * Wait for an environment a request asked for to answer: until its deploy serves or ends, or | |
| 40 | * `signal` aborts. Every caller waits on the one deploy; none starts another. | |
| 41 | */ | |
| 42 | export function served(rt: AppsRuntime, record: AppRecord, env: EnvName, signal: AbortSignal): Promise<void> { | |
| 43 | const state = rt.envs.get(key(record.id, env)); | |
| 44 | const coming = (): boolean => | |
| 45 | state !== undefined && | |
| 46 | state.live === undefined && | |
| 47 | (state.queued !== undefined || state.deploying !== undefined); | |
| 48 | if (!coming() || signal.aborted) return Promise.resolve(); | |
| 49 | return new Promise((resolve) => { | |
| 50 | const done = (): void => { | |
| 51 | off(); | |
| 52 | signal.removeEventListener("abort", done); | |
| 53 | resolve(); | |
| 54 | }; | |
| 55 | const off = rt.app.bus.subscribe((event) => { | |
| 56 | if (event.repoId === record.repoId && (event.kind === "app.serving" || event.kind === "run.finished") && !coming()) | |
| 57 | done(); | |
| 58 | }); | |
| 59 | signal.addEventListener("abort", done, { once: true }); | |
| 60 | }); | |
| 61 | } | |
| 62 | ||
| 63 | /** | |
| 64 | * Serve one commit now, at `build-<sha7>`. Asked again for a build already | |
| 65 | * here, it redeploys nothing and hands back the same environment. A new | |
| 66 | * one beyond `BUILD_PREVIEWS_MAX` takes the place of the oldest, which is | |
| 67 | * torn down with its data. Nothing of it outlives this process: a forge | |
| 68 | * coming back removes what a build preview left on disk. | |
| 69 | */ | |
| 70 | export async function previewBuild( | |
| 71 | rt: AppsRuntime, | |
| 72 | record: AppRecord, | |
| 73 | repo: Repo, | |
| 74 | commitOid: string, | |
| 75 | actorId?: string, | |
| 76 | ): Promise<{ env: EnvName; run: Run | undefined }> { | |
| 77 | const env = buildEnv(commitOid); | |
| 78 | const k = key(record.id, env); | |
| 79 | if (!rt.envs.has(k)) { | |
| 80 | // Claimed before anything waits, so two asks at once count each other. | |
| 81 | rt.envFor(record, env).buildOrder = ++rt.buildCount; | |
| 82 | const builds = [...rt.envs.values()] | |
| 83 | .filter((s) => s.app.id === record.id && s.buildOrder !== undefined) | |
| 84 | .sort((a, b) => b.buildOrder! - a.buildOrder!); | |
| 85 | for (const old of builds.slice(BUILD_PREVIEWS_MAX)) { | |
| 86 | log.info("build preview replaced", { app: record.slug, env: old.env, by: env }); | |
| 87 | await rt.teardown(record, old.env); | |
| 88 | } | |
| 89 | } | |
| 90 | // Torn down by a newer ask while the old ones went: nothing to deploy. | |
| 91 | if (!rt.envs.has(k)) return { env, run: undefined }; | |
| 92 | const run = rt.requestDeploy(record, repo, env, commitOid, "build", actorId); | |
| 93 | return { env, run }; | |
| 94 | } | |
| 95 | ||
| 96 | /** | |
| 97 | * Start (or move) a variant's environment at its branch's tip, now: the | |
| 98 | * ask that registers a variant, or saves its values. Nothing when the | |
| 99 | * branch has no tip. | |
| 100 | */ | |
| 101 | export async function deployVariant(rt: AppsRuntime, record: AppRecord, repo: Repo, label: string, actorId?: string): Promise<Run | undefined> { | |
| 102 | const variant = getVariant(rt.app.db, record.id, label); | |
| 103 | if (variant === undefined) return undefined; | |
| 104 | const dir = repoDir(rt.app.reposRoot, repo.ownerName, repo.name); | |
| 105 | const tip = await resolveOid(dir, `refs/heads/${variant.branch}`); | |
| 106 | if (tip === undefined) return undefined; | |
| 107 | return rt.requestDeploy(record, repo, variantEnv(label), tip, "variant", actorId); | |
| 108 | } | |
| 109 | ||
| 110 | export function pullIssueId(rt: AppsRuntime, repo: Repo, env: EnvName): string | undefined { | |
| 111 | const pull = getPull(rt.app.db, repo.id, Number(env.slice(5))); | |
| 112 | return pull?.id; | |
| 113 | } | |
| 114 | ||
| 115 | export function envFor(rt: AppsRuntime, record: AppRecord, env: EnvName): Environment { | |
| 116 | const k = key(record.id, env); | |
| 117 | let state = rt.envs.get(k); | |
| 118 | if (state === undefined) { | |
| 119 | state = { | |
| 120 | app: record, | |
| 121 | env, | |
| 122 | chain: Promise.resolve(), | |
| 123 | restarts: [], | |
| 124 | openStreams: 0, | |
| 125 | openRequests: 0, | |
| 126 | }; | |
| 127 | rt.envs.set(k, state); | |
| 128 | } | |
| 129 | return state; | |
| 130 | } | |
| 131 | ||
| 132 | /** | |
| 133 | * Previews of the account's apps that are up, `except` aside, and those | |
| 134 | * coming up whose run is older than `before`: of two deploying together | |
| 135 | * the older counts against the younger, never both against each other. | |
| 136 | * With no `before`, every one coming up counts (a wake has no run). | |
| 137 | */ | |
| 138 | export function livePreviews( | |
| 139 | rt: AppsRuntime, | |
| 140 | ownerId: string, | |
| 141 | except: Environment, | |
| 142 | before?: string, | |
| 143 | ): number { | |
| 144 | let n = 0; | |
| 145 | for (const other of rt.envs.values()) { | |
| 146 | if (other === except || other.env === "production") continue; | |
| 147 | const up = | |
| 148 | other.live !== undefined || | |
| 149 | (other.deploying !== undefined && | |
| 150 | (before === undefined || other.deploying.run.id < before)); | |
| 151 | if (!up) continue; | |
| 152 | if (getRepoById(rt.app.db, other.app.repoId)?.ownerId === ownerId) n++; | |
| 153 | } | |
| 154 | return n; | |
| 155 | } | |
| 156 | ||
| 157 | export async function execute( | |
| 158 | rt: AppsRuntime, | |
| 159 | state: Environment, | |
| 160 | queued: Repo, | |
| 161 | run: Run, | |
| 162 | ): Promise<void> { | |
| 163 | // The row as it is now, not as it was when the deploy was asked for: a | |
| 164 | // move (#570) between the two gives the repository a new owner and a | |
| 165 | // new place on disk, and a queued run follows it there. | |
| 166 | let repo = getRepoById(rt.app.db, queued.id) ?? queued; | |
| 167 | const claimed = claimRun(rt.app.db, run.id); | |
| 168 | if (!claimed) return; // cancelled meanwhile | |
| 169 | const lease = holdLease(rt.app.db, run.id); | |
| 170 | const abort = new AbortController(); | |
| 171 | state.deploying = { run: claimed, abort }; | |
| 172 | state.lastRunId = run.id; | |
| 173 | const write = (line: string): void => | |
| 174 | appendRunLog(rt.app.config.dataDir, run.id, line); | |
| 175 | announceRun(rt.app, claimed); | |
| 176 | // An account's live previews are bounded (`settings/quotas.ts`): a | |
| 177 | // preview past the limit does not start, and its run — the check the | |
| 178 | // pull shows — fails saying why, so the pull never waits for it. | |
| 179 | const refusal = | |
| 180 | state.env === "production" || state.live !== undefined | |
| 181 | ? undefined | |
| 182 | : runningLimitSentence( | |
| 183 | rt.app.db, | |
| 184 | "previews", | |
| 185 | "live previews", | |
| 186 | repo.ownerId, | |
| 187 | () => rt.livePreviews(repo.ownerId, state, run.id), | |
| 188 | ); | |
| 189 | if (refusal !== undefined) { | |
| 190 | state.deploying = undefined; | |
| 191 | lease(); | |
| 192 | const message = `not started: ${refusal}. Close a pull or stop a preview to make room.`; | |
| 193 | write(message); | |
| 194 | const done = finishRun(rt.app.db, run.id, { state: "failed", summary: message }); | |
| 195 | if (done) announceRun(rt.app, done); | |
| 196 | return; | |
| 197 | } | |
| 198 | write( | |
| 199 | `deploy ${state.env} of ${repo.ownerName}/${repo.name} at ${run.commitOid.slice(0, 7)} (${run.reason})`, | |
| 200 | ); | |
| 201 | ||
| 202 | // The clock starts at the claim, the instant the log calls "the deploy | |
| 203 | // started"; the first lap is the wait for a permit. Production goes | |
| 204 | // ahead of previews waiting for one: it is the environment a person | |
| 205 | // actually visits. | |
| 206 | const clock = new Stopwatch(); | |
| 207 | // A deploy that throws is a failed run, never a fallen forge: the chain | |
| 208 | // this runs on would otherwise reject unhandled and take the process. | |
| 209 | const outcome = await rt.gate | |
| 210 | .run( | |
| 211 | () => { | |
| 212 | clock.lap("turn"); | |
| 213 | // Read again after the wait for a permit: a move may land in it. | |
| 214 | repo = getRepoById(rt.app.db, repo.id) ?? repo; | |
| 215 | return rt.bringUp(state, repo, run, write, abort.signal, clock); | |
| 216 | }, | |
| 217 | { first: state.env === "production" }, | |
| 218 | ) | |
| 219 | .catch((error: unknown) => ({ | |
| 220 | ok: false as const, | |
| 221 | error: `deploy failed: ${error instanceof Error ? error.message : String(error)}`, | |
| 222 | })); | |
| 223 | ||
| 224 | if (!outcome.ok) { | |
| 225 | state.deploying = undefined; | |
| 226 | lease(); | |
| 227 | // The runtime itself stopping is what aborted the step in flight — | |
| 228 | // the forge's own business, never this commit's fault, and the same | |
| 229 | // honest reason the still-queued case already carries. A real | |
| 230 | // failure (a non-zero exit, a genuine timeout) still fails the run | |
| 231 | // below even if the runtime happens to be stopping at the same time: | |
| 232 | // only an abort this runtime raised earns the quieter outcome. | |
| 233 | if (rt.stopped && abort.signal.aborted) { | |
| 234 | const done = cancelRun(rt.app.db, run.id, STOPPED_SUMMARY); | |
| 235 | write(`cancelled: ${STOPPED_SUMMARY}`); | |
| 236 | if (done) announceRun(rt.app, done); | |
| 237 | return; | |
| 238 | } | |
| 239 | state.lastError = outcome.error; | |
| 240 | state.lastErrorCommitOid = run.commitOid; | |
| 241 | const done = finishRun(rt.app.db, run.id, { | |
| 242 | state: "failed", | |
| 243 | summary: outcome.error, | |
| 244 | }); | |
| 245 | write(`failed: ${outcome.error}`); | |
| 246 | if (state.live !== undefined) | |
| 247 | write( | |
| 248 | `the previous deployment (${state.live.commitOid.slice(0, 7)}) keeps serving`, | |
| 249 | ); | |
| 250 | if (done) announceRun(rt.app, done); | |
| 251 | return; | |
| 252 | } | |
| 253 | ||
| 254 | // A direct push to production is gated (#13709): the declared tests run on the new tree | |
| 255 | // before the switch, and while they fail the last green commit keeps serving. A merged | |
| 256 | // pull already passed its check, and a first deploy has nothing to keep, so both swap in | |
| 257 | // at once as before. | |
| 258 | let gated: Result<void, string> | undefined; | |
| 259 | if (gatesPush(state.env, run.reason, state.live !== undefined, outcome.value.manifest.test !== null)) { | |
| 260 | write(`gate: running the tests on ${run.commitOid.slice(0, 7)} before the switch`); | |
| 261 | gated = await withinBudget(outcome.tested, gateBudgetMs()); | |
| 262 | if (!gated.ok) { | |
| 263 | const kept = state.live!; | |
| 264 | await outcome.value.handle.stop(); | |
| 265 | revokeAppToken(rt.app.db, outcome.value.tokenId); | |
| 266 | if (outcome.value.dir !== kept.dir) await rm(outcome.value.dir, { recursive: true, force: true }); | |
| 267 | state.deploying = undefined; | |
| 268 | lease(); | |
| 269 | const summary = gateSummary(gated.error, kept.commitOid); | |
| 270 | state.lastError = summary; | |
| 271 | state.lastErrorCommitOid = run.commitOid; | |
| 272 | const done = finishRun(rt.app.db, run.id, { state: "failed", summary }); | |
| 273 | write(`failed: ${summary}`); | |
| 274 | if (done) announceRun(rt.app, done); | |
| 275 | rt.app.bus.append({ | |
| 276 | kind: "app.gated", | |
| 277 | repoId: repo.id, | |
| 278 | payload: { app: state.app.slug, env: state.env, commitOid: run.commitOid, keptOid: kept.commitOid, why: gated.error.slice(0, 300) }, | |
| 279 | }); | |
| 280 | await tellGated(rt.app.db, repo.ownerId, `${state.app.slug}: ${summary}\n${rt.urlFor(state.app, state.env)}`); | |
| 281 | return; | |
| 282 | } | |
| 283 | write("gate: the tests passed; switching"); | |
| 284 | } | |
| 285 | ||
| 286 | // It answers: swap it in now, before its tests are done. `state.deploying` | |
| 287 | // stays set — the run is still open, and a stop or a teardown must still | |
| 288 | // be able to reach in and kill the test step — until they finish below. | |
| 289 | // `servingAt` is stamped here, at the swap-in, the same instant the log | |
| 290 | // says so — never the moment the tests beside it finish. | |
| 291 | const previous = state.live; | |
| 292 | const rested = state.asleep; | |
| 293 | const served = markServing(rt.app.db, run.id); | |
| 294 | const servingAt = served?.servingAt ?? new Date().toISOString(); | |
| 295 | const deployStartedAt = claimed.startedAt; | |
| 296 | // The run was created the instant the push asked for it; it started when | |
| 297 | // the gate handed it a permit. Both travel with the deployment: the log | |
| 298 | // below is measured from the start, the studio's phrase from the push. | |
| 299 | const createdAt = PUSH_REASONS.includes(run.reason) | |
| 300 | ? run.createdAt | |
| 301 | : undefined; | |
| 302 | const live: Live = { | |
| 303 | ...outcome.value, | |
| 304 | servingAt, | |
| 305 | createdAt, | |
| 306 | deployStartedAt, | |
| 307 | }; | |
| 308 | state.live = live; | |
| 309 | rt.awake(state, servingAt); | |
| 310 | state.asleep = undefined; | |
| 311 | state.lastRequestAt = rt.now(); | |
| 312 | state.lastError = undefined; | |
| 313 | state.lastErrorCommitOid = undefined; | |
| 314 | // A checkout and its node_modules: tens of thousands of files, removed off this thread. | |
| 315 | if (rested !== undefined && rested.dir !== live.dir) | |
| 316 | await rm(rested.dir, { recursive: true, force: true }); | |
| 317 | if (previous !== undefined) { | |
| 318 | write( | |
| 319 | `stopping the previous deployment (${previous.commitOid.slice(0, 7)})`, | |
| 320 | ); | |
| 321 | revokeAppToken(rt.app.db, previous.tokenId); | |
| 322 | // Traffic already goes to the new one: the old one's sockets are told | |
| 323 | // to reconnect (1012), and what it was answering is answered (#4496). | |
| 324 | const unanswered = await drainPort(previous.handle.port); | |
| 325 | if (unanswered > 0) | |
| 326 | write(`${unanswered} request(s) to the previous deployment were still waiting after ${seconds(DRAIN_BUDGET_MS)}`); | |
| 327 | await previous.handle.stop(); | |
| 328 | if (previous.dir !== live.dir) | |
| 329 | await rm(previous.dir, { recursive: true, force: true }); | |
| 330 | } | |
| 331 | rt.watch(state, repo, live); | |
| 332 | const url = rt.urlFor(state.app, state.env); | |
| 333 | write( | |
| 334 | deployStartedAt | |
| 335 | ? `serving ${url} — ${elapsedSeconds(deployStartedAt, servingAt)} after the deploy started` | |
| 336 | : `serving ${url}`, | |
| 337 | ); | |
| 338 | await rt.noteChange(repo, state.app.id, state.env, live.commitOid); | |
| 339 | // The same statement the log line above just made, on the bus: a | |
| 340 | // subscriber (the looks service) must never be able to take the | |
| 341 | // deploy path down with it. | |
| 342 | try { | |
| 343 | const release = await releaseOfCommit( | |
| 344 | repoDir(rt.app.reposRoot, repo.ownerName, repo.name), | |
| 345 | state.env, | |
| 346 | live.commitOid, | |
| 347 | ); | |
| 348 | rt.app.bus.append({ | |
| 349 | kind: "app.serving", | |
| 350 | repoId: repo.id, | |
| 351 | payload: { | |
| 352 | app: state.app.slug, | |
| 353 | env: state.env, | |
| 354 | commitOid: live.commitOid, | |
| 355 | servingAt, | |
| 356 | ...(deployStartedAt === null ? {} : { deployStartedAt }), | |
| 357 | testsRunning: live.manifest.test !== null, | |
| 358 | ...(release === undefined ? {} : { version: release.version, pulls: release.pulls }), | |
| 359 | }, | |
| 360 | }); | |
| 361 | } catch (error) { | |
| 362 | log.error("could not publish app.serving", { error: String(error) }); | |
| 363 | } | |
| 364 | ||
| 365 | // A new version served by production is a release (#13707); never fails the deploy. | |
| 366 | if (recordsReleases(state.env, live.manifest)) | |
| 367 | await noteRelease(rt.app.db, { | |
| 368 | repoId: repo.id, | |
| 369 | repoDir: repoDir(rt.app.reposRoot, repo.ownerName, repo.name), | |
| 370 | source: live.manifest.version, | |
| 371 | port: live.handle.port, | |
| 372 | oid: live.commitOid, | |
| 373 | servingAt, | |
| 374 | pushedAt: createdAt, | |
| 375 | startedAt: deployStartedAt ?? undefined, | |
| 376 | tested: live.manifest.test !== null, | |
| 377 | }).catch((error: unknown) => log.error("could not record a release", { error: String(error) })); | |
| 378 | ||
| 379 | // A preview is held to the website specification once it is live, beside | |
| 380 | // its tests and after the swap-in (#2955). Report-only and unawaited: the | |
| 381 | // run neither waits for it nor can it fail. | |
| 382 | if (state.env !== "production") | |
| 383 | void checkPreview( | |
| 384 | rt.app.db, | |
| 385 | run.id, | |
| 386 | `http://127.0.0.1:${live.handle.port}`, | |
| 387 | live.manifest.looks, | |
| 388 | ); | |
| 389 | ||
| 390 | const tested = gated ?? (await outcome.tested); | |
| 391 | state.deploying = undefined; | |
| 392 | lease(); | |
| 393 | noteReleaseTests(rt.app.db, state.env, live.manifest, repo.id, live.commitOid, tested.ok); | |
| 394 | if (tested.ok) { | |
| 395 | const done = finishRun(rt.app.db, run.id, { | |
| 396 | state: "ok", | |
| 397 | summary: `running at ${url}${live.manifest.test === null ? NO_TESTS_TAIL : TESTS_PASSED_TAIL}`, | |
| 398 | }); | |
| 399 | write(`done: ${url}`); | |
| 400 | if (done && state.env === "production") await rt.noteCarried(repo, done); | |
| 401 | if (done) announceRun(rt.app, done); | |
| 402 | return; | |
| 403 | } | |
| 404 | // The tests failed, but the deployment that answered is what was asked | |
| 405 | // for: it keeps serving, and the run — the check a pull shows — fails. | |
| 406 | if (state.live === live) | |
| 407 | state.live = { ...live, testsFailed: tested.error }; | |
| 408 | const done = finishRun(rt.app.db, run.id, { | |
| 409 | state: "failed", | |
| 410 | summary: `test: ${tested.error}`, | |
| 411 | }); | |
| 412 | write(`failed: test: ${tested.error}`); | |
| 413 | write(`serving ${url} anyway — its tests failed`); | |
| 414 | if (done) announceRun(rt.app, done); | |
| 415 | } | |
| 416 | ||
| 417 | /** Note the merged pulls a finished production deploy carries; never fails the deploy. */ | |
| 418 | export async function noteCarried(rt: AppsRuntime, repo: Repo, run: Run): Promise<void> { | |
| 419 | try { | |
| 420 | const dir = await runDir(rt.app.db, rt.app.reposRoot, repo, null); | |
| 421 | await recordCarried(rt.app.db, { | |
| 422 | runId: run.id, | |
| 423 | repoId: repo.id, | |
| 424 | commitOid: run.commitOid, | |
| 425 | dir, | |
| 426 | }); | |
| 427 | } catch (error) { | |
| 428 | log.error("could not record what a deploy carries", { error: String(error) }); | |
| 429 | } | |
| 430 | } | |
| 431 | ||
| 432 | /** | |
| 433 | * The manifest a start of `env` runs under. A pull's preview takes the hosts it may reach and | |
| 434 | * its limits from the pull's base branch until the pull lands (`checks/held.ts`), and its log | |
| 435 | * says what it asked for beyond them; production, a build preview and a variant keep their | |
| 436 | * own. Then an agent's unclaimed account runs small, with no host but the registries (#10912). | |
| 437 | */ | |
| 438 | export async function heldFor(rt: AppsRuntime, repo: Repo, env: EnvName, own: AppManifest, write?: (line: string) => void): Promise<AppManifest> { | |
| 439 | const number = envPull(env); | |
| 440 | const pull = number === undefined ? undefined : getPull(rt.app.db, repo.id, number); | |
| 441 | let manifest = own; | |
| 442 | if (pull !== undefined) { | |
| 443 | const dir = repoDir(rt.app.reposRoot, repo.ownerName, repo.name); | |
| 444 | const tip = await resolveOid(dir, `refs/heads/${pull.baseBranch}`); | |
| 445 | const held = heldPull(own, tip === undefined ? undefined : await readFileAt(dir, tip, "forge.json"), "app"); | |
| 446 | manifest = held.manifest; | |
| 447 | if (held.asks !== undefined) write?.(`${asksLine(held.asks, pull.baseBranch)} ${ASKS_NEXT}`); | |
| 448 | } | |
| 449 | return heldManifest(rt.app.db, repo.ownerId, manifest); | |
| 450 | } |
src/apps/runtime_env.ts added +389
| @@ -0,0 +1,389 @@ | ||
| 1 | /** | |
| 2 | * The shapes and constants of `AppsRuntime` (`apps/runtime.ts`): its options, an | |
| 3 | * environment's record and the status read from it, and the small helpers its | |
| 4 | * modules share. Plain data; nothing here holds state. | |
| 5 | */ | |
| 6 | ||
| 7 | import type { Database } from "bun:sqlite"; | |
| 8 | import type { CertificateProbe } from "./certificate.ts"; | |
| 9 | import { listEgressGrants } from "../core/egress_grants.ts"; | |
| 10 | import type { AppRecord } from "../core/apps.ts"; | |
| 11 | import type { Run } from "../core/runs.ts"; | |
| 12 | import { Gate } from "../git/env.ts"; | |
| 13 | import { createLog } from "../kernel/log.ts"; | |
| 14 | import type { BusyReason } from "./busy.ts"; | |
| 15 | import { seconds } from "./timing.ts"; | |
| 16 | import type { DomainResolver } from "./domains.ts"; | |
| 17 | import { envVariant, envPull, type EnvName } from "./data.ts"; | |
| 18 | import type { AppManifest } from "./manifest.ts"; | |
| 19 | import type { CellSpec } from "./contain.ts"; | |
| 20 | import type { ProcessHandle, StartError } from "./process.ts"; | |
| 21 | ||
| 22 | export const log = createLog("apps"); | |
| 23 | ||
| 24 | export interface RuntimeOptions { | |
| 25 | /** The domain app hostnames hang off: `<slug>.<domain>`. */ | |
| 26 | readonly domain: string; | |
| 27 | /** `http` or `https` — what the public URL of an app starts with. */ | |
| 28 | readonly scheme: "http" | "https"; | |
| 29 | /** The port in public app URLs, or undefined for the scheme's default. */ | |
| 30 | readonly publicPort?: number | undefined; | |
| 31 | /** The forge's own public URL, for the app's `FORGE_URL` and links. */ | |
| 32 | readonly forgeUrl: string; | |
| 33 | /** How many deploys may run their heavy steps at once. @default 2 */ | |
| 34 | readonly concurrency?: number | undefined; | |
| 35 | /** | |
| 36 | * How long an environment serves nobody before its process is stopped | |
| 37 | * and it sleeps, to be woken by the next request. Unset (the forge's | |
| 38 | * own runtime) reads the profit slider's position on every sweep | |
| 39 | * (`apps/sleep_window.ts`); set, it is fixed — tests and nothing else. | |
| 40 | */ | |
| 41 | readonly sleepAfterMs?: | |
| 42 | | { readonly production: number; readonly preview: number } | |
| 43 | | undefined; | |
| 44 | /** | |
| 45 | * The sweep's clock; a test hands in one that runs ahead to prove a | |
| 46 | * window of minutes without waiting minutes. @default Date.now | |
| 47 | */ | |
| 48 | readonly clock?: (() => number) | undefined; | |
| 49 | /** How often the runtime looks for environments to put to sleep. */ | |
| 50 | readonly sweepEveryMs?: number | undefined; | |
| 51 | /** | |
| 52 | * Whose timers these are when this runtime serves a forge hosted in its | |
| 53 | * parent's process: `child:<name>`, so a stall they ran across names that | |
| 54 | * child and their slices are counted against it (`perf/child_time.ts`). | |
| 55 | */ | |
| 56 | readonly owner?: string | undefined; | |
| 57 | /** | |
| 58 | * How long a pull may go without a push or a comment before its preview | |
| 59 | * is stopped (not removed: its row, data and pictures stay) and reads as | |
| 60 | * asleep until someone asks for it. Unset follows the profit slider | |
| 61 | * (`policyAt(position).idlePullMs`, read each sweep; `null` is never); | |
| 62 | * 0 keeps every preview warm. | |
| 63 | */ | |
| 64 | readonly idlePullMs?: number | undefined; | |
| 65 | /** | |
| 66 | * How long past its idle window a busy environment (`apps/busy.ts`) is | |
| 67 | * left awake before it sleeps anyway — one crew run's length, so an app | |
| 68 | * that only claims to be busy cannot stay up forever. | |
| 69 | */ | |
| 70 | readonly busyGraceMs?: number | undefined; | |
| 71 | /** | |
| 72 | * What a domain check asks: the system resolver unless a test hands in | |
| 73 | * its own, so proving `app.example.com` needs no network in a test. | |
| 74 | */ | |
| 75 | readonly dns?: DomainResolver | undefined; | |
| 76 | /** | |
| 77 | * What the registrar adapter calls (`apps/dns_provider.ts`); the global `fetch` unless a test | |
| 78 | * hands in its own. | |
| 79 | */ | |
| 80 | readonly dnsFetch?: typeof fetch | undefined; | |
| 81 | /** | |
| 82 | * What a check asks of a verified domain to read its certificate's expiry: | |
| 83 | * a TLS handshake unless a test hands in its own. | |
| 84 | */ | |
| 85 | readonly certificate?: CertificateProbe | undefined; | |
| 86 | /** | |
| 87 | * What the cut-over's preview check asks of each side (`apps/parity_probe.ts`): a status per | |
| 88 | * URL, or null. The vetted outbound fetch unless a test hands in its own. | |
| 89 | */ | |
| 90 | readonly probe?: ((url: string) => Promise<number | null>) | undefined; | |
| 91 | /** | |
| 92 | * Whether a cloud body can take a burst (`spawn/placement.ts`). Without | |
| 93 | * one every environment is placed on owned metal and a burst queues. | |
| 94 | * @default CLOUD_BODY_CONFIGURED (none yet) | |
| 95 | */ | |
| 96 | readonly cloudBody?: boolean | undefined; | |
| 97 | /** | |
| 98 | * How many environments owned metal holds, read at each placement. | |
| 99 | * @default unbounded — no limit is known, so owned is never full | |
| 100 | */ | |
| 101 | readonly ownedCapacity?: (() => number) | undefined; | |
| 102 | /** How often storage is sampled for the meter. @default one hour */ | |
| 103 | readonly storageEveryMs?: number | undefined; | |
| 104 | /** | |
| 105 | * The gate deploys take their turn at. A hosted child's runtime is handed | |
| 106 | * its parent's, so a child's deploys count against the parent's capacity | |
| 107 | * instead of beside it. @default a gate of `concurrency` | |
| 108 | */ | |
| 109 | readonly gate?: Gate | undefined; | |
| 110 | } | |
| 111 | ||
| 112 | export type EnvState = | |
| 113 | | "stopped" | |
| 114 | | "deploying" | |
| 115 | | "running" | |
| 116 | | "asleep" | |
| 117 | | "failed"; | |
| 118 | ||
| 119 | /** How long an always-on app may go unopened while it keeps calling the forge. */ | |
| 120 | export const UNOPENED_MS = 24 * 60 * 60_000; | |
| 121 | ||
| 122 | /** Idle windows before an environment sleeps: the profit slider's 50 (`core/slider.ts`). */ | |
| 123 | export const SLEEP_AFTER_MS = { production: 24 * 3_600_000, preview: 30 * 60_000 }; | |
| 124 | /** Days a quiet pull keeps its preview by default: the slider's 50 too. */ | |
| 125 | export const IDLE_PULL_DAYS = 3; | |
| 126 | export const SWEEP_EVERY_MS = 60_000; | |
| 127 | export const BUSY_GRACE_MS = 45 * 60_000; | |
| 128 | ||
| 129 | /** What a finished deploy's summary ends with: the studio words its green | |
| 130 | * from these, so a pull says whether its tests ran (`words.ts`). */ | |
| 131 | export const TESTS_PASSED_TAIL = " · its tests passed"; | |
| 132 | export const NO_TESTS_TAIL = " · no tests declared, none run"; | |
| 133 | ||
| 134 | export interface EnvStatus { | |
| 135 | readonly env: EnvName; | |
| 136 | readonly state: EnvState; | |
| 137 | /** The commit serving, or being brought up. */ | |
| 138 | readonly commitOid: string; | |
| 139 | readonly url: string; | |
| 140 | readonly port?: number | undefined; | |
| 141 | /** The run that produced this state — a link to its log. */ | |
| 142 | readonly runId?: string | undefined; | |
| 143 | readonly since?: string | undefined; | |
| 144 | /** The instant this deployment started serving, set once at the swap-in. */ | |
| 145 | readonly servingAt?: string | undefined; | |
| 146 | /** | |
| 147 | * The instant the push created this deploy run — what a person means by | |
| 148 | * "after the push", and the reference `servedPhrase` measures from. Unset | |
| 149 | * for a deploy no push asked for (a redeploy someone pressed, a boot | |
| 150 | * reconcile), so the words for those never claim a push that never was. | |
| 151 | */ | |
| 152 | readonly createdAt?: string | undefined; | |
| 153 | /** The deploy run's own start — after however long it waited for a permit. */ | |
| 154 | readonly deployStartedAt?: string | undefined; | |
| 155 | readonly error?: string | undefined; | |
| 156 | /** | |
| 157 | * A running environment whose deploy is still checking itself: its tests | |
| 158 | * are still running, or ran and failed. Never set while `state` is | |
| 159 | * anything but `running` — a failed test never stops a deployment that | |
| 160 | * already answers. | |
| 161 | */ | |
| 162 | readonly note?: string | undefined; | |
| 163 | /** | |
| 164 | * The app's own `tty` command, once known live — `null` when its | |
| 165 | * `forge.json` declares none. Unset while never live, since a checkout | |
| 166 | * that has not started has not said either way. | |
| 167 | */ | |
| 168 | readonly tty?: string | null | undefined; | |
| 169 | /** | |
| 170 | * The pages `app.looks` names worth a picture, once known live — `["/"]` | |
| 171 | * when its `forge.json` says nothing. Unset while never live, the same | |
| 172 | * as `tty`. | |
| 173 | */ | |
| 174 | readonly looks?: readonly string[] | undefined; | |
| 175 | /** The pages of `looks` pictured as the owner (`app.looksAsOwner`), once known live. */ | |
| 176 | readonly looksAsOwner?: readonly string[] | undefined; | |
| 177 | /** `false` when its `forge.json` turns the forge pill off; unset is on. */ | |
| 178 | readonly toolbar?: boolean | undefined; | |
| 179 | /** | |
| 180 | * A deploy newer than the one serving that never came up — set only | |
| 181 | * while an older deployment keeps answering in its place. A later | |
| 182 | * successful deploy of this environment clears it (it is the newest | |
| 183 | * attempt's own outcome, never a history), and it is never set once | |
| 184 | * nothing serves at all: `state: "failed"` and `error` already say that. | |
| 185 | */ | |
| 186 | readonly failedDeploy?: | |
| 187 | | { | |
| 188 | readonly commitOid: string; | |
| 189 | readonly runId: string; | |
| 190 | readonly error: string; | |
| 191 | } | |
| 192 | | undefined; | |
| 193 | /** | |
| 194 | * Why a running environment is still awake past its idle window, in a | |
| 195 | * person's words (`BUSY_WORDS`) — set only while the sweep is sparing it. | |
| 196 | */ | |
| 197 | readonly busy?: string | undefined; | |
| 198 | /** Production whose `forge.json` sets `app.always_on: true`: it never sleeps. */ | |
| 199 | readonly alwaysOn?: true | undefined; | |
| 200 | /** | |
| 201 | * Set while a preview is stopped because its pull went quiet: when it was | |
| 202 | * stopped, and the last touch (push, comment, wake) it was measured from. | |
| 203 | * A stopped preview with this set is asleep, never failed. | |
| 204 | */ | |
| 205 | readonly dormant?: | |
| 206 | | { | |
| 207 | readonly since: string; | |
| 208 | readonly idleSince: string; | |
| 209 | /** Never started at this head: recorded asleep at the push (`apps/prepared.ts`). */ | |
| 210 | readonly prepared?: true | undefined; | |
| 211 | } | |
| 212 | | undefined; | |
| 213 | /** | |
| 214 | * Set while it comes up because a request asked for a preview that was stopped or not started. | |
| 215 | */ | |
| 216 | readonly waking?: true | undefined; | |
| 217 | } | |
| 218 | ||
| 219 | export interface Live { | |
| 220 | readonly commitOid: string; | |
| 221 | readonly handle: ProcessHandle; | |
| 222 | readonly dir: string; | |
| 223 | readonly runId: string; | |
| 224 | readonly since: string; | |
| 225 | /** How this checkout runs — its `tty` is what an ssh session opens. */ | |
| 226 | readonly manifest: AppManifest; | |
| 227 | /** Unset while tests are still running or there are none; the message once they fail. */ | |
| 228 | readonly testsFailed?: string | undefined; | |
| 229 | /** | |
| 230 | * The instant this deployment started serving, set once at the swap-in — | |
| 231 | * unset on the value `bringUp` returns, which does not yet know it. | |
| 232 | */ | |
| 233 | readonly servingAt?: string | undefined; | |
| 234 | /** The instant the push created this deploy run, when a push did. */ | |
| 235 | readonly createdAt?: string | undefined; | |
| 236 | /** The deploy run's own start — the reference the deploy log measures from. */ | |
| 237 | readonly deployStartedAt?: string | null | undefined; | |
| 238 | /** This process's forge token (`apps/token.ts`), revoked when it stops. */ | |
| 239 | readonly tokenId?: string | undefined; | |
| 240 | } | |
| 241 | ||
| 242 | /** | |
| 243 | * The deploy reasons a ref moving created: for these the run's `createdAt` | |
| 244 | * is the instant of the push that asked for the deploy, so the wait for a | |
| 245 | * permit sits inside the number the studio prints. A redeploy someone | |
| 246 | * pressed, a crash restart, and a boot or enable reconcile are not pushes. | |
| 247 | */ | |
| 248 | export const PUSH_REASONS: readonly string[] = ["push", "opened", "merge"]; | |
| 249 | ||
| 250 | /** | |
| 251 | * The deploy reasons that start the same commit again, even while it serves | |
| 252 | * or is coming up: someone pressed redeploy, or the repository moved to | |
| 253 | * another owner (#570) and what runs must start again as the new owner's. | |
| 254 | */ | |
| 255 | export const AGAIN: ReadonlySet<string> = new Set(["redeploy", "moved"]); | |
| 256 | ||
| 257 | /** How long between two instants, in seconds to one decimal place: `3.4s`. */ | |
| 258 | export function elapsedSeconds(fromIso: string, toIso: string): string { | |
| 259 | return seconds(Date.parse(toIso) - Date.parse(fromIso)); | |
| 260 | } | |
| 261 | ||
| 262 | /** | |
| 263 | * Whether a start failure is the process itself exiting before it ever | |
| 264 | * answered its health path — as opposed to install failing, or the process | |
| 265 | * staying up but never answering. Only this shape of failure is worth | |
| 266 | * retrying on empty data: it is what a build too old for the data it was | |
| 267 | * given looks like. | |
| 268 | */ | |
| 269 | export function diedBeforeAnswering(error: StartError): boolean { | |
| 270 | return ( | |
| 271 | error.stage === "start" && | |
| 272 | /^exited with code \d+ before answering/.test(error.message) | |
| 273 | ); | |
| 274 | } | |
| 275 | ||
| 276 | /** | |
| 277 | * What is wrong with an app's `/`, if anything: the line naming the status | |
| 278 | * when it answers 4xx or 5xx, `undefined` when it answers 200 or redirects. | |
| 279 | * It asks the process on its own port, so a private app's access door never | |
| 280 | * answers in its place: whatever status comes back is the app's own. | |
| 281 | */ | |
| 282 | export async function rootFault(port: number): Promise<string | undefined> { | |
| 283 | try { | |
| 284 | const response = await fetch(`http://127.0.0.1:${port}/`, { | |
| 285 | signal: AbortSignal.timeout(5_000), | |
| 286 | redirect: "manual", | |
| 287 | }); | |
| 288 | await response.body?.cancel(); | |
| 289 | if (response.status >= 400) | |
| 290 | return `/ answered ${response.status}; the app is up but has no page at its root`; | |
| 291 | } catch { | |
| 292 | // No answer is the health path's business, not this check's. | |
| 293 | } | |
| 294 | return undefined; | |
| 295 | } | |
| 296 | ||
| 297 | /** A running environment, as a terminal session needs it. */ | |
| 298 | /** | |
| 299 | * A checkout that served and was stopped for want of visitors, kept whole: | |
| 300 | * the next request starts it again, warm, in about the time a page takes. | |
| 301 | */ | |
| 302 | export interface Asleep { | |
| 303 | readonly commitOid: string; | |
| 304 | readonly dir: string; | |
| 305 | readonly manifest: AppManifest; | |
| 306 | readonly runId: string; | |
| 307 | readonly since: string; | |
| 308 | } | |
| 309 | ||
| 310 | export interface LiveEnv { | |
| 311 | readonly env: EnvName; | |
| 312 | readonly dir: string; | |
| 313 | readonly dataDir: string; | |
| 314 | readonly port: number; | |
| 315 | readonly commitOid: string; | |
| 316 | readonly url: string; | |
| 317 | readonly tty: string | null; | |
| 318 | /** The cell a terminal session runs in, the app's own; unset where cells cannot work. */ | |
| 319 | readonly cell?: CellSpec | undefined; | |
| 320 | } | |
| 321 | ||
| 322 | export interface Environment { | |
| 323 | readonly app: AppRecord; | |
| 324 | readonly env: EnvName; | |
| 325 | live?: Live | undefined; | |
| 326 | asleep?: Asleep | undefined; | |
| 327 | /** Stopped for want of a push or comment; asked for, it redeploys. */ | |
| 328 | dormant?: | |
| 329 | | { since: string; idleSince: string; commitOid: string; prepared?: true } | |
| 330 | | undefined; | |
| 331 | /** Why the last wake did not happen (the owner's preview limit), for the page a visitor sees. */ | |
| 332 | wakeRefused?: string | undefined; | |
| 333 | /** One wake at a time; every request that arrives meanwhile waits on it. */ | |
| 334 | waking?: Promise<void> | undefined; | |
| 335 | /** When something last came through the proxy for this environment. */ | |
| 336 | lastRequestAt?: number | undefined; | |
| 337 | /** Streams the proxy holds open to it: websockets, event streams. */ | |
| 338 | openStreams: number; | |
| 339 | /** Requests passed to it whose answer has not begun. */ | |
| 340 | openRequests: number; | |
| 341 | /** Why the last sweep spared it, while it still is. */ | |
| 342 | keptAwake?: BusyReason | undefined; | |
| 343 | /** The deploy in flight, if one is. */ | |
| 344 | deploying?: { run: Run; abort: AbortController } | undefined; | |
| 345 | /** The next deploy, waiting its turn; replaced by a newer request. */ | |
| 346 | queued?: Run | undefined; | |
| 347 | chain: Promise<void>; | |
| 348 | lastError?: string | undefined; | |
| 349 | lastRunId?: string | undefined; | |
| 350 | /** The commit `lastError` belongs to — set together, cleared together. */ | |
| 351 | lastErrorCommitOid?: string | undefined; | |
| 352 | /** Crash restarts in the current window, for backing off. */ | |
| 353 | restarts: number[]; | |
| 354 | /** A build preview's place in the order build previews were started, oldest lowest. */ | |
| 355 | buildOrder?: number | undefined; | |
| 356 | } | |
| 357 | ||
| 358 | /** | |
| 359 | * How many build previews (`build-<sha7>`) one app runs at once; a newer one takes the oldest's | |
| 360 | * place. | |
| 361 | */ | |
| 362 | export const BUILD_PREVIEWS_MAX = 3; | |
| 363 | ||
| 364 | /** The deploy reasons that mean a person touched the change. */ | |
| 365 | export const TOUCH_REASONS: readonly string[] = [ | |
| 366 | "push", | |
| 367 | "opened", | |
| 368 | "redeploy", | |
| 369 | "wake", | |
| 370 | ]; | |
| 371 | ||
| 372 | export const RESTART_WINDOW_MS = 10 * 60_000; | |
| 373 | export const RESTART_MAX = 3; | |
| 374 | export const RESTART_BACKOFF_MS = [2_000, 10_000, 30_000]; | |
| 375 | ||
| 376 | /** | |
| 377 | * Production first, then build previews, then previews by | |
| 378 | * pull number — a number, not a string, so `pull-10` follows `pull-9`. | |
| 379 | */ | |
| 380 | export const envRank = (env: EnvName): number => | |
| 381 | env === "production" ? -1 : envVariant(env) !== undefined ? -0.5 : (envPull(env) ?? 0); | |
| 382 | export const byEnv = (a: EnvStatus, b: EnvStatus): number => envRank(a.env) - envRank(b.env); | |
| 383 | ||
| 384 | export const key = (appId: string, env: EnvName): string => `${appId}/${env}`; | |
| 385 | ||
| 386 | /** The `host:port` destinations a site admin granted this repository's cells. */ | |
| 387 | export function grantedFor(db: Database, repoId: string): string[] { | |
| 388 | return listEgressGrants(db, repoId).map((g) => g.destination); | |
| 389 | }↵ |
src/apps/runtime_events.ts added +402
| @@ -0,0 +1,402 @@ | ||
| 1 | /** | |
| 2 | * What the outbox asks of the runtime (`AppsRuntime.onEvent`): a push, a pull | |
| 3 | * opening or closing, a repository leaving. Each turns into a deploy request | |
| 4 | * (`requestDeploy`), a prepared asleep preview (`prepare`) or a teardown. | |
| 5 | */ | |
| 6 | ||
| 7 | import { ASKER_WORDS, declaredAsker, type PushAsker } from "./prepared.ts"; | |
| 8 | import { lookPagesAt } from "../checks/look_pages.ts"; | |
| 9 | import { dropCheckouts, STRAY_CHECKOUT_AGE_MS } from "./dormant_checkout.ts"; | |
| 10 | import { commitLog } from "../git/read.ts"; | |
| 11 | import { noteLanded } from "./landed.ts"; | |
| 12 | import { pullDir } from "../core/pulls.ts"; | |
| 13 | import { accountActive } from "../core/accounts.ts"; | |
| 14 | import { existsSync, readdirSync, rmSync } from "node:fs"; | |
| 15 | import { join } from "node:path"; | |
| 16 | import { getAppByRepo, type AppRecord } from "../core/apps.ts"; | |
| 17 | import { variantOfBranch } from "../core/variants.ts"; | |
| 18 | import type { ForgeEvent } from "../core/events.ts"; | |
| 19 | import { announceRun } from "../actions/runs.ts"; | |
| 20 | import { checksFor } from "../checks/service.ts"; | |
| 21 | import { getPull, listPulls, type Pull } from "../core/pulls.ts"; | |
| 22 | import { canWrite, getRepoById, type Repo } from "../core/repos.ts"; | |
| 23 | import { getUserById } from "../core/users.ts"; | |
| 24 | import { cancelRun, createRun, type Run } from "../core/runs.ts"; | |
| 25 | import { resolveOid } from "../git/merge.ts"; | |
| 26 | import { repoDir } from "../git/repos.ts"; | |
| 27 | import { previewCommit } from "../git/combined.ts"; | |
| 28 | import { readFileAt } from "./checkout.ts"; | |
| 29 | import { appsRoot, variantEnv, envPull, isEnvName, previewEnv, type EnvName } from "./data.ts"; | |
| 30 | import type { AppsRuntime } from "./runtime.ts"; | |
| 31 | import { log, AGAIN, key } from "./runtime_env.ts"; | |
| 32 | ||
| 33 | /** | |
| 34 | * At boot, remove checkouts no deployment serves, rests on or is making: what a failed or | |
| 35 | * interrupted deploy left (#13772). Ones touched within `STRAY_CHECKOUT_AGE_MS` stay. | |
| 36 | */ | |
| 37 | export async function dropStrayCheckouts(rt: AppsRuntime, record: AppRecord): Promise<void> { | |
| 38 | const root = join(appsRoot(rt.app.config.dataDir), record.id); | |
| 39 | let names: string[]; | |
| 40 | try { | |
| 41 | names = readdirSync(root); | |
| 42 | } catch { | |
| 43 | return; | |
| 44 | } | |
| 45 | for (const name of names) { | |
| 46 | if (!isEnvName(name)) continue; | |
| 47 | const state = rt.envs.get(key(record.id, name)); | |
| 48 | const keep = [state?.live, state?.asleep, state?.deploying?.run, state?.queued] | |
| 49 | .map((x) => x?.commitOid) | |
| 50 | .filter((oid): oid is string => oid !== undefined && oid !== ""); | |
| 51 | try { | |
| 52 | await dropCheckouts(rt.app.config.dataDir, record.id, name, keep, STRAY_CHECKOUT_AGE_MS); | |
| 53 | } catch (error) { | |
| 54 | log.warn("could not drop stray checkouts", { app: record.slug, env: name, error: String(error) }); | |
| 55 | } | |
| 56 | } | |
| 57 | } | |
| 58 | ||
| 59 | /** | |
| 60 | * Note the subject of the commit an environment serves, for the toolbar's "what changed" | |
| 61 | * (`apps/landed.ts`): at the swap-in, and at boot for one adopted asleep or about to come back, | |
| 62 | * so a restart does not leave every app saying "no change yet". A nicety: git that cannot be | |
| 63 | * read is logged, never thrown into the deploy or the boot. | |
| 64 | */ | |
| 65 | export async function noteChange(rt: AppsRuntime, repo: Repo, appId: string, env: EnvName, oid: string): Promise<void> { | |
| 66 | try { | |
| 67 | const read = await commitLog(repoDir(rt.app.reposRoot, repo.ownerName, repo.name), oid, { limit: 1 }); | |
| 68 | noteLanded(rt.app, appId, env, oid, read.ok ? (read.value[0]?.subject ?? "") : ""); | |
| 69 | } catch (error) { | |
| 70 | log.warn("could not read the landed change", { repo: repo.name, env, commit: oid.slice(0, 7), error: String(error) }); | |
| 71 | } | |
| 72 | } | |
| 73 | ||
| 74 | /** Data on disk for apps that no longer exist. */ | |
| 75 | export function pruneDirs(rt: AppsRuntime, known: Set<string>): void { | |
| 76 | const root = appsRoot(rt.app.config.dataDir); | |
| 77 | if (!existsSync(root)) return; | |
| 78 | for (const name of readdirSync(root)) { | |
| 79 | if (!known.has(name)) | |
| 80 | rmSync(join(root, name), { recursive: true, force: true }); | |
| 81 | } | |
| 82 | } | |
| 83 | ||
| 84 | export async function pullHead(rt: AppsRuntime, repo: Repo, pull: Pull): Promise<string | undefined> { | |
| 85 | const dir = await pullDir(rt.app.db, rt.app.reposRoot, repo, pull); | |
| 86 | // A fork's pull combined on our side previews the combined build (#9012). | |
| 87 | return previewCommit(dir, pull); | |
| 88 | } | |
| 89 | ||
| 90 | // ---- events ---- | |
| 91 | ||
| 92 | export function pusherMayRunVariant(rt: AppsRuntime, repo: Repo, actorId: string | null): boolean { | |
| 93 | const who = actorId === null ? undefined : getUserById(rt.app.db, actorId); | |
| 94 | if (who === undefined || who.kind === "agent" || !canWrite(rt.app.db, repo, who)) { | |
| 95 | log.info("variant waits for a person who can write", { repo: repo.name }); | |
| 96 | return false; | |
| 97 | } | |
| 98 | return true; | |
| 99 | } | |
| 100 | ||
| 101 | export async function onEvent(rt: AppsRuntime, event: ForgeEvent): Promise<void> { | |
| 102 | if (rt.stopped || event.repoId === null) return; | |
| 103 | if (event.kind === "repo.deleted") { | |
| 104 | await rt.teardownAll(event.repoId); | |
| 105 | return; | |
| 106 | } | |
| 107 | const record = getAppByRepo(rt.app.db, event.repoId); | |
| 108 | if (!record) return; | |
| 109 | const repo = getRepoById(rt.app.db, event.repoId); | |
| 110 | if (!repo) return; | |
| 111 | const dir = repoDir(rt.app.reposRoot, repo.ownerName, repo.name); | |
| 112 | ||
| 113 | switch (event.kind) { | |
| 114 | case "app.created": | |
| 115 | await rt.reconcileApp(record, repo, "enabled"); | |
| 116 | return; | |
| 117 | case "app.deleted": | |
| 118 | await rt.teardownAll(event.repoId); | |
| 119 | return; | |
| 120 | case "push": { | |
| 121 | const updates = event.payload["updates"]; | |
| 122 | if (!Array.isArray(updates)) return; | |
| 123 | for (const update of updates as { ref?: unknown; newOid?: unknown }[]) { | |
| 124 | if ( | |
| 125 | typeof update.ref !== "string" || | |
| 126 | typeof update.newOid !== "string" | |
| 127 | ) | |
| 128 | continue; | |
| 129 | if (!update.ref.startsWith("refs/heads/")) continue; | |
| 130 | const branch = update.ref.slice("refs/heads/".length); | |
| 131 | const deleted = /^0+$/.test(update.newOid); | |
| 132 | if (branch === repo.defaultBranch && !deleted) { | |
| 133 | rt.requestDeploy( | |
| 134 | record, | |
| 135 | repo, | |
| 136 | "production", | |
| 137 | update.newOid, | |
| 138 | "push", | |
| 139 | event.actorId ?? undefined, | |
| 140 | ); | |
| 141 | } | |
| 142 | const variant = deleted ? undefined : variantOfBranch(rt.app.db, record.id, branch); | |
| 143 | // A variant runs with the app's sealed values and a copy of its data: | |
| 144 | // a push to its branch redeploys only when a person who can write made it. | |
| 145 | // An agent proposes; the writer's own "set values" deploys directly. | |
| 146 | if (variant !== undefined && rt.pusherMayRunVariant(repo, event.actorId)) | |
| 147 | rt.requestDeploy( | |
| 148 | record, | |
| 149 | repo, | |
| 150 | variantEnv(variant.label), | |
| 151 | update.newOid, | |
| 152 | "push", | |
| 153 | event.actorId ?? undefined, | |
| 154 | ); | |
| 155 | // Every open pull whose head is this branch previews the new tip. | |
| 156 | for (const pull of listPulls(rt.app.db, repo.id, { sealed: true, | |
| 157 | state: "open", | |
| 158 | limit: 200, | |
| 159 | })) { | |
| 160 | if (pull.crossRepo || pull.headBranch !== branch || deleted) | |
| 161 | continue; | |
| 162 | void rt.prepare(record, repo, pull, update.newOid, "push", event.actorId ?? undefined); | |
| 163 | } | |
| 164 | } | |
| 165 | return; | |
| 166 | } | |
| 167 | case "pull.pushed": { | |
| 168 | // A fork's branch moved: the base's snapshot followed, and so | |
| 169 | // does the preview. | |
| 170 | const number = event.payload["number"]; | |
| 171 | const headOid = event.payload["headOid"]; | |
| 172 | if (typeof number !== "number" || typeof headOid !== "string") return; | |
| 173 | const pull = getPull(rt.app.db, repo.id, number); | |
| 174 | if (pull) void rt.prepare(record, repo, pull, headOid, "push", event.actorId ?? undefined); | |
| 175 | return; | |
| 176 | } | |
| 177 | case "pull.opened": | |
| 178 | case "pull.reopened": { | |
| 179 | const pull = rt.pullFromEvent(repo, event); | |
| 180 | if (!pull) return; | |
| 181 | const head = await rt.pullHead(repo, pull); | |
| 182 | if (head !== undefined) | |
| 183 | void rt.prepare(record, repo, pull, head, "opened", event.actorId ?? undefined); | |
| 184 | return; | |
| 185 | } | |
| 186 | // An approval is what lets a reader's pull run: its head may be asked for from now. | |
| 187 | // A guest's approval restarts a preview that is up with the app's variables. | |
| 188 | case "pull.run_granted": | |
| 189 | case "pull.guest_approved": | |
| 190 | case "pull.reviewed": { | |
| 191 | // A writer asking for changes may withdraw what let the preview run: | |
| 192 | // if the code it serves may no longer run, it stops now, not at its | |
| 193 | // next sleep or restart. | |
| 194 | if ( | |
| 195 | event.kind === "pull.reviewed" && | |
| 196 | event.payload["state"] === "changes_requested" | |
| 197 | ) { | |
| 198 | const pull = rt.pullFromEvent(repo, event); | |
| 199 | if (!pull) return; | |
| 200 | const env = previewEnv(pull.number); | |
| 201 | const state = rt.envs.get(key(record.id, env)); | |
| 202 | const serving = | |
| 203 | state?.live?.commitOid ?? | |
| 204 | state?.deploying?.run.commitOid ?? | |
| 205 | state?.asleep?.commitOid; | |
| 206 | if (serving !== undefined && !rt.mayRun(record, repo, env, serving)) | |
| 207 | await rt.teardown(record, env); | |
| 208 | return; | |
| 209 | } | |
| 210 | if ( | |
| 211 | event.kind === "pull.reviewed" && | |
| 212 | event.payload["state"] !== "approved" | |
| 213 | ) | |
| 214 | return; | |
| 215 | const pull = rt.pullFromEvent(repo, event); | |
| 216 | if (!pull || pull.state !== "open") return; | |
| 217 | const head = await rt.pullHead(repo, pull); | |
| 218 | if (head !== undefined) | |
| 219 | void rt.prepare( | |
| 220 | record, | |
| 221 | repo, | |
| 222 | pull, | |
| 223 | head, | |
| 224 | // The same head may be serving already: start it again so it | |
| 225 | // comes up holding the variables it ran without. | |
| 226 | event.kind === "pull.guest_approved" ? "redeploy" : "approved", | |
| 227 | event.actorId ?? undefined, | |
| 228 | ); | |
| 229 | return; | |
| 230 | } | |
| 231 | // A move (#570): a process started before it holds the old owner's | |
| 232 | // token and limits, and a deploy in flight may have read the old | |
| 233 | // place on disk. Each starts again at the commit it serves, under the | |
| 234 | // new owner; an asleep one wakes as the new owner's anyway, and a | |
| 235 | // queued one reads the repository when its turn comes. | |
| 236 | case "repo.transferred": { | |
| 237 | for (const state of rt.envs.values()) { | |
| 238 | if (state.app.id !== record.id) continue; | |
| 239 | const commit = state.deploying?.run.commitOid ?? state.live?.commitOid; | |
| 240 | if (commit === undefined || state.queued !== undefined) continue; | |
| 241 | rt.requestDeploy(record, repo, state.env, commit, "moved", event.actorId ?? undefined); | |
| 242 | } | |
| 243 | return; | |
| 244 | } | |
| 245 | case "pull.closed": | |
| 246 | case "pull.merged": { | |
| 247 | const number = event.payload["number"]; | |
| 248 | if (typeof number === "number") | |
| 249 | await rt.teardown(record, previewEnv(number)); | |
| 250 | // A merge moves the default branch with `update-ref`, which runs no | |
| 251 | // hooks and so announces no push: production follows it from here. | |
| 252 | if (event.kind === "pull.merged") { | |
| 253 | const tip = await resolveOid(dir, `refs/heads/${repo.defaultBranch}`); | |
| 254 | if (tip !== undefined) | |
| 255 | rt.requestDeploy( | |
| 256 | record, | |
| 257 | repo, | |
| 258 | "production", | |
| 259 | tip, | |
| 260 | "merge", | |
| 261 | event.actorId ?? undefined, | |
| 262 | ); | |
| 263 | } | |
| 264 | return; | |
| 265 | } | |
| 266 | default: | |
| 267 | return; | |
| 268 | } | |
| 269 | } | |
| 270 | ||
| 271 | export function pullFromEvent(rt: AppsRuntime, repo: Repo, event: ForgeEvent): Pull | undefined { | |
| 272 | const number = event.payload["number"]; | |
| 273 | return typeof number === "number" | |
| 274 | ? getPull(rt.app.db, repo.id, number) | |
| 275 | : undefined; | |
| 276 | } | |
| 277 | ||
| 278 | // ---- deploying ---- | |
| 279 | ||
| 280 | /** | |
| 281 | * Ask for `env` to run `commitOid`. Returns the run that will do it — or | |
| 282 | * the one already serving that commit, when nothing needs to happen. | |
| 283 | */ | |
| 284 | export function requestDeploy( | |
| 285 | rt: AppsRuntime, | |
| 286 | record: AppRecord, | |
| 287 | repo: Repo, | |
| 288 | env: EnvName, | |
| 289 | commitOid: string, | |
| 290 | reason: string, | |
| 291 | actorId?: string, | |
| 292 | ): Run | undefined { | |
| 293 | // A suspended or removed owner's app does not deploy, nor does an | |
| 294 | // archived repository's: it is put away, its data kept. | |
| 295 | if ( | |
| 296 | rt.stopped || | |
| 297 | repo.archivedAt !== null || | |
| 298 | !accountActive(rt.app.db, repo.ownerId) | |
| 299 | ) | |
| 300 | return undefined; | |
| 301 | if (!rt.mayRun(record, repo, env, commitOid)) return undefined; | |
| 302 | const state = rt.envFor(record, env); | |
| 303 | state.dormant = undefined; | |
| 304 | if ( | |
| 305 | (state.live?.commitOid === commitOid || | |
| 306 | state.asleep?.commitOid === commitOid) && | |
| 307 | state.deploying === undefined && | |
| 308 | state.queued === undefined && | |
| 309 | reason !== "restart" && | |
| 310 | reason !== "oom" && | |
| 311 | !AGAIN.has(reason) | |
| 312 | ) { | |
| 313 | return undefined; | |
| 314 | } | |
| 315 | if (state.deploying?.run.commitOid === commitOid && !AGAIN.has(reason)) { | |
| 316 | return state.deploying.run; | |
| 317 | } | |
| 318 | const issue = | |
| 319 | envPull(env) === undefined ? undefined : rt.pullIssueId(repo, env); | |
| 320 | const run = createRun(rt.app.db, { | |
| 321 | repoId: repo.id, | |
| 322 | kind: "deploy", | |
| 323 | env, | |
| 324 | commitOid, | |
| 325 | reason, | |
| 326 | ...(issue === undefined ? {} : { issueId: issue }), | |
| 327 | ...(actorId === undefined ? {} : { actorId }), | |
| 328 | }); | |
| 329 | announceRun(rt.app, run, actorId); | |
| 330 | if (state.queued !== undefined) { | |
| 331 | cancelRun( | |
| 332 | rt.app.db, | |
| 333 | state.queued.id, | |
| 334 | `superseded by ${commitOid.slice(0, 7)}`, | |
| 335 | ); | |
| 336 | } | |
| 337 | state.queued = run; | |
| 338 | if (reason === "boot" && env === "production") rt.bootRuns.set(state, run.id); | |
| 339 | state.chain = state.chain.then(async () => { | |
| 340 | if (state.queued?.id !== run.id) return; // superseded | |
| 341 | state.queued = undefined; | |
| 342 | await rt.execute(state, repo, run); | |
| 343 | }); | |
| 344 | return run; | |
| 345 | } | |
| 346 | ||
| 347 | /** | |
| 348 | * A pull's preview at `head`, after a push, an opening or an approval. One that something asks | |
| 349 | * for at the push (`apps/prepared.ts`), or that is up or coming up already, deploys the | |
| 350 | * ordinary way. Any other costs nothing until it is asked for: it is recorded asleep at `head` | |
| 351 | * with no checkout and no process, and the first request builds and starts it (`rouse`). | |
| 352 | * Decided on the environment's chain, so a burst of pushes ends at the last head. | |
| 353 | */ | |
| 354 | export function prepare(rt: AppsRuntime, record: AppRecord, repo: Repo, pull: Pull, head: string, reason: string, actorId?: string): Promise<void> { | |
| 355 | const env = previewEnv(pull.number); | |
| 356 | if (rt.stopped || repo.archivedAt !== null || !accountActive(rt.app.db, repo.ownerId)) return Promise.resolve(); | |
| 357 | if (!rt.mayRun(record, repo, env, head)) return Promise.resolve(); | |
| 358 | const state = rt.envFor(record, env); | |
| 359 | const decide = async (): Promise<void> => { | |
| 360 | if (rt.stopped || rt.envs.get(key(record.id, env)) !== state) return; | |
| 361 | const up = | |
| 362 | state.live !== undefined || | |
| 363 | state.deploying !== undefined || | |
| 364 | state.queued !== undefined || | |
| 365 | state.waking !== undefined; | |
| 366 | // A repository that cannot be read is deployed, as it always was: the deploy says why. | |
| 367 | const asker = up ? "up" : await rt.pushAsker(repo, pull, head, reason !== "boot").catch(() => "check" as const); | |
| 368 | if (asker !== undefined) { | |
| 369 | rt.requestDeploy(record, repo, env, head, reason, actorId); | |
| 370 | return; | |
| 371 | } | |
| 372 | // Asleep at this very head already (a boot, an approval): it stays as it is. | |
| 373 | if (state.asleep?.commitOid === head || state.dormant?.commitOid === head) return; | |
| 374 | const now = new Date().toISOString(); | |
| 375 | state.asleep = undefined; | |
| 376 | state.dormant = { since: now, idleSince: now, commitOid: head, prepared: true }; | |
| 377 | state.lastError = undefined; | |
| 378 | state.lastErrorCommitOid = undefined; | |
| 379 | log.info("preview asleep until asked for", { app: record.slug, env, commit: head.slice(0, 7), reason }); | |
| 380 | // An older head's checkout is dead weight now: the first request checks this one out. | |
| 381 | await dropCheckouts(rt.app.config.dataDir, record.id, env); | |
| 382 | }; | |
| 383 | const decided = state.chain.then(decide).catch((error: unknown) => | |
| 384 | log.error("could not prepare a preview", { app: record.slug, env, error: String(error) }), | |
| 385 | ); | |
| 386 | state.chain = decided; | |
| 387 | return decided; | |
| 388 | } | |
| 389 | ||
| 390 | /** What asks for this pull's preview at the push, if anything does (`apps/prepared.ts`). */ | |
| 391 | export async function pushAsker(rt: AppsRuntime, repo: Repo, pull: Pull, head: string, pictured: boolean): Promise<PushAsker | undefined> { | |
| 392 | // A forge that runs no checks (`FORGE_CHECKS=0`) judges every pull by its deploy. | |
| 393 | if (checksFor(rt.app) === undefined) return "check"; | |
| 394 | const dir = await pullDir(rt.app.db, rt.app.reposRoot, repo, pull); | |
| 395 | const asker = | |
| 396 | declaredAsker(await readFileAt(dir, head, "forge.json")) ?? | |
| 397 | // A boot is no push: a head pictured before it has its pictures, and nothing new is owed one. | |
| 398 | (pictured && (await lookPagesAt(dir, pull.baseOid, head)).length > 0 ? ("look" as const) : undefined); | |
| 399 | if (asker !== undefined) | |
| 400 | log.info("preview deploys at the push", { repo: repo.name, pull: pull.number, why: ASKER_WORDS[asker] }); | |
| 401 | return asker; | |
| 402 | } |
src/apps/runtime_meter.ts added +142
| @@ -0,0 +1,142 @@ | ||
| 1 | /** | |
| 2 | * The awake meter (`AppsRuntime.awake`, `resting`): each interval an | |
| 3 | * environment serves, written as it opens and closes, and the storage sample | |
| 4 | * beside it. | |
| 5 | */ | |
| 6 | ||
| 7 | import { existsSync, readdirSync } from "node:fs"; | |
| 8 | import { join } from "node:path"; | |
| 9 | import { listApps } from "../core/apps.ts"; | |
| 10 | import { getRepoById } from "../core/repos.ts"; | |
| 11 | import { placeFor } from "../core/placement_policy.ts"; | |
| 12 | import { recordPlacement } from "../core/decisions.ts"; | |
| 13 | import { bodyTarget, CLOUD_BODY_CONFIGURED } from "../spawn/placement.ts"; | |
| 14 | import { repoDir } from "../git/repos.ts"; | |
| 15 | import { closeAllAwake, closeAwake, lastStorageAt, openAwake, recordStorage, SAMPLE_SPAN_MS } from "../core/meter.ts"; | |
| 16 | import { dirBytes } from "./bytes.ts"; | |
| 17 | import { appCacheSize } from "./build_cache.ts"; | |
| 18 | import { appsRoot, envDataDir, isEnvName } from "./data.ts"; | |
| 19 | import { policyAt } from "../core/slider.ts"; | |
| 20 | import { getEconomics, getOwnedNodeLabel } from "../settings/economics.ts"; | |
| 21 | import type { AppsRuntime } from "./runtime.ts"; | |
| 22 | import { log, type Environment } from "./runtime_env.ts"; | |
| 23 | ||
| 24 | // ---- the meter ---- | |
| 25 | ||
| 26 | /** | |
| 27 | * Run one write to the meter. What an app used is worth recording, never | |
| 28 | * worth taking a deploy, a sleep or a shutdown down with it. | |
| 29 | */ | |
| 30 | export function metered<T>(rt: AppsRuntime, write: () => T): T | undefined { | |
| 31 | if (rt.app.closing.signal.aborted) return undefined; | |
| 32 | try { | |
| 33 | return write(); | |
| 34 | } catch (error) { | |
| 35 | log.error("meter write failed", { error: String(error) }); | |
| 36 | return undefined; | |
| 37 | } | |
| 38 | } | |
| 39 | ||
| 40 | /** The forge is closing: stop the sweeper, end open intervals, then the meter goes quiet. */ | |
| 41 | export function closeMeter(rt: AppsRuntime): void { | |
| 42 | rt.stopped = true; | |
| 43 | if (rt.sweeper !== undefined) clearInterval(rt.sweeper); | |
| 44 | try { | |
| 45 | closeAllAwake(rt.app.db, new Date().toISOString()); | |
| 46 | } catch (error) { | |
| 47 | log.error("meter write failed", { error: String(error) }); | |
| 48 | } | |
| 49 | } | |
| 50 | ||
| 51 | /** | |
| 52 | * The environment began serving: its serving interval opens, if not open, | |
| 53 | * placed by the policy (`core/placement_policy.ts`) and with that decision | |
| 54 | * kept in the ledger, so cost knows which price applies and why. | |
| 55 | */ | |
| 56 | export function awake(rt: AppsRuntime, state: Environment, at: string): void { | |
| 57 | rt.metered(() => { | |
| 58 | const db = rt.app.db; | |
| 59 | const service = state.live?.manifest.service ?? null; | |
| 60 | const seen = { | |
| 61 | ownedUsed: rt.ownedUsed(state), | |
| 62 | ownedCapacity: rt.options.ownedCapacity?.() ?? Infinity, | |
| 63 | spillAtOwnedUse: policyAt(getEconomics(db).position).spillAtOwnedUse, | |
| 64 | alwaysOn: state.env === "production" || service?.alwaysOn === true, | |
| 65 | preference: service?.placement ?? null, | |
| 66 | cloudBody: rt.options.cloudBody ?? CLOUD_BODY_CONFIGURED, | |
| 67 | }; | |
| 68 | const placed = placeFor(seen); | |
| 69 | const opened = openAwake(db, { repoId: state.app.repoId, env: state.env, placement: placed.where, at }); | |
| 70 | if (!opened) return; | |
| 71 | recordPlacement(db, { | |
| 72 | repoId: state.app.repoId, | |
| 73 | env: state.env, | |
| 74 | where: placed.where, | |
| 75 | reason: placed.reason, | |
| 76 | // JSON has no Infinity: an unbounded capacity is kept as null. | |
| 77 | seen: { ...seen, ownedCapacity: Number.isFinite(seen.ownedCapacity) ? seen.ownedCapacity : null }, | |
| 78 | target: bodyTarget(placed.where, getOwnedNodeLabel(db)), | |
| 79 | }); | |
| 80 | }); | |
| 81 | } | |
| 82 | ||
| 83 | /** Environments up on this forge now, `except` aside: what owned metal holds. */ | |
| 84 | export function ownedUsed(rt: AppsRuntime, except: Environment): number { | |
| 85 | let n = 0; | |
| 86 | for (const other of rt.envs.values()) if (other !== except && other.live !== undefined) n++; | |
| 87 | return n; | |
| 88 | } | |
| 89 | ||
| 90 | /** The environment stopped serving: its serving interval closes. */ | |
| 91 | export function resting(rt: AppsRuntime, state: Environment, at: number = Date.now()): void { | |
| 92 | rt.metered(() => | |
| 93 | closeAwake(rt.app.db, { | |
| 94 | repoId: state.app.repoId, | |
| 95 | env: state.env, | |
| 96 | at: new Date(at).toISOString(), | |
| 97 | }), | |
| 98 | ); | |
| 99 | } | |
| 100 | ||
| 101 | /** | |
| 102 | * Once per `storageEveryMs`: the bytes of each app's `FORGE_APP_DATA`, | |
| 103 | * every environment summed, and of its bare repository. | |
| 104 | */ | |
| 105 | export async function sampleStorage(rt: AppsRuntime): Promise<void> { | |
| 106 | if (rt.stopped || rt.sampling) return; | |
| 107 | const every = rt.options.storageEveryMs ?? SAMPLE_SPAN_MS; | |
| 108 | const last = rt.metered(() => lastStorageAt(rt.app.db)); | |
| 109 | if (last !== undefined && Date.now() - Date.parse(last) < every) return; | |
| 110 | rt.sampling = true; | |
| 111 | try { | |
| 112 | const at = new Date().toISOString(); | |
| 113 | for (const record of listApps(rt.app.db)) { | |
| 114 | const repo = getRepoById(rt.app.db, record.repoId); | |
| 115 | if (!repo) continue; | |
| 116 | let dataBytes = 0; | |
| 117 | const root = join(appsRoot(rt.app.config.dataDir), record.id); | |
| 118 | if (existsSync(root)) | |
| 119 | for (const name of readdirSync(root)) | |
| 120 | if (isEnvName(name)) | |
| 121 | dataBytes += await dirBytes( | |
| 122 | envDataDir(rt.app.config.dataDir, record.id, name), | |
| 123 | ); | |
| 124 | // The build cache is the app's disk too (#10526). | |
| 125 | dataBytes += appCacheSize(rt.app.config.dataDir, record.id); | |
| 126 | const repoBytes = await dirBytes( | |
| 127 | repoDir(rt.app.reposRoot, repo.ownerName, repo.name), | |
| 128 | ); | |
| 129 | if (rt.stopped) return; | |
| 130 | rt.metered(() => | |
| 131 | recordStorage(rt.app.db, { | |
| 132 | repoId: record.repoId, | |
| 133 | dataBytes, | |
| 134 | repoBytes, | |
| 135 | at, | |
| 136 | }), | |
| 137 | ); | |
| 138 | } | |
| 139 | } finally { | |
| 140 | rt.sampling = false; | |
| 141 | } | |
| 142 | } |
src/apps/runtime_sleep.ts added +505
| @@ -0,0 +1,505 @@ | ||
| 1 | /** | |
| 2 | * Sleeping and waking (`AppsRuntime.wake`, `hibernate`, `sweep`): what is | |
| 3 | * wanted is touched, what is idle past its window sleeps, what is asleep | |
| 4 | * wakes on its first request. | |
| 5 | */ | |
| 6 | ||
| 7 | import { dropCheckouts } from "./dormant_checkout.ts"; | |
| 8 | import { accountActive } from "../core/accounts.ts"; | |
| 9 | import { existsSync, readFileSync } from "node:fs"; | |
| 10 | import { join } from "node:path"; | |
| 11 | import type { AppRecord } from "../core/apps.ts"; | |
| 12 | import { appTokenName, mintAppToken, renewAppToken, revokeAppToken } from "./token.ts"; | |
| 13 | import { askingOf } from "../perf/asking.ts"; | |
| 14 | import { getPull, type Pull } from "../core/pulls.ts"; | |
| 15 | import { isContributionFork } from "../core/contribute.ts"; | |
| 16 | import { previewMayRun } from "./preview_gate.ts"; | |
| 17 | import { getRepoById, type Repo } from "../core/repos.ts"; | |
| 18 | import { runningLimitSentence } from "../settings/quotas.ts"; | |
| 19 | import { listRuns } from "../core/runs.ts"; | |
| 20 | import { repoDir } from "../git/repos.ts"; | |
| 21 | import { headUnder } from "../git/combined.ts"; | |
| 22 | import { appendRunLog } from "../runs/log.ts"; | |
| 23 | import { beatAwake } from "../core/meter.ts"; | |
| 24 | import { dropPreviewBuildCache } from "./build_cache.ts"; | |
| 25 | import { BUSY_WORDS, busyReason } from "./busy.ts"; | |
| 26 | import { envBuild, envVariant, envDataDir, envDir, envPull, type EnvName } from "./data.ts"; | |
| 27 | import { parseManifest } from "./manifest.ts"; | |
| 28 | import { heldManifest } from "../core/starts.ts"; | |
| 29 | import { identityKey } from "./proxy.ts"; | |
| 30 | import { policyAt } from "../core/slider.ts"; | |
| 31 | import { getEconomics } from "../settings/economics.ts"; | |
| 32 | import { asleepWords, sleepWindow, type WindowSource } from "./sleep_window.ts"; | |
| 33 | import { startProcess } from "./process.ts"; | |
| 34 | import type { AppsRuntime } from "./runtime.ts"; | |
| 35 | import { log, UNOPENED_MS, BUSY_GRACE_MS, type Live, type LiveEnv, type Environment, TOUCH_REASONS, key } from "./runtime_env.ts"; | |
| 36 | ||
| 37 | // ---- tearing down ---- | |
| 38 | ||
| 39 | /** Stop an environment and remove everything it owned, data included. */ | |
| 40 | /** Something came through the proxy: the environment is wanted. */ | |
| 41 | export function touch(rt: AppsRuntime, record: AppRecord, env: EnvName): void { | |
| 42 | const state = rt.envs.get(key(record.id, env)); | |
| 43 | if (state === undefined) return; | |
| 44 | state.lastRequestAt = rt.now(); | |
| 45 | state.keptAwake = undefined; | |
| 46 | } | |
| 47 | ||
| 48 | /** | |
| 49 | * A stream through the proxy opened: the environment is busy until the | |
| 50 | * returned release is called — once; a second call does nothing. | |
| 51 | */ | |
| 52 | export function holdStream(rt: AppsRuntime, record: AppRecord, env: EnvName): () => void { | |
| 53 | const state = rt.envs.get(key(record.id, env)); | |
| 54 | if (state === undefined) return () => {}; | |
| 55 | state.openStreams++; | |
| 56 | let held = true; | |
| 57 | return () => { | |
| 58 | if (!held) return; | |
| 59 | held = false; | |
| 60 | state.openStreams--; | |
| 61 | state.lastRequestAt = rt.now(); | |
| 62 | }; | |
| 63 | } | |
| 64 | ||
| 65 | /** | |
| 66 | * A request passed to the app is waiting on its answer: the environment | |
| 67 | * is busy until the returned release is called — once; a second call | |
| 68 | * does nothing. Still inside the busy grace, so a hung app sleeps anyway. | |
| 69 | */ | |
| 70 | export function holdRequest(rt: AppsRuntime, record: AppRecord, env: EnvName): () => void { | |
| 71 | const state = rt.envs.get(key(record.id, env)); | |
| 72 | if (state === undefined) return () => {}; | |
| 73 | state.openRequests++; | |
| 74 | let held = true; | |
| 75 | return () => { | |
| 76 | if (!held) return; | |
| 77 | held = false; | |
| 78 | state.openRequests--; | |
| 79 | state.lastRequestAt = rt.now(); | |
| 80 | }; | |
| 81 | } | |
| 82 | ||
| 83 | /** | |
| 84 | * An environment asleep is started again, warm, from the checkout it | |
| 85 | * kept; one start serves every request that arrives while it happens. | |
| 86 | * Returns once it is live, or once it is clear it will not be. | |
| 87 | */ | |
| 88 | /** | |
| 89 | * A preview runs a pull's code: from someone who cannot write, only once | |
| 90 | * a writer has approved this exact head (`core/pull_trust.ts`). Asked by | |
| 91 | * every door that starts one — a deploy and a waking. | |
| 92 | */ | |
| 93 | export function mayRun( | |
| 94 | rt: AppsRuntime, | |
| 95 | record: AppRecord, | |
| 96 | repo: Repo, | |
| 97 | env: EnvName, | |
| 98 | commitOid: string, | |
| 99 | ): boolean { | |
| 100 | // A guest's fork has no app; were one ever to reach here, it starts nothing. | |
| 101 | if (isContributionFork(rt.app.db, repo.id)) return false; | |
| 102 | // Production is the default branch and a build preview a commit a writer asked to see | |
| 103 | // (`actions/build_preview.ts`): each is a writer's choice already. | |
| 104 | if (env === "production" || envBuild(env) !== undefined || envVariant(env) !== undefined) | |
| 105 | return true; | |
| 106 | const number = envPull(env); | |
| 107 | const pull = | |
| 108 | number === undefined ? undefined : getPull(rt.app.db, repo.id, number); | |
| 109 | // A combined preview runs under the head it carries, read from git (#9012). | |
| 110 | const head = | |
| 111 | pull === undefined | |
| 112 | ? commitOid | |
| 113 | : headUnder(repoDir(rt.app.reposRoot, repo.ownerName, repo.name), pull, commitOid); | |
| 114 | // An outside agent's branch (`contrib/…`) is previewed only once a person who can write has | |
| 115 | // approved the pull (`origins/guest.ts`); its check, in a cell, is what runs before that. | |
| 116 | if (pull !== undefined && previewMayRun(rt.app.db, repo, pull, head)) return true; | |
| 117 | log.info("preview waits for a writer's approval", { | |
| 118 | app: record.slug, | |
| 119 | env, | |
| 120 | commit: commitOid.slice(0, 7), | |
| 121 | }); | |
| 122 | return false; | |
| 123 | } | |
| 124 | ||
| 125 | export async function wake(rt: AppsRuntime, record: AppRecord, env: EnvName): Promise<LiveEnv | undefined> { | |
| 126 | const state = rt.envs.get(key(record.id, env)); | |
| 127 | if (state === undefined) return undefined; | |
| 128 | if (state.waking !== undefined) await state.waking; | |
| 129 | if (state.live !== undefined) return rt.liveFor(record, env); | |
| 130 | const asleep = state.asleep; | |
| 131 | if (asleep === undefined || rt.stopped) return undefined; | |
| 132 | const repo = getRepoById(rt.app.db, record.repoId); | |
| 133 | // Waking runs code without a run row: the owner's standing is asked here. | |
| 134 | if (!repo || !accountActive(rt.app.db, repo.ownerId)) return undefined; | |
| 135 | // So is whose code it is: a preview asleep at a head nobody who can | |
| 136 | // write approved — built before the gate, or approved by someone who | |
| 137 | // can no longer write — is taken down, not woken. Approving the head | |
| 138 | // deploys it again. | |
| 139 | if (!rt.mayRun(record, repo, env, asleep.commitOid)) { | |
| 140 | await rt.teardown(record, env); | |
| 141 | return undefined; | |
| 142 | } | |
| 143 | // A wake starts a process too: past the account's preview limit it stays asleep. | |
| 144 | const full = runningLimitSentence( | |
| 145 | rt.app.db, | |
| 146 | "previews", | |
| 147 | "live previews", | |
| 148 | repo.ownerId, | |
| 149 | () => rt.livePreviews(repo.ownerId, state), | |
| 150 | ); | |
| 151 | if (full !== undefined) { | |
| 152 | log.info("preview stays asleep", { app: record.slug, env, limit: full }); | |
| 153 | state.wakeRefused = `This preview stays asleep: its owner is at ${full}. Close a pull or stop a preview to make room.`; | |
| 154 | return undefined; | |
| 155 | } | |
| 156 | state.wakeRefused = undefined; | |
| 157 | state.waking = (async () => { | |
| 158 | const write = (line: string): void => { | |
| 159 | if (asleep.runId !== "") | |
| 160 | appendRunLog(rt.app.config.dataDir, asleep.runId, line); | |
| 161 | }; | |
| 162 | write("waking: a request came"); | |
| 163 | // Asked again at every start: a preview that slept wakes under what its base reaches now. | |
| 164 | const manifest = await rt.heldFor(repo, env, asleep.manifest); | |
| 165 | const token = mintAppToken(rt.app.db, repo.ownerId, record.slug, env); | |
| 166 | const started = await startProcess({ | |
| 167 | dir: asleep.dir, | |
| 168 | dataDir: envDataDir(rt.app.config.dataDir, record.id, env), | |
| 169 | manifest, | |
| 170 | env, | |
| 171 | url: rt.urlFor(record, env), | |
| 172 | commitOid: asleep.commitOid, | |
| 173 | forgeUrl: rt.options.forgeUrl, | |
| 174 | identityKey: identityKey(rt.app.config.dataDir, record.id), | |
| 175 | ssh: rt.sshFor(record, env), | |
| 176 | ...rt.secretsFor(repo, record.id, env, token.token, asleep.dir), | |
| 177 | feature: rt.featureFor(repo, record.id, env, asleep.dir), | |
| 178 | log: write, | |
| 179 | signal: new AbortController().signal, | |
| 180 | warm: true, | |
| 181 | cell: rt.cellFor(record, env), | |
| 182 | }).catch((error: unknown) => { | |
| 183 | revokeAppToken(rt.app.db, token.id); | |
| 184 | throw error; | |
| 185 | }); | |
| 186 | if (state.asleep !== asleep || !started.ok) | |
| 187 | revokeAppToken(rt.app.db, token.id); | |
| 188 | if (state.asleep !== asleep) { | |
| 189 | // Replaced while waking — a deploy landed. What we started is spare. | |
| 190 | if (started.ok) await started.value.handle.stop(); | |
| 191 | return; | |
| 192 | } | |
| 193 | if (!started.ok) { | |
| 194 | state.asleep = undefined; | |
| 195 | state.lastError = `did not wake — ${started.error.stage}: ${started.error.message}`; | |
| 196 | write(`failed: ${state.lastError}`); | |
| 197 | return; | |
| 198 | } | |
| 199 | const now = new Date().toISOString(); | |
| 200 | const live: Live = { | |
| 201 | commitOid: asleep.commitOid, | |
| 202 | handle: started.value.handle, | |
| 203 | dir: asleep.dir, | |
| 204 | runId: asleep.runId, | |
| 205 | since: now, | |
| 206 | manifest, | |
| 207 | servingAt: now, | |
| 208 | tokenId: token.id, | |
| 209 | }; | |
| 210 | state.asleep = undefined; | |
| 211 | state.live = live; | |
| 212 | rt.awake(state, now); | |
| 213 | state.lastRequestAt = rt.now(); | |
| 214 | rt.watch(state, repo, live); | |
| 215 | write("awake"); | |
| 216 | })().finally(() => { | |
| 217 | state.waking = undefined; | |
| 218 | }); | |
| 219 | await state.waking; | |
| 220 | return rt.liveFor(record, env); | |
| 221 | } | |
| 222 | ||
| 223 | /** | |
| 224 | * At boot, an environment whose checkout for this commit is still here | |
| 225 | * is registered asleep rather than deployed. True when it was. | |
| 226 | */ | |
| 227 | export function settleAsleep( | |
| 228 | rt: AppsRuntime, | |
| 229 | record: AppRecord, | |
| 230 | env: EnvName, | |
| 231 | commitOid: string, | |
| 232 | ): boolean { | |
| 233 | const dir = join( | |
| 234 | envDir(rt.app.config.dataDir, record.id, env), | |
| 235 | commitOid.slice(0, 7), | |
| 236 | ); | |
| 237 | if (!existsSync(join(dir, "package.json"))) return false; | |
| 238 | const read = (name: string): string | undefined => { | |
| 239 | try { | |
| 240 | return readFileSync(join(dir, name), "utf8"); | |
| 241 | } catch { | |
| 242 | return undefined; | |
| 243 | } | |
| 244 | }; | |
| 245 | const manifest = parseManifest(read("forge.json"), read("package.json")); | |
| 246 | if (!manifest.ok) return false; | |
| 247 | const owner = getRepoById(rt.app.db, record.repoId)?.ownerId; | |
| 248 | const state = rt.envFor(record, env); | |
| 249 | if (state.live !== undefined || state.deploying !== undefined) return true; | |
| 250 | // The deploy that built this checkout, in an earlier life: its log is | |
| 251 | // where the environment's sleeping and waking keep being written. | |
| 252 | const built = listRuns(rt.app.db, record.repoId, { | |
| 253 | kind: "deploy", | |
| 254 | limit: 100, | |
| 255 | }).find( | |
| 256 | (r) => r.env === env && r.state === "ok" && r.commitOid === commitOid, | |
| 257 | ); | |
| 258 | state.asleep = { | |
| 259 | commitOid, | |
| 260 | dir, | |
| 261 | manifest: owner === undefined ? manifest.value : heldManifest(rt.app.db, owner, manifest.value), | |
| 262 | runId: built?.id ?? "", | |
| 263 | since: new Date().toISOString(), | |
| 264 | }; | |
| 265 | state.lastRunId ??= built?.id; | |
| 266 | log.info("asleep from boot", { | |
| 267 | app: record.slug, | |
| 268 | env, | |
| 269 | commit: commitOid.slice(0, 7), | |
| 270 | }); | |
| 271 | return true; | |
| 272 | } | |
| 273 | ||
| 274 | /** | |
| 275 | * When a pull was last touched, if that is longer ago than the idle age: | |
| 276 | * the later of its `updatedAt` (comments, reviews) and the newest deploy | |
| 277 | * run a person caused — a push, an opening, a press of Redeploy, a waking. | |
| 278 | * Undefined when the feature is off, the pull was touched within the age, | |
| 279 | * or someone is deciding it at this moment (a queued or running run on it). | |
| 280 | */ | |
| 281 | export function idleSince(rt: AppsRuntime, repo: Repo, pull: Pull, env: EnvName): string | undefined { | |
| 282 | const age = rt.idlePullAge(); | |
| 283 | if (age === null || pull.state !== "open") return undefined; | |
| 284 | const now = Date.now(); | |
| 285 | let last = Date.parse(pull.updatedAt); | |
| 286 | if (now - last < age) return undefined; | |
| 287 | const runs = listRuns(rt.app.db, repo.id, { | |
| 288 | issueId: pull.id, | |
| 289 | limit: 50, | |
| 290 | }); | |
| 291 | if (runs.some((r) => r.state === "queued" || r.state === "running")) | |
| 292 | return undefined; | |
| 293 | for (const r of runs) { | |
| 294 | if (r.kind === "deploy" && r.env === env && TOUCH_REASONS.includes(r.reason)) | |
| 295 | last = Math.max(last, Date.parse(r.createdAt)); | |
| 296 | } | |
| 297 | return now - last < age ? undefined : new Date(last).toISOString(); | |
| 298 | } | |
| 299 | ||
| 300 | /** At boot: register a preview idle past the age as stopped, no process. */ | |
| 301 | export function hibernateAtBoot( | |
| 302 | rt: AppsRuntime, | |
| 303 | record: AppRecord, | |
| 304 | env: EnvName, | |
| 305 | commitOid: string, | |
| 306 | idleSince: string, | |
| 307 | ): void { | |
| 308 | const state = rt.envFor(record, env); | |
| 309 | if (state.live !== undefined || state.deploying !== undefined) return; | |
| 310 | state.dormant = { | |
| 311 | since: new Date().toISOString(), | |
| 312 | idleSince, | |
| 313 | commitOid, | |
| 314 | }; | |
| 315 | const built = listRuns(rt.app.db, record.repoId, { | |
| 316 | kind: "deploy", | |
| 317 | env, | |
| 318 | limit: 50, | |
| 319 | }).find((r) => r.state === "ok"); | |
| 320 | state.lastRunId ??= built?.id; | |
| 321 | void rt.dropDormantCheckouts(state); | |
| 322 | log.info("dormant from boot", { app: record.slug, env }); | |
| 323 | } | |
| 324 | ||
| 325 | /** | |
| 326 | * Give back a dormant preview's checkouts on the environment's deploy chain, so a waking | |
| 327 | * deploy starts only after the removal ends. Skipped when it was roused first; the commits | |
| 328 | * serving or coming up are spared. A failure is logged, never thrown. | |
| 329 | */ | |
| 330 | export function dropDormantCheckouts(rt: AppsRuntime, state: Environment): Promise<void> { | |
| 331 | const run = async (): Promise<void> => { | |
| 332 | if (state.dormant === undefined) return; | |
| 333 | const keep = [state.live, state.deploying?.run, state.queued] | |
| 334 | .map((x) => x?.commitOid) | |
| 335 | .filter((oid): oid is string => oid !== undefined && oid !== ""); | |
| 336 | try { | |
| 337 | await dropCheckouts(rt.app.config.dataDir, state.app.id, state.env, keep); | |
| 338 | if (state.dormant !== undefined && state.deploying === undefined && state.queued === undefined) | |
| 339 | dropPreviewBuildCache(rt.app.config.dataDir, state.app.id, state.env); | |
| 340 | } catch (error) { | |
| 341 | log.warn("could not drop dormant checkouts", { app: state.app.slug, env: state.env, error: String(error) }); | |
| 342 | } | |
| 343 | }; | |
| 344 | const done = state.chain.then(run); | |
| 345 | state.chain = done; | |
| 346 | return done; | |
| 347 | } | |
| 348 | ||
| 349 | /** A dormant preview someone asked for deploys again, the ordinary way. */ | |
| 350 | export function rouse(rt: AppsRuntime, record: AppRecord, env: EnvName): void { | |
| 351 | const state = rt.envs.get(key(record.id, env)); | |
| 352 | const dormant = state?.dormant; | |
| 353 | if (dormant === undefined) return; | |
| 354 | const repo = getRepoById(rt.app.db, record.repoId); | |
| 355 | if (repo) rt.requestDeploy(record, repo, env, dormant.commitOid, "wake"); | |
| 356 | } | |
| 357 | ||
| 358 | /** Stop the previews whose pulls have gone quiet; keep everything else. */ | |
| 359 | export async function hibernate(rt: AppsRuntime): Promise<void> { | |
| 360 | if (rt.idlePullAge() === null) return; | |
| 361 | for (const state of [...rt.envs.values()]) { | |
| 362 | const number = envPull(state.env); | |
| 363 | if ( | |
| 364 | number === undefined || | |
| 365 | state.dormant !== undefined || | |
| 366 | state.deploying !== undefined || | |
| 367 | state.queued !== undefined || | |
| 368 | state.waking !== undefined || | |
| 369 | (state.live === undefined && state.asleep === undefined) | |
| 370 | ) | |
| 371 | continue; | |
| 372 | const repo = getRepoById(rt.app.db, state.app.repoId); | |
| 373 | const pull = repo && getPull(rt.app.db, repo.id, number); | |
| 374 | if (!repo || !pull) continue; | |
| 375 | const idleSince = rt.idleSince(repo, pull, state.env); | |
| 376 | if (idleSince === undefined) continue; | |
| 377 | const commitOid = state.live?.commitOid ?? state.asleep!.commitOid; | |
| 378 | const runId = state.live?.runId ?? state.asleep!.runId; | |
| 379 | await rt.stopEnv(state.app, state.env); | |
| 380 | if (rt.stopped || state.live !== undefined || state.queued !== undefined) | |
| 381 | continue; | |
| 382 | state.dormant = { | |
| 383 | since: new Date().toISOString(), | |
| 384 | idleSince, | |
| 385 | commitOid, | |
| 386 | }; | |
| 387 | state.lastError = undefined; | |
| 388 | state.lastErrorCommitOid = undefined; | |
| 389 | if (runId !== "") | |
| 390 | appendRunLog( | |
| 391 | rt.app.config.dataDir, | |
| 392 | runId, | |
| 393 | `asleep: nobody pushed to or commented on pull #${number} since ${idleSince}; asking for it, or Redeploy, brings it back`, | |
| 394 | ); | |
| 395 | await rt.dropDormantCheckouts(state); | |
| 396 | log.info("dormant", { app: state.app.slug, env: state.env }); | |
| 397 | } | |
| 398 | } | |
| 399 | ||
| 400 | /** | |
| 401 | * How long a pull may sit untouched before its preview stops; `null` is | |
| 402 | * never. A fixed option wins (0 is off); unset follows the slider, read | |
| 403 | * at this moment — so moving it changes the next idle pass, with no restart. | |
| 404 | */ | |
| 405 | export function idlePullAge(rt: AppsRuntime): number | null { | |
| 406 | const fixed = rt.options.idlePullMs; | |
| 407 | if (fixed !== undefined) return fixed > 0 ? fixed : null; | |
| 408 | return policyAt(getEconomics(rt.app.db).position).idlePullMs; | |
| 409 | } | |
| 410 | ||
| 411 | /** Where the idle windows of this pass come from: a fixed option, else the slider as it stands. */ | |
| 412 | export function windowSource(rt: AppsRuntime): WindowSource { | |
| 413 | const fixed = rt.options.sleepAfterMs; | |
| 414 | return fixed !== undefined | |
| 415 | ? { kind: "fixed", sleepAfterMs: fixed } | |
| 416 | : { kind: "slider", position: getEconomics(rt.app.db).position }; | |
| 417 | } | |
| 418 | ||
| 419 | /** Stop what nobody has asked for lately, and keep it ready to start again on the next request. */ | |
| 420 | export async function sweep(rt: AppsRuntime): Promise<void> { | |
| 421 | if (rt.stopped) return; | |
| 422 | await rt.hibernate(); | |
| 423 | const source = rt.windowSource(); | |
| 424 | const grace = rt.options.busyGraceMs ?? BUSY_GRACE_MS; | |
| 425 | const now = rt.now(); | |
| 426 | rt.metered(() => beatAwake(rt.app.db, new Date(now).toISOString())); | |
| 427 | for (const state of rt.envs.values()) { | |
| 428 | const live = state.live; | |
| 429 | // Still running: its credential is still wanted for another window. | |
| 430 | if (live?.tokenId !== undefined) | |
| 431 | renewAppToken(rt.app.db, live.tokenId, now); | |
| 432 | const spared = state.keptAwake; | |
| 433 | state.keptAwake = undefined; | |
| 434 | if ( | |
| 435 | live === undefined || | |
| 436 | state.deploying !== undefined || | |
| 437 | state.queued !== undefined || | |
| 438 | state.waking !== undefined | |
| 439 | ) | |
| 440 | continue; | |
| 441 | const window = sleepWindow( | |
| 442 | source, | |
| 443 | state.env === "production" || envVariant(state.env) !== undefined, | |
| 444 | live.manifest.alwaysOn === true, | |
| 445 | ); | |
| 446 | const last = state.lastRequestAt ?? Date.parse(live.since); | |
| 447 | // An always-on app nobody has opened for a day, still calling the forge, is idle like any | |
| 448 | // other: its own calls are not a visit (`touch` is the proxy's), and they do not keep it | |
| 449 | // awake (#12265). | |
| 450 | const limit = | |
| 451 | window.ms ?? | |
| 452 | (now - last >= UNOPENED_MS && askingOf(rt.app).calling(appTokenName(state.app.slug, state.env), 60 * 60_000, now) | |
| 453 | ? UNOPENED_MS | |
| 454 | : null); | |
| 455 | if (limit === null) continue; | |
| 456 | if (now - last < limit) continue; | |
| 457 | // Idle by the clock; asked whether it is mid-work, and spared while | |
| 458 | // it is — but only for one grace past the window, never forever. | |
| 459 | const busy = busyReason( | |
| 460 | live.handle.pid, | |
| 461 | state.openStreams, | |
| 462 | state.openRequests, | |
| 463 | ); | |
| 464 | const overdue = now - last - limit >= grace; | |
| 465 | if (busy !== undefined && !overdue) { | |
| 466 | if (spared !== busy && live.runId !== "") | |
| 467 | appendRunLog( | |
| 468 | rt.app.config.dataDir, | |
| 469 | live.runId, | |
| 470 | `kept awake: ${BUSY_WORDS[busy]}`, | |
| 471 | ); | |
| 472 | state.keptAwake = busy; | |
| 473 | continue; | |
| 474 | } | |
| 475 | state.live = undefined; | |
| 476 | rt.resting(state, now); | |
| 477 | state.asleep = { | |
| 478 | commitOid: live.commitOid, | |
| 479 | dir: live.dir, | |
| 480 | manifest: live.manifest, | |
| 481 | runId: live.runId, | |
| 482 | since: new Date(now).toISOString(), | |
| 483 | }; | |
| 484 | if (live.runId !== "") | |
| 485 | appendRunLog( | |
| 486 | rt.app.config.dataDir, | |
| 487 | live.runId, | |
| 488 | busy === undefined | |
| 489 | ? asleepWords({ ms: limit, why: window.ms === null ? "always_on, but nobody opened it and it kept calling the forge" : window.why }) | |
| 490 | : `asleep: still busy ${Math.round(grace / 60_000)} minutes past its idle window, which is as long as busy keeps an app awake; the next request wakes it`, | |
| 491 | ); | |
| 492 | log.info("asleep", { app: state.app.slug, env: state.env }); | |
| 493 | revokeAppToken(rt.app.db, live.tokenId); | |
| 494 | await live.handle.stop(); | |
| 495 | } | |
| 496 | await rt.sampleStorage(); | |
| 497 | } | |
| 498 | ||
| 499 | /** | |
| 500 | * The sweep's clock. Requests stamp `lastRequestAt` with it too, so a | |
| 501 | * window measured by the sweep starts again at a request on the same clock. | |
| 502 | */ | |
| 503 | export function now(rt: AppsRuntime): number { | |
| 504 | return rt.options.clock?.() ?? Date.now(); | |
| 505 | } |
src/apps/runtime_status.ts added +253
| @@ -0,0 +1,253 @@ | ||
| 1 | /** | |
| 2 | * How an environment reads from outside (`AppsRuntime.describe`, `liveFor`, | |
| 3 | * `cellFor`, `startingCount`): its status, its live process and its cell, | |
| 4 | * computed from the in-memory record without starting anything. | |
| 5 | */ | |
| 6 | ||
| 7 | import { cellHides, grantsHere } from "../spawn/hosted-mark.ts"; | |
| 8 | import { VERIFYING_NOTE } from "./env_phrase.ts"; | |
| 9 | import { join } from "node:path"; | |
| 10 | import type { AppRecord } from "../core/apps.ts"; | |
| 11 | import { getRun, noteRunProgress } from "../core/runs.ts"; | |
| 12 | import { BUSY_WORDS } from "./busy.ts"; | |
| 13 | import { envDataDir, envDir, type EnvName } from "./data.ts"; | |
| 14 | import { noteReached } from "./egress_report.ts"; | |
| 15 | import { cellHooks } from "./egress_refused.ts"; | |
| 16 | import { egressSetting } from "../core/capacity.ts"; | |
| 17 | import { resolveLimits } from "./contain.ts"; | |
| 18 | import type { CellBase } from "./process.ts"; | |
| 19 | import type { AppsRuntime } from "./runtime.ts"; | |
| 20 | import { type EnvStatus, type LiveEnv, type Environment, key, grantedFor } from "./runtime_env.ts"; | |
| 21 | ||
| 22 | export function describe( | |
| 23 | rt: AppsRuntime, | |
| 24 | state: Environment, | |
| 25 | hosts?: ReadonlyMap<string, string>, | |
| 26 | ): EnvStatus { | |
| 27 | const url = rt.urlFor(state.app, state.env, hosts); | |
| 28 | // Once a deploy's own commit has answered and swapped in, it is | |
| 29 | // "running" from here — even though its run stays open, checking its | |
| 30 | // tests, until they finish. Only a build still short of that point (no | |
| 31 | // live process yet, or the live one is an older commit) is "deploying". | |
| 32 | const swapped = | |
| 33 | state.deploying !== undefined && | |
| 34 | state.live?.commitOid === state.deploying.run.commitOid; | |
| 35 | if (state.deploying !== undefined && !swapped) { | |
| 36 | return { | |
| 37 | env: state.env, | |
| 38 | state: "deploying", | |
| 39 | commitOid: state.deploying.run.commitOid, | |
| 40 | url, | |
| 41 | ...(state.live === undefined && state.deploying.run.reason === "wake" ? { waking: true as const } : {}), | |
| 42 | ...(state.live === undefined | |
| 43 | ? {} | |
| 44 | : { | |
| 45 | port: state.live.handle.port, | |
| 46 | since: state.live.since, | |
| 47 | servingAt: state.live.servingAt, | |
| 48 | createdAt: state.live.createdAt, | |
| 49 | tty: state.live.manifest.tty, | |
| 50 | looks: state.live.manifest.looks, | |
| 51 | looksAsOwner: state.live.manifest.looksAsOwner, | |
| 52 | toolbar: state.live.manifest.toolbar, | |
| 53 | ...(state.live.deployStartedAt === null | |
| 54 | ? {} | |
| 55 | : { deployStartedAt: state.live.deployStartedAt }), | |
| 56 | }), | |
| 57 | runId: state.deploying.run.id, | |
| 58 | }; | |
| 59 | } | |
| 60 | if (state.live !== undefined) { | |
| 61 | // The most recent deploy attempt failed, and it was not this one — a | |
| 62 | // newer commit that never came up, while this older deployment kept | |
| 63 | // serving. `swapped` already covers the alternative (the newest | |
| 64 | // attempt is this one and it succeeded, however its tests read). | |
| 65 | const failedDeploy = | |
| 66 | !swapped && | |
| 67 | state.lastError !== undefined && | |
| 68 | state.lastErrorCommitOid !== undefined && | |
| 69 | state.lastErrorCommitOid !== state.live.commitOid && | |
| 70 | state.lastRunId !== undefined | |
| 71 | ? { | |
| 72 | commitOid: state.lastErrorCommitOid, | |
| 73 | runId: state.lastRunId, | |
| 74 | error: state.lastError, | |
| 75 | } | |
| 76 | : undefined; | |
| 77 | return { | |
| 78 | env: state.env, | |
| 79 | state: "running", | |
| 80 | commitOid: state.live.commitOid, | |
| 81 | url, | |
| 82 | port: state.live.handle.port, | |
| 83 | runId: swapped ? state.deploying!.run.id : state.live.runId, | |
| 84 | since: state.live.since, | |
| 85 | servingAt: state.live.servingAt, | |
| 86 | createdAt: state.live.createdAt, | |
| 87 | tty: state.live.manifest.tty, | |
| 88 | looks: state.live.manifest.looks, | |
| 89 | looksAsOwner: state.live.manifest.looksAsOwner, | |
| 90 | toolbar: state.live.manifest.toolbar, | |
| 91 | ...(state.env === "production" && state.live.manifest.alwaysOn === true | |
| 92 | ? { alwaysOn: true as const } | |
| 93 | : {}), | |
| 94 | ...(state.live.deployStartedAt === null | |
| 95 | ? {} | |
| 96 | : { deployStartedAt: state.live.deployStartedAt }), | |
| 97 | ...(swapped ? { note: VERIFYING_NOTE } : {}), | |
| 98 | ...(state.live.testsFailed === undefined | |
| 99 | ? {} | |
| 100 | : { error: `serving · tests failed: ${state.live.testsFailed}` }), | |
| 101 | ...(failedDeploy === undefined ? {} : { failedDeploy }), | |
| 102 | ...(state.keptAwake === undefined | |
| 103 | ? {} | |
| 104 | : { busy: BUSY_WORDS[state.keptAwake] }), | |
| 105 | }; | |
| 106 | } | |
| 107 | // Nothing is live and nothing is in flight yet, but a deploy already | |
| 108 | // waits its turn on the chain — a restart's own reconcile queues one | |
| 109 | // the instant it boots. That is "starting", not the stale error (if | |
| 110 | // any) a cancelled attempt before it left behind. | |
| 111 | if (state.queued !== undefined) { | |
| 112 | return { | |
| 113 | env: state.env, | |
| 114 | state: "deploying", | |
| 115 | commitOid: state.queued.commitOid, | |
| 116 | url, | |
| 117 | runId: state.queued.id, | |
| 118 | ...(state.queued.reason === "wake" ? { waking: true as const } : {}), | |
| 119 | }; | |
| 120 | } | |
| 121 | if (state.dormant !== undefined) { | |
| 122 | return { | |
| 123 | env: state.env, | |
| 124 | state: "stopped", | |
| 125 | commitOid: state.dormant.commitOid, | |
| 126 | url, | |
| 127 | runId: state.lastRunId, | |
| 128 | dormant: { | |
| 129 | since: state.dormant.since, | |
| 130 | idleSince: state.dormant.idleSince, | |
| 131 | ...(state.dormant.prepared === true ? { prepared: true as const } : {}), | |
| 132 | }, | |
| 133 | }; | |
| 134 | } | |
| 135 | if (state.asleep !== undefined) { | |
| 136 | return { | |
| 137 | env: state.env, | |
| 138 | state: "asleep", | |
| 139 | commitOid: state.asleep.commitOid, | |
| 140 | url, | |
| 141 | runId: state.asleep.runId, | |
| 142 | since: state.asleep.since, | |
| 143 | error: state.wakeRefused, | |
| 144 | }; | |
| 145 | } | |
| 146 | return { | |
| 147 | env: state.env, | |
| 148 | state: state.lastError === undefined ? "stopped" : "failed", | |
| 149 | commitOid: "", | |
| 150 | url, | |
| 151 | runId: state.lastRunId, | |
| 152 | error: state.lastError, | |
| 153 | }; | |
| 154 | } | |
| 155 | ||
| 156 | /** | |
| 157 | * How many of the productions this boot asked for are neither serving | |
| 158 | * nor settled yet. Until it reads 0 the forge is not ready (`/readyz`), | |
| 159 | * so in a rollout the forge it replaces keeps proxying to its own copy | |
| 160 | * of each app and no request finds an app that is still starting (#4496). | |
| 161 | */ | |
| 162 | export function startingCount(rt: AppsRuntime): number { | |
| 163 | let count = 0; | |
| 164 | for (const [state, runId] of rt.bootRuns) { | |
| 165 | const run = state.live?.runId === runId ? undefined : getRun(rt.app.db, runId); | |
| 166 | const pending = run?.state === "queued" || run?.state === "running"; | |
| 167 | if (pending) count++; | |
| 168 | else rt.bootRuns.delete(state); | |
| 169 | } | |
| 170 | return count; | |
| 171 | } | |
| 172 | ||
| 173 | /** | |
| 174 | * What an environment's processes are held to: the forge's data and home | |
| 175 | * hidden, the environment's own home. `undefined` where cells cannot work. | |
| 176 | */ | |
| 177 | export function cellFor(rt: AppsRuntime, record: AppRecord, env: EnvName): CellBase | undefined { | |
| 178 | if (!rt.cells.ok) return undefined; | |
| 179 | const dataDir = rt.app.config.dataDir; | |
| 180 | return { | |
| 181 | hide: cellHides(rt.app), | |
| 182 | home: join(envDir(dataDir, record.id, env), "home"), | |
| 183 | onUndeclared: (host, port) => | |
| 184 | noteReached(dataDir, record.repoId, host, port), | |
| 185 | // Asked on every connection: an owner's Allow reaches the running cell. | |
| 186 | // Said once per host: the first refusal is an event, the rest a count. | |
| 187 | ...cellHooks(dataDir, record.repoId, env, (host, port) => | |
| 188 | rt.app.bus.append({ kind: "app.egress.refused", repoId: record.repoId, payload: { host, port } }), | |
| 189 | (line) => rt.app.log.warn(line), | |
| 190 | record.slug, | |
| 191 | ), | |
| 192 | // Asked at every start: an admin's grant reaches the next cell. | |
| 193 | granted: grantsHere(rt.app, () => grantedFor(rt.app.db, record.repoId)), | |
| 194 | // Likewise an admin's flip of the egress mode. | |
| 195 | egressMode: egressSetting(rt.app.db), | |
| 196 | label: `app:${record.slug}/${env}`, | |
| 197 | repoId: record.repoId, | |
| 198 | }; | |
| 199 | } | |
| 200 | ||
| 201 | /** Say on the run, once, whether its processes were held. */ | |
| 202 | export function noteCell(rt: AppsRuntime, runId: string, write: (line: string) => void): void { | |
| 203 | noteRunProgress(rt.app.db, runId, { | |
| 204 | sandbox: { | |
| 205 | enforced: rt.cells.ok, | |
| 206 | note: rt.cells.ok ? "" : rt.cells.reason, | |
| 207 | }, | |
| 208 | }); | |
| 209 | if (!rt.cells.ok) | |
| 210 | write(`uncontained here: ${rt.cells.reason}; the app's processes run unrestricted`); | |
| 211 | } | |
| 212 | ||
| 213 | /** The running environment a terminal session would attach to, if any. */ | |
| 214 | export function liveFor(rt: AppsRuntime, record: AppRecord, env: EnvName): LiveEnv | undefined { | |
| 215 | const state = rt.envs.get(key(record.id, env)); | |
| 216 | const live = state?.live; | |
| 217 | if (state === undefined || live === undefined) return undefined; | |
| 218 | const dataDir = envDataDir(rt.app.config.dataDir, record.id, env); | |
| 219 | const base = rt.cellFor(record, env); | |
| 220 | return { | |
| 221 | env, | |
| 222 | dir: live.dir, | |
| 223 | dataDir, | |
| 224 | port: live.handle.port, | |
| 225 | commitOid: live.commitOid, | |
| 226 | url: rt.urlFor(record, env), | |
| 227 | tty: live.manifest.tty, | |
| 228 | cell: | |
| 229 | base === undefined | |
| 230 | ? undefined | |
| 231 | : { | |
| 232 | dir: live.dir, | |
| 233 | writable: [dataDir], | |
| 234 | home: base.home, | |
| 235 | hide: base.hide, | |
| 236 | egress: live.manifest.egress ?? [], | |
| 237 | granted: base.granted, | |
| 238 | // A terminal is the app's own client: it reaches the app on | |
| 239 | // its port, as it would beside it. | |
| 240 | reach: [ | |
| 241 | rt.options.forgeUrl, | |
| 242 | `http://127.0.0.1:${live.handle.port}`, | |
| 243 | ], | |
| 244 | limits: resolveLimits("app", live.manifest.limits), | |
| 245 | terminal: true, | |
| 246 | onUndeclared: base.onUndeclared, | |
| 247 | allowed: base.allowed, | |
| 248 | onRefused: base.onRefused, | |
| 249 | egressMode: base.egressMode, | |
| 250 | label: base.label, | |
| 251 | }, | |
| 252 | }; | |
| 253 | } |
src/apps/runtime_teardown.ts added +69
| @@ -0,0 +1,69 @@ | ||
| 1 | /** | |
| 2 | * Tearing down (`AppsRuntime.teardown`, `stopEnv`): stop an environment and | |
| 3 | * remove what it owned, or only stop it. | |
| 4 | */ | |
| 5 | ||
| 6 | import { forgetLanded } from "./landed.ts"; | |
| 7 | import type { AppRecord } from "../core/apps.ts"; | |
| 8 | import { revokeAppToken } from "./token.ts"; | |
| 9 | import { cancelRun } from "../core/runs.ts"; | |
| 10 | import { removeApp, removeEnv, type EnvName } from "./data.ts"; | |
| 11 | import type { AppsRuntime } from "./runtime.ts"; | |
| 12 | import { key } from "./runtime_env.ts"; | |
| 13 | ||
| 14 | export async function teardown(rt: AppsRuntime, record: AppRecord, env: EnvName): Promise<void> { | |
| 15 | const k = key(record.id, env); | |
| 16 | const state = rt.envs.get(k); | |
| 17 | if (state !== undefined) { | |
| 18 | rt.envs.delete(k); | |
| 19 | forgetLanded(rt.app, record.id, env); | |
| 20 | if (state.queued) | |
| 21 | cancelRun(rt.app.db, state.queued.id, "the environment was removed"); | |
| 22 | state.deploying?.abort.abort(); | |
| 23 | await state.chain.catch(() => {}); | |
| 24 | // Null the reference before stopping it: `watch()`'s exit handler | |
| 25 | // only no-ops on a deliberate stop when `state.live` no longer points | |
| 26 | // at the process it is watching. Leaving it set here (as `stopEnv` | |
| 27 | // knows not to) had the exit handler read this as a crash and | |
| 28 | // schedule a restart — resurrecting a torn-down preview seconds | |
| 29 | // after its pull closed, checkout, cloned data and all. | |
| 30 | const live = state.live; | |
| 31 | state.live = undefined; | |
| 32 | rt.resting(state); | |
| 33 | revokeAppToken(rt.app.db, live?.tokenId); | |
| 34 | if (live) await live.handle.stop(); | |
| 35 | } | |
| 36 | removeEnv(rt.app.config.dataDir, record.id, env); | |
| 37 | } | |
| 38 | ||
| 39 | export async function teardownAll(rt: AppsRuntime, repoId: string): Promise<void> { | |
| 40 | const states = [...rt.envs.values()].filter( | |
| 41 | (s) => s.app.repoId === repoId, | |
| 42 | ); | |
| 43 | for (const state of states) await rt.teardown(state.app, state.env); | |
| 44 | for (const state of states) | |
| 45 | removeApp(rt.app.config.dataDir, state.app.id); | |
| 46 | } | |
| 47 | ||
| 48 | /** Stop every environment of a repository's app; the data stays. */ | |
| 49 | export async function stopRepo(rt: AppsRuntime, repoId: string): Promise<void> { | |
| 50 | const states = [...rt.envs.values()].filter( | |
| 51 | (s) => s.app.repoId === repoId, | |
| 52 | ); | |
| 53 | for (const state of states) await rt.stopEnv(state.app, state.env); | |
| 54 | } | |
| 55 | ||
| 56 | /** Stop an environment's process without removing its data. */ | |
| 57 | export async function stopEnv(rt: AppsRuntime, record: AppRecord, env: EnvName): Promise<void> { | |
| 58 | const state = rt.envs.get(key(record.id, env)); | |
| 59 | if (state === undefined) return; | |
| 60 | if (state.queued) cancelRun(rt.app.db, state.queued.id, "stopped"); | |
| 61 | state.deploying?.abort.abort(); | |
| 62 | await state.chain.catch(() => {}); | |
| 63 | const live = state.live; | |
| 64 | state.live = undefined; | |
| 65 | rt.resting(state); | |
| 66 | state.asleep = undefined; | |
| 67 | revokeAppToken(rt.app.db, live?.tokenId); | |
| 68 | if (live) await live.handle.stop(); | |
| 69 | } |
src/apps/runtime_vars.ts added +88
| @@ -0,0 +1,88 @@ | ||
| 1 | /** | |
| 2 | * The variables an app's process starts with (`AppsRuntime.featureFor`, | |
| 3 | * `sealedFor`, `secretsFor`): its feature flags, the forge's own sealed | |
| 4 | * values and the owner's secrets. | |
| 5 | */ | |
| 6 | ||
| 7 | import { readFileSync } from "node:fs"; | |
| 8 | import { join } from "node:path"; | |
| 9 | import { resolveAppEnv } from "../core/app_env.ts"; | |
| 10 | import { neededNames, resolveAppSecrets } from "../core/app_secrets.ts"; | |
| 11 | import { featureVarsAt } from "./feature_env.ts"; | |
| 12 | import { readVariantValues } from "./variant_file.ts"; | |
| 13 | import { getPull } from "../core/pulls.ts"; | |
| 14 | import { guestWaits } from "../origins/guest.ts"; | |
| 15 | import type { Repo } from "../core/repos.ts"; | |
| 16 | import { envVariant, envPull, type EnvName } from "./data.ts"; | |
| 17 | import type { AppsRuntime } from "./runtime.ts"; | |
| 18 | ||
| 19 | /** | |
| 20 | * An absorbed app's settings for a process about to start in `dir`, read | |
| 21 | * from that checkout's forge.json; nothing for an app not absorbed. A | |
| 22 | * guest's preview nobody has approved holds none, as it holds no secrets, | |
| 23 | * and only production renames the menu entry: a pull's forge.json cannot | |
| 24 | * relabel the forge's menu for every reader. | |
| 25 | */ | |
| 26 | export function featureFor(rt: AppsRuntime, repo: Repo, appId: string, env: EnvName, dir: string): Record<string, string> { | |
| 27 | const number = envPull(env); | |
| 28 | const pull = number === undefined ? undefined : getPull(rt.app.db, repo.id, number); | |
| 29 | if (pull !== undefined && guestWaits(rt.app.db, pull)) return {}; | |
| 30 | let forgeJson: string | undefined; | |
| 31 | try { | |
| 32 | forgeJson = readFileSync(join(dir, "forge.json"), "utf8"); | |
| 33 | } catch { | |
| 34 | forgeJson = undefined; | |
| 35 | } | |
| 36 | return featureVarsAt(rt.app.db, rt.app.config.dataDir, appId, forgeJson); | |
| 37 | } | |
| 38 | ||
| 39 | /** | |
| 40 | * The app's sealed secrets for a start: its own, and each granted name its forge.json asks | |
| 41 | * for. | |
| 42 | */ | |
| 43 | export function sealedFor(rt: AppsRuntime, repo: Repo, appId: string, env: EnvName, dir?: string, start = true): Record<string, string> { | |
| 44 | let forgeJson: string | undefined; | |
| 45 | try { | |
| 46 | forgeJson = dir === undefined ? undefined : readFileSync(join(dir, "forge.json"), "utf8"); | |
| 47 | } catch { | |
| 48 | forgeJson = undefined; | |
| 49 | } | |
| 50 | return resolveAppSecrets(rt.app.db, rt.app.config.dataDir, rt.app.bus, { | |
| 51 | appId, | |
| 52 | repoId: repo.id, | |
| 53 | ownerId: repo.ownerId, | |
| 54 | env, | |
| 55 | needs: neededNames(forgeJson), | |
| 56 | start, | |
| 57 | }); | |
| 58 | } | |
| 59 | ||
| 60 | /** | |
| 61 | * What a process for `env` starts holding: the operator's variables, read | |
| 62 | * at start, and the forge token. A preview of a guest pull nobody has | |
| 63 | * approved yet (`origins/guest.ts`) holds neither. The one reader of an | |
| 64 | * app's variables for a process: its terminal (`apps/tty.ts`) asks here | |
| 65 | * too, with no token. | |
| 66 | */ | |
| 67 | export function secretsFor( | |
| 68 | rt: AppsRuntime, | |
| 69 | repo: Repo, | |
| 70 | appId: string, | |
| 71 | env: EnvName, | |
| 72 | forgeToken: string | undefined, | |
| 73 | dir?: string, | |
| 74 | start = true, | |
| 75 | ): { vars: Record<string, string>; forgeToken: string | undefined } { | |
| 76 | const number = envPull(env); | |
| 77 | const pull = | |
| 78 | number === undefined ? undefined : getPull(rt.app.db, repo.id, number); | |
| 79 | if (pull !== undefined && guestWaits(rt.app.db, pull)) | |
| 80 | return { vars: {}, forgeToken: undefined }; | |
| 81 | // A variant's own values (`variant.json` on its branch) go over the | |
| 82 | // app's defaults; they are plain settings, never secrets. | |
| 83 | const own = dir !== undefined && envVariant(env) !== undefined ? readVariantValues(dir) : {}; | |
| 84 | return { | |
| 85 | vars: { ...rt.sealedFor(repo, appId, env, dir, start), ...resolveAppEnv(rt.app.db, rt.app.config.dataDir, appId, env), ...own }, | |
| 86 | forgeToken, | |
| 87 | }; | |
| 88 | } |
test/apps-runtime-split.test.ts added +62
| @@ -0,0 +1,62 @@ | ||
| 1 | /** | |
| 2 | * #14010: `apps/runtime.ts` is cut into modules named for one concept, and | |
| 3 | * the one registry of runtimes (`runtimes`, the WeakMap behind `appsRuntime` | |
| 4 | * and `envsFor`) stays in exactly one of them. Were it made twice, the runtime | |
| 5 | * a constructor registered would be one map's and the reader another's. | |
| 6 | */ | |
| 7 | ||
| 8 | import { describe, expect, test } from "bun:test"; | |
| 9 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 10 | import { readdirSync, readFileSync } from "node:fs"; | |
| 11 | import { tmpdir } from "node:os"; | |
| 12 | import { join } from "node:path"; | |
| 13 | import { createApp } from "../src/app.ts"; | |
| 14 | import { AppsRuntime, appsRuntime, envsFor } from "../src/apps/runtime.ts"; | |
| 15 | import * as status from "../src/apps/runtime_status.ts"; | |
| 16 | ||
| 17 | const root = new URL("..", import.meta.url).pathname; | |
| 18 | const parts = readdirSync(join(root, "src/apps")).filter((f) => /^runtime(_[a-z]+)?\.ts$/.test(f)); | |
| 19 | ||
| 20 | describe("the runtime's modules", () => { | |
| 21 | test("the runtime a constructor registers is the one appsRuntime and envsFor read", async () => { | |
| 22 | const dataDir = await mkdtemp(join(tmpdir(), "forge-runtime-split-")); | |
| 23 | const app = createApp({ dataDir, port: 0, host: "forge.test" }); | |
| 24 | try { | |
| 25 | expect(appsRuntime(app)).toBeUndefined(); | |
| 26 | const runtime = new AppsRuntime(app, { domain: "apps.test", scheme: "http", forgeUrl: "http://forge.test" }); | |
| 27 | expect(appsRuntime(app)).toBe(runtime); | |
| 28 | const record = { id: "a1", repoId: "r1", slug: "notes", access: "public" } as Parameters<typeof envsFor>[2]; | |
| 29 | // Nothing deployed: the registered runtime answers with its (empty) status, not the "no | |
| 30 | // runtime" []. | |
| 31 | let asked = 0; | |
| 32 | const own = runtime.status.bind(runtime); | |
| 33 | runtime.status = (r) => (asked++, own(r)); | |
| 34 | expect(envsFor(app, undefined, record)).toEqual([]); | |
| 35 | expect(asked).toBe(1); | |
| 36 | } finally { | |
| 37 | app.close(); | |
| 38 | await rm(dataDir, { recursive: true, force: true }); | |
| 39 | } | |
| 40 | }); | |
| 41 | ||
| 42 | test("a method handed to its module acts on the instance it is handed, not a copy", () => { | |
| 43 | const bootRuns = new Map<unknown, string>([[{ live: { runId: "run-1" } }, "run-1"]]); | |
| 44 | const fake = { bootRuns, app: {} } as unknown as AppsRuntime; | |
| 45 | // Its boot run is the one serving: not starting, and dropped from the instance's own map. | |
| 46 | expect(status.startingCount(fake)).toBe(0); | |
| 47 | expect(bootRuns.size).toBe(0); | |
| 48 | }); | |
| 49 | ||
| 50 | test("each module is under 600 lines, and only runtime.ts holds module-level state", () => { | |
| 51 | expect(parts.length).toBeGreaterThan(5); | |
| 52 | const held: string[] = []; | |
| 53 | for (const file of parts) { | |
| 54 | const text = readFileSync(join(root, "src/apps", file), "utf8"); | |
| 55 | expect(`${file}: ${text.split("\n").length}`).toBe(`${file}: ${Math.min(text.split("\n").length, 599)}`); | |
| 56 | for (const line of text.split("\n")) | |
| 57 | if (/^(?:export\s+)?(?:const|let)\s+\w+\s*=\s*new\s+(?:Weak)?(?:Map|Set)\b|^(?:export\s+)?let\s/.test(line)) held.push(`${file}: ${line}`); | |
| 58 | } | |
| 59 | expect(held).toEqual(["runtime.ts: const runtimes = new WeakMap<App, AppsRuntime>();"]); | |
| 60 | }); | |
| 61 | }); | |
| 62 |
new above
offline