Public: anyone can read this; the crew can change it.
App builds can name a pinned toolchain (Rust first), with a per-app build cache #10483
scout/forge:scout/10426-app-builds-can-name-a-pinned-toolchain-r → main 8 commits fork
15 files
changed +455
−14
docs/REFERENCE.md +1
| @@ -500,6 +500,7 @@ This page is the front door. Everything else is a file of its own under | ||
| 500 | 500 | **Using it** |
| 501 | 501 | |
| 502 | 502 | - [](VISION.md) — what the forge is for, and the order of work. |
| 503 | - [](TOOLCHAINS.md) — pinned toolchains for app builds (Rust first) and the per-app build cache. | |
| 503 | 504 | - [](STUDIO.md) — the front door in full: the five words and where each is decided, the box, the three lists, a tell, a run and a change each on its own page, and the board. |
| 504 | 505 | - [](TOUR.md) — every other page and screen, with each of its doors: the terminal over SSH, the home and its figures, review, signing in, the page-view beacon, the board's notes, long threads, who can see a repository, archive, where a repository came from, big directories, search, sharing to the forge, link-preview cards and choosing among variants. |
| 505 | 506 | - [](ASKS.md) — what waits on you and how to answer it: waiting pulls, mutes, wave proposals, feed chips and the decider. |
docs/TOOLCHAINS.md added +27
| @@ -0,0 +1,27 @@ | ||
| 1 | # Pinned toolchains and the build cache | |
| 2 | ||
| 3 | An app's `forge.json` may name tools its install and test steps need: | |
| 4 | ||
| 5 | json | |
| 6 | { "app": { "run": "./server", "install": "cargo build --release" }, | |
| 7 | "toolchain": { "rust": "1.85.0" } } | |
| 8 | ||
| 9 | ||
| 10 | - Only exact versions of known tools (`rust`) are accepted. | |
| 11 | - The forge never downloads a toolchain at build time. The operator installs | |
| 12 | one once at `<FORGE_TOOLCHAINS or $FORGE_DATA/toolchains>/rust/1.85.0/bin` | |
| 13 | (a `rustup` or nix-built tree); a build only reads it, read-only, and puts its | |
| 14 | `bin` first on `PATH`. A version the forge lacks is refused with one line: | |
| 15 | `this forge does not provide rust 1.99.0: ask its operator to install it, or name a version it has`. | |
| 16 | - **Build cache** (`src/apps/build_cache.ts`): `<data>/build-cache/<app>/<key>`, | |
| 17 | key = hash of `Cargo.lock`, `bun.lock`, `package-lock.json` and the pins. Steps | |
| 18 | get `FORGE_BUILD_CACHE`, `CARGO_HOME` and `CARGO_TARGET_DIR` inside it, so cargo's | |
| 19 | registry and target dir (and a framework checkout kept under `$FORGE_BUILD_CACHE`) | |
| 20 | survive deploys. The two newest keys are kept. The app's admin clears it from | |
| 21 | the app page (**Build cache → Clear**). | |
| 22 | - **Limits**: a build with a toolchain gets 15 minutes for each of install and test; | |
| 23 | when it hits them the failure reads "the build used its 15 minutes". | |
| 24 | ||
| 25 | Left out: CPU and memory caps beyond the cell's existing limits, and the | |
| 26 | cache is not yet shown on the page when the time limit is hit beyond the | |
| 27 | deploy's failure line. |
scripts/unreached-allow.json −1
| @@ -15,7 +15,6 @@ | ||
| 15 | 15 | "src/flow/review_rounds.ts", |
| 16 | 16 | "src/flow/shared_red.ts", |
| 17 | 17 | "src/flow/stale_fold.ts", |
| 18 | "src/kernel/why_now.ts", | |
| 19 | 18 | "src/spawn/pod.ts", |
| 20 | 19 | "src/tui/gallery.ts", |
| 21 | 20 | "src/tui/ui/qr.ts", |
src/actions/build_cache.ts added +30
| @@ -0,0 +1,30 @@ | ||
| 1 | /** | |
| 2 | * An app's build cache, from the owner's side (`apps/build_cache.ts`): how | |
| 3 | * much is kept, and clearing it. Admin of the repository only; a reader or | |
| 4 | * a stranger is told `not_found`, as everywhere. | |
| 5 | */ | |
| 6 | import type { App } from "../app.ts"; | |
| 7 | import { clearBuildCache, buildCacheBytes } from "../apps/build_cache.ts"; | |
| 8 | import { getAppByRepo } from "../core/apps.ts"; | |
| 9 | import type { Repo } from "../core/repos.ts"; | |
| 10 | import type { User } from "../core/users.ts"; | |
| 11 | import { err, ok, type Result } from "../kernel/result.ts"; | |
| 12 | import { requireRepoAdmin } from "./access.ts"; | |
| 13 | ||
| 14 | /** Bytes the app's build cache holds; `not_found` for anyone who may not clear it. */ | |
| 15 | export function buildCacheSize(app: App, actor: User | undefined, repo: Repo): Result<number, "not_found"> { | |
| 16 | const admin = requireRepoAdmin(app, actor, repo); | |
| 17 | if (!admin.ok) return admin; | |
| 18 | const record = getAppByRepo(app.db, repo.id); | |
| 19 | if (!record) return err("not_found"); | |
| 20 | return ok(buildCacheBytes(app.config.dataDir, record.id)); | |
| 21 | } | |
| 22 | ||
| 23 | /** Clear the app's build cache; the next build starts cold. */ | |
| 24 | export function clearAppBuildCache(app: App, actor: User | undefined, repo: Repo): Result<boolean, "not_found"> { | |
| 25 | const admin = requireRepoAdmin(app, actor, repo); | |
| 26 | if (!admin.ok) return admin; | |
| 27 | const record = getAppByRepo(app.db, repo.id); | |
| 28 | if (!record) return err("not_found"); | |
| 29 | return ok(clearBuildCache(app.config.dataDir, record.id)); | |
| 30 | } |
src/apps/build_cache.ts added +83
| @@ -0,0 +1,83 @@ | ||
| 1 | /** | |
| 2 | * An app's build cache: what a build would otherwise redo every deploy — | |
| 3 | * cargo's registry and target directory, a framework's checkout. One per | |
| 4 | * app and per key, `<data>/build-cache/<app>/<key>`, where the key is a | |
| 5 | * hash of the lockfiles and the toolchain pins: change either and the build | |
| 6 | * starts on a fresh directory, the old one kept until it is the third | |
| 7 | * oldest. Kept across deploys, cleared by the app's admin from its page. | |
| 8 | */ | |
| 9 | ||
| 10 | import { createHash } from "node:crypto"; | |
| 11 | import { existsSync, mkdirSync, readdirSync, rmSync, statSync, utimesSync } from "node:fs"; | |
| 12 | import { join } from "node:path"; | |
| 13 | import type { ToolchainSpec } from "./toolchain.ts"; | |
| 14 | ||
| 15 | /** The lockfiles a key reads, in this order. */ | |
| 16 | export const LOCKFILES = ["Cargo.lock", "bun.lock", "package-lock.json"] as const; | |
| 17 | ||
| 18 | /** How many keys an app keeps; older ones are removed when a new one is made. */ | |
| 19 | export const KEYS_KEPT = 2; | |
| 20 | ||
| 21 | const safe = (id: string): string => id.replace(/[^A-Za-z0-9_-]/g, "_"); | |
| 22 | ||
| 23 | export function buildCacheRoot(dataDir: string, appId: string): string { | |
| 24 | return join(dataDir, "build-cache", safe(appId)); | |
| 25 | } | |
| 26 | ||
| 27 | /** The key for these lockfile contents (`undefined` for an absent file) and pins. */ | |
| 28 | export function buildCacheKey( | |
| 29 | files: Readonly<Record<string, string | undefined>>, | |
| 30 | pins: ToolchainSpec, | |
| 31 | ): string { | |
| 32 | const hash = createHash("sha256"); | |
| 33 | for (const name of LOCKFILES) hash.update(`${name}\0${files[name] ?? ""}\0`); | |
| 34 | for (const [tool, version] of Object.entries(pins).sort()) hash.update(`${tool}=${version}\0`); | |
| 35 | return hash.digest("hex").slice(0, 16); | |
| 36 | } | |
| 37 | ||
| 38 | /** Make (or reuse) the directory for a key, drop the oldest beyond {@link KEYS_KEPT}, return it. */ | |
| 39 | export function openBuildCache(dataDir: string, appId: string, key: string): string { | |
| 40 | const root = buildCacheRoot(dataDir, appId); | |
| 41 | const dir = join(root, key); | |
| 42 | mkdirSync(dir, { recursive: true }); | |
| 43 | const now = new Date(); | |
| 44 | utimesSync(dir, now, now); | |
| 45 | const others = readdirSync(root) | |
| 46 | .filter((name) => name !== key) | |
| 47 | .map((name) => ({ name, at: statSync(join(root, name)).mtimeMs })) | |
| 48 | .sort((a, b) => b.at - a.at); | |
| 49 | for (const old of others.slice(KEYS_KEPT - 1)) rmSync(join(root, old.name), { recursive: true, force: true }); | |
| 50 | return dir; | |
| 51 | } | |
| 52 | ||
| 53 | /** What a step is told about its cache. */ | |
| 54 | export function buildCacheEnv(dir: string): Record<string, string> { | |
| 55 | return { | |
| 56 | FORGE_BUILD_CACHE: dir, | |
| 57 | CARGO_HOME: join(dir, "cargo"), | |
| 58 | CARGO_TARGET_DIR: join(dir, "target"), | |
| 59 | }; | |
| 60 | } | |
| 61 | ||
| 62 | /** Remove everything cached for an app. Returns whether there was anything. */ | |
| 63 | export function clearBuildCache(dataDir: string, appId: string): boolean { | |
| 64 | const root = buildCacheRoot(dataDir, appId); | |
| 65 | if (!existsSync(root)) return false; | |
| 66 | rmSync(root, { recursive: true, force: true }); | |
| 67 | return true; | |
| 68 | } | |
| 69 | ||
| 70 | /** Bytes on disk under an app's cache (files only, no follow of links). */ | |
| 71 | export function buildCacheBytes(dataDir: string, appId: string): number { | |
| 72 | const walk = (dir: string): number => { | |
| 73 | let total = 0; | |
| 74 | for (const entry of readdirSync(dir, { withFileTypes: true })) { | |
| 75 | const path = join(dir, entry.name); | |
| 76 | if (entry.isDirectory()) total += walk(path); | |
| 77 | else if (entry.isFile()) total += statSync(path).size; | |
| 78 | } | |
| 79 | return total; | |
| 80 | }; | |
| 81 | const root = buildCacheRoot(dataDir, appId); | |
| 82 | return existsSync(root) ? walk(root) : 0; | |
| 83 | } |
src/apps/manifest.ts +9
| @@ -17,6 +17,7 @@ import type { DeclaredLimits } from "./contain.ts"; | ||
| 17 | 17 | import { isPublicAddress } from "./egress.ts"; |
| 18 | 18 | import { isLaunch, LAUNCHES, type Launch } from "../git/builds.ts"; |
| 19 | 19 | import { parseAdapter, type AdapterSpec } from "./adapter.ts"; |
| 20 | import { parseToolchain, type ToolchainSpec } from "./toolchain.ts"; | |
| 20 | 21 | |
| 21 | 22 | export interface AppManifest { |
| 22 | 23 | /** The command that serves on `$PORT`. Run through `sh -c`, `exec`-ed. */ |
| @@ -82,6 +83,11 @@ export interface AppManifest { | ||
| 82 | 83 | * schedule (`apps/adapter.ts`). Unset is not an adapter. |
| 83 | 84 | */ |
| 84 | 85 | readonly adapter?: AdapterSpec | undefined; |
| 86 | /** | |
| 87 | * `toolchain` (top level of `forge.json`): pinned tools the install and | |
| 88 | * test steps get, with a build cache (`apps/toolchain.ts`). Unset is none. | |
| 89 | */ | |
| 90 | readonly toolchain?: ToolchainSpec | undefined; | |
| 85 | 91 | } |
| 86 | 92 | |
| 87 | 93 | export type ManifestError = |
| @@ -185,6 +191,8 @@ export function parseManifest( | ||
| 185 | 191 | return invalid("forge.json: toolbar must be true or false"); |
| 186 | 192 | const adapter = parseAdapter(parsed.value["adapter"]); |
| 187 | 193 | if (!adapter.ok) return invalid(adapter.error); |
| 194 | const toolchain = parseToolchain(parsed.value["toolchain"]); | |
| 195 | if (!toolchain.ok) return invalid(toolchain.error); | |
| 188 | 196 | // Only `egress` opens a host (docs/ADAPTERS.md): a source host it does |
| 189 | 197 | // not list would be refused by the proxy, so the block is invalid. |
| 190 | 198 | const unopened = adapter.value?.source?.hosts.find( |
| @@ -226,6 +234,7 @@ export function parseManifest( | ||
| 226 | 234 | limits: limits.value, |
| 227 | 235 | toolbar: toolbar ?? true, |
| 228 | 236 | ...(adapter.value === undefined ? {} : { adapter: adapter.value }), |
| 237 | ...(toolchain.value === undefined ? {} : { toolchain: toolchain.value }), | |
| 229 | 238 | }); |
| 230 | 239 | } |
| 231 | 240 |
src/apps/process.ts +79 −9
| @@ -22,6 +22,9 @@ import { spellings, stripForgeDirs } from "../runs/paths.ts"; | ||
| 22 | 22 | import { seconds } from "./timing.ts"; |
| 23 | 23 | import { notFoundHint, shellCommand } from "./run_command.ts"; |
| 24 | 24 | import { cellInstall, meteredInstall, prefillInstallCache, withLock } from "../checks/install.ts"; |
| 25 | import { existsSync, readFileSync } from "node:fs"; | |
| 26 | import { LOCKFILES, buildCacheEnv, buildCacheKey, openBuildCache } from "./build_cache.ts"; | |
| 27 | import { resolveToolchain, toolchainRoot, withToolchainPath } from "./toolchain.ts"; | |
| 25 | 28 | import { |
| 26 | 29 | commandPid, |
| 27 | 30 | killTree, |
| @@ -86,6 +89,44 @@ export interface StartInput { | ||
| 86 | 89 | * fill will not trust (no `bun.lock`) installs as before, into its home. |
| 87 | 90 | */ |
| 88 | 91 | readonly installCache?: string | undefined; |
| 92 | /** | |
| 93 | * The forge's data directory and the app's id, which a pinned toolchain | |
| 94 | * and its build cache are found by (`apps/toolchain.ts`, `build_cache.ts`). | |
| 95 | */ | |
| 96 | readonly forgeData?: string | undefined; | |
| 97 | readonly appId?: string | undefined; | |
| 98 | } | |
| 99 | ||
| 100 | /** What a build with a pinned toolchain adds to its steps. */ | |
| 101 | export interface BuildKit { | |
| 102 | readonly env: Record<string, string>; | |
| 103 | readonly reveal: readonly string[]; | |
| 104 | readonly writable: readonly string[]; | |
| 105 | } | |
| 106 | ||
| 107 | /** A toolchain build's time: named in its failure, so the app page can say it. */ | |
| 108 | export const BUILD_MINUTES = 15; | |
| 109 | ||
| 110 | /** | |
| 111 | * The toolchain and cache an app's steps get, `undefined` when it names no | |
| 112 | * toolchain, or the one line saying the forge does not provide it. | |
| 113 | */ | |
| 114 | export function buildKit(input: StartInput, path: string): Result<BuildKit | undefined, string> { | |
| 115 | const spec = input.manifest.toolchain; | |
| 116 | if (spec === undefined || input.forgeData === undefined || input.appId === undefined) return ok(undefined); | |
| 117 | const provided = resolveToolchain(toolchainRoot(input.forgeData), spec); | |
| 118 | if (!provided.ok) return provided; | |
| 119 | const files: Record<string, string | undefined> = {}; | |
| 120 | for (const name of LOCKFILES) { | |
| 121 | const at = `${input.dir}/${name}`; | |
| 122 | files[name] = existsSync(at) ? readFileSync(at, "utf8") : undefined; | |
| 123 | } | |
| 124 | const cache = openBuildCache(input.forgeData, input.appId, buildCacheKey(files, spec)); | |
| 125 | return ok({ | |
| 126 | env: { ...buildCacheEnv(cache), PATH: withToolchainPath(path, provided.value) }, | |
| 127 | reveal: provided.value.dirs, | |
| 128 | writable: [cache], | |
| 129 | }); | |
| 89 | 130 | } |
| 90 | 131 | |
| 91 | 132 | /** What a caller knows about a cell; the rest comes from the manifest. */ |
| @@ -314,6 +355,8 @@ export async function runStep( | ||
| 314 | 355 | timeoutMs?: number; |
| 315 | 356 | /** Run inside a cell (`apps/contain.ts`) instead of as a plain child. */ |
| 316 | 357 | cell?: CellSpec | undefined; |
| 358 | /** What a timeout says it used, instead of `took longer than Ns`. */ | |
| 359 | timeoutWords?: string | undefined; | |
| 317 | 360 | }, |
| 318 | 361 | ): Promise<Result<void, string>> { |
| 319 | 362 | options.log(`$ ${command}`); |
| @@ -357,7 +400,7 @@ export async function runStep( | ||
| 357 | 400 | // is sent, so a step in flight never outlives its own cancellation. |
| 358 | 401 | const stopReading = new AbortController(); |
| 359 | 402 | const timer = setTimeout(() => { |
| 360 | why = `${name} took longer than ${Math.round(timeoutMs / 1000)}s`; | |
| 403 | why = options.timeoutWords ?? `${name} took longer than ${Math.round(timeoutMs / 1000)}s`; | |
| 361 | 404 | kill(); |
| 362 | 405 | stopReading.abort(); |
| 363 | 406 | }, timeoutMs); |
| @@ -398,13 +441,24 @@ export async function runStep( | ||
| 398 | 441 | } |
| 399 | 442 | } |
| 400 | 443 | |
| 444 | /** A toolchain build gets {@link BUILD_MINUTES}, and says so when it uses them. */ | |
| 445 | function limitsFor(kit: BuildKit | undefined): { timeoutMs?: number; timeoutWords?: string } { | |
| 446 | return kit === undefined | |
| 447 | ? {} | |
| 448 | : { | |
| 449 | timeoutMs: BUILD_MINUTES * 60_000, | |
| 450 | timeoutWords: `the build used its ${BUILD_MINUTES} minutes (the time limit for a build with a toolchain)`, | |
| 451 | }; | |
| 452 | } | |
| 453 | ||
| 401 | 454 | /** The cell a step of this deploy runs in: the checkout and the home, the registry and what the app declared. */ |
| 402 | export function stepCell(input: StartInput): CellSpec | undefined { | |
| 455 | export function stepCell(input: StartInput, kit?: BuildKit): CellSpec | undefined { | |
| 403 | 456 | const base = input.cell; |
| 404 | 457 | if (base === undefined) return undefined; |
| 405 | 458 | return { |
| 406 | 459 | dir: input.dir, |
| 407 | writable: [], | |
| 460 | writable: kit?.writable ?? [], | |
| 461 | ...(kit === undefined ? {} : { reveal: kit.reveal }), | |
| 408 | 462 | home: base.home, |
| 409 | 463 | hide: base.hide, |
| 410 | 464 | egress: [...STEP_HOSTS, ...(input.manifest.egress ?? [])], |
| @@ -443,9 +497,10 @@ export function installPlan( | ||
| 443 | 497 | input: StartInput, |
| 444 | 498 | env: Record<string, string>, |
| 445 | 499 | filled: boolean, |
| 500 | kit?: BuildKit, | |
| 446 | 501 | ): InstallPlan { |
| 447 | 502 | const cache = input.installCache; |
| 448 | const cell = stepCell(input); | |
| 503 | const cell = stepCell(input, kit); | |
| 449 | 504 | if (cache === undefined || !filled) return { command, env, cell, lock: undefined }; |
| 450 | 505 | if (cell !== undefined) |
| 451 | 506 | return { ...cellInstall(command, env, cache), cell: { ...cell, reveal: [...(cell.reveal ?? []), cache] }, lock: undefined }; |
| @@ -461,6 +516,7 @@ async function install( | ||
| 461 | 516 | command: string, |
| 462 | 517 | input: StartInput, |
| 463 | 518 | env: Record<string, string>, |
| 519 | kit?: BuildKit, | |
| 464 | 520 | ): Promise<Result<void, string>> { |
| 465 | 521 | let filled = false; |
| 466 | 522 | if (input.installCache !== undefined) { |
| @@ -475,9 +531,16 @@ async function install( | ||
| 475 | 531 | if (fill.ok) filled = fill.value; |
| 476 | 532 | else input.log(`cache not filled: ${fill.error}`); |
| 477 | 533 | } |
| 478 | const plan = installPlan(command, input, env, filled); | |
| 534 | const plan = installPlan(command, input, env, filled, kit); | |
| 479 | 535 | const step = (): Promise<Result<void, string>> => |
| 480 | runStep("install", plan.command, { cwd: input.dir, env: plan.env, log: input.log, signal: input.signal, cell: plan.cell }); | |
| 536 | runStep("install", plan.command, { | |
| 537 | cwd: input.dir, | |
| 538 | env: plan.env, | |
| 539 | log: input.log, | |
| 540 | signal: input.signal, | |
| 541 | cell: plan.cell, | |
| 542 | ...limitsFor(kit), | |
| 543 | }); | |
| 481 | 544 | if (plan.lock !== undefined) { |
| 482 | 545 | const held = await withLock(plan.lock, step, { signal: input.signal }); |
| 483 | 546 | return held.ok ? held.value : held; |
| @@ -507,8 +570,14 @@ export async function startProcess( | ||
| 507 | 570 | ...forgeRegistry(), |
| 508 | 571 | }); |
| 509 | 572 | |
| 573 | // A pinned toolchain and its cache, or the one line that refuses the build. | |
| 574 | const provided = input.warm === true ? ok(undefined) : buildKit(input, stepEnv["PATH"] ?? ""); | |
| 575 | if (!provided.ok) return err({ stage: "install", message: provided.error }); | |
| 576 | const kit = provided.value; | |
| 577 | const stepEnvKit = kit === undefined ? stepEnv : { ...stepEnv, ...kit.env }; | |
| 578 | ||
| 510 | 579 | if (manifest.install !== null && input.warm !== true) { |
| 511 | const installed = await install(manifest.install, input, stepEnv); | |
| 580 | const installed = await install(manifest.install, input, stepEnvKit, kit); | |
| 512 | 581 | if (!installed.ok) |
| 513 | 582 | return err({ stage: "install", message: installed.error }); |
| 514 | 583 | } |
| @@ -529,10 +598,11 @@ export async function startProcess( | ||
| 529 | 598 | ? Promise.resolve(ok()) |
| 530 | 599 | : runStep("test", manifest.test, { |
| 531 | 600 | cwd: input.dir, |
| 532 | env: stepEnv, | |
| 601 | env: stepEnvKit, | |
| 533 | 602 | log, |
| 534 | 603 | signal: testAbort.signal, |
| 535 | cell: stepCell(input), | |
| 604 | cell: stepCell(input, kit), | |
| 605 | ...limitsFor(kit), | |
| 536 | 606 | }); |
| 537 | 607 | |
| 538 | 608 | // The port is chosen, then released, then bound by the app: another |
src/apps/runtime.ts +2
| @@ -1510,6 +1510,8 @@ export class AppsRuntime { | ||
| 1510 | 1510 | // Every environment, production too, installs from the cache the |
| 1511 | 1511 | // repository's checks fill, so a new preview fetches nothing it has. |
| 1512 | 1512 | installCache: checkInstallCache(this.app.config.dataDir, repo.id), |
| 1513 | forgeData: this.app.config.dataDir, | |
| 1514 | appId: state.app.id, | |
| 1513 | 1515 | }; |
| 1514 | 1516 | this.noteCell(run.id, write); |
| 1515 | 1517 | clock.lap("setup"); |
src/apps/toolchain.ts added +78
| @@ -0,0 +1,78 @@ | ||
| 1 | /** | |
| 2 | * A pinned toolchain for an app's build and test steps: `toolchain` at the | |
| 3 | * top level of `forge.json`, `{"rust": "1.85.0"}`. The forge never fetches | |
| 4 | * one at build time. A toolchain is a directory its operator installed once | |
| 5 | * — `<toolchains>/<tool>/<version>/bin` — and a build only reads it, so a | |
| 6 | * version the forge does not hold is refused with one plain line, not | |
| 7 | * downloaded. `<toolchains>` is `FORGE_TOOLCHAINS`, else `<data>/toolchains`. | |
| 8 | */ | |
| 9 | ||
| 10 | import { existsSync, statSync } from "node:fs"; | |
| 11 | import { join } from "node:path"; | |
| 12 | import { err, ok, type Result } from "../kernel/result.ts"; | |
| 13 | ||
| 14 | /** The tools a manifest may name; a closed set. */ | |
| 15 | export const TOOLCHAINS = ["rust"] as const; | |
| 16 | export type Tool = (typeof TOOLCHAINS)[number]; | |
| 17 | ||
| 18 | /** `{ rust: "1.85.0" }`: each named tool with its exact version. */ | |
| 19 | export type ToolchainSpec = { readonly [T in Tool]?: string }; | |
| 20 | ||
| 21 | const VERSION_RE = /^\d{1,3}\.\d{1,3}(\.\d{1,3})?$/; | |
| 22 | ||
| 23 | /** The manifest's `toolchain` value, validated; `undefined` when absent. */ | |
| 24 | export function parseToolchain( | |
| 25 | value: unknown, | |
| 26 | ): Result<ToolchainSpec | undefined, string> { | |
| 27 | if (value === undefined) return ok(undefined); | |
| 28 | if (typeof value !== "object" || value === null || Array.isArray(value)) | |
| 29 | return err('forge.json: toolchain must be an object like {"rust": "1.85.0"}'); | |
| 30 | const out: { [T in Tool]?: string } = {}; | |
| 31 | for (const [tool, version] of Object.entries(value)) { | |
| 32 | if (!(TOOLCHAINS as readonly string[]).includes(tool)) | |
| 33 | return err(`forge.json: toolchain takes ${TOOLCHAINS.join(", ")}, not ${tool.slice(0, 40)}`); | |
| 34 | if (typeof version !== "string" || !VERSION_RE.test(version)) | |
| 35 | return err(`forge.json: toolchain.${tool} must be an exact version like "1.85.0"`); | |
| 36 | out[tool as Tool] = version; | |
| 37 | } | |
| 38 | return ok(Object.keys(out).length === 0 ? undefined : out); | |
| 39 | } | |
| 40 | ||
| 41 | /** Where the operator's pinned toolchains live. */ | |
| 42 | export function toolchainRoot(dataDir: string): string { | |
| 43 | const named = process.env["FORGE_TOOLCHAINS"]; | |
| 44 | return named !== undefined && named !== "" ? named : join(dataDir, "toolchains"); | |
| 45 | } | |
| 46 | ||
| 47 | /** What a step is given for the toolchain: directories to read, and the PATH prefix. */ | |
| 48 | export interface Provided { | |
| 49 | /** Each toolchain's directory, shown read-only inside the cell. */ | |
| 50 | readonly dirs: readonly string[]; | |
| 51 | /** `bin` directories put in front of PATH. */ | |
| 52 | readonly bins: readonly string[]; | |
| 53 | } | |
| 54 | ||
| 55 | /** The toolchain directories for a spec, or the one line saying which is missing. */ | |
| 56 | export function resolveToolchain( | |
| 57 | root: string, | |
| 58 | spec: ToolchainSpec, | |
| 59 | ): Result<Provided, string> { | |
| 60 | const dirs: string[] = []; | |
| 61 | const bins: string[] = []; | |
| 62 | for (const tool of TOOLCHAINS) { | |
| 63 | const version = spec[tool]; | |
| 64 | if (version === undefined) continue; | |
| 65 | const dir = join(root, tool, version); | |
| 66 | const bin = join(dir, "bin"); | |
| 67 | if (!existsSync(bin) || !statSync(bin).isDirectory()) | |
| 68 | return err(`this forge does not provide ${tool} ${version}: ask its operator to install it, or name a version it has`); | |
| 69 | dirs.push(dir); | |
| 70 | bins.push(bin); | |
| 71 | } | |
| 72 | return ok({ dirs, bins }); | |
| 73 | } | |
| 74 | ||
| 75 | /** `PATH` with the toolchain's bins first. */ | |
| 76 | export function withToolchainPath(path: string, provided: Provided): string { | |
| 77 | return [...provided.bins, path].filter((p) => p !== "").join(":"); | |
| 78 | } |
src/http/server.ts +2
| @@ -225,6 +225,7 @@ import { | ||
| 225 | 225 | newIssueSubmit, |
| 226 | 226 | } from "../web/pages/issues.ts"; |
| 227 | 227 | import { adapterPull } from "../web/studio/adapter.ts"; |
| 228 | import { buildCacheClear } from "../web/studio/build_cache.ts"; | |
| 228 | 229 | import { noteModerate } from "../web/studio/notes.ts"; |
| 229 | 230 | import { tellPage } from "../web/studio/tell.ts"; |
| 230 | 231 | import { issuePicture } from "../web/studio/picture.ts"; |
| @@ -1310,6 +1311,7 @@ export function buildRouter(): Router<Handler> { | ||
| 1310 | 1311 | router.add("GET", "/apps/open", appsOpen); |
| 1311 | 1312 | router.add("POST", "/:owner/:name/app/notes", noteModerate); |
| 1312 | 1313 | router.add("POST", "/:owner/:name/app/pull", adapterPull); |
| 1314 | router.add("POST", "/:owner/:name/app/build-cache/clear", buildCacheClear); | |
| 1313 | 1315 | router.add("POST", "/:owner/:name/app/:env/redeploy", appRedeploy); |
| 1314 | 1316 | router.add("POST", "/:owner/:name/app/:env/stop", appStop); |
| 1315 | 1317 | router.add("GET", "/:owner/:name/events", eventsList); |
src/web/studio/app.ts +2 −1
| @@ -57,6 +57,7 @@ import { nowCard, nowRegion } from "./now.ts"; | ||
| 57 | 57 | import { repoAsks } from "./asks.ts"; |
| 58 | 58 | import { notesCard } from "./notes.ts"; |
| 59 | 59 | import { adapterCard } from "./adapter.ts"; |
| 60 | import { buildCacheCard } from "./build_cache.ts"; | |
| 60 | 61 | import { appPanels } from "./previews.ts"; |
| 61 | 62 | import { noCrewLine, pausedWords, servingHistory, type WordCtx } from "./words.ts"; |
| 62 | 63 | import { crewKeyNote } from "./crew.ts"; |
| @@ -515,7 +516,7 @@ export async function studioApp( | ||
| 515 | 516 | ${notesCard(ctx, repo, record)} |
| 516 | 517 | </div> |
| 517 | 518 | <aside class="stack"> |
| 518 | ${adapter} ${crewCard} ${weekCard} ${ownCard} ${more} | |
| 519 | ${adapter} ${ownerFacts ? buildCacheCard(ctx, repo) : null} ${crewCard} ${weekCard} ${ownCard} ${more} | |
| 519 | 520 | </aside> |
| 520 | 521 | </div> |
| 521 | 522 | ${liveMount({ repo: base, app: true })}`; |
src/web/studio/build_cache.ts added +36
| @@ -0,0 +1,36 @@ | ||
| 1 | /** | |
| 2 | * The build cache card on an app's page: what a toolchain build keeps | |
| 3 | * between deploys, and the owner's button to clear it. Shown to the app's | |
| 4 | * admins when there is something cached. | |
| 5 | */ | |
| 6 | import { buildCacheSize, clearAppBuildCache } from "../../actions/build_cache.ts"; | |
| 7 | import type { Repo } from "../../core/repos.ts"; | |
| 8 | import { formatBytes } from "../../core/storage.ts"; | |
| 9 | import type { Ctx } from "../../http/context.ts"; | |
| 10 | import { loadRepo, notFoundPage, repoBase } from "../chrome.ts"; | |
| 11 | import { readForm } from "../form.ts"; | |
| 12 | import { html, type Html } from "../html.ts"; | |
| 13 | import { flashTo } from "../layout.ts"; | |
| 14 | import { btn } from "../ui.ts"; | |
| 15 | ||
| 16 | export function buildCacheCard(ctx: Ctx, repo: Repo): Html | null { | |
| 17 | const size = buildCacheSize(ctx.app, ctx.auth.user, repo); | |
| 18 | if (!size.ok || size.value === 0) return null; | |
| 19 | return html`<section class="build-cache"> | |
| 20 | <h2>Build cache</h2> | |
| 21 | <p class="muted">Kept across deploys, keyed by the lockfiles and the toolchain pins: ${formatBytes(size.value)}.</p> | |
| 22 | <form method="post" action="${repoBase(repo)}/app/build-cache/clear">${btn("Clear build cache", { size: "sm", variant: "outline" })}</form> | |
| 23 | </section>`; | |
| 24 | } | |
| 25 | ||
| 26 | // POST /:owner/:name/app/build-cache/clear | |
| 27 | export async function buildCacheClear(ctx: Ctx): Promise<Response> { | |
| 28 | const repo = loadRepo(ctx); | |
| 29 | if (!repo) return notFoundPage(ctx); | |
| 30 | const size = buildCacheSize(ctx.app, ctx.auth.user, repo); | |
| 31 | if (!size.ok) return notFoundPage(ctx); | |
| 32 | const form = await readForm(ctx, "rw"); | |
| 33 | if (form instanceof Response) return form; | |
| 34 | clearAppBuildCache(ctx.app, ctx.auth.user, repo); | |
| 35 | return flashTo(ctx, repoBase(repo), { message: "Build cache cleared: the next build starts cold." }); | |
| 36 | } |
test/build-cart.test.ts +8 −2
| @@ -77,8 +77,14 @@ describe("a pin's cart", () => { | ||
| 77 | 77 | expect(res.status).toBe(202); |
| 78 | 78 | expect(await res.json()).toMatchObject({ demo: { name: "fiesta-kart", version: "1", launch: "file", page: "/", showcase: true } }); |
| 79 | 79 | // karts is no app, so there is no build to capture from: the state says why. |
| 80 | await Bun.sleep(50); | |
| 81 | expect(await (await api("karts/builds/v1/cart")).json()).toMatchObject({ capture: { state: "failed", error: "not_an_app" } }); | |
| 80 | // Wait for the capture to settle, so it cannot land after the next test's import. | |
| 81 | let state: unknown = {}; | |
| 82 | for (let i = 0; i < 100; i++) { | |
| 83 | state = await (await api("karts/builds/v1/cart")).json(); | |
| 84 | if ((state as { capture: { state: string } }).capture.state === "failed") break; | |
| 85 | await Bun.sleep(50); | |
| 86 | } | |
| 87 | expect(state).toMatchObject({ capture: { state: "failed", error: "not_an_app" } }); | |
| 82 | 88 | }); |
| 83 | 89 | |
| 84 | 90 | test("a captured cart plays sandboxed, with its picture, and the builds page links it", async () => { |
test/child-quota.test.ts +1 −1
| @@ -101,7 +101,7 @@ describe("child quota", () => { | ||
| 101 | 101 | |
| 102 | 102 | test("room for small but not large is refused only for large", async () => { |
| 103 | 103 | clear(); |
| 104 | process.env["FORGE_CHILD_MEMORY_MIB"] = "2000"; | |
| 104 | process.env["FORGE_CHILD_MEMORY_MIB"] = "4096"; | |
| 105 | 105 | expect(quotaRefusal(dir, "small")).toBeUndefined(); |
| 106 | 106 | expect(quotaRefusal(dir, "large")).toContain("memory limit"); |
| 107 | 107 | }); |
test/toolchain.test.ts added +97
| @@ -0,0 +1,97 @@ | ||
| 1 | import { expect, test } from "bun:test"; | |
| 2 | import { chmod, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; | |
| 3 | import { existsSync } from "node:fs"; | |
| 4 | import { tmpdir } from "node:os"; | |
| 5 | import { join } from "node:path"; | |
| 6 | import { startProcess } from "../src/apps/process.ts"; | |
| 7 | import { parseManifest } from "../src/apps/manifest.ts"; | |
| 8 | import { parseToolchain } from "../src/apps/toolchain.ts"; | |
| 9 | import { buildCacheKey, clearBuildCache, openBuildCache, buildCacheBytes } from "../src/apps/build_cache.ts"; | |
| 10 | ||
| 11 | test("forge.json names a toolchain as exact versions of known tools", () => { | |
| 12 | expect(parseToolchain(undefined)).toEqual({ ok: true, value: undefined }); | |
| 13 | expect(parseToolchain({ rust: "1.85.0" })).toEqual({ ok: true, value: { rust: "1.85.0" } }); | |
| 14 | for (const bad of [{ rust: "latest" }, { rust: 1 }, { go: "1.2.3" }, "rust", [], { rust: "1.85; rm" }]) | |
| 15 | expect(parseToolchain(bad).ok).toBe(false); | |
| 16 | const m = parseManifest(JSON.stringify({ app: { run: "bun x.ts" }, toolchain: { rust: "1.85.0" } }), undefined); | |
| 17 | expect(m.ok && m.value.toolchain).toEqual({ rust: "1.85.0" }); | |
| 18 | }); | |
| 19 | ||
| 20 | test("the cache key follows lockfiles and pins", () => { | |
| 21 | const a = buildCacheKey({ "Cargo.lock": "a" }, { rust: "1.85.0" }); | |
| 22 | expect(buildCacheKey({ "Cargo.lock": "a" }, { rust: "1.85.0" })).toBe(a); | |
| 23 | expect(buildCacheKey({ "Cargo.lock": "b" }, { rust: "1.85.0" })).not.toBe(a); | |
| 24 | expect(buildCacheKey({ "Cargo.lock": "a" }, { rust: "1.86.0" })).not.toBe(a); | |
| 25 | }); | |
| 26 | ||
| 27 | test("the build cache is kept per key, bounded, and cleared", async () => { | |
| 28 | const data = await mkdtemp(join(tmpdir(), "forge-bc-")); | |
| 29 | try { | |
| 30 | const one = openBuildCache(data, "app1", "k1"); | |
| 31 | await writeFile(join(one, "f"), "12345"); | |
| 32 | expect(buildCacheBytes(data, "app1")).toBe(5); | |
| 33 | await Bun.sleep(15); | |
| 34 | openBuildCache(data, "app1", "k2"); | |
| 35 | await Bun.sleep(15); | |
| 36 | openBuildCache(data, "app1", "k3"); | |
| 37 | expect(existsSync(one)).toBe(false); | |
| 38 | expect(clearBuildCache(data, "app1")).toBe(true); | |
| 39 | expect(buildCacheBytes(data, "app1")).toBe(0); | |
| 40 | } finally { | |
| 41 | await rm(data, { recursive: true, force: true }); | |
| 42 | } | |
| 43 | }); | |
| 44 | ||
| 45 | const rustApp = async (version: string, install: string) => { | |
| 46 | const data = await mkdtemp(join(tmpdir(), "forge-tc-")); | |
| 47 | const dir = join(data, "checkout"); | |
| 48 | await mkdir(dir); | |
| 49 | // A stand-in cargo in a pinned toolchain: writes into the cache's target dir. | |
| 50 | const bin = join(data, "toolchains", "rust", "1.85.0", "bin"); | |
| 51 | await mkdir(bin, { recursive: true }); | |
| 52 | await writeFile(join(bin, "cargo"), '#!/bin/sh\nmkdir -p "$CARGO_TARGET_DIR" && echo hello-rust > "$CARGO_TARGET_DIR/out" && cp "$CARGO_TARGET_DIR/out" built.txt\n'); | |
| 53 | await chmod(join(bin, "cargo"), 0o755); | |
| 54 | await writeFile(join(dir, "server.ts"), "Bun.serve({port:Number(process.env.PORT),hostname:'127.0.0.1',fetch:()=>new Response(require('fs').readFileSync('built.txt','utf8'))})"); | |
| 55 | const lines: string[] = []; | |
| 56 | const started = await startProcess({ | |
| 57 | dir, | |
| 58 | dataDir: data, | |
| 59 | forgeData: data, | |
| 60 | appId: "app9", | |
| 61 | manifest: { run: "bun server.ts", install, health: "/", ready: 8, toolchain: { rust: version } } as never, | |
| 62 | env: "production", | |
| 63 | url: "http://app.test", | |
| 64 | commitOid: "0".repeat(40), | |
| 65 | forgeUrl: "http://forge.test", | |
| 66 | identityKey: "key", | |
| 67 | log: (l) => void lines.push(l), | |
| 68 | signal: new AbortController().signal, | |
| 69 | }); | |
| 70 | return { data, started, lines }; | |
| 71 | }; | |
| 72 | ||
| 73 | test("a Rust app builds with its pinned toolchain, caches, and serves", async () => { | |
| 74 | const { data, started } = await rustApp("1.85.0", "cargo build"); | |
| 75 | try { | |
| 76 | expect(started.ok).toBe(true); | |
| 77 | if (!started.ok) return; | |
| 78 | const res = await fetch(`http://127.0.0.1:${started.value.handle.port}/`); | |
| 79 | expect(await res.text()).toContain("hello-rust"); | |
| 80 | expect(buildCacheBytes(data, "app9")).toBeGreaterThan(0); | |
| 81 | await started.value.handle.stop(); | |
| 82 | await started.value.handle.exited; | |
| 83 | } finally { | |
| 84 | await rm(data, { recursive: true, force: true }); | |
| 85 | } | |
| 86 | }, 30_000); | |
| 87 | ||
| 88 | test("an app naming a toolchain the forge lacks is refused with one line", async () => { | |
| 89 | const { data, started } = await rustApp("1.99.0", "cargo build"); | |
| 90 | try { | |
| 91 | expect(started.ok).toBe(false); | |
| 92 | if (started.ok) return; | |
| 93 | expect(started.error.message).toBe("this forge does not provide rust 1.99.0: ask its operator to install it, or name a version it has"); | |
| 94 | } finally { | |
| 95 | await rm(data, { recursive: true, force: true }); | |
| 96 | } | |
| 97 | }); |
new above
offline