Public: anyone can read this; the crew can change it.
Adapter app shape: sync an external system into the forge, with a contract the forge can audit #10034
scout/forge:scout/437-adapter-app-shape-sync-an-external-syste → main 8 commits fork
6 files
changed +300
−0
docs/ADAPTERS.md added +53
| @@ -0,0 +1,53 @@ | ||
| 1 | # Adapters | |
| 2 | ||
| 3 | An adapter is an app that syncs one external system into its own data | |
| 4 | under `FORGE_APP_DATA`. It declares itself in `forge.json`, so the forge can | |
| 5 | schedule, inspect and audit it the same way for every source, and so an | |
| 6 | agent can write one by filling in a template (#437). | |
| 7 | ||
| 8 | ## The block | |
| 9 | ||
| 10 | json | |
| 11 | { | |
| 12 | "app": { "run": "bun run src/server.ts" }, | |
| 13 | "egress": ["api.ledger.example"], | |
| 14 | "adapter": { | |
| 15 | "source": { "name": "ledger", "hosts": ["api.ledger.example"] }, | |
| 16 | "schedule": { "every": 900 }, | |
| 17 | "config": { | |
| 18 | "account": { "type": "string", "default": "main" }, | |
| 19 | "token": { "type": "secret", "ref": "LEDGER_TOKEN" } | |
| 20 | } | |
| 21 | } | |
| 22 | } | |
| 23 | ||
| 24 | ||
| 25 | `src/apps/adapter.ts` validates it, and nothing else does. A block that is | |
| 26 | present and wrong fails the deploy, and the error names the field. | |
| 27 | ||
| 28 | - `source.name`: lowercase letters, digits, `-` or `_`. | |
| 29 | - `source.hosts`: 1 to 8 bare host names, with no scheme, port, path or | |
| 30 | wildcard. **Each one must also be in the top-level `egress`.** The cell's | |
| 31 | proxy (`apps/egress.ts`) refuses every host not in `egress`, so this rule | |
| 32 | means a source host the app cannot reach is refused at deploy time. | |
| 33 | Unknown hosts are refused on every request. | |
| 34 | - `schedule.every`: whole seconds between pulls, from 10 to 604800 (one | |
| 35 | week). | |
| 36 | - `config`: at most 32 named fields, each `string`, `number`, `boolean` or | |
| 37 | `secret`. A plain field may set a `default` of its own type. A `secret` | |
| 38 | takes `ref`, the name of an environment variable that is set (sealed) in | |
| 39 | the app's environment settings. A secret never takes a value: a | |
| 40 | `default` on a secret is refused, so a secret cannot end up in the | |
| 41 | repository. | |
| 42 | ||
| 43 | ## The contract (to come) | |
| 44 | ||
| 45 | The adapter answers `POST /pull` with `{pulled, skipped, cursor, errors}`. | |
| 46 | The pull is idempotent: the same cursor gives the same result, and a | |
| 47 | replay writes nothing new. The adapter keeps its cursor and records under | |
| 48 | `FORGE_APP_DATA`. The forge will call `/pull` every `schedule.every` | |
| 49 | seconds and record each call as a run (counts, cursor, duration). The app | |
| 50 | page will show the last pull, the next pull and the cursor. A template | |
| 51 | adapter will come with its own tests. None of this exists yet: today the | |
| 52 | forge validates the block and enforces the host rule, and nothing calls | |
| 53 | `/pull`. |
docs/APPS.md +5
| @@ -564,6 +564,11 @@ are capped by the machine (`FORGE_CELL_MEMORY_MAX_MIB`, default 4096; the | ||
| 564 | 564 | number of cores; `FORGE_CELL_DISK_MAX_MIB`, default 16384). A cell's `/tmp` |
| 565 | 565 | holds at most 256 MiB. |
| 566 | 566 | |
| 567 | `adapter`, at the top level, declares the app an adapter that syncs one | |
| 568 | external system into its own data: its source (each host also named in | |
| 569 | `egress`), how often it pulls, and its typed config, secrets by reference. | |
| 570 | [](ADAPTERS.md) has the block and the contract. | |
| 571 | ||
| 567 | 572 | Memory is held twice. The watch reads every cell's resident memory once a |
| 568 | 573 | second and stops one over its limit. Where the forge holds a delegated |
| 569 | 574 | cgroup v2 (`apps/cgroup.ts`), the kernel holds it as well: each cell runs |
docs/REFERENCE.md +1
| @@ -516,6 +516,7 @@ This page is the front door. Everything else is a file of its own under | ||
| 516 | 516 | |
| 517 | 517 | **Apps and the crew** |
| 518 | 518 | |
| 519 | - [](ADAPTERS.md) — the `adapter` block of `forge.json`: an app that syncs one external system, its source hosts, schedule and typed config. | |
| 519 | 520 | - [](APPS.md) — the contract for apps, previews and runs: making an app, the vocabulary, and the rules that are mechanical rather than advisory. |
| 520 | 521 | - [](CONTRACT.md) — who may do what, as a table: an app's `contract.json`, the one evaluator every app decides with, the audit that reads it from git, the probe that knocks on the running app, and the wall a person alone crosses. |
| 521 | 522 | - [](WEBSPEC-PREVIEWS.md) — the website specification run over every preview, report-only. |
src/apps/adapter.ts added +149
| @@ -0,0 +1,149 @@ | ||
| 1 | /** | |
| 2 | * The adapter block of `forge.json`: an app that declares itself a process | |
| 3 | * that syncs one external system into its own data, which the forge can | |
| 4 | * schedule, inspect and audit (docs/ADAPTERS.md). Validated here and nowhere | |
| 5 | * else, failing closed like the rest of the manifest — a block that is | |
| 6 | * present and wrong fails the deploy with the field named. | |
| 7 | * | |
| 8 | * "egress": ["api.ledger.example"], | |
| 9 | * "adapter": { | |
| 10 | * "source": { "name": "ledger", "hosts": ["api.ledger.example"] }, | |
| 11 | * "schedule": { "every": 900 }, | |
| 12 | * "config": { "account": { "type": "string", "default": "main" }, | |
| 13 | * "token": { "type": "secret", "ref": "LEDGER_TOKEN" } } | |
| 14 | * } | |
| 15 | * | |
| 16 | * A secret is a reference to one of the app's environment variables (set, | |
| 17 | * sealed, in the app's environment settings); its value is never in the | |
| 18 | * repository, and a `default` on a secret is refused. Every source host must | |
| 19 | * also be named in the top-level `egress`: the cell's proxy is what refuses | |
| 20 | * every other host, so the declared source and what the process can reach | |
| 21 | * are the same list, checked here. | |
| 22 | */ | |
| 23 | ||
| 24 | import { err, ok, type Result } from "../kernel/result.ts"; | |
| 25 | ||
| 26 | export type AdapterFieldType = "string" | "number" | "boolean" | "secret"; | |
| 27 | ||
| 28 | export interface AdapterField { | |
| 29 | readonly type: AdapterFieldType; | |
| 30 | /** The environment variable a `secret` is read from. Only for secrets. */ | |
| 31 | readonly ref: string | null; | |
| 32 | /** The value when the operator sets none. Never for a secret. */ | |
| 33 | readonly default: string | number | boolean | null; | |
| 34 | } | |
| 35 | ||
| 36 | export interface AdapterManifest { | |
| 37 | readonly source: { readonly name: string; readonly hosts: readonly string[] }; | |
| 38 | /** Seconds between pulls. */ | |
| 39 | readonly every: number; | |
| 40 | readonly config: Readonly<Record<string, AdapterField>>; | |
| 41 | } | |
| 42 | ||
| 43 | export const EVERY_MIN = 10; | |
| 44 | export const EVERY_MAX = 7 * 24 * 3600; | |
| 45 | const HOSTS_MAX = 8; | |
| 46 | const FIELDS_MAX = 32; | |
| 47 | const STRING_MAX = 1_000; | |
| 48 | const NAME = /^[a-z][a-z0-9_-]{0,63}$/; | |
| 49 | const FIELD = /^[a-z][a-z0-9_]{0,63}$/; | |
| 50 | const REF = /^[A-Z][A-Z0-9_]{0,127}$/; | |
| 51 | /** A bare DNS name: labels of letters, digits, hyphens; no port, scheme or wildcard. */ | |
| 52 | const HOST = | |
| 53 | /^(?=.{1,253}$)[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?(\.[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)*$/; | |
| 54 | const TYPES: readonly AdapterFieldType[] = ["string", "number", "boolean", "secret"]; | |
| 55 | ||
| 56 | type Json = Record<string, unknown>; | |
| 57 | const isObject = (v: unknown): v is Json => | |
| 58 | typeof v === "object" && v !== null && !Array.isArray(v); | |
| 59 | const bad = (message: string): Result<never, string> => | |
| 60 | err(`forge.json: adapter${message}`); | |
| 61 | ||
| 62 | /** | |
| 63 | * The block `forge.json` carries, `null` when it declares none. `egress` is | |
| 64 | * the manifest's already-validated top-level list: each source host must be | |
| 65 | * one of its entries, exactly. | |
| 66 | */ | |
| 67 | export function parseAdapter( | |
| 68 | value: unknown, | |
| 69 | egress: readonly string[], | |
| 70 | ): Result<AdapterManifest | null, string> { | |
| 71 | if (value === undefined) return ok(null); | |
| 72 | if (!isObject(value)) return bad(" must be an object"); | |
| 73 | for (const key of Object.keys(value)) | |
| 74 | if (!["source", "schedule", "config"].includes(key)) | |
| 75 | return bad(`.${key.slice(0, 40)} is not a known field`); | |
| 76 | ||
| 77 | const source = value["source"]; | |
| 78 | if (!isObject(source)) return bad(".source is required and must be an object"); | |
| 79 | const name = source["name"]; | |
| 80 | if (typeof name !== "string" || !NAME.test(name)) | |
| 81 | return bad(".source.name must be lowercase letters, digits, - or _"); | |
| 82 | const hosts = source["hosts"]; | |
| 83 | if (!Array.isArray(hosts) || hosts.length < 1 || hosts.length > HOSTS_MAX) | |
| 84 | return bad(`.source.hosts must list 1–${HOSTS_MAX} host names`); | |
| 85 | const seen: string[] = []; | |
| 86 | for (const host of hosts) { | |
| 87 | if (typeof host !== "string" || !HOST.test(host)) | |
| 88 | return bad( | |
| 89 | ".source.hosts entries must be bare lowercase host names (no scheme, port, path or wildcard)", | |
| 90 | ); | |
| 91 | if (!egress.includes(host)) | |
| 92 | return bad( | |
| 93 | `.source.hosts: ${host} must also be named in the top-level egress, which is what the app may reach`, | |
| 94 | ); | |
| 95 | if (!seen.includes(host)) seen.push(host); | |
| 96 | } | |
| 97 | ||
| 98 | const schedule = value["schedule"]; | |
| 99 | if (!isObject(schedule)) return bad(".schedule is required and must be an object"); | |
| 100 | const every = schedule["every"]; | |
| 101 | if ( | |
| 102 | typeof every !== "number" || | |
| 103 | !Number.isInteger(every) || | |
| 104 | every < EVERY_MIN || | |
| 105 | every > EVERY_MAX | |
| 106 | ) | |
| 107 | return bad(`.schedule.every must be a whole number of seconds, ${EVERY_MIN}–${EVERY_MAX}`); | |
| 108 | ||
| 109 | const rawConfig = value["config"] ?? {}; | |
| 110 | if (!isObject(rawConfig)) return bad(".config must be an object"); | |
| 111 | const names = Object.keys(rawConfig); | |
| 112 | if (names.length > FIELDS_MAX) return bad(`.config may name at most ${FIELDS_MAX} fields`); | |
| 113 | const config: Record<string, AdapterField> = {}; | |
| 114 | for (const field of names) { | |
| 115 | if (!FIELD.test(field)) return bad(`.config.${field.slice(0, 40)} is not a valid field name`); | |
| 116 | const parsed = fieldOf(field, rawConfig[field]); | |
| 117 | if (!parsed.ok) return parsed; | |
| 118 | config[field] = parsed.value; | |
| 119 | } | |
| 120 | return ok({ source: { name, hosts: seen }, every, config }); | |
| 121 | } | |
| 122 | ||
| 123 | function fieldOf(name: string, raw: unknown): Result<AdapterField, string> { | |
| 124 | const at = `.config.${name}`; | |
| 125 | if (!isObject(raw)) return bad(`${at} must be an object`); | |
| 126 | for (const key of Object.keys(raw)) | |
| 127 | if (!["type", "ref", "default"].includes(key)) | |
| 128 | return bad(`${at}.${key.slice(0, 40)} is not a known field`); | |
| 129 | const type = raw["type"]; | |
| 130 | if (typeof type !== "string" || !TYPES.includes(type as AdapterFieldType)) | |
| 131 | return bad(`${at}.type must be one of ${TYPES.join(", ")}`); | |
| 132 | if (type === "secret") { | |
| 133 | if (raw["default"] !== undefined) | |
| 134 | return bad(`${at} is a secret: it takes a ref, never a value`); | |
| 135 | const ref = raw["ref"]; | |
| 136 | if (typeof ref !== "string" || !REF.test(ref)) | |
| 137 | return bad(`${at}.ref must name an environment variable, like MY_TOKEN`); | |
| 138 | return ok({ type, ref, default: null }); | |
| 139 | } | |
| 140 | if (raw["ref"] !== undefined) return bad(`${at}.ref is only for secrets`); | |
| 141 | const def = raw["default"]; | |
| 142 | if (def === undefined) return ok({ type: type as AdapterFieldType, ref: null, default: null }); | |
| 143 | const matches = | |
| 144 | (type === "string" && typeof def === "string" && def.length <= STRING_MAX) || | |
| 145 | (type === "number" && typeof def === "number" && Number.isFinite(def)) || | |
| 146 | (type === "boolean" && typeof def === "boolean"); | |
| 147 | if (!matches) return bad(`${at}.default must be a ${type}`); | |
| 148 | return ok({ type: type as AdapterFieldType, ref: null, default: def as string | number | boolean }); | |
| 149 | } |
src/apps/manifest.ts +9
| @@ -16,6 +16,7 @@ import { parseService, type ServiceSpec } from "./service.ts"; | ||
| 16 | 16 | import type { DeclaredLimits } from "./contain.ts"; |
| 17 | 17 | import { isPublicAddress } from "./egress.ts"; |
| 18 | 18 | import { isLaunch, LAUNCHES, type Launch } from "../git/builds.ts"; |
| 19 | import { parseAdapter, type AdapterManifest } from "./adapter.ts"; | |
| 19 | 20 | |
| 20 | 21 | export interface AppManifest { |
| 21 | 22 | /** The command that serves on `$PORT`. Run through `sh -c`, `exec`-ed. */ |
| @@ -70,6 +71,11 @@ export interface AppManifest { | ||
| 70 | 71 | * out of the app's HTML pages. Unset is on. |
| 71 | 72 | */ |
| 72 | 73 | readonly toolbar?: boolean | undefined; |
| 74 | /** | |
| 75 | * `adapter` (top level of `forge.json`): set when the app declares itself | |
| 76 | * an adapter syncing one external source (`apps/adapter.ts`). Unset is none. | |
| 77 | */ | |
| 78 | readonly adapter?: AdapterManifest | undefined; | |
| 73 | 79 | } |
| 74 | 80 | |
| 75 | 81 | export type ManifestError = |
| @@ -163,6 +169,8 @@ export function parseManifest( | ||
| 163 | 169 | return invalid(`forge.json: app.launch must be one of ${LAUNCHES.map((l) => `"${l}"`).join(", ")}`); |
| 164 | 170 | const egress = egressField(parsed.value); |
| 165 | 171 | if (!egress.ok) return egress; |
| 172 | const adapter = parseAdapter(parsed.value["adapter"], egress.value); | |
| 173 | if (!adapter.ok) return invalid(adapter.error); | |
| 166 | 174 | const limits = limitsField(app, "app"); |
| 167 | 175 | if (!limits.ok) return limits; |
| 168 | 176 | const toolbar = parsed.value["toolbar"]; |
| @@ -200,6 +208,7 @@ export function parseManifest( | ||
| 200 | 208 | egress: egress.value, |
| 201 | 209 | limits: limits.value, |
| 202 | 210 | toolbar: toolbar ?? true, |
| 211 | ...(adapter.value === null ? {} : { adapter: adapter.value }), | |
| 203 | 212 | }); |
| 204 | 213 | } |
| 205 | 214 |
test/adapter.test.ts added +83
| @@ -0,0 +1,83 @@ | ||
| 1 | /** The `adapter` block of forge.json: validated at the boundary, in one module (`apps/adapter.ts`). */ | |
| 2 | ||
| 3 | import { describe, expect, test } from "bun:test"; | |
| 4 | import { parseAdapter } from "../src/apps/adapter.ts"; | |
| 5 | import { parseManifest } from "../src/apps/manifest.ts"; | |
| 6 | ||
| 7 | const HOSTS = ["api.ledger.example"]; | |
| 8 | const good = { | |
| 9 | source: { name: "ledger", hosts: HOSTS }, | |
| 10 | schedule: { every: 900 }, | |
| 11 | config: { | |
| 12 | account: { type: "string", default: "main" }, | |
| 13 | batch: { type: "number", default: 50 }, | |
| 14 | dry: { type: "boolean" }, | |
| 15 | token: { type: "secret", ref: "LEDGER_TOKEN" }, | |
| 16 | }, | |
| 17 | }; | |
| 18 | ||
| 19 | const message = (block: unknown, egress: readonly string[] = HOSTS): string => { | |
| 20 | const r = parseAdapter(block, egress); | |
| 21 | if (r.ok) throw new Error("expected a refusal"); | |
| 22 | return r.error; | |
| 23 | }; | |
| 24 | ||
| 25 | describe("parseAdapter", () => { | |
| 26 | test("absent is no adapter; a good block parses", () => { | |
| 27 | expect(parseAdapter(undefined, [])).toEqual({ ok: true, value: null }); | |
| 28 | const r = parseAdapter(good, HOSTS); | |
| 29 | expect(r.ok && r.value).toMatchObject({ source: { name: "ledger", hosts: HOSTS }, every: 900 }); | |
| 30 | expect(r.ok && r.value?.config["token"]).toEqual({ type: "secret", ref: "LEDGER_TOKEN", default: null }); | |
| 31 | expect(r.ok && r.value?.config["batch"]).toEqual({ type: "number", ref: null, default: 50 }); | |
| 32 | }); | |
| 33 | ||
| 34 | test("a secret is a reference, never a value", () => { | |
| 35 | expect(message({ ...good, config: { token: { type: "secret", ref: "T", default: "hunter2" } } })).toContain("never a value"); | |
| 36 | expect(message({ ...good, config: { token: { type: "secret" } } })).toContain("ref"); | |
| 37 | expect(message({ ...good, config: { token: { type: "secret", ref: "lower" } } })).toContain("ref"); | |
| 38 | expect(message({ ...good, config: { a: { type: "string", ref: "X" } } })).toContain("only for secrets"); | |
| 39 | }); | |
| 40 | ||
| 41 | test("source hosts are bare names, and each is one the app may reach", () => { | |
| 42 | for (const host of ["https://a.example", "a.example:8080", "*.example", "a.example/x", "", "A.Example", "a b"]) | |
| 43 | expect(message({ ...good, source: { name: "x", hosts: [host] } }, [host])).toContain("hosts"); | |
| 44 | expect(message({ ...good, source: { name: "x", hosts: [] } })).toContain("hosts"); | |
| 45 | expect(message({ ...good, source: { name: "X y", hosts: HOSTS } })).toContain("name"); | |
| 46 | // A host the source names but egress does not is refused: the proxy would refuse it anyway. | |
| 47 | expect(message(good, [])).toContain("egress"); | |
| 48 | expect(message(good, ["other.example"])).toContain("egress"); | |
| 49 | }); | |
| 50 | ||
| 51 | test("schedule and config are bounded and typed", () => { | |
| 52 | for (const every of [0, 9, 1.5, "900", 8 * 24 * 3600, undefined]) | |
| 53 | expect(message({ ...good, schedule: { every } })).toContain("every"); | |
| 54 | expect(message({ ...good, schedule: undefined })).toContain("schedule"); | |
| 55 | expect(message({ ...good, config: { n: { type: "number", default: "1" } } })).toContain("number"); | |
| 56 | expect(message({ ...good, config: { n: { type: "date" } } })).toContain("type"); | |
| 57 | expect(message({ ...good, config: { "Bad Name": { type: "string" } } })).toContain("field name"); | |
| 58 | expect(message({ ...good, config: { n: { type: "string", extra: 1 } } })).toContain("not a known field"); | |
| 59 | expect(message({ ...good, extra: 1 })).toContain("not a known field"); | |
| 60 | expect(message("x")).toContain("object"); | |
| 61 | }); | |
| 62 | }); | |
| 63 | ||
| 64 | describe("forge.json carries it", () => { | |
| 65 | const manifest = (extra: object) => | |
| 66 | parseManifest(JSON.stringify({ app: { run: "bun run src/server.ts" }, ...extra }), undefined); | |
| 67 | ||
| 68 | test("a declared adapter lands on the manifest; none leaves it unset", () => { | |
| 69 | const r = manifest({ egress: HOSTS, adapter: good }); | |
| 70 | expect(r.ok && r.value.adapter?.source.name).toBe("ledger"); | |
| 71 | expect(r.ok && r.value.adapter?.every).toBe(900); | |
| 72 | const none = manifest({}); | |
| 73 | expect(none.ok && none.value.adapter).toBeUndefined(); | |
| 74 | }); | |
| 75 | ||
| 76 | test("a wrong block fails the deploy with the field named", () => { | |
| 77 | const r = manifest({ adapter: good }); | |
| 78 | expect(r.ok).toBe(false); | |
| 79 | expect(!r.ok && r.error.kind === "invalid" && r.error.message).toContain("forge.json: adapter.source.hosts"); | |
| 80 | const secret = manifest({ egress: HOSTS, adapter: { ...good, config: { t: { type: "secret", ref: "T", default: "v" } } } }); | |
| 81 | expect(!secret.ok && secret.error.kind === "invalid" && secret.error.message).toContain("never a value"); | |
| 82 | }); | |
| 83 | }); |
new above
offline