Public: anyone can read this; the crew can change it.

Adapter app shape: sync an external system into the forge, with a contract the forge can audit #10034

Closed opened this
Closed without merging

scout/forge:scout/437-adapter-app-shape-sync-an-external-syste → main 8 commits fork

  • Closed without merging. Nothing was applied to main.

Review

Sign in to review this change.
6 files changed +300 −0
docs/ADAPTERS.md added +53
@@ -0,0 +1,53 @@
1 # Adapters
2
3 An adapter is an app that syncs one external system into its own data
4 under `FORGE_APP_DATA`. It declares itself in `forge.json`, so the forge can
5 schedule, inspect and audit it the same way for every source, and so an
6 agent can write one by filling in a template (#437).
7
8 ## The block
9
10 ```json
11 {
12 "app": { "run": "bun run src/server.ts" },
13 "egress": ["api.ledger.example"],
14 "adapter": {
15 "source": { "name": "ledger", "hosts": ["api.ledger.example"] },
16 "schedule": { "every": 900 },
17 "config": {
18 "account": { "type": "string", "default": "main" },
19 "token": { "type": "secret", "ref": "LEDGER_TOKEN" }
20 }
21 }
22 }
23 ```
24
25 `src/apps/adapter.ts` validates it, and nothing else does. A block that is
26 present and wrong fails the deploy, and the error names the field.
27
28 - `source.name`: lowercase letters, digits, `-` or `_`.
29 - `source.hosts`: 1 to 8 bare host names, with no scheme, port, path or
30 wildcard. **Each one must also be in the top-level `egress`.** The cell's
31 proxy (`apps/egress.ts`) refuses every host not in `egress`, so this rule
32 means a source host the app cannot reach is refused at deploy time.
33 Unknown hosts are refused on every request.
34 - `schedule.every`: whole seconds between pulls, from 10 to 604800 (one
35 week).
36 - `config`: at most 32 named fields, each `string`, `number`, `boolean` or
37 `secret`. A plain field may set a `default` of its own type. A `secret`
38 takes `ref`, the name of an environment variable that is set (sealed) in
39 the app's environment settings. A secret never takes a value: a
40 `default` on a secret is refused, so a secret cannot end up in the
41 repository.
42
43 ## The contract (to come)
44
45 The adapter answers `POST /pull` with `{pulled, skipped, cursor, errors}`.
46 The pull is idempotent: the same cursor gives the same result, and a
47 replay writes nothing new. The adapter keeps its cursor and records under
48 `FORGE_APP_DATA`. The forge will call `/pull` every `schedule.every`
49 seconds and record each call as a run (counts, cursor, duration). The app
50 page will show the last pull, the next pull and the cursor. A template
51 adapter will come with its own tests. None of this exists yet: today the
52 forge validates the block and enforces the host rule, and nothing calls
53 `/pull`.
docs/APPS.md +5
@@ -564,6 +564,11 @@ are capped by the machine (`FORGE_CELL_MEMORY_MAX_MIB`, default 4096; the
564564 number of cores; `FORGE_CELL_DISK_MAX_MIB`, default 16384). A cell's `/tmp`
565565 holds at most 256 MiB.
566566
567 `adapter`, at the top level, declares the app an adapter that syncs one
568 external system into its own data: its source (each host also named in
569 `egress`), how often it pulls, and its typed config, secrets by reference.
570 [ADAPTERS.md](ADAPTERS.md) has the block and the contract.
571
567572 Memory is held twice. The watch reads every cell's resident memory once a
568573 second and stops one over its limit. Where the forge holds a delegated
569574 cgroup v2 (`apps/cgroup.ts`), the kernel holds it as well: each cell runs
docs/REFERENCE.md +1
@@ -516,6 +516,7 @@ This page is the front door. Everything else is a file of its own under
516516
517517 **Apps and the crew**
518518
519 - [`docs/ADAPTERS.md`](ADAPTERS.md) — the `adapter` block of `forge.json`: an app that syncs one external system, its source hosts, schedule and typed config.
519520 - [`docs/APPS.md`](APPS.md) — the contract for apps, previews and runs: making an app, the vocabulary, and the rules that are mechanical rather than advisory.
520521 - [`docs/CONTRACT.md`](CONTRACT.md) — who may do what, as a table: an app's `contract.json`, the one evaluator every app decides with, the audit that reads it from git, the probe that knocks on the running app, and the wall a person alone crosses.
521522 - [`docs/WEBSPEC-PREVIEWS.md`](WEBSPEC-PREVIEWS.md) — the website specification run over every preview, report-only.
src/apps/adapter.ts added +149
@@ -0,0 +1,149 @@
1 /**
2 * The adapter block of `forge.json`: an app that declares itself a process
3 * that syncs one external system into its own data, which the forge can
4 * schedule, inspect and audit (docs/ADAPTERS.md). Validated here and nowhere
5 * else, failing closed like the rest of the manifest — a block that is
6 * present and wrong fails the deploy with the field named.
7 *
8 * "egress": ["api.ledger.example"],
9 * "adapter": {
10 * "source": { "name": "ledger", "hosts": ["api.ledger.example"] },
11 * "schedule": { "every": 900 },
12 * "config": { "account": { "type": "string", "default": "main" },
13 * "token": { "type": "secret", "ref": "LEDGER_TOKEN" } }
14 * }
15 *
16 * A secret is a reference to one of the app's environment variables (set,
17 * sealed, in the app's environment settings); its value is never in the
18 * repository, and a `default` on a secret is refused. Every source host must
19 * also be named in the top-level `egress`: the cell's proxy is what refuses
20 * every other host, so the declared source and what the process can reach
21 * are the same list, checked here.
22 */
23
24 import { err, ok, type Result } from "../kernel/result.ts";
25
26 export type AdapterFieldType = "string" | "number" | "boolean" | "secret";
27
28 export interface AdapterField {
29 readonly type: AdapterFieldType;
30 /** The environment variable a `secret` is read from. Only for secrets. */
31 readonly ref: string | null;
32 /** The value when the operator sets none. Never for a secret. */
33 readonly default: string | number | boolean | null;
34 }
35
36 export interface AdapterManifest {
37 readonly source: { readonly name: string; readonly hosts: readonly string[] };
38 /** Seconds between pulls. */
39 readonly every: number;
40 readonly config: Readonly<Record<string, AdapterField>>;
41 }
42
43 export const EVERY_MIN = 10;
44 export const EVERY_MAX = 7 * 24 * 3600;
45 const HOSTS_MAX = 8;
46 const FIELDS_MAX = 32;
47 const STRING_MAX = 1_000;
48 const NAME = /^[a-z][a-z0-9_-]{0,63}$/;
49 const FIELD = /^[a-z][a-z0-9_]{0,63}$/;
50 const REF = /^[A-Z][A-Z0-9_]{0,127}$/;
51 /** A bare DNS name: labels of letters, digits, hyphens; no port, scheme or wildcard. */
52 const HOST =
53 /^(?=.{1,253}$)[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?(\.[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)*$/;
54 const TYPES: readonly AdapterFieldType[] = ["string", "number", "boolean", "secret"];
55
56 type Json = Record<string, unknown>;
57 const isObject = (v: unknown): v is Json =>
58 typeof v === "object" && v !== null && !Array.isArray(v);
59 const bad = (message: string): Result<never, string> =>
60 err(`forge.json: adapter${message}`);
61
62 /**
63 * The block `forge.json` carries, `null` when it declares none. `egress` is
64 * the manifest's already-validated top-level list: each source host must be
65 * one of its entries, exactly.
66 */
67 export function parseAdapter(
68 value: unknown,
69 egress: readonly string[],
70 ): Result<AdapterManifest | null, string> {
71 if (value === undefined) return ok(null);
72 if (!isObject(value)) return bad(" must be an object");
73 for (const key of Object.keys(value))
74 if (!["source", "schedule", "config"].includes(key))
75 return bad(`.${key.slice(0, 40)} is not a known field`);
76
77 const source = value["source"];
78 if (!isObject(source)) return bad(".source is required and must be an object");
79 const name = source["name"];
80 if (typeof name !== "string" || !NAME.test(name))
81 return bad(".source.name must be lowercase letters, digits, - or _");
82 const hosts = source["hosts"];
83 if (!Array.isArray(hosts) || hosts.length < 1 || hosts.length > HOSTS_MAX)
84 return bad(`.source.hosts must list 1–${HOSTS_MAX} host names`);
85 const seen: string[] = [];
86 for (const host of hosts) {
87 if (typeof host !== "string" || !HOST.test(host))
88 return bad(
89 ".source.hosts entries must be bare lowercase host names (no scheme, port, path or wildcard)",
90 );
91 if (!egress.includes(host))
92 return bad(
93 `.source.hosts: ${host} must also be named in the top-level egress, which is what the app may reach`,
94 );
95 if (!seen.includes(host)) seen.push(host);
96 }
97
98 const schedule = value["schedule"];
99 if (!isObject(schedule)) return bad(".schedule is required and must be an object");
100 const every = schedule["every"];
101 if (
102 typeof every !== "number" ||
103 !Number.isInteger(every) ||
104 every < EVERY_MIN ||
105 every > EVERY_MAX
106 )
107 return bad(`.schedule.every must be a whole number of seconds, ${EVERY_MIN}–${EVERY_MAX}`);
108
109 const rawConfig = value["config"] ?? {};
110 if (!isObject(rawConfig)) return bad(".config must be an object");
111 const names = Object.keys(rawConfig);
112 if (names.length > FIELDS_MAX) return bad(`.config may name at most ${FIELDS_MAX} fields`);
113 const config: Record<string, AdapterField> = {};
114 for (const field of names) {
115 if (!FIELD.test(field)) return bad(`.config.${field.slice(0, 40)} is not a valid field name`);
116 const parsed = fieldOf(field, rawConfig[field]);
117 if (!parsed.ok) return parsed;
118 config[field] = parsed.value;
119 }
120 return ok({ source: { name, hosts: seen }, every, config });
121 }
122
123 function fieldOf(name: string, raw: unknown): Result<AdapterField, string> {
124 const at = `.config.${name}`;
125 if (!isObject(raw)) return bad(`${at} must be an object`);
126 for (const key of Object.keys(raw))
127 if (!["type", "ref", "default"].includes(key))
128 return bad(`${at}.${key.slice(0, 40)} is not a known field`);
129 const type = raw["type"];
130 if (typeof type !== "string" || !TYPES.includes(type as AdapterFieldType))
131 return bad(`${at}.type must be one of ${TYPES.join(", ")}`);
132 if (type === "secret") {
133 if (raw["default"] !== undefined)
134 return bad(`${at} is a secret: it takes a ref, never a value`);
135 const ref = raw["ref"];
136 if (typeof ref !== "string" || !REF.test(ref))
137 return bad(`${at}.ref must name an environment variable, like MY_TOKEN`);
138 return ok({ type, ref, default: null });
139 }
140 if (raw["ref"] !== undefined) return bad(`${at}.ref is only for secrets`);
141 const def = raw["default"];
142 if (def === undefined) return ok({ type: type as AdapterFieldType, ref: null, default: null });
143 const matches =
144 (type === "string" && typeof def === "string" && def.length <= STRING_MAX) ||
145 (type === "number" && typeof def === "number" && Number.isFinite(def)) ||
146 (type === "boolean" && typeof def === "boolean");
147 if (!matches) return bad(`${at}.default must be a ${type}`);
148 return ok({ type: type as AdapterFieldType, ref: null, default: def as string | number | boolean });
149 }
src/apps/manifest.ts +9
@@ -16,6 +16,7 @@ import { parseService, type ServiceSpec } from "./service.ts";
1616 import type { DeclaredLimits } from "./contain.ts";
1717 import { isPublicAddress } from "./egress.ts";
1818 import { isLaunch, LAUNCHES, type Launch } from "../git/builds.ts";
19 import { parseAdapter, type AdapterManifest } from "./adapter.ts";
1920
2021 export interface AppManifest {
2122 /** The command that serves on `$PORT`. Run through `sh -c`, `exec`-ed. */
@@ -70,6 +71,11 @@ export interface AppManifest {
7071 * out of the app's HTML pages. Unset is on.
7172 */
7273 readonly toolbar?: boolean | undefined;
74 /**
75 * `adapter` (top level of `forge.json`): set when the app declares itself
76 * an adapter syncing one external source (`apps/adapter.ts`). Unset is none.
77 */
78 readonly adapter?: AdapterManifest | undefined;
7379 }
7480
7581 export type ManifestError =
@@ -163,6 +169,8 @@ export function parseManifest(
163169 return invalid(`forge.json: app.launch must be one of ${LAUNCHES.map((l) => `"${l}"`).join(", ")}`);
164170 const egress = egressField(parsed.value);
165171 if (!egress.ok) return egress;
172 const adapter = parseAdapter(parsed.value["adapter"], egress.value);
173 if (!adapter.ok) return invalid(adapter.error);
166174 const limits = limitsField(app, "app");
167175 if (!limits.ok) return limits;
168176 const toolbar = parsed.value["toolbar"];
@@ -200,6 +208,7 @@ export function parseManifest(
200208 egress: egress.value,
201209 limits: limits.value,
202210 toolbar: toolbar ?? true,
211 ...(adapter.value === null ? {} : { adapter: adapter.value }),
203212 });
204213 }
205214
test/adapter.test.ts added +83
@@ -0,0 +1,83 @@
1 /** The `adapter` block of forge.json: validated at the boundary, in one module (`apps/adapter.ts`). */
2
3 import { describe, expect, test } from "bun:test";
4 import { parseAdapter } from "../src/apps/adapter.ts";
5 import { parseManifest } from "../src/apps/manifest.ts";
6
7 const HOSTS = ["api.ledger.example"];
8 const good = {
9 source: { name: "ledger", hosts: HOSTS },
10 schedule: { every: 900 },
11 config: {
12 account: { type: "string", default: "main" },
13 batch: { type: "number", default: 50 },
14 dry: { type: "boolean" },
15 token: { type: "secret", ref: "LEDGER_TOKEN" },
16 },
17 };
18
19 const message = (block: unknown, egress: readonly string[] = HOSTS): string => {
20 const r = parseAdapter(block, egress);
21 if (r.ok) throw new Error("expected a refusal");
22 return r.error;
23 };
24
25 describe("parseAdapter", () => {
26 test("absent is no adapter; a good block parses", () => {
27 expect(parseAdapter(undefined, [])).toEqual({ ok: true, value: null });
28 const r = parseAdapter(good, HOSTS);
29 expect(r.ok && r.value).toMatchObject({ source: { name: "ledger", hosts: HOSTS }, every: 900 });
30 expect(r.ok && r.value?.config["token"]).toEqual({ type: "secret", ref: "LEDGER_TOKEN", default: null });
31 expect(r.ok && r.value?.config["batch"]).toEqual({ type: "number", ref: null, default: 50 });
32 });
33
34 test("a secret is a reference, never a value", () => {
35 expect(message({ ...good, config: { token: { type: "secret", ref: "T", default: "hunter2" } } })).toContain("never a value");
36 expect(message({ ...good, config: { token: { type: "secret" } } })).toContain("ref");
37 expect(message({ ...good, config: { token: { type: "secret", ref: "lower" } } })).toContain("ref");
38 expect(message({ ...good, config: { a: { type: "string", ref: "X" } } })).toContain("only for secrets");
39 });
40
41 test("source hosts are bare names, and each is one the app may reach", () => {
42 for (const host of ["https://a.example", "a.example:8080", "*.example", "a.example/x", "", "A.Example", "a b"])
43 expect(message({ ...good, source: { name: "x", hosts: [host] } }, [host])).toContain("hosts");
44 expect(message({ ...good, source: { name: "x", hosts: [] } })).toContain("hosts");
45 expect(message({ ...good, source: { name: "X y", hosts: HOSTS } })).toContain("name");
46 // A host the source names but egress does not is refused: the proxy would refuse it anyway.
47 expect(message(good, [])).toContain("egress");
48 expect(message(good, ["other.example"])).toContain("egress");
49 });
50
51 test("schedule and config are bounded and typed", () => {
52 for (const every of [0, 9, 1.5, "900", 8 * 24 * 3600, undefined])
53 expect(message({ ...good, schedule: { every } })).toContain("every");
54 expect(message({ ...good, schedule: undefined })).toContain("schedule");
55 expect(message({ ...good, config: { n: { type: "number", default: "1" } } })).toContain("number");
56 expect(message({ ...good, config: { n: { type: "date" } } })).toContain("type");
57 expect(message({ ...good, config: { "Bad Name": { type: "string" } } })).toContain("field name");
58 expect(message({ ...good, config: { n: { type: "string", extra: 1 } } })).toContain("not a known field");
59 expect(message({ ...good, extra: 1 })).toContain("not a known field");
60 expect(message("x")).toContain("object");
61 });
62 });
63
64 describe("forge.json carries it", () => {
65 const manifest = (extra: object) =>
66 parseManifest(JSON.stringify({ app: { run: "bun run src/server.ts" }, ...extra }), undefined);
67
68 test("a declared adapter lands on the manifest; none leaves it unset", () => {
69 const r = manifest({ egress: HOSTS, adapter: good });
70 expect(r.ok && r.value.adapter?.source.name).toBe("ledger");
71 expect(r.ok && r.value.adapter?.every).toBe(900);
72 const none = manifest({});
73 expect(none.ok && none.value.adapter).toBeUndefined();
74 });
75
76 test("a wrong block fails the deploy with the field named", () => {
77 const r = manifest({ adapter: good });
78 expect(r.ok).toBe(false);
79 expect(!r.ok && r.error.kind === "invalid" && r.error.message).toContain("forge.json: adapter.source.hosts");
80 const secret = manifest({ egress: HOSTS, adapter: { ...good, config: { t: { type: "secret", ref: "T", default: "v" } } } });
81 expect(!secret.ok && secret.error.kind === "invalid" && secret.error.message).toContain("never a value");
82 });
83 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs