Public: anyone can read this; the crew can change it.

/oembed?url= answers a public app or repository link with a rich card for other tools #15079

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/14455-oembed-url-answers-a-public-app-or-repos → main 15 commits fork 3 threads · 3 unresolved

  • Merged. scout/14455-oembed-url-answers-a-public-app-or-repos was merged into main as b707483.

Review

Sign in to review this change.
12 files changed +177 −2
docs/STANDARDS.md +1
@@ -12,6 +12,7 @@ its own mark. Nothing here is configured, and nothing here is optional.
1212 | `/.well-known/change-password` | where a password manager sends a person | `/settings/password` |
1313 | `/robots.txt` | crawlers stay out of accounts, search and the API; the sitemap | — |
1414 | `/sitemap.xml` | the front page and every public repository, never a private one | `listReposVisibleTo(db, nobody)` |
15 | `/oembed?url=<page>&format=json` | the oEmbed 1.0 `link` card of a public repository or app page (title, author, provider, thumbnail); 404 alike for private and unknown, 400 for another host, 501 for `xml`; public pages advertise it with `<link rel="alternate" type="application/json+oembed">` | `getRepo` + `canRead(db, repo, undefined)`; the url is parsed, never fetched |
1516 | `/manifest.webmanifest` | the forge as an installable app | — |
1617 | `/llms.txt` | the forge for an agent (llmstxt.org): how to report, how to offer a change, the API and MCP, the public repositories | `listReposVisibleTo(db, nobody)`, the report limits |
1718 | `/icons/forge-<size>.png` | the mark at 192, 512, 512-maskable and 180 (iOS), painted at boot | `design/mark.ts`, the favicon's two colours |
src/contract/openapi/routes/getOembed.ts added +3
@@ -0,0 +1,3 @@
1 import type { Route } from "../shared.ts";
2
3 export const route: Route = { method: "get", path: "/oembed", summary: "The oEmbed 1.0 link card of a public repository or app page", detail: "oEmbed 1.0 `link` answer for the page `url` (a link on this forge; only its first two path segments, owner and name, are read, and it is never fetched) as { version, type, title, author_name, author_url, provider_name, provider_url, cache_age, thumbnail_url }. `format=json` (the default) is the only format: `xml` is a 501. A private repository and an unknown one are the same 404; a url on another host, or none, is a 400. Counted by the anonymous per-source limit; cacheable for five minutes. Public pages advertise it as `<link rel=\"alternate\" type=\"application/json+oembed\">`", auth: false, role: "anonymous", query: { url: "string", format: "string" }, errorStatus: { 400: "url is missing, too long, or on another host" }, refusals: { bad_request: "400, url is missing, too long, or on another host", not_found: "404: the page names no repository you can read, anonymously", not_implemented: "501, format is not json", rate_limited: "429, too many reads from one source without a token" } };
src/contract/openapi/routes/index.ts +1
@@ -262,6 +262,7 @@ export { route as getApiV1Version } from "./getApiV1Version.ts";
262262 export { route as getContractOpenapiJson } from "./getContractOpenapiJson.ts";
263263 export { route as getHealth } from "./getHealth.ts";
264264 export { route as getMetrics } from "./getMetrics.ts";
265 export { route as getOembed } from "./getOembed.ts";
265266 export { route as getOwnerNamePullsNumberCheckTxt } from "./getOwnerNamePullsNumberCheckTxt.ts";
266267 export { route as patchApiV1AdminSettings } from "./patchApiV1AdminSettings.ts";
267268 export { route as patchApiV1AgentsUsername } from "./patchApiV1AgentsUsername.ts";
src/contract/tags.ts +1
@@ -6,6 +6,7 @@
66
77 /** A path segment that names a resource under another tag. */
88 const ALIAS: Readonly<Record<string, string>> = {
9 oembed: "meta",
910 app: "apps",
1011 timers: "apps",
1112 schedules: "apps",
src/core/reserved.ts +1
@@ -129,6 +129,7 @@ export const SYSTEM_NAMES: readonly string[] = [
129129 "signin",
130130 "signup",
131131 "sitemap.xml",
132 "oembed",

Keep the list in its existing order: place "oembed" alphabetically, not after sitemap.xml.

132133 "releases",
133134 "releases.json",
134135 "templates.json",
src/http/marks/oembed.json added +4
@@ -0,0 +1,4 @@
1 {
2 "head": "/oembed",
3 "mark": "open"
4 }
src/http/server.ts +3
@@ -231,6 +231,7 @@ import {
231231 agentCardJson,
232232 forgeJson,
233233 sitemapXml,
234 oembed,
234235 webManifest,
235236 } from "../web/standards.ts";
236237 import {
@@ -630,6 +631,7 @@ import { isProbe } from "../kernel/probe.ts";
630631 import { strangerCode } from "./stranger_code.ts";
631632 import { sourceArchive } from "./archive.ts";
632633 import { agentsMd } from "./agents_md.ts";
634
633635 import { takeStrangerRead } from "./stranger_reads.ts";

Stray blank line added in the import block; remove it.

634636 import { parseCidrs } from "../kernel/cidr.ts";
635637 import {
@@ -898,6 +900,7 @@ export function buildRouter(): Router<Handler> {
898900 router.add("GET", "/agents", agentsPage);
899901 router.add("GET", "/.well-known/change-password", changePassword);
900902 router.add("GET", "/robots.txt", robotsTxt);
903 router.add("GET", "/oembed", oembed);
901904 router.add("GET", "/releases.json", releasesJson);
902905 router.add("GET", "/sitemap.xml", sitemapXml);
903906 router.add("GET", "/manifest.webmanifest", webManifest);
src/web/standards.ts +78 −1
@@ -13,6 +13,7 @@
1313 * /.well-known/change-password where a password manager sends someone
1414 * /robots.txt what a crawler may index, and the sitemap
1515 * /sitemap.xml every public repository
16 * /oembed?url= the oEmbed link card of a public repository page
1617 * /releases.json the releases this forge has shipped, newest
1718 * first, for a child forge to read
1819 * /manifest.webmanifest the forge as an installable app
@@ -47,7 +48,8 @@ import { llmsRepos, newestLanding, sitemapRepos, type LlmsListing } from "../cor
4748 import { canRead, getRepo, type Repo } from "../core/repos.ts";
4849 import { publicForgeRepo } from "../core/forge_repo.ts";
4950 import type { Ctx } from "../http/context.ts";
50 import { notFound } from "../http/respond.ts";
51 import { apiError, json, notFound } from "../http/respond.ts";
52 import { takeStrangerRead } from "../http/stranger_reads.ts";
5153 import { SHARE_TARGET } from "./share_target.ts";
5254 import { offerTable } from "./offer.ts";
5355 import { getPlans } from "../settings/plans.ts";
@@ -719,6 +721,13 @@ export function pageLinks(ctx: Ctx): PageLink[] {
719721 href: `/${repo.ownerName}/${repo.name}/report`,
720722 title: REPORT_LINK_TITLE,
721723 });
724 // A public page unfurls elsewhere through oEmbed; a private one advertises nothing.
725 links.push({
726 rel: "alternate",
727 href: oembedHref(ctx.url.origin, ctx.url.pathname),
728 title: "oEmbed",
729 type: OEMBED_TYPE,
730 });
722731 // The same door for an agent: the endpoint that takes the JSON.
723732 links.push({
724733 rel: "help",
@@ -746,3 +755,71 @@ export function linkHeader(links: readonly PageLink[]): string {
746755 )
747756 .join(", ");
748757 }
758
759 // ---- /oembed: the oEmbed 1.0 link card of a public repository or app page (never fetches the url)
760
761 export const OEMBED_TYPE = "application/json+oembed";
762 const URL_MAX = 2048;
763
764 /** The `<link>` a public page advertises: where its oEmbed answer is. */
765 export function oembedHref(origin: string, pathname: string): string {
766 return `${origin}/oembed?url=${encodeURIComponent(`${origin}${pathname}`)}&format=json`;
767 }
768
769 /**
770 * `[owner, name]` from a url on `origin`; undefined when it is another host or names no repository.

The comment says undefined for another host, but the function returns "foreign" for that and undefined for no repository. Fix the comment and the return type description.

771 */
772 export function repoOfUrl(raw: string, origin: string): readonly [string, string] | "foreign" | undefined {
773 let u: URL;
774 try {
775 u = new URL(raw);
776 } catch {
777 return "foreign";
778 }
779 if (u.origin !== origin) return "foreign";
780 const parts = u.pathname.split("/").filter((s) => s !== "");
781 if (parts.length < 2) return undefined;
782 try {
783 return [decodeURIComponent(parts[0]!), decodeURIComponent(parts[1]!)];
784 } catch {
785 return undefined;
786 }
787 }
788
789 export function oembed(ctx: Ctx): Response {
790 const verdict = takeStrangerRead(ctx.app.db, ctx.request, false);
791 if (verdict !== undefined && !verdict.allowed)
792 return json(
793 { error: "rate_limited", message: `too many requests: ${verdict.limit} per minute from one source — retry in ${verdict.retryAfter}s` },
794 429,
795 { "retry-after": String(verdict.retryAfter) },
796 );
797 const format = ctx.url.searchParams.get("format") ?? "json";
798 if (format !== "json") return apiError("not_implemented", 501, "only format=json is served");
799 const raw = ctx.url.searchParams.get("url") ?? "";
800 if (raw === "" || raw.length > URL_MAX) return apiError("bad_request", 400, "url must be a link to a page on this forge");
801 const at = repoOfUrl(raw, ctx.url.origin);
802 if (at === "foreign") return apiError("bad_request", 400, "url must be a link to a page on this forge");
803 if (at === undefined) return notFound();
804 const repo = getRepo(ctx.app.db, at[0], at[1]);
805 if (repo === undefined || !canRead(ctx.app.db, repo, undefined)) return notFound();
806 const origin = ctx.url.origin;
807 const slug = `${repo.ownerName}/${repo.name}`;
808 return json(
809 {
810 version: "1.0",
811 type: "link",
812 title: slug,
813 author_name: repo.ownerName,
814 author_url: `${origin}/${encodeURIComponent(repo.ownerName)}`,
815 provider_name: "Open Platform Forge",
816 provider_url: origin,
817 cache_age: 300,
818 thumbnail_url: `${origin}/og/repo/${encodeURIComponent(repo.ownerName)}~${encodeURIComponent(repo.name)}.png`,
819 thumbnail_width: 1200,
820 thumbnail_height: 630,
821 },
822 200,
823 { "cache-control": "public, max-age=300" },
824 );
825 }
test/design-audit.test.ts +1 −1
@@ -17,7 +17,7 @@ const doc = readFileSync("docs/design/audit.md", "utf8");
1717
1818 /** Routes that answer a file, a fragment, a redirect hop or a callback. */
1919 const NOT_PAGES =
20 /^\/(assets\/|\.well-known\/|icons\/|og\/|live\/|__forge\/|go\/|new\/issue$|events$|system\/stream$|talk\/stream$|settings\/teams\/stream$|sw\.js$|offline$|robots\.txt$|favicon\.ico$|apple-touch-icon(-precomposed)?\.png$|sitemap\.xml$|manifest\.webmanifest$|opensearch\.xml$|llms(-full)?\.txt$|openapi\.json$|releases\.json$|templates\.json$|maintenance\.json$|changelog\.atom$|explore\.atom$|feed\.json$|docs(\/[^/]*)?$|audit\.csv$|settings\/traffic\/scrapes\/export$|fleet\/graph\/live$|fleet\/graph\.csv$|forges\/export\.csv$|settings\/origins\/callback$|broker\/origins\/start$|home\/archived\/|settings\/backups\/live$|api\/v1(\/|$))/;
20 /^\/(assets\/|\.well-known\/|icons\/|og\/|live\/|__forge\/|go\/|new\/issue$|events$|system\/stream$|talk\/stream$|settings\/teams\/stream$|sw\.js$|offline$|robots\.txt$|favicon\.ico$|apple-touch-icon(-precomposed)?\.png$|sitemap\.xml$|oembed$|manifest\.webmanifest$|opensearch\.xml$|llms(-full)?\.txt$|openapi\.json$|releases\.json$|templates\.json$|maintenance\.json$|changelog\.atom$|explore\.atom$|feed\.json$|docs(\/[^/]*)?$|audit\.csv$|settings\/traffic\/scrapes\/export$|fleet\/graph\/live$|fleet\/graph\.csv$|forges\/export\.csv$|settings\/origins\/callback$|broker\/origins\/start$|home\/archived\/|settings\/backups\/live$|api\/v1(\/|$))/;
2121
2222 const getRoutes = [
2323 ...server.matchAll(/router\.add\(\s*"GET",\s*"([^"]+)"/g),
test/forge-looks.test.ts +1
@@ -53,6 +53,7 @@ const NOT_A_PAGE: Record<string, string> = {
5353 "/explore.atom": "Atom feed of /explore",
5454 "/feed.json": "JSON Feed of /feed",
5555 "/sitemap.xml": "XML for crawlers",
56 "/oembed": "JSON for other sites to build a link card; covered by test/oembed.test.ts",
5657 "/manifest.webmanifest": "the install manifest, JSON",
5758 "/opensearch.xml": "the browser search description, XML",
5859 "/sw.js": "the service worker script",
test/oembed.test.ts added +82
@@ -0,0 +1,82 @@
1 /** /oembed: a public repository unfurls; a private one and an unknown one are the same 404. */
2 import { afterAll, beforeAll, expect, test } from "bun:test";
3 import { mkdtemp, rm } from "node:fs/promises";
4 import { tmpdir } from "node:os";
5 import { join } from "node:path";
6 import { createApp, type App } from "../src/app.ts";
7 import { createRepo } from "../src/actions/repos.ts";
8 import { createUser } from "../src/core/users.ts";
9 import { serve, type Servers } from "../src/http/server.ts";
10
11 let dataDir: string;
12 let app: App;
13 let servers: Servers;
14
15 const ask = (url: string, extra = "&format=json") => fetch(`${servers.url}/oembed?url=${encodeURIComponent(url)}${extra}`);
16
17 beforeAll(async () => {
18 dataDir = await mkdtemp(join(tmpdir(), "forge-oembed-"));
19 app = createApp({ dataDir, port: 0 });
20 servers = await serve(app);
21 const u = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
22 if (!u.ok) throw new Error(u.error);
23 for (const [name, visibility] of [["open", "public"], ["secret", "private"]] as const) {
24 const r = await createRepo(app, u.value, { name, visibility });
25 if (!r.ok) throw new Error(String(r.error));
26 }
27 });
28
29 afterAll(async () => {
30 servers.stop();
31 app.close();
32 await rm(dataDir, { recursive: true, force: true });
33 });
34
35 test("a public repository answers the oEmbed link type, with the security floor", async () => {
36 const res = await ask(`${servers.url}/alice/open`);
37 expect(res.status).toBe(200);
38 expect(res.headers.get("content-type")).toContain("application/json");
39 expect(res.headers.get("x-content-type-options")).toBe("nosniff");
40 expect(res.headers.get("referrer-policy")).not.toBeNull();
41 expect(res.headers.get("content-security-policy")).not.toBeNull();
42 const body = (await res.json()) as Record<string, unknown>;
43 expect(body.version).toBe("1.0");
44 expect(body.type).toBe("link");
45 expect(body.title).toBe("alice/open");
46 expect(body.author_name).toBe("alice");
47 expect(body.provider_name).toBe("Open Platform Forge");
48 expect(body.thumbnail_url).toBe(`${servers.url}/og/repo/alice~open.png`);
49 // A deeper page of the same repository is the same card; format may be left out.
50 expect((await ask(`${servers.url}/alice/open/issues`, "")).status).toBe(200);
51 });
52
53 test("a private repository answers the same 404 bytes as an unknown one", async () => {
54 const priv = await ask(`${servers.url}/alice/secret`);
55 const none = await ask(`${servers.url}/alice/nothing-here`);
56 expect(priv.status).toBe(404);
57 expect(none.status).toBe(404);
58 const a = await priv.text();
59 expect(a).toBe(await none.text());
60 expect(a).not.toContain("secret");
61 });
62
63 test("another host is a 400, xml a 501, and a hostile url is never echoed", async () => {
64 expect((await ask("https://example.com/alice/open")).status).toBe(400);
65 expect((await ask("not a url")).status).toBe(400);
66 expect((await fetch(`${servers.url}/oembed`)).status).toBe(400);
67 expect((await ask(`${servers.url}/alice/open`, "&format=xml")).status).toBe(501);
68 const evil = await ask(`${servers.url}/alice/<script>alert(1)</script>`);
69 expect(evil.status).toBe(404);
70 expect(await evil.text()).not.toContain("<script>");
71 const foreign = await ask("https://evil.test/<script>");
72 expect(await foreign.text()).not.toContain("<script>");
73 });
74
75 test("a public page advertises the card; a private page advertises none", async () => {
76 const open = await (await fetch(`${servers.url}/alice/open`)).text();
77 expect(open).toContain('type="application/json+oembed"');
78 expect(open).toContain(encodeURIComponent(`${servers.url}/alice/open`));
79 const secret = await fetch(`${servers.url}/alice/secret`);
80 expect(await secret.text()).not.toContain("oembed");
81 expect(secret.headers.get("link") ?? "").not.toContain("oembed");
82 });
test/openapi-drift.test.ts +1
@@ -13,6 +13,7 @@ const NOT_ROUTED: Readonly<Record<string, string>> = {
1313 "GET /health": "routed outside /api/v1 (src/http/api.ts), the monitor's alias of /api/v1/health (#13191)",
1414 "GET /{}/{}/pulls/{}/check.txt": "routed outside /api/v1 (src/http/server.ts, check_text.ts); test/openapi-check-txt.test.ts pins it to server.ts",
1515 "POST /mcp": "routed outside /api/v1 (src/http/mcp.ts); the MCP has its own card and tool list",
16 "GET /oembed": "routed outside /api/v1 (src/http/server.ts, web/standards.ts): the oEmbed endpoint other tools discover from a page's head",
1617 };
1718 const NOT_DESCRIBED: Readonly<Record<string, string>> = {};
1819

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs