feat: schema map endpoint for agent/crawler discovery (v0.7.0)

jdevalk · claude · jdevalk · commit 591a5e76feea · 2026-04-13T15:22:52.000+02:00
Adds a new public plugin route `schema/map` that enumerates every published URL backed by schema markup, returning `{items: [{url, collection, updatedAt}]}`. Site owners wire it to `/schemamap.xml` at the site root via a small Astro endpoint (snippet in README). Per-URL schema endpoints (`/schema/<slug>.json`) are deferred pending an upstream helper to build PublicPageContext from a plugin route (emdash-cms/emdash#527) — avoids reimplementing core's page builder and the drift risk that would come with it. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.7.0] - 2026-04-13
+
+### Added
+
+- **Schema map (experimental).** New public plugin route `schema/map` returning the list of every published URL backed by schema markup (`{ items: [{ url, collection, updatedAt }] }`). Wire it to `/schemamap.xml` at your site root with the Astro snippet in the README so agents and crawlers can enumerate structured-data URLs without scraping HTML. Per-URL schema endpoints (`/schema/<slug>.json`) are deferred pending an upstream helper for building page contexts from plugin routes.
+
 ## [0.6.0] - 2026-04-13
 
 ### Added
diff --git a/README.md b/README.md
@@ -27,6 +27,7 @@ An SEO plugin for [EmDash CMS](https://github.com/emdash-cms/emdash) that genera
 - **Breadcrumbs** — derived from the URL path by default, with segment label overrides (`/blog/` → "Blog") and per-`pageType` rule overrides both editable in the admin UI. `@id` scheme matches [joost.blog](https://joost.blog) via `@jdevalk/seo-graph-core`
 - **hreflang alternates** — for multilingual EmDash sites (Astro `i18n` + `translation_group`), one `<link rel="alternate" hreflang="…">` per published sibling plus an automatic `x-default`, with BCP 47 tag normalization (`fr-ca` → `fr-CA`). Zero cost on single-locale sites
 - **llms.txt** *(experimental)* — exposes an index of published content at the plugin's `llms/txt` route, following the small-form [llms.txt spec](https://llmstxt.org). Enabled by default; flip the setting to disable. Only the plain `llms.txt` file is supported; the `llms-full.txt` variant is not implemented
+- **Schema map** *(experimental)* — exposes a list of every published URL backed by schema markup at the plugin's `schema/map` route, ready to be wired to a `/schemamap.xml` Astro endpoint for agent/crawler discovery
 - **IndexNow** — on publish/unpublish transitions, submits the affected URL to [IndexNow](https://www.indexnow.org) so Bing, Yandex, Seznam, Naver, and Yep recrawl immediately. Opt-in via a single toggle in the settings UI; the key is generated and persisted automatically on first use
 - **Admin settings UI** — auto-generated from `settingsSchema` for configuring Person/Organization identity, social profiles, title separator, and default description
 
@@ -186,6 +187,69 @@ Alternatively, import `buildLlmsTxt` from this plugin and assemble the
 body yourself from `getEmDashCollection()` results if you want full
 control over sectioning, ordering, or filtering.
 
+## Schema map (experimental)
+
+> **Experimental.** Shape and exposed API may change in a minor
+> release. Per-URL schema endpoints (`/schema/<slug>.json`) are not
+> implemented yet — they depend on a core helper being discussed
+> upstream.
+
+Every published page on an EmDash site carries a JSON-LD schema graph
+in its `<head>`. Agents and crawlers that want to enumerate those pages
+without scraping every HTML document need a *schema map* — the same
+idea as `sitemap.xml`, but scoped to "URLs that have structured data."
+
+The plugin exposes the raw list at `schema/map` as JSON:
+
+```json
+{
+  "items": [
+    { "url": "https://example.com/blog/hello/", "collection": "blog", "updatedAt": "2026-02-01T00:00:00Z" },
+    { "url": "https://example.com/about/",      "collection": "pages", "updatedAt": "2026-01-03T00:00:00Z" }
+  ]
+}
+```
+
+Wire it to `/schemamap.xml` at your site root with a small Astro
+endpoint. The route is public — no auth needed on the fetch:
+
+```ts
+// src/pages/schemamap.xml.ts
+import type { APIRoute } from "astro";
+
+interface SchemaMapEntry {
+  url: string;
+  collection: string;
+  updatedAt: string;
+}
+
+export const GET: APIRoute = async ({ request }) => {
+  const origin = new URL(request.url).origin;
+  const res = await fetch(`${origin}/_emdash/api/plugins/seo/schema/map`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: "{}",
+  });
+  const { items } = (await res.json()) as { items: SchemaMapEntry[] };
+
+  const urls = items
+    .map(
+      ({ url, updatedAt }) =>
+        `  <url><loc>${url}</loc><lastmod>${updatedAt}</lastmod></url>`,
+    )
+    .join("\n");
+
+  const xml = `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+${urls}
+</urlset>`;
+
+  return new Response(xml, {
+    headers: { "Content-Type": "application/xml; charset=utf-8" },
+  });
+};
+```
+
 ## Requirements
 
 Requires EmDash with support for running `page:metadata` hooks on public pages for anonymous visitors (fixed in [emdash-cms/emdash#119](https://github.com/emdash-cms/emdash/pull/119)).
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@jdevalk/emdash-plugin-seo",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "description": "SEO plugin for EmDash CMS — meta tags, Open Graph, canonical URLs, robots directives, and JSON-LD schema markup",
   "type": "module",
   "exports": {
diff --git a/src/index.ts b/src/index.ts
@@ -7,11 +7,12 @@ import {
   handleIndexNowTransition,
 } from "./indexnow.js";
 import { generateLlmsTxt } from "./llms.js";
+import { listSchemaEntries } from "./schema/endpoints.js";
 
 export function seoPlugin(): PluginDescriptor {
   return {
     id: "seo",
-    version: "0.6.0",
+    version: "0.7.0",
     format: "native",
     entrypoint: new URL("./index.ts", import.meta.url).pathname,
     adminEntry: new URL("./admin.tsx", import.meta.url).pathname,
@@ -25,7 +26,7 @@ export function seoPlugin(): PluginDescriptor {
 export function createPlugin() {
   return definePlugin({
     id: "seo",
-    version: "0.6.0",
+    version: "0.7.0",
     capabilities: ["read:content", "page:inject", "network:fetch"],
     allowedHosts: ["api.indexnow.org"],
 
@@ -77,6 +78,13 @@ export function createPlugin() {
           return { enabled: body !== null, body: body ?? "" };
         },
       },
+      "schema/map": {
+        public: true,
+        handler: async (ctx: RouteContext) => {
+          const items = await listSchemaEntries(ctx);
+          return { items };
+        },
+      },
     },
 
     admin: {
diff --git a/src/schema/endpoints.ts b/src/schema/endpoints.ts
@@ -0,0 +1,85 @@
+import type { PluginContext } from "emdash";
+import { buildPageUrl } from "../urls.js";
+
+/**
+ * One entry in the schema map — a URL that has a published content
+ * record backing it, with the last-modified timestamp used by crawlers
+ * and agents to decide whether to refetch.
+ */
+export interface SchemaMapEntry {
+  /** Absolute URL of the live page. */
+  url: string;
+  /** Collection slug (e.g. "posts", "pages"). */
+  collection: string;
+  /** ISO-8601 last-modified timestamp (updatedAt, falling back to createdAt). */
+  updatedAt: string;
+}
+
+/**
+ * Enumerate every published URL the site exposes — the data a
+ * `schemamap.xml` endpoint needs.
+ *
+ * This mirrors `generateLlmsTxt`'s iteration: walk every collection
+ * with a `urlPattern`, paginate through `ctx.content.list`, filter to
+ * `status === "published"`, and project each item to a `(url,
+ * collection, updatedAt)` triple.
+ *
+ * Returns an empty array (not null) when the site has no publishable
+ * content — callers should treat that as "serve an empty but valid
+ * schema map," not "404." Collections without a `urlPattern` are
+ * silently skipped; they have no public URL to advertise.
+ */
+export async function listSchemaEntries(ctx: PluginContext): Promise<SchemaMapEntry[]> {
+  if (!ctx.content) return [];
+  const siteUrl = ctx.site.url;
+  if (!siteUrl) return [];
+
+  const { SchemaRegistry, isI18nEnabled, getI18nConfig } = await import("emdash");
+  const { getDb } = await import("emdash/runtime");
+  const db = await getDb();
+  const registry = new SchemaRegistry(db);
+  const collections = await registry.listCollections();
+
+  const cfg =
+    isI18nEnabled() && getI18nConfig()
+      ? getI18nConfig()!
+      : { locales: ["en"], defaultLocale: "en", prefixDefaultLocale: false };
+
+  const entries: SchemaMapEntry[] = [];
+
+  for (const collection of collections) {
+    if (!collection.urlPattern) continue;
+
+    let cursor: string | undefined;
+    do {
+      const page = await ctx.content.list(collection.slug, { limit: 100, cursor });
+      for (const item of page.items) {
+        const data = item.data as Record<string, unknown>;
+        if (data.status !== "published") continue;
+        const slug = typeof data.slug === "string" ? data.slug : null;
+        if (!slug) continue;
+        const locale =
+          typeof data.locale === "string" && data.locale ? data.locale : cfg.defaultLocale;
+
+        const url = buildPageUrl({
+          locale,
+          slug,
+          siteUrl,
+          cfg,
+          urlPattern: collection.urlPattern,
+        });
+        if (!url) continue;
+
+        const updatedAt =
+          (typeof item.updatedAt === "string" && item.updatedAt) ||
+          (typeof item.createdAt === "string" && item.createdAt) ||
+          new Date(0).toISOString();
+
+        entries.push({ url, collection: collection.slug, updatedAt });
+      }
+      cursor = page.cursor;
+    } while (cursor);
+  }
+
+  return entries;
+}
diff --git a/test/schema-endpoints.test.ts b/test/schema-endpoints.test.ts

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "@jdevalk/emdash-plugin-seo",`
`3`		`- "version": "0.6.0",`
	`3`	`+ "version": "0.7.0",`
`4`	`4`	`"description": "SEO plugin for EmDash CMS — meta tags, Open Graph, canonical URLs, robots directives, and JSON-LD schema markup",`
`5`	`5`	`"type": "module",`
`6`	`6`	`"exports": {`