[website] generate llms.txt and llms-full.txt files (#8345)

An attempt to generate llms.txt - #8162 

Info on llms.txt: https://llmstxt.org/

This PR adds the following:
- `llms.txt` in several locations
- `llms-full.txt` in several locations
- `llms.json` in docs root - this is for easier consumption of llms.txt
from our MCP tool as we won't have to parse markdown to get links
- generate markdown files of all `docs`. Excludes blogs and release
notes
- Updates `tspd` to add some llmstxt frontmatter

This PR introduces `llms.txt` at a few different entry points:
- `https://typespec.io/docs/llms.txt`
- `https://typespec.io/docs/language-basics/llms.txt`
- `https://typespec.io/docs/standard-library/llms.txt`
- `https://typespec.io/docs/libraries/[name]/llms.txt`

Each one of these files contains links to markdown documents in
descendant paths. The root version will contain the same links as all
the other llms.txt files.

`llms-full.txt` is an unofficial but common practice that is meant to be
the concatenation of the docs `llms.txt` points to. I'm generating these
for possible use in our own MCP server (to be served up like
typespec-mcp's `learn` tool) - but I am concerned with how many tokens
these files take up. My hand-written versions for TypeSpec Azure were
much smaller (about 1/10th the size) - so I think these will need to be
tuned further.

One other potential thing to change in this PR - make `site`
configurable. For `llms.txt`, the links should be full URLs. This means
PR checks/localhost will still cause the embedded links to point to
`https://typespec.io`. Can add a new build step to `website` for dev to
point to `localhost`, but not sure how to make it work with the PR
checks yet.

---------

Co-authored-by: Christopher Radek <[email protected]>

Author: chrisradek

.chronus/changes/gen-llmstxt-2025-8-8-9-27-39.md

@@ -0,0 +1,7 @@
+---
+changeKind: feature
+packages:
+  - "@typespec/tspd"
+---
+
+Adds `llmstxt` frontmatter to generated reference docs to enable inclusion in llms.txt. Opt-in: specify `--llmstxt` to enable

.chronus/changes/gen-llmstxt-2025-8-8-9-28-15.md

@@ -0,0 +1,15 @@
+---
+changeKind: internal
+packages:
+  - "@typespec/events"
+  - "@typespec/http"
+  - "@typespec/json-schema"
+  - "@typespec/openapi"
+  - "@typespec/openapi3"
+  - "@typespec/protobuf"
+  - "@typespec/rest"
+  - "@typespec/streams"
+  - "@typespec/versioning"
+---
+
+Updated doc generation to generate `llmstxt` frontmatter

cspell.yaml

@@ -129,6 +129,8 @@ words:
   - LINUXOS
   - LINUXVMIMAGE
   - ljust
+  - llms
+  - llmstxt
   - lmazuel
   - lropaging
   - lstrip

packages/astro-utils/package.json

@@ -10,7 +10,9 @@
     "./components/*": "./src/components/*",
     "./utils/*": "./src/utils/*.ts",
     "./css/*": "./src/css/*",
-    "./expressive-code/*": "./dist/expressive-code/*.js"
+    "./expressive-code/*": "./dist/expressive-code/*.js",
+    "./llmstxt": "./src/llmstxt/index.ts",
+    "./llmstxt/schema": "./src/llmstxt/schema.ts"
   },
   "files": [
     "src",

packages/astro-utils/src/llmstxt/generators.ts

@@ -0,0 +1,90 @@
+import { mergeSiteWithPath, type DocEntry, type LlmsTxtAsJson } from "./index";
+
+/**
+ * Generates the Markdown path following the llms.tst specification.
+ * @param docId The document ID from Astro content collections.
+ */
+export function generateMarkdownPath(docId: string): string {
+  // If the final path fragment does not include a file extension, use `index.html.md`
+  if (docId.endsWith("/")) {
+    return `${docId}index.html.md`;
+  }
+
+  const finalPathFragment = docId.split("/").pop() ?? "";
+  if (!finalPathFragment.includes(".")) {
+    return `${docId}/index.html.md`;
+  }
+
+  return `${docId}.md`;
+}
+
+/**
+ * Generates the LLMs text following the llms.tst specification.
+ * @param llmsData The pre-processed LLMs JSON data.
+ * @see `processDocsForLlmsTxt`
+ */
+export function generateLlmstxt(llmsData: LlmsTxtAsJson): string {
+  const contents: string[] = [];
+  contents.push(`# ${llmsData.title}`);
+  contents.push(`> ${llmsData.description}`);
+
+  for (const [name, topic] of Object.entries(llmsData.topics)) {
+    if (!topic.length) continue;
+    const section: string[] = [];
+    section.push(`## ${name}\n`);
+    for (const { title, url, description } of topic) {
+      section.push(`- [${title}](${url}): ${description}`);
+    }
+    contents.push(section.join("\n"));
+  }
+  return contents.join("\n\n");
+}
+
+/**
+ * Generates the full LLMs text - the combined markdown documentation referenced from an llms.txt.
+ * @param title The title for the LLMs text file.
+ * @param docs The collection of documentation entries to include.
+ */
+export function generateLlmstxtFull(title: string, docs: DocEntry[]): string {
+  const contents: string[] = [];
+  contents.push(`# ${title}`);
+
+  for (const doc of docs) {
+    if (!doc.body) continue;
+
+    const docTitle = doc.data.title;
+    const docDescription = doc.data.description ?? "";
+    contents.push(`# ${docTitle}`);
+    if (docDescription) contents.push(docDescription);
+    contents.push(doc.body);
+  }
+
+  return contents.join("\n\n");
+}
+
+export type GenerateLlmsJsonTopicDetails = {
+  id: string;
+  description: string;
+  pathPrefix: string;
+};
+
+export type LlmsJson = {
+  topic: string;
+  description: string;
+  contentUrl: string;
+}[];
+
+/**
+ * Generates the `llms.json` version of `llms.txt`.
+ * This is meant for easier consumption by our tools.
+ */
+export function generateLlmsJson(
+  topicDetails: GenerateLlmsJsonTopicDetails[],
+  siteHref: string,
+): LlmsJson {
+  return topicDetails.map(({ id, description, pathPrefix }) => ({
+    topic: id,
+    description,
+    contentUrl: mergeSiteWithPath(siteHref, pathPrefix, "llms-full.txt"),
+  }));
+}

packages/astro-utils/src/llmstxt/index.ts

@@ -0,0 +1,137 @@
+import type { z } from "astro:content";
+import { generateMarkdownPath } from "./generators";
+import type { llmstxtSchema } from "./schema";
+
+export * from "./generators";
+export * from "./routes";
+export * from "./topics";
+export interface DocEntry {
+  id: string;
+  data: {
+    title: string;
+    description?: string;
+    llmstxt?: z.infer<typeof llmstxtSchema>;
+  };
+  body?: string;
+}
+
+export interface LlmsTxtAsJson {
+  title: string;
+  description: string;
+  topics: Record<
+    string,
+    {
+      title: string;
+      description?: string;
+      url: string;
+    }[]
+  >;
+}
+
+export interface ProcessDocsProps {
+  /**
+   * Title for the LLMs text file.
+   */
+  title: string;
+  /**
+   * Description for the LLMs text file.
+   */
+  description: string;
+  /**
+   * The site URL, used to generate full URLs for documentation entries.
+   */
+  site?: URL;
+  /**
+   * The collection of documentation entries to process.
+   * Each entry must include a valid LLMs text schema.
+   * See `import("astro:content").getCollection`
+   */
+  docs: DocEntry[];
+  /**
+   * Name of the llmstxt section and the pathPrefix to match against doc IDs.
+   * If a doc matches multiple pathPrefixes, it will be assigned to the first matching section.
+   */
+  llmsSections: { name: string; pathPrefix: string }[];
+}
+
+/**
+ * Processes astro content collection docs and metadata for easy `llms.txt` generation.
+ */
+export async function processDocsForLlmsTxt({
+  title,
+  description,
+  site,
+  docs,
+  llmsSections,
+}: ProcessDocsProps) {
+  const sections = organizeDocsIntoSections(docs, llmsSections);
+  const result: LlmsTxtAsJson = { title, description, topics: {} };
+
+  const siteHref = site?.href ?? "";
+  for (const [sectionName, sectionDocs] of Object.entries(sections)) {
+    if (sectionDocs.length === 0) continue;
+
+    const topic = sectionName;
+    const topics = sectionDocs.map((doc) => {
+      const title = doc.data.title;
+      const desc = doc.data.description ?? "";
+      const path = generateMarkdownPath(doc.id);
+      const url = mergeSiteWithPath(siteHref, path);
+      return { title, description: desc, url };
+    });
+
+    result.topics[topic] = topics;
+  }
+
+  return result;
+}
+
+function organizeDocsIntoSections(
+  docs: DocEntry[],
+  llmsSections: ProcessDocsProps["llmsSections"],
+) {
+  docs.sort((a, b) => (a.id > b.id ? 1 : -1));
+  const seenDocs = new Set<DocEntry>();
+  const sections: Record<string, DocEntry[]> = {};
+
+  for (const { name, pathPrefix } of llmsSections) {
+    sections[name] = docs.filter((doc) => {
+      if (seenDocs.has(doc)) return false;
+      if (doc.id.startsWith(pathPrefix)) {
+        seenDocs.add(doc);
+        return true;
+      }
+      return false;
+    });
+  }
+
+  return sections;
+}
+
+/**
+ * Merges a site URL with path parts.
+ * Used when needing to create full URLs when working with astro content collections.
+ * @param siteHref The base URL of the site.
+ * @param pathParts The path parts to merge with the site URL.
+ * @returns The merged URL.
+ */
+export function mergeSiteWithPath(siteHref: string, ...pathParts: string[]): string {
+  let result = siteHref;
+
+  for (const part of pathParts) {
+    if (!part) continue; // Skip empty parts
+
+    const resultTrailingSlash = result.endsWith("/");
+    const partLeadingSlash = part.startsWith("/");
+
+    if (resultTrailingSlash && partLeadingSlash) {
+      result = `${result}${part.slice(1)}`;
+    } else if (!resultTrailingSlash && !partLeadingSlash) {
+      result = `${result}/${part}`;
+    } else {
+      result = `${result}${part}`;
+    }
+  }
+
+  return result;
+}

packages/astro-utils/src/llmstxt/routes.ts

@@ -0,0 +1,54 @@
+import type { APIRoute } from "astro";
+import {
+  generateLlmstxt,
+  generateLlmstxtFull,
+  processDocsForLlmsTxt,
+  type DocEntry,
+  type TopicProps,
+} from "./index";
+
+export type RouteParams = { path: string; llms_type: "llms" | "llms-full" };
+export type RouteProps = Pick<Required<TopicProps>, "title" | "description" | "docs">;
+
+export const spreadLlmsTxtRoute: APIRoute<RouteProps, RouteParams> = async ({
+  props,
+  params,
+  site,
+}) => {
+  const { title, docs, description } = props;
+  const { llms_type } = params;
+
+  if (llms_type === "llms") {
+    const llmsData = await processDocsForLlmsTxt({
+      title,
+      description,
+      docs,
+      // Use blank pathPrefix to include all docs in the llms.txt
+      llmsSections: [{ name: "Docs", pathPrefix: "" }],
+      site,
+    });
+
+    const llmstxt = generateLlmstxt(llmsData);
+    return new Response(llmstxt, {
+      headers: {
+        "Content-Type": "text/markdown; charset=utf-8",
+      },
+    });
+  } else {
+    const llmstxt = generateLlmstxtFull(description, docs);
+    return new Response(llmstxt, {
+      headers: {
+        "Content-Type": "text/markdown; charset=utf-8",
+      },
+    });
+  }
+};
+
+export const markdownRoute: APIRoute<{ doc: DocEntry }> = async ({ props }) => {
+  const { doc } = props;
+  return new Response(doc.body ?? "", {
+    headers: {
+      "Content-Type": "text/markdown; charset=utf-8",
+    },
+  });
+};

packages/astro-utils/src/llmstxt/schema.ts

@@ -0,0 +1,3 @@
+import { z } from "astro:content";
+
+export const llmstxtSchema = z.boolean().optional();

packages/astro-utils/src/llmstxt/topics.ts

@@ -0,0 +1,34 @@
+import type { DocEntry } from ".";
+
+export interface TopicProps {
+  title: string;
+  description: string;
+  pathPrefix: string;
+  docs: DocEntry[];
+  id: string;
+}
+
+/**
+ *
+ */
+export function populateTopicDocs(
+  topics: Omit<TopicProps, "docs">[],
+  docs: DocEntry[],
+): TopicProps[] {
+  docs.sort((a, b) => (a.id > b.id ? 1 : -1));
+  const seenDocs = new Set<DocEntry>();
+
+  return topics.map((topic) => {
+    return {
+      ...topic,
+      docs: docs.filter((doc) => {
+        if (seenDocs.has(doc)) return false;
+        if (doc.id.startsWith(topic.pathPrefix)) {
+          seenDocs.add(doc);
+          return true;
+        }
+        return false;
+      }),
+    };
+  });
+}

packages/events/package.json

@@ -45,7 +45,7 @@
     "test-official": "vitest run --coverage --reporter=junit --reporter=default --no-file-parallelism",
     "lint": "eslint . --ext .ts --max-warnings=0",
     "lint:fix": "eslint . --fix --ext .ts",
-    "regen-docs": "tspd doc .  --enable-experimental  --output-dir ../../website/src/content/docs/docs/libraries/events/reference"
+    "regen-docs": "tspd doc .  --enable-experimental --llmstxt --output-dir ../../website/src/content/docs/docs/libraries/events/reference"
   },
   "files": [
     "lib/*.tsp",