[docs] generate llms.txt files (#3239)

chrisradek · Christopher Radek · web-flow · commit a6ffc7f59a74 · 2025-09-08T20:21:58.000Z
This is the sibling to microsoft/typespec#8345 This PR updates website doc generation to generate llms.txt and llms-full.txt files for public consumption, as well as the structured llms.json file for our own MCP use. --------- Co-authored-by: Christopher Radek <Christopher.Radek@microsoft.com>
diff --git a/.chronus/changes/gen-llmstxt-2025-8-8-12-38-4.md b/.chronus/changes/gen-llmstxt-2025-8-8-12-38-4.md
@@ -0,0 +1,9 @@
+---
+changeKind: internal
+packages:
+  - "@azure-tools/typespec-azure-core"
+  - "@azure-tools/typespec-azure-resource-manager"
+  - "@azure-tools/typespec-client-generator-core"
+---
+
+Update doc generation to include llmstxt frontmatter
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -308,6 +308,19 @@ Depending on the package where the fix needs to go do this on the `Microsoft/typ
 1. Find the backmerge branch [here](https://github.com/Azure/typespec-azure/branches) and click "New pull request".
 1. Rebase merge the new backmerge PR into main.
 
+## llms.txt
+
+The website build generates `llms.txt` and `llms-full.txt` files based on 2 factors:
+
+1. Presence of `llms.txt: true` frontmatter. See [example](website/src/content/docs/docs/getstarted/azure-core/step01.md) for an example of this in practice.
+1. The source document exists in a defined "topic". See [topic configuration](website/src/utils/llmstxt.ts) for where to define topics. Each topic appears as its own "section" in the root `llms.txt` file, and also generates its own `llms.txt` file at the specified `pathPrefix`.
+
+For libraries, update the `regen-docs` npm script to include the `--llmstxt` flag to opt into llms.txt generation. See [example](packages/typespec-azure-core/package.json) for an example:
+
+> "regen-docs": "tspd doc . --enable-experimental --llmstxt --output-dir ../../website/src/content/docs/docs/libraries/azure-core/reference"
+
+For now the guidance is to prioritize including documentation for TypeSpec users as opposed to including documentation for emitter/library authors.
+
 ## Labels
 
 TypeSpec repo use labels to help categorize and manage issues and PRs. The following is a list of labels and their descriptions.
diff --git a/core b/core
@@ -1 +1 @@
-Subproject commit 890c6ad35c904afe5dc156f25c72f19fca4656a3
+Subproject commit 3ef37e134780367fb62bf4c7a9928db90ad6f729
diff --git a/packages/typespec-azure-core/package.json b/packages/typespec-azure-core/package.json
@@ -49,7 +49,7 @@
     "test:ci": "vitest run --coverage  --reporter=junit --reporter=default",
     "lint": "eslint .  --max-warnings=0",
     "lint:fix": "eslint . --fix ",
-    "regen-docs": "tspd doc .  --enable-experimental  --output-dir ../../website/src/content/docs/docs/libraries/azure-core/reference"
+    "regen-docs": "tspd doc .  --enable-experimental --llmstxt --output-dir ../../website/src/content/docs/docs/libraries/azure-core/reference"
   },
   "files": [
     "lib/*.tsp",
diff --git a/packages/typespec-azure-resource-manager/package.json b/packages/typespec-azure-resource-manager/package.json
@@ -46,7 +46,7 @@
     "test:ci": "vitest run --coverage  --reporter=junit --reporter=default",
     "lint": "eslint .  --max-warnings=0",
     "lint:fix": "eslint . --fix ",
-    "regen-docs": "tspd doc .  --enable-experimental  --output-dir ../../website/src/content/docs/docs/libraries/azure-resource-manager/reference"
+    "regen-docs": "tspd doc .  --enable-experimental --llmstxt  --output-dir ../../website/src/content/docs/docs/libraries/azure-resource-manager/reference"
   },
   "files": [
     "lib/**/*.tsp",
diff --git a/packages/typespec-client-generator-core/package.json b/packages/typespec-client-generator-core/package.json
@@ -47,7 +47,7 @@
     "test:ci": "vitest run --coverage  --reporter=junit --reporter=default",
     "lint": "eslint .  --max-warnings=0",
     "lint:fix": "eslint . --fix ",
-    "regen-docs": "tspd doc .  --enable-experimental  --output-dir ../../website/src/content/docs/docs/libraries/typespec-client-generator-core/reference"
+    "regen-docs": "tspd doc .  --enable-experimental --llmstxt --output-dir ../../website/src/content/docs/docs/libraries/typespec-client-generator-core/reference"
   },
   "files": [
     "lib/*.tsp",
diff --git a/website/src/content.config.ts b/website/src/content.config.ts
@@ -1,7 +1,11 @@
 import { docsLoader } from "@astrojs/starlight/loaders";
 import { docsSchema } from "@astrojs/starlight/schema";
-import { defineCollection } from "astro:content";
+import { llmstxtSchema } from "@typespec/astro-utils/llmstxt/schema";
+import { defineCollection, z } from "astro:content";
 
 export const collections = {
-  docs: defineCollection({ loader: docsLoader(), schema: docsSchema() }),
+  docs: defineCollection({
+    loader: docsLoader(),
+    schema: docsSchema({ extend: z.object({ llmstxt: llmstxtSchema.optional() }) }),
+  }),
 };
diff --git a/website/src/content/docs/docs/emitters/typespec-autorest/reference/decorators.md b/website/src/content/docs/docs/emitters/typespec-autorest/reference/decorators.md
@@ -1,5 +1,6 @@
 ---
 title: "Decorators"
+description: "Decorators exported by @azure-tools/typespec-autorest"
 toc_min_heading_level: 2
 toc_max_heading_level: 3
 ---
diff --git a/website/src/content/docs/docs/getstarted/azure-core/step01.md b/website/src/content/docs/docs/getstarted/azure-core/step01.md
@@ -1,5 +1,7 @@
 ---
 title: 1. Writing Your First Service
+description: Creating a data-plane service
+llmstxt: true
 ---
 
 The Azure Data Plane Service template will create a very basic TypeSpec file in `main.tsp`:
diff --git a/website/src/content/docs/docs/getstarted/azure-core/step02.md b/website/src/content/docs/docs/getstarted/azure-core/step02.md
@@ -1,5 +1,7 @@
 ---
 title: 2. Create the service namespace
+description: Describing a data-plane service
+llmstxt: true
 ---
 
 To describe a service, you first need to define a "blockless" (file-level, no curly braces) namespace and use the `@service` decorator to mark it as the service namespace:
diff --git a/website/src/content/docs/docs/getstarted/azure-core/step03.md b/website/src/content/docs/docs/getstarted/azure-core/step03.md
@@ -1,5 +1,7 @@
 ---
 title: 3. Using the versioned Azure.Core types
+description: Using versioned Azure.Core
+llmstxt: true
 ---
 
 Before you can use the models and operations defined in the `Azure.Core` namespace, you will need to specify the API version of the `Azure.Core` library that your service uses. You can do this by adding the `@useDependency` decorator to the `Contoso.WidgetManager` namespace as seen here:
diff --git a/website/src/content/docs/docs/getstarted/azure-core/step04.md b/website/src/content/docs/docs/getstarted/azure-core/step04.md
@@ -1,5 +1,7 @@
 ---
 title: 4. Defining your first resource
+description: Defining data-plane resources
+llmstxt: true
 ---
 
 In the context of your service, a "resource" is a fundamental entity that your service manages. For our `WidgetService`, the most basic entity we need is a `Widget`.
diff --git a/website/src/content/docs/docs/getstarted/azure-core/step05.md b/website/src/content/docs/docs/getstarted/azure-core/step05.md
@@ -1,5 +1,7 @@
 ---
 title: 5. Defining standard resource operations
+description: Defining standard data-plane resource operations
+llmstxt: true
 ---
 
 The `Azure.Core` namespace provides a variety of [standard lifecycle operations](https://azure.github.io/typespec-azure/docs/libraries/azure-core/reference/interfaces#Azure.Core.ResourceOperations) for resource types. These operations adhere to the requirements of the [Azure REST API Guidelines](https://github.com/microsoft/api-guidelines/blob/vNext/azure/Guidelines.md).
diff --git a/website/src/content/docs/docs/getstarted/azure-core/step06.md b/website/src/content/docs/docs/getstarted/azure-core/step06.md
@@ -1,5 +1,7 @@
 ---
 title: 6. Defining long-running resource operations
+description: Defining long-running resource operations
+llmstxt: true
 ---
 
 If your service uses any long-running operations (LROs; see [our guidelines](https://github.com/microsoft/api-guidelines/blob/vNext/azure/Guidelines.md#long-running-operations--jobs) for specifics), you will need to define a "status monitor" operation which can report the status of the operation.
diff --git a/website/src/content/docs/docs/getstarted/azure-core/step07.md b/website/src/content/docs/docs/getstarted/azure-core/step07.md
@@ -1,5 +1,7 @@
 ---
-title: 7. Defining child resources
+title: 7. Defining child resources\
+description: Defining child resources
+llmstxt: true
 ---
 
 Sometimes your resource types will need to have child resources that relate to their parent types. You can identify that a resource type is the child of another resource by using the `@parentResource` decorator.
diff --git a/website/src/content/docs/docs/getstarted/azure-core/step08.md b/website/src/content/docs/docs/getstarted/azure-core/step08.md
@@ -1,5 +1,7 @@
 ---
 title: 8. Defining custom resource actions
+description: Defining custom resource actions
+llmstxt: true
 ---
 
 Often your resource types will need additional operations that are not covered by the standard resource operation shapes. For this, there are a set of operation signatures for defining _resource actions_ at the instance and collection level.
diff --git a/website/src/content/docs/docs/getstarted/azure-core/step09.md b/website/src/content/docs/docs/getstarted/azure-core/step09.md
@@ -1,5 +1,7 @@
 ---
 title: 9. Customizing operations with traits
+description: Customizing operations with traits
+llmstxt: true
 ---
 
 For all standard lifecycle operations you can customize the operation parameters and response body by passing a special model type to the `Traits` parameter of the operation template, typically the second parameter of the operation template. You can also customize the whole set of resource operations by passing traits to the `ResourceOperations` interface.
diff --git a/website/src/content/docs/docs/getstarted/azure-core/step10.md b/website/src/content/docs/docs/getstarted/azure-core/step10.md
@@ -1,5 +1,7 @@
 ---
 title: 10. Versioning
+description: Versioning your data-plane service
+llmstxt: true
 ---
 
 ## Versioning your service
diff --git a/website/src/content/docs/docs/getstarted/azure-core/step12.md b/website/src/content/docs/docs/getstarted/azure-core/step12.md
@@ -1,5 +1,7 @@
 ---
 title: Advanced Topics
+description: Defining singleton resources
+llmstxt: true
 ---
 
 Once you have written your first service with `Azure.Core`, you might be interested to try the following features:
diff --git a/website/src/content/docs/docs/getstarted/azure-resource-manager/step00.md b/website/src/content/docs/docs/getstarted/azure-resource-manager/step00.md
@@ -1,5 +1,7 @@
 ---
 title: 1. Installing Tools
+description: Creating an ARM service
+llmstxt: true
 ---
 
 ## Installing IDE Tools
diff --git a/website/src/content/docs/docs/getstarted/azure-resource-manager/step01.md b/website/src/content/docs/docs/getstarted/azure-resource-manager/step01.md
@@ -1,5 +1,7 @@
 ---
 title: 2. Defining the Service
+description: Defining the ARM service
+llmstxt: true
 ---
 
 To define an Azure Resource Manager service, the first thing you will need to do is define the service namespace and decorate it with the `service` and `armProviderNamespace` decorators:
diff --git a/website/src/content/docs/docs/getstarted/azure-resource-manager/step02.md b/website/src/content/docs/docs/getstarted/azure-resource-manager/step02.md
@@ -1,5 +1,7 @@
 ---
 title: 3. Defining the Resources
+description: Defining ARM resources
+llmstxt: true
 ---
 
 An ARM resource provider is composed of resources. The TypeSpec Azure Resource Manager library makes it much easier to define the structure and endpoints of such resources.
diff --git a/website/src/content/docs/docs/getstarted/azure-resource-manager/step03.md b/website/src/content/docs/docs/getstarted/azure-resource-manager/step03.md
@@ -1,5 +1,7 @@
 ---
 title: 4. Defining Child Resource Types
+description: Defining ARM child resources
+llmstxt: true
 ---
 
 You can create parent/child relationships between resource types by using the `@parentResource` decorator when defining a resource type.
diff --git a/website/src/content/docs/docs/getstarted/azure-resource-manager/step04.md b/website/src/content/docs/docs/getstarted/azure-resource-manager/step04.md
@@ -1,5 +1,7 @@
 ---
 title: 5. Defining Custom Actions
+description: Defining custom actions on ARM resources
+llmstxt: true
 ---
 
 Some resources will provide more than the standard CRUD operations and will need to define a custom action endpoint. Additional resource operations can be added to the `interface` where you defined standard resource operations, using the `ArmResourceAction` templates.
diff --git a/website/src/content/docs/docs/getstarted/azure-resource-manager/step05.md b/website/src/content/docs/docs/getstarted/azure-resource-manager/step05.md
@@ -1,5 +1,7 @@
 ---
 title: 6. Complete Example and Generate OpenApi 2.0 spec
+description: Complete ARM example
+llmstxt: true
 ---
 
 To generate an OpenAPI v2 (Swagger) specification from the service definition, run the following command inside of the project folder:
diff --git a/website/src/content/docs/docs/howtos/ARM/change-provider-namespace.md b/website/src/content/docs/docs/howtos/ARM/change-provider-namespace.md
@@ -1,5 +1,7 @@
 ---
 title: Change the provider namespace
+description: Using a different ARM provider namespace than TypeSpec namespace
+llmstxt: true
 ---
 
 If you want to have a different provider namespace than the TypeSpec namespec you'll have to change the following
diff --git a/website/src/content/docs/docs/howtos/ARM/resource-operations.md b/website/src/content/docs/docs/howtos/ARM/resource-operations.md
@@ -1,5 +1,7 @@
 ---
 title: ARM Resource Operations
+description: Recommended and required operations
+llmstxt: true
 ---
 
 ## Recommended and Required Operations
diff --git a/website/src/content/docs/docs/howtos/ARM/resource-type.md b/website/src/content/docs/docs/howtos/ARM/resource-type.md
@@ -1,5 +1,7 @@
 ---
 title: ARM Resource Types
+description: Choosing and modeling ARM resources
+llmstxt: true
 ---
 
 ## Introductions
diff --git a/website/src/content/docs/docs/howtos/ARM/versioning.md b/website/src/content/docs/docs/howtos/ARM/versioning.md
@@ -1,5 +1,7 @@
 ---
 title: Versioning
+description: Managing versioning for ARM services
+llmstxt: true
 ---
 
 This document explains how to manage versioning in TypeSpec projects, including how to add, remove, or modify resources, operations, and properties across API versions.
diff --git a/website/src/content/docs/docs/howtos/Azure Core/content-negotiation.md b/website/src/content/docs/docs/howtos/Azure Core/content-negotiation.md
@@ -1,5 +1,6 @@
 ---
 title: Content Type Negotiation
+llmstxt: true
 ---
 
 See [Http documentation on content type negotiation](https://typespec.io/docs/libraries/http/content-types/#content-type-negotiation) for the general concept
diff --git a/website/src/content/docs/docs/howtos/Azure Core/long-running-operations.md b/website/src/content/docs/docs/howtos/Azure Core/long-running-operations.md
@@ -1,5 +1,6 @@
 ---
 title: "Deep Dive: Long-running (Asynchronous) Operations"
+llmstxt: true
 ---
 
 Long-running, or asynchronous operations (LROs for short) are Http Operations that are not complete in a single request/response sequence. Such operations require multiple requests and responses, usually to different endpoints before the operation is complete. Some common patterns that services use for LROs include:
diff --git a/website/src/content/docs/docs/howtos/Generate client libraries/00howtogen.mdx b/website/src/content/docs/docs/howtos/Generate client libraries/00howtogen.mdx
@@ -1,5 +1,6 @@
 ---
 title: How to generate client libraries
+llmstxt: true
 ---
 
 import { ClientTabs, ClientTabItem } from "@components/client-tabs";
diff --git a/website/src/content/docs/docs/howtos/Generate client libraries/01setup.mdx b/website/src/content/docs/docs/howtos/Generate client libraries/01setup.mdx
@@ -1,5 +1,6 @@
 ---
 title: Setup for SDK Customization
+llmstxt: true
 ---
 
 import { ClientTabs, ClientTabItem } from "@components/client-tabs";
diff --git a/website/src/content/docs/docs/howtos/Generate client libraries/02package.mdx b/website/src/content/docs/docs/howtos/Generate client libraries/02package.mdx
@@ -1,5 +1,6 @@
 ---
 title: Common Behavior for Client Packages
+llmstxt: true
 ---
 
 import { ClientTabs, ClientTabItem } from "@components/client-tabs";
diff --git a/website/src/content/docs/docs/howtos/Generate client libraries/03client.mdx b/website/src/content/docs/docs/howtos/Generate client libraries/03client.mdx
@@ -1,5 +1,6 @@
 ---
 title: Clients
+llmstxt: true
 ---
 
 import { ClientTabs, ClientTabItem } from "@components/client-tabs";
diff --git a/website/src/content/docs/docs/howtos/Generate client libraries/04method.mdx b/website/src/content/docs/docs/howtos/Generate client libraries/04method.mdx
@@ -1,5 +1,6 @@
 ---
 title: Basic methods
+llmstxt: true
 ---
 
 import { ClientTabs, ClientTabItem } from "@components/client-tabs";
diff --git a/website/src/content/docs/docs/howtos/Generate client libraries/05pagingOperations.mdx b/website/src/content/docs/docs/howtos/Generate client libraries/05pagingOperations.mdx
@@ -1,5 +1,6 @@
 ---
 title: Paging Operations
+llmstxt: true
 ---
 
 import { ClientTabs, ClientTabItem } from "@components/client-tabs";
diff --git a/website/src/content/docs/docs/howtos/Generate client libraries/06longRunningOperations.mdx b/website/src/content/docs/docs/howtos/Generate client libraries/06longRunningOperations.mdx
@@ -1,5 +1,6 @@
 ---
 title: Long-Running Operations
+llmstxt: true
 ---
 
 import { ClientTabs, ClientTabItem } from "@components/client-tabs";
diff --git a/website/src/content/docs/docs/howtos/Generate client libraries/07multipart.mdx b/website/src/content/docs/docs/howtos/Generate client libraries/07multipart.mdx
@@ -1,5 +1,6 @@
 ---
 title: Multipart Operations
+llmstxt: true
 ---
 
 import { ClientTabs, ClientTabItem } from "@components/client-tabs";
diff --git a/website/src/content/docs/docs/howtos/Generate client libraries/08types.mdx b/website/src/content/docs/docs/howtos/Generate client libraries/08types.mdx
@@ -1,5 +1,6 @@
 ---
 title: Generated Types
+llmstxt: true
 ---
 
 import { ClientTabs, ClientTabItem } from "@components/client-tabs";
diff --git a/website/src/content/docs/docs/howtos/Generate client libraries/09renaming.mdx b/website/src/content/docs/docs/howtos/Generate client libraries/09renaming.mdx
@@ -1,5 +1,6 @@
 ---
 title: Renaming Types
+llmstxt: true
 ---
 
 import { ClientTabs, ClientTabItem } from "@components/client-tabs";
diff --git a/website/src/content/docs/docs/howtos/Generate client libraries/10versioning.mdx b/website/src/content/docs/docs/howtos/Generate client libraries/10versioning.mdx
@@ -1,5 +1,6 @@
 ---
 title: Versioning
+llmstxt: true
 ---
 
 import { ClientTabs, ClientTabItem } from "@components/client-tabs";
diff --git a/website/src/content/docs/docs/howtos/Versioning/preview-version.md b/website/src/content/docs/docs/howtos/Versioning/preview-version.md
@@ -1,5 +1,6 @@
 ---
 title: How to define a preview version
+llmstxt: true
 ---
 
 See [`@typespec/versioning` documentation](https://typespec.io/docs/libraries/versioning/guide) for the general versioning concept. This guide expands on how Azure Services should define Preview versions.
diff --git a/website/src/content/docs/docs/libraries/azure-core/reference/data-types.md b/website/src/content/docs/docs/libraries/azure-core/reference/data-types.md
@@ -1,5 +1,7 @@
 ---
 title: "Data types"
+description: "Data types exported by @azure-tools/typespec-azure-core"
+llmstxt: true
 ---
 
 ## Azure.Core
diff --git a/website/src/content/docs/docs/libraries/azure-core/reference/decorators.md b/website/src/content/docs/docs/libraries/azure-core/reference/decorators.md
@@ -1,7 +1,9 @@
 ---
 title: "Decorators"
+description: "Decorators exported by @azure-tools/typespec-azure-core"
 toc_min_heading_level: 2
 toc_max_heading_level: 3
+llmstxt: true
 ---
 
 ## Azure.Core
diff --git a/website/src/content/docs/docs/libraries/azure-core/reference/interfaces.md b/website/src/content/docs/docs/libraries/azure-core/reference/interfaces.md
@@ -1,5 +1,7 @@
 ---
 title: "Interfaces and Operations"
+description: "Interfaces and Operations exported by @azure-tools/typespec-azure-core"
+llmstxt: true
 ---
 
 ## Azure.Core
diff --git a/website/src/content/docs/docs/libraries/azure-portal-core/reference/data-types.md b/website/src/content/docs/docs/libraries/azure-portal-core/reference/data-types.md
@@ -1,5 +1,6 @@
 ---
 title: "Data types"
+description: "Data types exported by @azure-tools/typespec-azure-portal-core"
 ---
 
 ## Azure.Portal
diff --git a/website/src/content/docs/docs/libraries/azure-portal-core/reference/decorators.md b/website/src/content/docs/docs/libraries/azure-portal-core/reference/decorators.md
@@ -1,5 +1,6 @@
 ---
 title: "Decorators"
+description: "Decorators exported by @azure-tools/typespec-azure-portal-core"
 toc_min_heading_level: 2
 toc_max_heading_level: 3
 ---
diff --git a/website/src/content/docs/docs/libraries/azure-resource-manager/reference/data-types.md b/website/src/content/docs/docs/libraries/azure-resource-manager/reference/data-types.md
@@ -1,5 +1,7 @@
 ---
 title: "Data types"
+description: "Data types exported by @azure-tools/typespec-azure-resource-manager"
+llmstxt: true
 ---
 
 ## Azure.ResourceManager
diff --git a/website/src/content/docs/docs/libraries/azure-resource-manager/reference/decorators.md b/website/src/content/docs/docs/libraries/azure-resource-manager/reference/decorators.md
diff --git a/website/src/content/docs/docs/libraries/azure-resource-manager/reference/interfaces.md b/website/src/content/docs/docs/libraries/azure-resource-manager/reference/interfaces.md
diff --git a/website/src/content/docs/docs/libraries/typespec-client-generator-core/reference/data-types.md b/website/src/content/docs/docs/libraries/typespec-client-generator-core/reference/data-types.md
diff --git a/website/src/content/docs/docs/libraries/typespec-client-generator-core/reference/decorators.md b/website/src/content/docs/docs/libraries/typespec-client-generator-core/reference/decorators.md
diff --git a/website/src/pages/docs/[...path]/[llms_type].txt.ts b/website/src/pages/docs/[...path]/[llms_type].txt.ts
diff --git a/website/src/pages/docs/[...slug].md.ts b/website/src/pages/docs/[...slug].md.ts
diff --git a/website/src/pages/docs/llms-full.txt.ts b/website/src/pages/docs/llms-full.txt.ts
diff --git a/website/src/pages/docs/llms.json.ts b/website/src/pages/docs/llms.json.ts
diff --git a/website/src/pages/docs/llms.txt.ts b/website/src/pages/docs/llms.txt.ts
diff --git a/website/src/utils/llmstxt.ts b/website/src/utils/llmstxt.ts
