diff --git a/.chronus/changes/gen-llmstxt-2025-8-8-9-27-39.md b/.chronus/changes/gen-llmstxt-2025-8-8-9-27-39.md new file mode 100644 index 00000000000..fa201ef166d --- /dev/null +++ b/.chronus/changes/gen-llmstxt-2025-8-8-9-27-39.md @@ -0,0 +1,7 @@ +--- +changeKind: feature +packages: + - "@typespec/tspd" +--- + +Adds `llmstxt` frontmatter to generated reference docs to enable inclusion in llms.txt. Opt-in: specify `--llmstxt` to enable diff --git a/.chronus/changes/gen-llmstxt-2025-8-8-9-28-15.md b/.chronus/changes/gen-llmstxt-2025-8-8-9-28-15.md new file mode 100644 index 00000000000..0f293659bc7 --- /dev/null +++ b/.chronus/changes/gen-llmstxt-2025-8-8-9-28-15.md @@ -0,0 +1,15 @@ +--- +changeKind: internal +packages: + - "@typespec/events" + - "@typespec/http" + - "@typespec/json-schema" + - "@typespec/openapi" + - "@typespec/openapi3" + - "@typespec/protobuf" + - "@typespec/rest" + - "@typespec/streams" + - "@typespec/versioning" +--- + +Updated doc generation to generate `llmstxt` frontmatter \ No newline at end of file diff --git a/cspell.yaml b/cspell.yaml index 6a40f459aa5..860e59d3485 100644 --- a/cspell.yaml +++ b/cspell.yaml @@ -129,6 +129,8 @@ words: - LINUXOS - LINUXVMIMAGE - ljust + - llms + - llmstxt - lmazuel - lropaging - lstrip diff --git a/packages/astro-utils/package.json b/packages/astro-utils/package.json index f9bdd76d0c5..37a1a299fd2 100644 --- a/packages/astro-utils/package.json +++ b/packages/astro-utils/package.json @@ -10,7 +10,9 @@ "./components/*": "./src/components/*", "./utils/*": "./src/utils/*.ts", "./css/*": "./src/css/*", - "./expressive-code/*": "./dist/expressive-code/*.js" + "./expressive-code/*": "./dist/expressive-code/*.js", + "./llmstxt": "./src/llmstxt/index.ts", + "./llmstxt/schema": "./src/llmstxt/schema.ts" }, "files": [ "src", diff --git a/packages/astro-utils/src/llmstxt/generators.ts b/packages/astro-utils/src/llmstxt/generators.ts new file mode 100644 index 00000000000..d1f1113acad --- /dev/null +++ b/packages/astro-utils/src/llmstxt/generators.ts @@ -0,0 +1,90 @@ +import { mergeSiteWithPath, type DocEntry, type LlmsTxtAsJson } from "./index"; + +/** + * Generates the Markdown path following the llms.tst specification. + * @param docId The document ID from Astro content collections. + */ +export function generateMarkdownPath(docId: string): string { + // If the final path fragment does not include a file extension, use `index.html.md` + if (docId.endsWith("/")) { + return `${docId}index.html.md`; + } + + const finalPathFragment = docId.split("/").pop() ?? ""; + if (!finalPathFragment.includes(".")) { + return `${docId}/index.html.md`; + } + + return `${docId}.md`; +} + +/** + * Generates the LLMs text following the llms.tst specification. + * @param llmsData The pre-processed LLMs JSON data. + * @see `processDocsForLlmsTxt` + */ +export function generateLlmstxt(llmsData: LlmsTxtAsJson): string { + const contents: string[] = []; + contents.push(`# ${llmsData.title}`); + contents.push(`> ${llmsData.description}`); + + for (const [name, topic] of Object.entries(llmsData.topics)) { + if (!topic.length) continue; + const section: string[] = []; + section.push(`## ${name}\n`); + for (const { title, url, description } of topic) { + section.push(`- [${title}](${url}): ${description}`); + } + contents.push(section.join("\n")); + } + return contents.join("\n\n"); +} + +/** + * Generates the full LLMs text - the combined markdown documentation referenced from an llms.txt. + * @param title The title for the LLMs text file. + * @param docs The collection of documentation entries to include. + */ +export function generateLlmstxtFull(title: string, docs: DocEntry[]): string { + const contents: string[] = []; + contents.push(`# ${title}`); + + for (const doc of docs) { + if (!doc.body) continue; + + const docTitle = doc.data.title; + const docDescription = doc.data.description ?? ""; + contents.push(`# ${docTitle}`); + if (docDescription) contents.push(docDescription); + contents.push(doc.body); + } + + return contents.join("\n\n"); +} + +export type GenerateLlmsJsonTopicDetails = { + id: string; + description: string; + pathPrefix: string; +}; + +export type LlmsJson = { + topic: string; + description: string; + contentUrl: string; +}[]; + +/** + * Generates the `llms.json` version of `llms.txt`. + * This is meant for easier consumption by our tools. + */ +export function generateLlmsJson( + topicDetails: GenerateLlmsJsonTopicDetails[], + siteHref: string, +): LlmsJson { + return topicDetails.map(({ id, description, pathPrefix }) => ({ + topic: id, + description, + contentUrl: mergeSiteWithPath(siteHref, pathPrefix, "llms-full.txt"), + })); +} diff --git a/packages/astro-utils/src/llmstxt/index.ts b/packages/astro-utils/src/llmstxt/index.ts new file mode 100644 index 00000000000..0653cbed1d6 --- /dev/null +++ b/packages/astro-utils/src/llmstxt/index.ts @@ -0,0 +1,137 @@ +import type { z } from "astro:content"; +import { generateMarkdownPath } from "./generators"; +import type { llmstxtSchema } from "./schema"; + +export * from "./generators"; +export * from "./routes"; +export * from "./topics"; +export interface DocEntry { + id: string; + data: { + title: string; + description?: string; + llmstxt?: z.infer; + }; + body?: string; +} + +export interface LlmsTxtAsJson { + title: string; + description: string; + topics: Record< + string, + { + title: string; + description?: string; + url: string; + }[] + >; +} + +export interface ProcessDocsProps { + /** + * Title for the LLMs text file. + */ + title: string; + /** + * Description for the LLMs text file. + */ + description: string; + /** + * The site URL, used to generate full URLs for documentation entries. + */ + site?: URL; + /** + * The collection of documentation entries to process. + * Each entry must include a valid LLMs text schema. + * See `import("astro:content").getCollection` + */ + docs: DocEntry[]; + /** + * Name of the llmstxt section and the pathPrefix to match against doc IDs. + * If a doc matches multiple pathPrefixes, it will be assigned to the first matching section. + */ + llmsSections: { name: string; pathPrefix: string }[]; +} + +/** + * Processes astro content collection docs and metadata for easy `llms.txt` generation. + */ +export async function processDocsForLlmsTxt({ + title, + description, + site, + docs, + llmsSections, +}: ProcessDocsProps) { + const sections = organizeDocsIntoSections(docs, llmsSections); + const result: LlmsTxtAsJson = { title, description, topics: {} }; + + const siteHref = site?.href ?? ""; + for (const [sectionName, sectionDocs] of Object.entries(sections)) { + if (sectionDocs.length === 0) continue; + + const topic = sectionName; + const topics = sectionDocs.map((doc) => { + const title = doc.data.title; + const desc = doc.data.description ?? ""; + const path = generateMarkdownPath(doc.id); + const url = mergeSiteWithPath(siteHref, path); + return { title, description: desc, url }; + }); + + result.topics[topic] = topics; + } + + return result; +} + +function organizeDocsIntoSections( + docs: DocEntry[], + llmsSections: ProcessDocsProps["llmsSections"], +) { + docs.sort((a, b) => (a.id > b.id ? 1 : -1)); + const seenDocs = new Set(); + const sections: Record = {}; + + for (const { name, pathPrefix } of llmsSections) { + sections[name] = docs.filter((doc) => { + if (seenDocs.has(doc)) return false; + if (doc.id.startsWith(pathPrefix)) { + seenDocs.add(doc); + return true; + } + return false; + }); + } + + return sections; +} + +/** + * Merges a site URL with path parts. + * Used when needing to create full URLs when working with astro content collections. + * @param siteHref The base URL of the site. + * @param pathParts The path parts to merge with the site URL. + * @returns The merged URL. + */ +export function mergeSiteWithPath(siteHref: string, ...pathParts: string[]): string { + let result = siteHref; + + for (const part of pathParts) { + if (!part) continue; // Skip empty parts + + const resultTrailingSlash = result.endsWith("/"); + const partLeadingSlash = part.startsWith("/"); + + if (resultTrailingSlash && partLeadingSlash) { + result = `${result}${part.slice(1)}`; + } else if (!resultTrailingSlash && !partLeadingSlash) { + result = `${result}/${part}`; + } else { + result = `${result}${part}`; + } + } + + return result; +} diff --git a/packages/astro-utils/src/llmstxt/routes.ts b/packages/astro-utils/src/llmstxt/routes.ts new file mode 100644 index 00000000000..4a13e809ccc --- /dev/null +++ b/packages/astro-utils/src/llmstxt/routes.ts @@ -0,0 +1,54 @@ +import type { APIRoute } from "astro"; +import { + generateLlmstxt, + generateLlmstxtFull, + processDocsForLlmsTxt, + type DocEntry, + type TopicProps, +} from "./index"; + +export type RouteParams = { path: string; llms_type: "llms" | "llms-full" }; +export type RouteProps = Pick, "title" | "description" | "docs">; + +export const spreadLlmsTxtRoute: APIRoute = async ({ + props, + params, + site, +}) => { + const { title, docs, description } = props; + const { llms_type } = params; + + if (llms_type === "llms") { + const llmsData = await processDocsForLlmsTxt({ + title, + description, + docs, + // Use blank pathPrefix to include all docs in the llms.txt + llmsSections: [{ name: "Docs", pathPrefix: "" }], + site, + }); + + const llmstxt = generateLlmstxt(llmsData); + return new Response(llmstxt, { + headers: { + "Content-Type": "text/markdown; charset=utf-8", + }, + }); + } else { + const llmstxt = generateLlmstxtFull(description, docs); + return new Response(llmstxt, { + headers: { + "Content-Type": "text/markdown; charset=utf-8", + }, + }); + } +}; + +export const markdownRoute: APIRoute<{ doc: DocEntry }> = async ({ props }) => { + const { doc } = props; + return new Response(doc.body ?? "", { + headers: { + "Content-Type": "text/markdown; charset=utf-8", + }, + }); +}; diff --git a/packages/astro-utils/src/llmstxt/schema.ts b/packages/astro-utils/src/llmstxt/schema.ts new file mode 100644 index 00000000000..43eb6c1c880 --- /dev/null +++ b/packages/astro-utils/src/llmstxt/schema.ts @@ -0,0 +1,3 @@ +import { z } from "astro:content"; + +export const llmstxtSchema = z.boolean().optional(); diff --git a/packages/astro-utils/src/llmstxt/topics.ts b/packages/astro-utils/src/llmstxt/topics.ts new file mode 100644 index 00000000000..41a1049ab86 --- /dev/null +++ b/packages/astro-utils/src/llmstxt/topics.ts @@ -0,0 +1,34 @@ +import type { DocEntry } from "."; + +export interface TopicProps { + title: string; + description: string; + pathPrefix: string; + docs: DocEntry[]; + id: string; +} + +/** + * + */ +export function populateTopicDocs( + topics: Omit[], + docs: DocEntry[], +): TopicProps[] { + docs.sort((a, b) => (a.id > b.id ? 1 : -1)); + const seenDocs = new Set(); + + return topics.map((topic) => { + return { + ...topic, + docs: docs.filter((doc) => { + if (seenDocs.has(doc)) return false; + if (doc.id.startsWith(topic.pathPrefix)) { + seenDocs.add(doc); + return true; + } + return false; + }), + }; + }); +} diff --git a/packages/events/package.json b/packages/events/package.json index 56ac5995d2c..af35433da18 100644 --- a/packages/events/package.json +++ b/packages/events/package.json @@ -45,7 +45,7 @@ "test-official": "vitest run --coverage --reporter=junit --reporter=default --no-file-parallelism", "lint": "eslint . --ext .ts --max-warnings=0", "lint:fix": "eslint . --fix --ext .ts", - "regen-docs": "tspd doc . --enable-experimental --output-dir ../../website/src/content/docs/docs/libraries/events/reference" + "regen-docs": "tspd doc . --enable-experimental --llmstxt --output-dir ../../website/src/content/docs/docs/libraries/events/reference" }, "files": [ "lib/*.tsp", diff --git a/packages/http/package.json b/packages/http/package.json index 35e1feb1ba2..7e2540ac69d 100644 --- a/packages/http/package.json +++ b/packages/http/package.json @@ -66,7 +66,7 @@ "test:ci": "vitest run --coverage --reporter=junit --reporter=default", "lint": "eslint . --max-warnings=0", "lint:fix": "eslint . --fix", - "regen-docs": "tspd doc . --enable-experimental --typekits --output-dir ../../website/src/content/docs/docs/libraries/http/reference" + "regen-docs": "tspd doc . --enable-experimental --typekits --llmstxt --output-dir ../../website/src/content/docs/docs/libraries/http/reference" }, "files": [ "lib/**/*.tsp", diff --git a/packages/json-schema/package.json b/packages/json-schema/package.json index 2dddef23500..937c3f2ac4f 100644 --- a/packages/json-schema/package.json +++ b/packages/json-schema/package.json @@ -45,7 +45,7 @@ "test:ci": "vitest run --coverage --reporter=junit --reporter=default", "lint": "eslint . --max-warnings=0", "lint:fix": "eslint . --fix", - "regen-docs": "tspd doc . --enable-experimental --output-dir ../../website/src/content/docs/docs/emitters/json-schema/reference", + "regen-docs": "tspd doc . --enable-experimental --llmstxt --output-dir ../../website/src/content/docs/docs/emitters/json-schema/reference", "api-extractor": "api-extractor run --local --verbose" }, "files": [ diff --git a/packages/openapi/package.json b/packages/openapi/package.json index 043a0a9b181..1a7712bfcdf 100644 --- a/packages/openapi/package.json +++ b/packages/openapi/package.json @@ -45,7 +45,7 @@ "test:ci": "vitest run --coverage --reporter=junit --reporter=default", "lint": "eslint . --max-warnings=0", "lint:fix": "eslint . --fix", - "regen-docs": "tspd doc . --enable-experimental --output-dir ../../website/src/content/docs/docs/libraries/openapi/reference", + "regen-docs": "tspd doc . --enable-experimental --llmstxt --output-dir ../../website/src/content/docs/docs/libraries/openapi/reference", "api-extractor": "api-extractor run --local --verbose" }, "files": [ diff --git a/packages/openapi3/package.json b/packages/openapi3/package.json index 5292fc0eb94..3ce2178d40c 100644 --- a/packages/openapi3/package.json +++ b/packages/openapi3/package.json @@ -48,7 +48,7 @@ "test:ci": "vitest run --coverage --reporter=junit --reporter=default", "lint": "eslint . --max-warnings=0", "lint:fix": "eslint . --fix", - "regen-docs": "tspd doc . --enable-experimental --output-dir ../../website/src/content/docs/docs/emitters/openapi3/reference", + "regen-docs": "tspd doc . --enable-experimental --llmstxt --output-dir ../../website/src/content/docs/docs/emitters/openapi3/reference", "regen-specs": "cross-env RECORD=true vitest run", "gen-version": "node scripts/generate-version.js", "api-extractor": "api-extractor run --local --verbose" diff --git a/packages/protobuf/package.json b/packages/protobuf/package.json index a5bbfc28aa4..0e05f57a544 100644 --- a/packages/protobuf/package.json +++ b/packages/protobuf/package.json @@ -38,7 +38,7 @@ "test:ci": "vitest run --coverage --reporter=junit --reporter=default", "lint": "eslint . --max-warnings=0", "lint:fix": "eslint . --fix", - "regen-docs": "tspd doc . --enable-experimental --output-dir ../../website/src/content/docs/docs/emitters/protobuf/reference" + "regen-docs": "tspd doc . --enable-experimental --llmstxt --output-dir ../../website/src/content/docs/docs/emitters/protobuf/reference" }, "peerDependencies": { "@typespec/compiler": "workspace:^" diff --git a/packages/rest/package.json b/packages/rest/package.json index b8a98ef8c78..296b19be990 100644 --- a/packages/rest/package.json +++ b/packages/rest/package.json @@ -45,7 +45,7 @@ "test:ci": "vitest run --coverage --reporter=junit --reporter=default", "lint": "eslint . --max-warnings=0", "lint:fix": "eslint . --fix", - "regen-docs": "tspd doc . --enable-experimental --output-dir ../../website/src/content/docs/docs/libraries/rest/reference" + "regen-docs": "tspd doc . --enable-experimental --llmstxt --output-dir ../../website/src/content/docs/docs/libraries/rest/reference" }, "files": [ "lib/*.tsp", diff --git a/packages/streams/package.json b/packages/streams/package.json index 7424fad1a8b..2b3b378e789 100644 --- a/packages/streams/package.json +++ b/packages/streams/package.json @@ -45,7 +45,7 @@ "test-official": "vitest run --coverage --reporter=junit --reporter=default --no-file-parallelism", "lint": "eslint . --ext .ts --max-warnings=0", "lint:fix": "eslint . --fix --ext .ts", - "regen-docs": "tspd doc . --enable-experimental --output-dir ../../website/src/content/docs/docs/libraries/streams/reference" + "regen-docs": "tspd doc . --enable-experimental --llmstxt --output-dir ../../website/src/content/docs/docs/libraries/streams/reference" }, "files": [ "lib/*.tsp", diff --git a/packages/tspd/src/cli.ts b/packages/tspd/src/cli.ts index e21b5087fbb..4ecdd3fa5d9 100644 --- a/packages/tspd/src/cli.ts +++ b/packages/tspd/src/cli.ts @@ -86,6 +86,11 @@ async function main() { .option("typekits", { description: "Generate typekit docs. Currently targeted for use with Astro Starlight.", type: "boolean", + }) + .option("llmstxt", { + description: + "Add llmstxt frontmatter to generated docs to aide in generating llms.txt files.", + type: "boolean", }); }, async (args) => { @@ -97,6 +102,7 @@ async function main() { { skipJSApi: args["skip-js"], typekits: args["typekits"], + llmstxt: args["llmstxt"], }, ); // const diagnostics = await generateExternSignatures(host, resolvedRoot); diff --git a/packages/tspd/src/ref-doc/emitters/starlight.ts b/packages/tspd/src/ref-doc/emitters/starlight.ts index 2187c2d168c..ab43db0a671 100644 --- a/packages/tspd/src/ref-doc/emitters/starlight.ts +++ b/packages/tspd/src/ref-doc/emitters/starlight.ts @@ -4,7 +4,6 @@ import { RefDocEntity, TypeSpecLibraryRefDoc, TypeSpecRefDoc, - TypeSpecRefDocBase, } from "../types.js"; import { MarkdownDoc, @@ -16,26 +15,33 @@ import { } from "../utils/markdown.js"; import { MarkdownRenderer, groupByNamespace } from "./markdown.js"; +export interface RenderToStarlightMarkdownOptions { + llmstxt?: boolean; +} + /** * Render doc to a markdown using docusaurus addons. */ -export function renderToAstroStarlightMarkdown(refDoc: TypeSpecRefDoc): Record { +export function renderToAstroStarlightMarkdown( + refDoc: TypeSpecRefDoc, + options: RenderToStarlightMarkdownOptions = {}, +): Record { const renderer = new StarlightRenderer(refDoc); const files: Record = { "index.mdx": renderIndexFile(renderer, refDoc), }; - const decoratorFile = renderDecoratorFile(renderer, refDoc); + const decoratorFile = renderDecoratorFile(renderer, refDoc, { llmstxt: options.llmstxt }); if (decoratorFile) { files["decorators.md"] = decoratorFile; } - const interfaceFile = renderInterfacesFile(renderer, refDoc); + const interfaceFile = renderInterfacesFile(renderer, refDoc, { llmstxt: options.llmstxt }); if (interfaceFile) { files["interfaces.md"] = interfaceFile; } - const dataTypes = renderDataTypes(renderer, refDoc); + const dataTypes = renderDataTypes(renderer, refDoc, { llmstxt: options.llmstxt }); if (dataTypes) { files["data-types.md"] = dataTypes; } @@ -94,37 +100,63 @@ function renderIndexFile(renderer: StarlightRenderer, refDoc: TypeSpecLibraryRef export type DecoratorRenderOptions = { title?: string; + llmstxt?: boolean; }; export function renderDecoratorFile( renderer: StarlightRenderer, - refDoc: TypeSpecRefDocBase, + refDoc: TypeSpecRefDoc, options?: DecoratorRenderOptions, ): string | undefined { if (!refDoc.namespaces.some((x) => x.decorators.length > 0)) { return undefined; } const title = options?.title ?? "Decorators"; + const name = refDoc.name ?? refDoc.namespaces[0]?.name ?? ""; const content: MarkdownDoc = [ "---", `title: "${title}"`, + `description: "Decorators ${name ? `exported by ${name}` : ""}"`, "toc_min_heading_level: 2", "toc_max_heading_level: 3", - "---", ]; + if (options?.llmstxt) { + content.push("llmstxt: true"); + } + + content.push("---"); + content.push(renderer.decoratorsSection(refDoc)); return renderMarkdowDoc(content, 2); } +export type InterfacesRenderOptions = { + llmstxt?: boolean; +}; + function renderInterfacesFile( renderer: StarlightRenderer, refDoc: TypeSpecRefDoc, + options?: InterfacesRenderOptions, ): string | undefined { if (!refDoc.namespaces.some((x) => x.operations.length > 0 || x.interfaces.length > 0)) { return undefined; } - const content: MarkdownDoc = ["---", `title: "Interfaces and Operations"`, "---"]; + + const title = "Interfaces and Operations"; + const name = refDoc.name ?? refDoc.namespaces[0]?.name ?? ""; + const content: MarkdownDoc = [ + "---", + `title: "${title}"`, + `description: "Interfaces and Operations ${name ? `exported by ${name}` : ""}"`, + ]; + + if (options?.llmstxt) { + content.push("llmstxt: true"); + } + + content.push("---"); content.push( groupByNamespace(refDoc.namespaces, (namespace) => { @@ -149,6 +181,7 @@ function renderInterfacesFile( export type DataTypeRenderOptions = { title?: string; + llmstxt?: boolean; }; export function renderDataTypes( @@ -160,7 +193,18 @@ export function renderDataTypes( return undefined; } const title = options?.title ?? "Data types"; - const content: MarkdownDoc = ["---", `title: "${title}"`, "---"]; + const name = refDoc.name ?? refDoc.namespaces[0]?.name ?? ""; + const content: MarkdownDoc = [ + "---", + `title: "${title}"`, + `description: "Data types ${name ? `exported by ${name}` : ""}"`, + ]; + + if (options?.llmstxt) { + content.push("llmstxt: true"); + } + + content.push("---"); content.push( groupByNamespace(refDoc.namespaces, (namespace) => { diff --git a/packages/tspd/src/ref-doc/experimental.ts b/packages/tspd/src/ref-doc/experimental.ts index 7f5f3297d15..a7cd57538bc 100644 --- a/packages/tspd/src/ref-doc/experimental.ts +++ b/packages/tspd/src/ref-doc/experimental.ts @@ -18,6 +18,7 @@ import { readPackageJson } from "./utils/misc.js"; export interface GenerateLibraryDocsOptions { typekits?: boolean; skipJSApi?: boolean; + llmstxt?: boolean; } /** * @experimental this is for experimental and is for internal use only. Breaking change to this API can happen at anytime. @@ -30,7 +31,7 @@ export async function generateLibraryDocs( const diagnostics = createDiagnosticCollector(); const pkgJson = await readPackageJson(libraryPath); const refDoc = diagnostics.pipe(await extractLibraryRefDocs(libraryPath)); - const files = renderToAstroStarlightMarkdown(refDoc); + const files = renderToAstroStarlightMarkdown(refDoc, options); await mkdir(outputDir, { recursive: true }); const config = await prettier.resolveConfig(libraryPath); for (const [name, content] of Object.entries(files)) { diff --git a/packages/versioning/package.json b/packages/versioning/package.json index 05b22db2ee9..61e714b0704 100644 --- a/packages/versioning/package.json +++ b/packages/versioning/package.json @@ -44,7 +44,7 @@ "test:ci": "vitest run --coverage --reporter=junit --reporter=default", "lint": "eslint . --max-warnings=0", "lint:fix": "eslint . --fix", - "regen-docs": "tspd doc . --enable-experimental --output-dir ../../website/src/content/docs/docs/libraries/versioning/reference" + "regen-docs": "tspd doc . --enable-experimental --llmstxt --output-dir ../../website/src/content/docs/docs/libraries/versioning/reference" }, "files": [ "lib/*.tsp", diff --git a/website/src/content.config.ts b/website/src/content.config.ts index ac86999553c..4b65bf1675e 100644 --- a/website/src/content.config.ts +++ b/website/src/content.config.ts @@ -1,5 +1,6 @@ import { docsLoader } from "@astrojs/starlight/loaders"; import { docsSchema } from "@astrojs/starlight/schema"; +import { llmstxtSchema } from "@typespec/astro-utils/llmstxt/schema"; import { defineCollection, z } from "astro:content"; const authorSchema = z.object({ @@ -20,6 +21,7 @@ export const collections = { .describe( "A date string or YAML date that is compatible with JavaScript's `new Date()` constructor.", ), + llmstxt: llmstxtSchema.optional(), }), }), }), diff --git a/website/src/content/docs/docs/emitters/json-schema/reference/data-types.md b/website/src/content/docs/docs/emitters/json-schema/reference/data-types.md index ba37a279705..070494b0b85 100644 --- a/website/src/content/docs/docs/emitters/json-schema/reference/data-types.md +++ b/website/src/content/docs/docs/emitters/json-schema/reference/data-types.md @@ -1,5 +1,7 @@ --- title: "Data types" +description: "Data types exported by @typespec/json-schema" +llmstxt: true --- ## TypeSpec.JsonSchema diff --git a/website/src/content/docs/docs/emitters/json-schema/reference/decorators.md b/website/src/content/docs/docs/emitters/json-schema/reference/decorators.md index 4178ba3dfee..7f5e56e9a07 100644 --- a/website/src/content/docs/docs/emitters/json-schema/reference/decorators.md +++ b/website/src/content/docs/docs/emitters/json-schema/reference/decorators.md @@ -1,7 +1,9 @@ --- title: "Decorators" +description: "Decorators exported by @typespec/json-schema" toc_min_heading_level: 2 toc_max_heading_level: 3 +llmstxt: true --- ## TypeSpec.JsonSchema diff --git a/website/src/content/docs/docs/emitters/openapi3/reference/decorators.md b/website/src/content/docs/docs/emitters/openapi3/reference/decorators.md index c057a1ad57c..efc0e2c0795 100644 --- a/website/src/content/docs/docs/emitters/openapi3/reference/decorators.md +++ b/website/src/content/docs/docs/emitters/openapi3/reference/decorators.md @@ -1,7 +1,9 @@ --- title: "Decorators" +description: "Decorators exported by @typespec/openapi3" toc_min_heading_level: 2 toc_max_heading_level: 3 +llmstxt: true --- ## TypeSpec.OpenAPI diff --git a/website/src/content/docs/docs/emitters/protobuf/reference/data-types.md b/website/src/content/docs/docs/emitters/protobuf/reference/data-types.md index 29a527ee876..1e1149a1cd5 100644 --- a/website/src/content/docs/docs/emitters/protobuf/reference/data-types.md +++ b/website/src/content/docs/docs/emitters/protobuf/reference/data-types.md @@ -1,5 +1,7 @@ --- title: "Data types" +description: "Data types exported by @typespec/protobuf" +llmstxt: true --- ## TypeSpec.Protobuf diff --git a/website/src/content/docs/docs/emitters/protobuf/reference/decorators.md b/website/src/content/docs/docs/emitters/protobuf/reference/decorators.md index 551e61e91ee..cdbd04a4fcb 100644 --- a/website/src/content/docs/docs/emitters/protobuf/reference/decorators.md +++ b/website/src/content/docs/docs/emitters/protobuf/reference/decorators.md @@ -1,7 +1,9 @@ --- title: "Decorators" +description: "Decorators exported by @typespec/protobuf" toc_min_heading_level: 2 toc_max_heading_level: 3 +llmstxt: true --- ## TypeSpec.Protobuf diff --git a/website/src/content/docs/docs/getting-started/getting-started-rest/01-setup-basic-syntax.mdx b/website/src/content/docs/docs/getting-started/getting-started-rest/01-setup-basic-syntax.mdx index 2d86d6085a7..40267bc99b8 100644 --- a/website/src/content/docs/docs/getting-started/getting-started-rest/01-setup-basic-syntax.mdx +++ b/website/src/content/docs/docs/getting-started/getting-started-rest/01-setup-basic-syntax.mdx @@ -1,6 +1,8 @@ --- id: 01-setup-basic-syntax title: Getting Started with TypeSpec For REST APIs +description: Getting started with REST - 01 setup basic syntax +llmstxt: true --- import { FileTree } from "@astrojs/starlight/components"; diff --git a/website/src/content/docs/docs/getting-started/getting-started-rest/02-operations-responses.md b/website/src/content/docs/docs/getting-started/getting-started-rest/02-operations-responses.md index fcef43e4ed3..18650cc2109 100644 --- a/website/src/content/docs/docs/getting-started/getting-started-rest/02-operations-responses.md +++ b/website/src/content/docs/docs/getting-started/getting-started-rest/02-operations-responses.md @@ -1,6 +1,8 @@ --- id: 02-operations-responses title: Operations and Responses +description: Getting started with REST - 02 defining CRUD operations +llmstxt: true --- ## Introduction diff --git a/website/src/content/docs/docs/getting-started/getting-started-rest/03-handling-errors.md b/website/src/content/docs/docs/getting-started/getting-started-rest/03-handling-errors.md index 6a465733313..6aa893c46a8 100644 --- a/website/src/content/docs/docs/getting-started/getting-started-rest/03-handling-errors.md +++ b/website/src/content/docs/docs/getting-started/getting-started-rest/03-handling-errors.md @@ -1,5 +1,7 @@ --- title: Handling Errors +description: Getting started with REST - 03 handling errors in your API +llmstxt: true --- ## Introduction diff --git a/website/src/content/docs/docs/getting-started/getting-started-rest/04-common-parameters.md b/website/src/content/docs/docs/getting-started/getting-started-rest/04-common-parameters.md index 07fb40d3fcd..665421e8d0a 100644 --- a/website/src/content/docs/docs/getting-started/getting-started-rest/04-common-parameters.md +++ b/website/src/content/docs/docs/getting-started/getting-started-rest/04-common-parameters.md @@ -1,5 +1,7 @@ --- title: Common Parameters +description: Getting started with REST - 04 reuse common parameters +llmstxt: true --- ## Introduction diff --git a/website/src/content/docs/docs/getting-started/getting-started-rest/05-authentication.md b/website/src/content/docs/docs/getting-started/getting-started-rest/05-authentication.md index 5bed24a31dc..f1937fb8798 100644 --- a/website/src/content/docs/docs/getting-started/getting-started-rest/05-authentication.md +++ b/website/src/content/docs/docs/getting-started/getting-started-rest/05-authentication.md @@ -1,5 +1,7 @@ --- title: Authentication +description: Getting started with REST - 05 adding authentication +llmstxt: true --- ## Introduction diff --git a/website/src/content/docs/docs/getting-started/getting-started-rest/06-versioning.mdx b/website/src/content/docs/docs/getting-started/getting-started-rest/06-versioning.mdx index 19ee7e2ea18..41d61f9c0a3 100644 --- a/website/src/content/docs/docs/getting-started/getting-started-rest/06-versioning.mdx +++ b/website/src/content/docs/docs/getting-started/getting-started-rest/06-versioning.mdx @@ -1,5 +1,7 @@ --- title: Versioning +description: Getting started with REST - 06 versioning your REST API +llmstxt: true --- import { FileTree } from "@astrojs/starlight/components"; diff --git a/website/src/content/docs/docs/getting-started/getting-started-rest/07-custom-response-models.md b/website/src/content/docs/docs/getting-started/getting-started-rest/07-custom-response-models.md index d6052c34d87..f26fadc8231 100644 --- a/website/src/content/docs/docs/getting-started/getting-started-rest/07-custom-response-models.md +++ b/website/src/content/docs/docs/getting-started/getting-started-rest/07-custom-response-models.md @@ -1,5 +1,7 @@ --- title: Custom Response Models +description: Getting started with REST - 07 customize API response models +llmstxt: true --- ## Introduction diff --git a/website/src/content/docs/docs/getting-started/typespec-for-openapi-dev.md b/website/src/content/docs/docs/getting-started/typespec-for-openapi-dev.md index aafeeca96d3..ca02a4d3ac5 100644 --- a/website/src/content/docs/docs/getting-started/typespec-for-openapi-dev.md +++ b/website/src/content/docs/docs/getting-started/typespec-for-openapi-dev.md @@ -1,5 +1,7 @@ --- title: TypeSpec for OpenAPI Developers +description: Getting started with TypeSpec for Open API developers +llmstxt: true --- This guide introduces TypeSpec using concepts familiar to developers who build or use API definitions in OpenAPI v2 or v3. diff --git a/website/src/content/docs/docs/language-basics/alias.md b/website/src/content/docs/docs/language-basics/alias.md index d726d4a108c..ee3eded0ac0 100644 --- a/website/src/content/docs/docs/language-basics/alias.md +++ b/website/src/content/docs/docs/language-basics/alias.md @@ -1,5 +1,7 @@ --- title: Aliases +description: "Language basics - aliases" +llmstxt: true --- Aliases are a convenient way to define shorthand for types, especially when dealing with complex expressions. They simplify the syntax but don't have a representation in the type graph. As a result, you can't decorate aliases. If you need to give an alternate name to a model, use [`model is`](./models.md/#is). diff --git a/website/src/content/docs/docs/language-basics/built-in-types.md b/website/src/content/docs/docs/language-basics/built-in-types.md index e46fe701932..e79aa5608e5 100644 --- a/website/src/content/docs/docs/language-basics/built-in-types.md +++ b/website/src/content/docs/docs/language-basics/built-in-types.md @@ -1,6 +1,8 @@ --- id: built-in-types title: Built-in types +description: "Language basics - built-in types" +llmstxt: true --- TypeSpec Standard Library provide some built-in types that can be used to build more complex types. diff --git a/website/src/content/docs/docs/language-basics/decorators.md b/website/src/content/docs/docs/language-basics/decorators.md index a82f9d3de92..92c820853a0 100644 --- a/website/src/content/docs/docs/language-basics/decorators.md +++ b/website/src/content/docs/docs/language-basics/decorators.md @@ -1,6 +1,8 @@ --- id: decorators title: Decorators +description: "Language basics - using decorators" +llmstxt: true --- Decorators in TypeSpec allow developers to attach metadata to types within a TypeSpec program. They can also be used to compute types based on their inputs. Decorators form the core of TypeSpec's extensibility, providing the flexibility to describe a wide variety of APIs and associated metadata such as documentation, constraints, samples, and more. diff --git a/website/src/content/docs/docs/language-basics/documentation.md b/website/src/content/docs/docs/language-basics/documentation.md index 5632b6ece8c..f321f2d21d9 100644 --- a/website/src/content/docs/docs/language-basics/documentation.md +++ b/website/src/content/docs/docs/language-basics/documentation.md @@ -1,6 +1,8 @@ --- id: documentation title: Documentation +description: "Language basics - documenting APIs" +llmstxt: true --- Documentation is a vital aspect of any API. TypeSpec offers several ways to document your API, including doc comments and decorators. diff --git a/website/src/content/docs/docs/language-basics/enums.md b/website/src/content/docs/docs/language-basics/enums.md index 829b5262f2a..cdfab7c02da 100644 --- a/website/src/content/docs/docs/language-basics/enums.md +++ b/website/src/content/docs/docs/language-basics/enums.md @@ -1,6 +1,8 @@ --- id: enums title: Enums +description: "Language basics - enums" +llmstxt: true --- Enums, short for enumerations, provide a way for developers to define a collection of named constants. They are useful for documenting the purpose of the code or for establishing a set of distinct scenarios. Enums can be either numeric or string-based. For other data types, consider using [unions](./unions.md). diff --git a/website/src/content/docs/docs/language-basics/identifiers.md b/website/src/content/docs/docs/language-basics/identifiers.md index 5de21c94045..d225e0ffcf4 100644 --- a/website/src/content/docs/docs/language-basics/identifiers.md +++ b/website/src/content/docs/docs/language-basics/identifiers.md @@ -1,5 +1,7 @@ --- title: Identifiers +description: "Language basics - identifiers" +llmstxt: true --- Identifiers are used to name models, enums, properties, and other entities in TypeSpec. An identifier is a sequence of one or more characters that must start with a letter, emoji, underscore, or dollar sign, and be followed by letters, numbers, emoji, underscores, or dollar signs. TypeSpec implements [UAX31-R1b stable identifiers](http://www.unicode.org/reports/tr31/#R1b) with the [emoji profile](http://www.unicode.org/reports/tr31/#Emoji_Profile). diff --git a/website/src/content/docs/docs/language-basics/imports.md b/website/src/content/docs/docs/language-basics/imports.md index de6ff9462e4..163b0b0633e 100644 --- a/website/src/content/docs/docs/language-basics/imports.md +++ b/website/src/content/docs/docs/language-basics/imports.md @@ -1,6 +1,8 @@ --- id: imports title: Imports +description: "Language basics - importing files and libraries" +llmstxt: true --- Imports are used to include files or libraries into your TypeSpec program. When compiling a TypeSpec file, you specify the path to your root TypeSpec file, typically named "main.tsp". From this root file, any imported files are added to your program. If a directory is imported, TypeSpec will search for a `main.tsp` file within that directory. diff --git a/website/src/content/docs/docs/language-basics/interfaces.md b/website/src/content/docs/docs/language-basics/interfaces.md index 28df25ffd64..7f06dde404e 100644 --- a/website/src/content/docs/docs/language-basics/interfaces.md +++ b/website/src/content/docs/docs/language-basics/interfaces.md @@ -1,6 +1,8 @@ --- id: interfaces title: Interfaces +description: "Language basics - grouping operations with interfaces" +llmstxt: true --- Interfaces are useful for grouping and reusing [operations](./operations.md). diff --git a/website/src/content/docs/docs/language-basics/intersections.md b/website/src/content/docs/docs/language-basics/intersections.md index 6023eaf4511..835fe415df1 100644 --- a/website/src/content/docs/docs/language-basics/intersections.md +++ b/website/src/content/docs/docs/language-basics/intersections.md @@ -1,6 +1,8 @@ --- id: intersections title: Intersections +description: "Language basics - composing models with intersections" +llmstxt: true --- Intersections in programming define a type that must encompass all the constituents of the intersection. You can declare an intersection using the `&` operator. diff --git a/website/src/content/docs/docs/language-basics/models.md b/website/src/content/docs/docs/language-basics/models.md index 4aff4b426d7..23c4eb0b9a6 100644 --- a/website/src/content/docs/docs/language-basics/models.md +++ b/website/src/content/docs/docs/language-basics/models.md @@ -1,6 +1,8 @@ --- id: models title: Models +description: "Language basics - defining schemas with models" +llmstxt: true --- Models in TypeSpec are utilized to define the structure or schema of data. diff --git a/website/src/content/docs/docs/language-basics/namespaces.md b/website/src/content/docs/docs/language-basics/namespaces.md index 6f4d6baa1b2..a80d243fda3 100644 --- a/website/src/content/docs/docs/language-basics/namespaces.md +++ b/website/src/content/docs/docs/language-basics/namespaces.md @@ -1,6 +1,8 @@ --- id: namespaces title: Namespaces +description: "Language basics - grouping declarations with namespaces" +llmstxt: true --- Namespaces in TypeSpec allow you to group related types together. This organization makes your types easier to locate and helps avoid naming conflicts. Namespaces are merged across files, enabling you to reference any type from anywhere in your TypeSpec program using its namespace. diff --git a/website/src/content/docs/docs/language-basics/operations.md b/website/src/content/docs/docs/language-basics/operations.md index 66518a7200c..c2571cdc96f 100644 --- a/website/src/content/docs/docs/language-basics/operations.md +++ b/website/src/content/docs/docs/language-basics/operations.md @@ -1,6 +1,8 @@ --- id: operations title: Operations +description: "Language basics - defining endpoints with operations" +llmstxt: true --- Operations are essentially service endpoints, characterized by an operation name, parameters, and a return type. diff --git a/website/src/content/docs/docs/language-basics/overview.md b/website/src/content/docs/docs/language-basics/overview.md index 0ab9329cb7c..df0a9385a9b 100644 --- a/website/src/content/docs/docs/language-basics/overview.md +++ b/website/src/content/docs/docs/language-basics/overview.md @@ -1,6 +1,8 @@ --- id: overview title: Overview +description: "Language basics - overview" +llmstxt: true --- This document provides a concise overview of the language concepts in TypeSpec. It serves as a quick reference guide rather than an in-depth tutorial. diff --git a/website/src/content/docs/docs/language-basics/scalars.md b/website/src/content/docs/docs/language-basics/scalars.md index 84ec6ffad77..090729b41f7 100644 --- a/website/src/content/docs/docs/language-basics/scalars.md +++ b/website/src/content/docs/docs/language-basics/scalars.md @@ -1,5 +1,7 @@ --- title: Scalars +description: "Language basics - custom scalars" +llmstxt: true --- Scalars are simple types that don't have any fields. Examples of these include `string`, `int32`, `boolean`, and so on. diff --git a/website/src/content/docs/docs/language-basics/templates.md b/website/src/content/docs/docs/language-basics/templates.md index f6507fac8b1..8bfff7a8053 100644 --- a/website/src/content/docs/docs/language-basics/templates.md +++ b/website/src/content/docs/docs/language-basics/templates.md @@ -1,6 +1,8 @@ --- id: templates title: Templates +description: "Language basics - reusability with templates" +llmstxt: true --- Templates are a powerful tool that allow users to customize certain aspects of a type. Similar to generics in other programming languages, templates define template parameters that users can specify when referencing the type. diff --git a/website/src/content/docs/docs/language-basics/type-literals.md b/website/src/content/docs/docs/language-basics/type-literals.md index e52575ba7c4..53eebd04aeb 100644 --- a/website/src/content/docs/docs/language-basics/type-literals.md +++ b/website/src/content/docs/docs/language-basics/type-literals.md @@ -1,6 +1,8 @@ --- id: type-literals title: Type Literals +description: "Language basics - literal types" +llmstxt: true --- When designing APIs, it's common to define the structure of the API in terms of specific literal values. For instance, an operation might return a specific integer status code, or a model member might be one of a few specific string values. It's also useful to pass specific literal values to decorators. TypeSpec supports string, number, and boolean literal values to cater to these needs. diff --git a/website/src/content/docs/docs/language-basics/type-relations.md b/website/src/content/docs/docs/language-basics/type-relations.md index 217b66b997c..1b12c86d69c 100644 --- a/website/src/content/docs/docs/language-basics/type-relations.md +++ b/website/src/content/docs/docs/language-basics/type-relations.md @@ -1,6 +1,8 @@ --- id: type-relations title: Type Relations +description: Language basics - type relations +llmstxt: true --- ## Type hierarchy diff --git a/website/src/content/docs/docs/language-basics/unions.md b/website/src/content/docs/docs/language-basics/unions.md index 9050918896e..8a0bbfbbc84 100644 --- a/website/src/content/docs/docs/language-basics/unions.md +++ b/website/src/content/docs/docs/language-basics/unions.md @@ -1,6 +1,8 @@ --- id: unions title: Unions +description: "Language basics - unions" +llmstxt: true --- Unions define a type that must be exactly one of several possible variants. There are two types of unions: diff --git a/website/src/content/docs/docs/language-basics/values.md b/website/src/content/docs/docs/language-basics/values.md index 4276e9b143b..879c95bf949 100644 --- a/website/src/content/docs/docs/language-basics/values.md +++ b/website/src/content/docs/docs/language-basics/values.md @@ -1,6 +1,8 @@ --- id: values title: Values +description: "Language basics - working with values" +llmstxt: true --- TypeSpec can define values in addition to types. Values are useful in an API description to define default values for types or provide example values. They are also useful when passing data to decorators, and for template parameters that are ultimately passed to decorators or used as default values. diff --git a/website/src/content/docs/docs/language-basics/visibility.md b/website/src/content/docs/docs/language-basics/visibility.md index 18f44970ef1..38f8ead170c 100644 --- a/website/src/content/docs/docs/language-basics/visibility.md +++ b/website/src/content/docs/docs/language-basics/visibility.md @@ -1,6 +1,8 @@ --- id: visibility title: Visibility +description: "Language basics - defining views of a model across operations with visibility" +llmstxt: true --- **Visibility** is a language feature that allows you to share a model between multiple operations and define in which contexts diff --git a/website/src/content/docs/docs/libraries/events/reference/decorators.md b/website/src/content/docs/docs/libraries/events/reference/decorators.md index 302ea7e5a3f..1df2148655d 100644 --- a/website/src/content/docs/docs/libraries/events/reference/decorators.md +++ b/website/src/content/docs/docs/libraries/events/reference/decorators.md @@ -1,7 +1,9 @@ --- title: "Decorators" +description: "Decorators exported by @typespec/events" toc_min_heading_level: 2 toc_max_heading_level: 3 +llmstxt: true --- ## TypeSpec.Events diff --git a/website/src/content/docs/docs/libraries/http/authentication.md b/website/src/content/docs/docs/libraries/http/authentication.md index 790644602cc..5b6532f7ae3 100644 --- a/website/src/content/docs/docs/libraries/http/authentication.md +++ b/website/src/content/docs/docs/libraries/http/authentication.md @@ -1,5 +1,7 @@ --- title: Authentication +description: Configuring HTTP authentication +llmstxt: true --- ## Configure diff --git a/website/src/content/docs/docs/libraries/http/content-types.md b/website/src/content/docs/docs/libraries/http/content-types.md index 8f6277be797..9de9e89769e 100644 --- a/website/src/content/docs/docs/libraries/http/content-types.md +++ b/website/src/content/docs/docs/libraries/http/content-types.md @@ -1,5 +1,7 @@ --- title: Content types +description: Working with HTTP content-types +llmstxt: true --- ## Default behavior diff --git a/website/src/content/docs/docs/libraries/http/encoding.md b/website/src/content/docs/docs/libraries/http/encoding.md index 020c3817535..c3ad510215d 100644 --- a/website/src/content/docs/docs/libraries/http/encoding.md +++ b/website/src/content/docs/docs/libraries/http/encoding.md @@ -1,5 +1,7 @@ --- title: Encoding of types +description: How HTTP handles encoding types +llmstxt: true --- This document describe how the http library interpret TypeSpec built-in types and how to configure diff --git a/website/src/content/docs/docs/libraries/http/examples.md b/website/src/content/docs/docs/libraries/http/examples.md index c1ea6cec2e0..7c53d7cf620 100644 --- a/website/src/content/docs/docs/libraries/http/examples.md +++ b/website/src/content/docs/docs/libraries/http/examples.md @@ -1,5 +1,7 @@ --- title: Examples +description: Defining HTTP operation examples +llmstxt: true --- This document provides examples specific to the HTTP library in TypeSpec. For general information about how examples work in TypeSpec, see the [examples docs](../../standard-library/examples.md). diff --git a/website/src/content/docs/docs/libraries/http/files.md b/website/src/content/docs/docs/libraries/http/files.md index 5d4530b44f0..ed7c07044f1 100644 --- a/website/src/content/docs/docs/libraries/http/files.md +++ b/website/src/content/docs/docs/libraries/http/files.md @@ -1,5 +1,7 @@ --- title: Files +description: Working with Http.File +llmstxt: true --- ## About `Http.File` diff --git a/website/src/content/docs/docs/libraries/http/multipart.md b/website/src/content/docs/docs/libraries/http/multipart.md index 031520c0d7f..9484f277528 100644 --- a/website/src/content/docs/docs/libraries/http/multipart.md +++ b/website/src/content/docs/docs/libraries/http/multipart.md @@ -1,5 +1,7 @@ --- title: Multipart requests +description: Using Http multipart requests +llmstxt: true --- Multipart requests combine one or more sets of data into a single body, separated by boundaries. This is commonly used to upload files. diff --git a/website/src/content/docs/docs/libraries/http/operations.md b/website/src/content/docs/docs/libraries/http/operations.md index 9052afbdd3d..667f858fbab 100644 --- a/website/src/content/docs/docs/libraries/http/operations.md +++ b/website/src/content/docs/docs/libraries/http/operations.md @@ -1,5 +1,7 @@ --- title: Operations +description: Defining HTTP endpoints +llmstxt: true --- ## Operation verb diff --git a/website/src/content/docs/docs/libraries/http/reference/data-types.md b/website/src/content/docs/docs/libraries/http/reference/data-types.md index 776282945d8..ca606b378d2 100644 --- a/website/src/content/docs/docs/libraries/http/reference/data-types.md +++ b/website/src/content/docs/docs/libraries/http/reference/data-types.md @@ -1,5 +1,7 @@ --- title: "Data types" +description: "Data types exported by @typespec/http" +llmstxt: true --- ## TypeSpec.Http diff --git a/website/src/content/docs/docs/libraries/http/reference/decorators.md b/website/src/content/docs/docs/libraries/http/reference/decorators.md index f30212194fa..921fdbfaef4 100644 --- a/website/src/content/docs/docs/libraries/http/reference/decorators.md +++ b/website/src/content/docs/docs/libraries/http/reference/decorators.md @@ -1,7 +1,9 @@ --- title: "Decorators" +description: "Decorators exported by @typespec/http" toc_min_heading_level: 2 toc_max_heading_level: 3 +llmstxt: true --- ## TypeSpec.Http diff --git a/website/src/content/docs/docs/libraries/openapi/reference/data-types.md b/website/src/content/docs/docs/libraries/openapi/reference/data-types.md index e0126e204d4..cb03c688b29 100644 --- a/website/src/content/docs/docs/libraries/openapi/reference/data-types.md +++ b/website/src/content/docs/docs/libraries/openapi/reference/data-types.md @@ -1,5 +1,7 @@ --- title: "Data types" +description: "Data types exported by @typespec/openapi" +llmstxt: true --- ## TypeSpec.OpenAPI diff --git a/website/src/content/docs/docs/libraries/openapi/reference/decorators.md b/website/src/content/docs/docs/libraries/openapi/reference/decorators.md index c5668dbbe4d..cd223411b0d 100644 --- a/website/src/content/docs/docs/libraries/openapi/reference/decorators.md +++ b/website/src/content/docs/docs/libraries/openapi/reference/decorators.md @@ -1,7 +1,9 @@ --- title: "Decorators" +description: "Decorators exported by @typespec/openapi" toc_min_heading_level: 2 toc_max_heading_level: 3 +llmstxt: true --- ## TypeSpec.OpenAPI diff --git a/website/src/content/docs/docs/libraries/rest/reference/data-types.md b/website/src/content/docs/docs/libraries/rest/reference/data-types.md index 70379e1e0fa..5ef54891046 100644 --- a/website/src/content/docs/docs/libraries/rest/reference/data-types.md +++ b/website/src/content/docs/docs/libraries/rest/reference/data-types.md @@ -1,5 +1,7 @@ --- title: "Data types" +description: "Data types exported by @typespec/rest" +llmstxt: true --- ## TypeSpec.Rest diff --git a/website/src/content/docs/docs/libraries/rest/reference/decorators.md b/website/src/content/docs/docs/libraries/rest/reference/decorators.md index c9862cab6b0..62a7179c5e1 100644 --- a/website/src/content/docs/docs/libraries/rest/reference/decorators.md +++ b/website/src/content/docs/docs/libraries/rest/reference/decorators.md @@ -1,7 +1,9 @@ --- title: "Decorators" +description: "Decorators exported by @typespec/rest" toc_min_heading_level: 2 toc_max_heading_level: 3 +llmstxt: true --- ## TypeSpec.Rest diff --git a/website/src/content/docs/docs/libraries/rest/reference/interfaces.md b/website/src/content/docs/docs/libraries/rest/reference/interfaces.md index 7e668394ac9..c92a88a690b 100644 --- a/website/src/content/docs/docs/libraries/rest/reference/interfaces.md +++ b/website/src/content/docs/docs/libraries/rest/reference/interfaces.md @@ -1,5 +1,7 @@ --- title: "Interfaces and Operations" +description: "Interfaces and Operations exported by @typespec/rest" +llmstxt: true --- ## TypeSpec.Rest.Resource diff --git a/website/src/content/docs/docs/libraries/rest/resource-routing.md b/website/src/content/docs/docs/libraries/rest/resource-routing.md index edb80457d17..403a46f284e 100644 --- a/website/src/content/docs/docs/libraries/rest/resource-routing.md +++ b/website/src/content/docs/docs/libraries/rest/resource-routing.md @@ -1,5 +1,7 @@ --- title: Resource and routes +description: Defining REST resources and endpoints +llmstxt: true --- Resources are operations that are grouped in a namespace. You declare such a namespace by adding the `@route` decorator to provide the path to that resource: diff --git a/website/src/content/docs/docs/libraries/sse/reference/data-types.md b/website/src/content/docs/docs/libraries/sse/reference/data-types.md index b957a43819d..3cb66e653b6 100644 --- a/website/src/content/docs/docs/libraries/sse/reference/data-types.md +++ b/website/src/content/docs/docs/libraries/sse/reference/data-types.md @@ -1,5 +1,6 @@ --- title: "Data types" +description: "Data types exported by @typespec/sse" --- ## TypeSpec.SSE diff --git a/website/src/content/docs/docs/libraries/sse/reference/decorators.md b/website/src/content/docs/docs/libraries/sse/reference/decorators.md index 072c6d53bb5..2577f232a39 100644 --- a/website/src/content/docs/docs/libraries/sse/reference/decorators.md +++ b/website/src/content/docs/docs/libraries/sse/reference/decorators.md @@ -1,5 +1,6 @@ --- title: "Decorators" +description: "Decorators exported by @typespec/sse" toc_min_heading_level: 2 toc_max_heading_level: 3 --- diff --git a/website/src/content/docs/docs/libraries/streams/reference/data-types.md b/website/src/content/docs/docs/libraries/streams/reference/data-types.md index 0208bbe5cde..29b9e7d13af 100644 --- a/website/src/content/docs/docs/libraries/streams/reference/data-types.md +++ b/website/src/content/docs/docs/libraries/streams/reference/data-types.md @@ -1,5 +1,7 @@ --- title: "Data types" +description: "Data types exported by @typespec/streams" +llmstxt: true --- ## TypeSpec.Streams diff --git a/website/src/content/docs/docs/libraries/streams/reference/decorators.md b/website/src/content/docs/docs/libraries/streams/reference/decorators.md index ebca14470b5..d227bef8f4c 100644 --- a/website/src/content/docs/docs/libraries/streams/reference/decorators.md +++ b/website/src/content/docs/docs/libraries/streams/reference/decorators.md @@ -1,7 +1,9 @@ --- title: "Decorators" +description: "Decorators exported by @typespec/streams" toc_min_heading_level: 2 toc_max_heading_level: 3 +llmstxt: true --- ## TypeSpec.Streams diff --git a/website/src/content/docs/docs/libraries/versioning/guide.md b/website/src/content/docs/docs/libraries/versioning/guide.md index 72e2e88798d..b7645cd483e 100644 --- a/website/src/content/docs/docs/libraries/versioning/guide.md +++ b/website/src/content/docs/docs/libraries/versioning/guide.md @@ -1,5 +1,7 @@ --- title: Tutorial +description: "Guide to versioning APIs authored in TypeSpec" +llmstxt: true --- ## Implementing versioned APIs diff --git a/website/src/content/docs/docs/libraries/versioning/reference/decorators.md b/website/src/content/docs/docs/libraries/versioning/reference/decorators.md index 7630dff3dd5..fd2cd1d88c9 100644 --- a/website/src/content/docs/docs/libraries/versioning/reference/decorators.md +++ b/website/src/content/docs/docs/libraries/versioning/reference/decorators.md @@ -1,7 +1,9 @@ --- title: "Decorators" +description: "Decorators exported by @typespec/versioning" toc_min_heading_level: 2 toc_max_heading_level: 3 +llmstxt: true --- ## TypeSpec.Versioning diff --git a/website/src/content/docs/docs/libraries/xml/reference/decorators.md b/website/src/content/docs/docs/libraries/xml/reference/decorators.md index c13536f91f9..d6775f560f7 100644 --- a/website/src/content/docs/docs/libraries/xml/reference/decorators.md +++ b/website/src/content/docs/docs/libraries/xml/reference/decorators.md @@ -1,5 +1,6 @@ --- title: "Decorators" +description: "Decorators exported by @typespec/xml" toc_min_heading_level: 2 toc_max_heading_level: 3 --- diff --git a/website/src/content/docs/docs/standard-library/built-in-data-types.md b/website/src/content/docs/docs/standard-library/built-in-data-types.md index 814f9fa63b6..74f9c5156f1 100644 --- a/website/src/content/docs/docs/standard-library/built-in-data-types.md +++ b/website/src/content/docs/docs/standard-library/built-in-data-types.md @@ -1,5 +1,6 @@ --- title: "Built-in Data types" +description: "Data types exported by TypeSpec" --- ## TypeSpec ### `Array` {#Array} diff --git a/website/src/content/docs/docs/standard-library/built-in-decorators.md b/website/src/content/docs/docs/standard-library/built-in-decorators.md index 98d7014f4f4..cdef4e9f1b0 100644 --- a/website/src/content/docs/docs/standard-library/built-in-decorators.md +++ b/website/src/content/docs/docs/standard-library/built-in-decorators.md @@ -1,5 +1,6 @@ --- title: "Built-in Decorators" +description: "Decorators exported by TypeSpec" toc_min_heading_level: 2 toc_max_heading_level: 3 --- diff --git a/website/src/content/docs/docs/standard-library/discriminated-types.md b/website/src/content/docs/docs/standard-library/discriminated-types.md index 308b833302d..fc3daa4fe09 100644 --- a/website/src/content/docs/docs/standard-library/discriminated-types.md +++ b/website/src/content/docs/docs/standard-library/discriminated-types.md @@ -1,5 +1,7 @@ --- title: Discriminated Types +description: Standard library reference to describing discriminated types +llmstxt: true --- TypeSpec allows for the expression of unions and inheritance. However, when transmitting types over the network, many languages require a mechanism to distinguish between different union variants or models within an inheritance hierarchy. diff --git a/website/src/content/docs/docs/standard-library/encoded-names.md b/website/src/content/docs/docs/standard-library/encoded-names.md index 1059d4a6be0..c704f584b67 100644 --- a/website/src/content/docs/docs/standard-library/encoded-names.md +++ b/website/src/content/docs/docs/standard-library/encoded-names.md @@ -1,6 +1,8 @@ --- id: encoded-names title: Encoded names +description: Standard library reference to encoding names sent over the wire +llmstxt: true --- There is some cases where the name you have in TypeSpec might differ from the name over the wire or for a certain language. diff --git a/website/src/content/docs/docs/standard-library/examples.md b/website/src/content/docs/docs/standard-library/examples.md index fe852205695..af8e17ea3da 100644 --- a/website/src/content/docs/docs/standard-library/examples.md +++ b/website/src/content/docs/docs/standard-library/examples.md @@ -1,5 +1,7 @@ --- title: Examples +description: Standard library reference to defining API examples +llmstxt: true --- TypeSpec provide 2 decorators `@example` and `@opExample` to provide some examples for the types and operations. diff --git a/website/src/content/docs/docs/standard-library/pagination.md b/website/src/content/docs/docs/standard-library/pagination.md index e2caeddc4de..1e47ca4d10f 100644 --- a/website/src/content/docs/docs/standard-library/pagination.md +++ b/website/src/content/docs/docs/standard-library/pagination.md @@ -1,5 +1,7 @@ --- title: Pagination +description: Standard library reference to using built-in pagination patterns +llmstxt: true --- TypeSpec provide built-in support for some of the common pagination pattern used. diff --git a/website/src/pages/docs/[...path]/[llms_type].txt.ts b/website/src/pages/docs/[...path]/[llms_type].txt.ts new file mode 100644 index 00000000000..da7fae1de67 --- /dev/null +++ b/website/src/pages/docs/[...path]/[llms_type].txt.ts @@ -0,0 +1,32 @@ +/* eslint-disable unicorn/filename-case */ +import type { RouteParams, RouteProps } from "@typespec/astro-utils/llmstxt"; +import { collectLlmsDocs, generateLlmsTopics } from "../../../utils/llmstxt"; + +export { spreadLlmsTxtRoute as GET } from "@typespec/astro-utils/llmstxt"; + +export async function getStaticPaths() { + const { docs, libraryNames } = await collectLlmsDocs(); + + const topics = generateLlmsTopics({ libraryNames, docs }); + + const staticPathsResults: { params: RouteParams; props: RouteProps }[] = []; + for (const topic of topics) { + if (!topic.docs || !topic.docs.length) continue; + staticPathsResults.push({ + params: { path: generateFullPath(topic.pathPrefix), llms_type: "llms" }, + props: { title: topic.title, description: topic.description, docs: topic.docs }, + }); + staticPathsResults.push({ + params: { path: generateFullPath(topic.pathPrefix), llms_type: "llms-full" }, + props: { title: topic.title, description: topic.description, docs: topic.docs }, + }); + } + + return staticPathsResults; +} + +function generateFullPath(docId: string): string { + // All of the docs in the collection have `docs` prepended to the path - we need to + // remove this since it'll be prepended again given this route is in the `docs` folder. + return docId.startsWith("docs") ? docId.slice(5) : docId; +} diff --git a/website/src/pages/docs/[...slug].md.ts b/website/src/pages/docs/[...slug].md.ts new file mode 100644 index 00000000000..13884a9b872 --- /dev/null +++ b/website/src/pages/docs/[...slug].md.ts @@ -0,0 +1,36 @@ +import { getCollection } from "astro:content"; + +export { markdownRoute as GET } from "@typespec/astro-utils/llmstxt"; + +export async function getStaticPaths() { + const docs = await getCollection("docs"); + + return docs + .filter((doc) => { + // Exclude release notes + if (doc.id.includes("/release-notes/")) return false; + return true; + }) + .map((doc) => ({ + params: { slug: generateMarkdownSlug(doc.id) }, + props: { doc }, + })); +} + +function generateMarkdownSlug(docId: string): string { + // All of the docs in the collection have `docs` prepended to the path - we need to + // remove this since it'll be prepended again given this route is in the `docs` folder. + const path = docId.startsWith("docs") ? docId.slice(5) : docId; + + // If the final path fragment does not include a file extension, use `index.html.md` + if (path.endsWith("/")) { + return `${path}index.html`; + } + + const finalPathFragment = path.split("/").pop() ?? ""; + if (!finalPathFragment.includes(".")) { + return `${path}/index.html`; + } + + return path; +} diff --git a/website/src/pages/docs/llms-full.txt.ts b/website/src/pages/docs/llms-full.txt.ts new file mode 100644 index 00000000000..146142a4cac --- /dev/null +++ b/website/src/pages/docs/llms-full.txt.ts @@ -0,0 +1,17 @@ +import { collectLlmsDocs } from "@site/src/utils/llmstxt"; +import { generateLlmstxtFull } from "@typespec/astro-utils/llmstxt"; +import type { APIRoute } from "astro"; + +export const GET: APIRoute = async () => { + const { docs } = await collectLlmsDocs(); + // Sort docs so that subjects are grouped together based on their paths + docs.sort((a, b) => (a.id > b.id ? 1 : -1)); + + const llmstxt = generateLlmstxtFull("TypeSpec Documentation", docs); + + return new Response(llmstxt, { + headers: { + "Content-Type": "text/markdown; charset=utf-8", + }, + }); +}; diff --git a/website/src/pages/docs/llms.json.ts b/website/src/pages/docs/llms.json.ts new file mode 100644 index 00000000000..55097161430 --- /dev/null +++ b/website/src/pages/docs/llms.json.ts @@ -0,0 +1,18 @@ +import { generateLlmsJson } from "@typespec/astro-utils/llmstxt"; +import type { APIRoute } from "astro"; +import { collectLlmsDocs, generateLlmsTopics } from "../../utils/llmstxt"; + +export const GET: APIRoute = async ({ site }) => { + const siteHref = site?.href ?? ""; + const { libraryNames } = await collectLlmsDocs(); + + const topicsDetails = generateLlmsTopics({ libraryNames, skipPopulateDocs: true }); + + const llmsJson = generateLlmsJson(topicsDetails, siteHref); + + return new Response(JSON.stringify(llmsJson, null, 2), { + headers: { + "Content-Type": "application/json; charset=utf-8", + }, + }); +}; diff --git a/website/src/pages/docs/llms.txt.ts b/website/src/pages/docs/llms.txt.ts new file mode 100644 index 00000000000..949de074f51 --- /dev/null +++ b/website/src/pages/docs/llms.txt.ts @@ -0,0 +1,28 @@ +import { generateLlmstxt, processDocsForLlmsTxt } from "@typespec/astro-utils/llmstxt"; +import type { APIRoute } from "astro"; +import { collectLlmsDocs, generateLlmsTopics } from "../../utils/llmstxt"; + +export const GET: APIRoute = async ({ site }) => { + const { docs, libraryNames } = await collectLlmsDocs(); + + const topics = generateLlmsTopics({ libraryNames, docs }); + const llmsData = await processDocsForLlmsTxt({ + title: "TypeSpec Documentation", + description: + "TypeSpec is an open-source language for designing APIs in a clear, extensible, and reusable way. It enables developers to define APIs as a single source of truth and generate client/server code, documentation, and OpenAPI specifications using emitters.", + docs, + llmsSections: topics.map((topic) => ({ + name: topic.title, + pathPrefix: topic.pathPrefix, + })), + site, + }); + + const llmstxt = generateLlmstxt(llmsData); + + return new Response(llmstxt, { + headers: { + "Content-Type": "text/markdown; charset=utf-8", + }, + }); +}; diff --git a/website/src/utils/llmstxt.ts b/website/src/utils/llmstxt.ts new file mode 100644 index 00000000000..b721b9fc40a --- /dev/null +++ b/website/src/utils/llmstxt.ts @@ -0,0 +1,102 @@ +import { populateTopicDocs, type DocEntry, type TopicProps } from "@typespec/astro-utils/llmstxt"; +import { getCollection } from "astro:content"; + +export type LlmsDocsDetails = { + docs: DocEntry[]; + libraryNames: Set; +}; + +export async function collectLlmsDocs() { + const libraryNames = new Set(); + const docs = await getCollection("docs", (entry) => { + if (!entry.data.llmstxt) return false; + + const libraryName = getLibraryName(entry.id); + if (libraryName) { + libraryNames.add(libraryName); + } + + return true; + }); + + return { + docs, + libraryNames, + }; +} + +export type GenerateLlmsTopicsProps = { + libraryNames: Set; +} & ( + | { + docs: DocEntry[]; + skipPopulateDocs?: boolean; + } + | { + docs?: never; + skipPopulateDocs: true; + } +); + +export function generateLlmsTopics( + props: { + libraryNames: Set; + } & { + docs: DocEntry[]; + skipPopulateDocs?: boolean; + }, +): TopicProps[]; +export function generateLlmsTopics( + props: { + libraryNames: Set; + } & { + docs?: never; + skipPopulateDocs: true; + }, +): Omit[]; +export function generateLlmsTopics({ + libraryNames, + docs, + skipPopulateDocs, +}: GenerateLlmsTopicsProps): TopicProps[] | Omit[] { + const topics = [ + { + title: "TypeSpec Language Basics", + id: "typespec-language-basics", + description: "Documentation for the TypeSpec language", + pathPrefix: "docs/language-basics/", + }, + { + title: "TypeSpec Standard Library", + id: "typespec-standard-library", + description: "Documentation for the TypeSpec standard library", + pathPrefix: "docs/standard-library/", + }, + { + title: "TypeSpec Getting Started", + id: "typespec-getting-started", + description: "Overview of getting started with TypeSpec", + pathPrefix: "docs/getting-started/", + }, + ]; + + for (const libraryName of libraryNames) { + topics.push({ + title: `@typespec/${libraryName}`, + id: `typespec-${libraryName}`, + description: `Documentation for the @typespec/${libraryName} library.`, + pathPrefix: `docs/libraries/${libraryName}/`, + }); + } + + if (skipPopulateDocs) { + return topics; + } + + return populateTopicDocs(topics, docs); +} + +function getLibraryName(id: string): string | undefined { + const match = id.match(/docs\/libraries\/([^/]+)\//); + return match ? match[1] : undefined; +}