Site search crawl-indexer: crawl, index, enrich elastic.co

.vhs/site-index-dry-run.tape

@@ -0,0 +1,13 @@
+Output .vhs/site-index-dry-run.gif
+
+Set Shell "zsh"
+Set FontSize 14
+Set Width 1400
+Set Height 900
+Set Padding 20
+Set Theme "Catppuccin Mocha"
+Set TypingSpeed 40ms
+
+Type "dotnet run --project src/tooling/crawl-indexer/ -- site index --fair --dry-run --max-pages 100 --rps 10 --no-ai"
+Enter
+Sleep 120s

Directory.Packages.props

@@ -39,6 +39,7 @@
     <PackageVersion Include="AWSSDK.SQS" Version="4.0.2.14" />
     <PackageVersion Include="AWSSDK.S3" Version="4.0.18.6" />
     <PackageVersion Include="Elastic.OpenTelemetry" Version="1.1.0" />
+    <PackageVersion Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="10.0.0" />
     <PackageVersion Include="Microsoft.Extensions.Configuration.UserSecrets" Version="10.0.0" />
     <PackageVersion Include="Microsoft.Extensions.Configuration.Abstractions" Version="10.0.3" />
     <PackageVersion Include="Microsoft.Extensions.Telemetry.Abstractions" Version="10.0.0" />
@@ -48,8 +49,8 @@
     <PackageVersion Include="Elastic.Aspire.Hosting.Elasticsearch" Version="9.3.0" />
     <PackageVersion Include="Elastic.Clients.Elasticsearch" Version="9.3.0" />
     <PackageVersion Include="FakeItEasy" Version="9.0.1" />
-    <PackageVersion Include="Elastic.Ingest.Elasticsearch" Version="0.34.5" />
-    <PackageVersion Include="Elastic.Mapping" Version="0.34.5" />
+    <PackageVersion Include="Elastic.Ingest.Elasticsearch" Version="0.37.0" />
+    <PackageVersion Include="Elastic.Mapping" Version="0.37.0" />
     <PackageVersion Include="InMemoryLogger" Version="1.0.66" />
     <PackageVersion Include="MartinCostello.Logging.XUnit.v3" Version="0.7.0" />
     <PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.3" />
@@ -73,6 +74,7 @@
     <PackageVersion Include="Crayon" Version="2.0.69" />
     <PackageVersion Include="DotNet.Glob" Version="3.1.3" />
     <PackageVersion Include="Errata" Version="0.15.0" />
+    <PackageVersion Include="Spectre.Console" Version="0.54.0" />
     <PackageVersion Include="Github.Actions.Core" Version="9.0.0" />
     <PackageVersion Include="Microsoft.Extensions.Logging" Version="10.0.0" />
     <PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="10.0.0" />
@@ -128,4 +130,4 @@
     </PackageVersion>
     <PackageVersion Include="xunit.v3" Version="2.0.2" />
   </ItemGroup>
-</Project>
+</Project>

config/changelog.example.yml

@@ -10,6 +10,14 @@
 #     - value3
 # Both forms produce the same result. Choose whichever is more readable for your use case.
 
+# Filename strategy for generated changelog files.
+# Controls how files created by 'changelog add' are named.
+#   pr        — use the PR number (e.g., 12345.yaml).
+#   issue     — use the issue number (e.g., 67890.yaml).
+#   timestamp — use a Unix timestamp with a title slug (e.g., 1735689600-fix-search.yaml). Default.
+# Can be overridden per invocation with --use-pr-number or --use-issue-number CLI flags.
+filename: timestamp
+
 # Products configuration (optional)
 # If not specified, all products from products.yml are allowed
 products:

config/versions.yml

@@ -53,7 +53,7 @@ versioning_systems:
     current: 4.15.0
   apm-agent-php:
     base: 1.0
-    current: 1.15.1
+    current: 1.16.0
   apm-agent-python:
     base: 6.0
     current: 6.25.0
@@ -155,7 +155,7 @@ versioning_systems:
     current: 9.3.4
   elasticsearch-client-dotnet:
     base: 9.0
-    current: 9.3.1
+    current: 9.3.2
   elasticsearch-client-php:
     base: 9.0
     current: 9.3.0

docs-builder.slnx

@@ -57,6 +57,7 @@
     <Project Path="src/Elastic.Documentation.Svg/Elastic.Documentation.Svg.csproj" />
     <Project Path="src/Elastic.Documentation/Elastic.Documentation.csproj" />
     <Project Path="src/Elastic.Markdown/Elastic.Markdown.csproj" />
+    <Project Path="src/Elastic.Documentation.Tooling/Elastic.Documentation.Tooling.csproj" />
   </Folder>
   <Folder Name="/src/api/">
     <Project Path="src/api/Elastic.Documentation.Api.Core/Elastic.Documentation.Api.Core.csproj" />
@@ -80,6 +81,7 @@
   </Folder>
   <Folder Name="/src/tooling/">
     <File Path="src/tooling/Directory.Build.props" />
+    <Project Path="src/tooling/crawl-indexer/crawl-indexer.csproj" />
     <Project Path="src/tooling/docs-builder/docs-builder.csproj" />
   </Folder>
   <Folder Name="/tests-integration/">

docs/cli/release/changelog-add.md

@@ -123,10 +123,10 @@ docs-builder changelog add [options...] [-h|--help]
 :   The valid types are listed in [ChangelogConfiguration.cs](https://github.com/elastic/docs-builder/blob/main/src/services/Elastic.Documentation.Services/Changelog/ChangelogConfiguration.cs).
 
 `--use-pr-number`
-:   Optional: Use PR numbers for filenames instead of timestamp-slug. With both `--prs` (which creates one changelog per specified PR) and `--issues` (which creates one changelog per specified issue), each changelog filename will be derived from its PR numbers. Requires `--prs` or `--issues`. Mutually exclusive with `--use-issue-number`.
+:   Optional: Use PR numbers for filenames instead of the configured `filename` strategy. With both `--prs` (which creates one changelog per specified PR) and `--issues` (which creates one changelog per specified issue), each changelog filename will be derived from its PR numbers. Requires `--prs` or `--issues`. Mutually exclusive with `--use-issue-number`.
 
 `--use-issue-number`
-:   Optional: Use issue numbers for filenames instead of timestamp-slug. With both `--prs` (which creates one changelog per specified PR) and `--issues` (which creates one changelog per specified issue), each changelog filename will be derived from its issues. Requires `--prs` or `--issues`. Mutually exclusive with `--use-pr-number`.
+:   Optional: Use issue numbers for filenames instead of the configured `filename` strategy. With both `--prs` (which creates one changelog per specified PR) and `--issues` (which creates one changelog per specified issue), each changelog filename will be derived from its issues. Requires `--prs` or `--issues`. Mutually exclusive with `--use-pr-number`.
 
 ## CI auto-detection [ci-auto-detection]
 
@@ -142,7 +142,7 @@ When running inside GitHub Actions, `changelog add` automatically reads the foll
 
 **Precedence**: explicit CLI arguments always take priority over environment variables. Environment variables are only used when the corresponding CLI argument is not provided.
 
-When `CHANGELOG_PR_NUMBER` is set and `--prs` is not provided, `--use-pr-number` is also implicitly enabled so the generated filename uses the PR number.
+The filename strategy is controlled by the `filename` option in `changelog.yml` (defaulting to `timestamp`). Refer to [changelog.example.yml](https://github.com/elastic/docs-builder/blob/main/config/changelog.example.yml) for details.
 
 This allows the CI action to invoke `changelog add` with a minimal command line:
 

docs/cli/release/changelog-evaluate-pr.md

@@ -68,6 +68,7 @@ docs-builder changelog evaluate-pr [options...] [-h|--help]
 | `title` | Resolved PR title |
 | `type` | Resolved changelog type |
 | `label-table` | Markdown table of configured label-to-type mappings |
+| `existing-changelog-filename` | Filename of a previously committed changelog for this PR (if any) |
 
 ## Environment variables
 

docs/contribute/changelog.md

@@ -313,12 +313,23 @@ Examples:
 
 ### Filenames
 
-By default, the `docs-builder changelog add` command generates filenames using a timestamp and a sanitized version of the title:
-`{timestamp}-{sanitized-title}.yaml`
+The `docs-builder changelog add` command names generated files according to the `filename` strategy in `changelog.yml`:
 
-For example: `1735689600-fixes-enrich-and-lookup-join-resolution.yaml`
+| Strategy | Example filename | Description |
+|---|---|---|
+| `timestamp` (default) | `1735689600-fixes-enrich-and-lookup-join-resolution.yaml` | Uses a Unix timestamp with a sanitized title slug. |
+| `pr` | `137431.yaml` | Uses the PR number. |
+| `issue` | `2571.yaml` | Uses the issue number. |
 
-If you want to use PR numbers for filenames, add the `--use-pr-number` option. With both `--prs` (which creates one changelog per specified PR) and `--issues` (which creates one changelog per specified issue), each changelog filename will be derived from its PR numbers:
+Set the strategy in your changelog configuration file:
+
+```yaml
+filename: timestamp
+```
+
+Refer to [changelog.example.yml](https://github.com/elastic/docs-builder/blob/main/config/changelog.example.yml) for full documentation.
+
+You can override the configured strategy per invocation with the `--use-pr-number` or `--use-issue-number` CLI flags:
 
 ```sh
 docs-builder changelog add \
@@ -330,13 +341,13 @@ docs-builder changelog add \
   --issues https://github.com/elastic/docs-builder/issues/2571 \
   --products "elasticsearch 9.3.0" \
   --config docs/changelog.yml \
-  --use-pr-number
+  --use-issue-number
 ```
 
-For filenames that match issue numbers instead of PR numbers, specify `--use-issue-number`.
-
 :::{important}
 `--use-pr-number` and `--use-issue-number` are mutually exclusive; specify only one. Each requires `--prs` or `--issues`. The numbers are extracted from the URLs or identifiers you provide, or from linked references in the issue or PR body when extraction is enabled.
+
+**Precedence**: CLI flags (`--use-pr-number` / `--use-issue-number`) > `filename` in `changelog.yml` > default (`timestamp`).
 :::
 
 ### Examples

src/Elastic.ApiExplorer/Operations/OperationView.cshtml

@@ -75,7 +75,7 @@
 			<span class="beta-badge">Beta</span>
 		}
 		@{
-			var versionInfo = operation.Extensions?.TryGetValue("x-state", out var stateValue) == true && stateValue is System.Text.Json.Nodes.JsonNode stateNode ? stateNode.ToString() : null;
+			var versionInfo = (operation.Extensions?.TryGetValue("x-state", out var stateValue) == true && stateValue is JsonNodeExtension stateExt) ? stateExt.Node.GetValue<string>() : null;
 		}
 		@if (!string.IsNullOrEmpty(versionInfo))
 		{

src/Elastic.ApiExplorer/Shared/_PropertyItem.cshtml

@@ -128,7 +128,7 @@
 			}
 			@if (ctx.ShowVersionInfo)
 			{
-				var propVersionInfo = propSchema.Extensions?.TryGetValue("x-state", out var propStateValue) == true && propStateValue is System.Text.Json.Nodes.JsonNode propStateNode ? propStateNode.ToString() : null;
+				var propVersionInfo = (propSchema.Extensions?.TryGetValue("x-state", out var propStateValue) == true && propStateValue is JsonNodeExtension propStateExt) ? propStateExt.Node.GetValue<string>() : null;
 				if (!string.IsNullOrEmpty(propVersionInfo))
 				{
 					<span class="version-badge">@propVersionInfo</span>

src/Elastic.Documentation.Configuration/Changelog/ChangelogConfiguration.cs

@@ -111,6 +111,12 @@ public record ChangelogConfiguration
 	/// </summary>
 	public ProductsConfig? ProductsConfiguration { get; init; }
 
+	/// <summary>
+	/// Filename strategy for generated changelog files.
+	/// Controls how files created by 'changelog add' are named.
+	/// </summary>
+	public FilenameStrategy Filename { get; init; } = FilenameStrategy.Timestamp;
+
 	/// <summary>
 	/// Bundle configuration with profiles and defaults.
 	/// </summary>

src/Elastic.Documentation.Configuration/Changelog/FilenameStrategy.cs

@@ -0,0 +1,25 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.ComponentModel.DataAnnotations;
+using NetEscapades.EnumGenerators;
+
+namespace Elastic.Documentation.Configuration.Changelog;
+
+/// <summary>Controls how changelog files created by 'changelog add' are named.</summary>
+[EnumExtensions]
+public enum FilenameStrategy
+{
+	/// <summary>Use the PR number as filename (e.g., 12345.yaml).</summary>
+	[Display(Name = "pr")]
+	Pr,
+
+	/// <summary>Use the issue number as filename (e.g., 67890.yaml).</summary>
+	[Display(Name = "issue")]
+	Issue,
+
+	/// <summary>Use a Unix timestamp with a title slug (e.g., 1735689600-fix-search.yaml).</summary>
+	[Display(Name = "timestamp")]
+	Timestamp
+}

src/Elastic.Documentation.Tooling/Elastic.Documentation.Tooling.csproj

@@ -0,0 +1,21 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net10.0</TargetFramework>
+    <RootNamespace>Elastic.Documentation.Tooling</RootNamespace>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Github.Actions.Core" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" />
+    <PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" />
+    <PackageReference Include="Microsoft.Extensions.ServiceDiscovery" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Elastic.Documentation\Elastic.Documentation.csproj" />
+    <ProjectReference Include="..\Elastic.Documentation.Configuration\Elastic.Documentation.Configuration.csproj" />
+  </ItemGroup>
+
+</Project>