From 2df97c5c75d3228e952e64402fe5fc1ac7d09eb2 Mon Sep 17 00:00:00 2001 From: Ashley Van Spankeren <25673124+ashleyvansp@users.noreply.github.com> Date: Fri, 4 Jul 2025 17:50:36 +0000 Subject: [PATCH 01/23] Create model for workload group + policy + request limits policy --- KustoSchemaTools/Model/WorkloadGroup.cs | 177 ++++++++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 KustoSchemaTools/Model/WorkloadGroup.cs diff --git a/KustoSchemaTools/Model/WorkloadGroup.cs b/KustoSchemaTools/Model/WorkloadGroup.cs new file mode 100644 index 0000000..f32cedf --- /dev/null +++ b/KustoSchemaTools/Model/WorkloadGroup.cs @@ -0,0 +1,177 @@ +using Newtonsoft.Json; + +namespace KustoSchemaTools.Model +{ + public class WorkloadGroup : IEquatable + { + public required string WorkloadGroupName { get; set; } + + public WorkloadGroupPolicy WorkloadGroupPolicy { get; set; } + public bool Equals(WorkloadGroup? other) + { + if (other is null) return false; + if (ReferenceEquals(this, other)) return true; + return + WorkloadGroupName == other.WorkloadGroupName && + EqualityComparer.Default.Equals(WorkloadGroupPolicy, other.WorkloadGroupPolicy); + } + + public override bool Equals(object? obj) => Equals(obj as WorkloadGroup); + public override int GetHashCode() + { + var hc = new HashCode(); + hc.Add(WorkloadGroupName); + hc.Add(WorkloadGroupPolicy); + return hc.ToHashCode(); + } + + public string ToUpdateScript() + { + var workloadGroupPolicyJson = WorkloadGroupPolicy.ToJson(); + var script = $".alter-merge workload_group {WorkloadGroupName} ```{workloadGroupPolicyJson}```"; + return script; + } + } + + public class WorkloadGroupPolicy : IEquatable + { + [JsonProperty("RequestLimitsPolicy")] + public RequestLimitsPolicy? RequestLimitsPolicy { get; set; } + + public bool Equals(WorkloadGroupPolicy? other) + { + if (other is null) return false; + if (ReferenceEquals(this, other)) return true; + return EqualityComparer.Default.Equals(RequestLimitsPolicy, other.RequestLimitsPolicy); + } + + public override bool Equals(object? obj) => Equals(obj as WorkloadGroupPolicy); + + public override int GetHashCode() + { + return RequestLimitsPolicy?.GetHashCode() ?? 0; + } + + public string ToJson() + { + return JsonConvert.SerializeObject(this, new JsonSerializerSettings + { + NullValueHandling = NullValueHandling.Ignore, + Formatting = Formatting.Indented + }); + } + } + + public class PolicyValue : IEquatable> + { + [JsonProperty("IsRelaxable")] + public bool IsRelaxable { get; set; } + + [JsonProperty("Value")] + public T? Value { get; set; } + + public bool Equals(PolicyValue? other) + { + if (other is null) return false; + if (ReferenceEquals(this, other)) return true; + return IsRelaxable == other.IsRelaxable && + EqualityComparer.Default.Equals(Value, other.Value); + } + + public override bool Equals(object? obj) => Equals(obj as PolicyValue); + + public override int GetHashCode() + { + return HashCode.Combine(IsRelaxable, Value); + } + } + + public class RequestLimitsPolicy : IEquatable + { + /// + /// Limits the data scope that the query is allowed to reference + /// + [JsonProperty("DataScope")] + public PolicyValue? DataScope { get; set; } + + /// + /// Maximum amount of memory a single query operator may allocate per node (in bytes) + /// + [JsonProperty("MaxMemoryPerQueryPerNode")] + public PolicyValue? MaxMemoryPerQueryPerNode { get; set; } + + /// + /// Maximum amount of memory a single query operator iterator can allocate (in bytes) + /// + [JsonProperty("MaxMemoryPerIterator")] + public PolicyValue? MaxMemoryPerIterator { get; set; } + + /// + /// Maximum percentage of total fanout threads in the cluster that a query can utilize + /// + [JsonProperty("MaxFanoutThreadsPercentage")] + public PolicyValue? MaxFanoutThreadsPercentage { get; set; } + + /// + /// Maximum percentage of nodes in the cluster that a query can fanout to + /// + [JsonProperty("MaxFanoutNodesPercentage")] + public PolicyValue? MaxFanoutNodesPercentage { get; set; } + + /// + /// Maximum number of records a query is allowed to return to the caller + /// + [JsonProperty("MaxResultRecords")] + public PolicyValue? MaxResultRecords { get; set; } + + /// + /// Maximum amount of data a query is allowed to return to the caller (in bytes) + /// + [JsonProperty("MaxResultBytes")] + public PolicyValue? MaxResultBytes { get; set; } + + /// + /// Maximum amount of time a request may execute + /// + [JsonProperty("MaxExecutionTime")] + public PolicyValue? MaxExecutionTime { get; set; } + + /// + /// How frequently progress of query results is reported (only takes effect if query results are progressive) + /// + [JsonProperty("QueryResultsProgressiveUpdatePeriod")] + public PolicyValue? QueryResultsProgressiveUpdatePeriod { get; set; } + + public bool Equals(RequestLimitsPolicy? other) + { + if (other is null) return false; + if (ReferenceEquals(this, other)) return true; + return EqualityComparer?>.Default.Equals(DataScope, other.DataScope) && + EqualityComparer?>.Default.Equals(MaxMemoryPerQueryPerNode, other.MaxMemoryPerQueryPerNode) && + EqualityComparer?>.Default.Equals(MaxMemoryPerIterator, other.MaxMemoryPerIterator) && + EqualityComparer?>.Default.Equals(MaxFanoutThreadsPercentage, other.MaxFanoutThreadsPercentage) && + EqualityComparer?>.Default.Equals(MaxFanoutNodesPercentage, other.MaxFanoutNodesPercentage) && + EqualityComparer?>.Default.Equals(MaxResultRecords, other.MaxResultRecords) && + EqualityComparer?>.Default.Equals(MaxResultBytes, other.MaxResultBytes) && + EqualityComparer?>.Default.Equals(MaxExecutionTime, other.MaxExecutionTime) && + EqualityComparer?>.Default.Equals(QueryResultsProgressiveUpdatePeriod, other.QueryResultsProgressiveUpdatePeriod); + } + + public override bool Equals(object? obj) => Equals(obj as RequestLimitsPolicy); + + public override int GetHashCode() + { + var hc = new HashCode(); + hc.Add(DataScope); + hc.Add(MaxMemoryPerQueryPerNode); + hc.Add(MaxMemoryPerIterator); + hc.Add(MaxFanoutThreadsPercentage); + hc.Add(MaxFanoutNodesPercentage); + hc.Add(MaxResultRecords); + hc.Add(MaxResultBytes); + hc.Add(MaxExecutionTime); + hc.Add(QueryResultsProgressiveUpdatePeriod); + return hc.ToHashCode(); + } + } +} \ No newline at end of file From 1daec7bf545fabcc12c6a10ab2f7b7ffcd8ef81b Mon Sep 17 00:00:00 2001 From: Ashley Van Spankeren <25673124+ashleyvansp@users.noreply.github.com> Date: Fri, 4 Jul 2025 18:34:59 +0000 Subject: [PATCH 02/23] Add RequestRateLimitPolicies to WorkloadGroup model --- KustoSchemaTools/Model/WorkloadGroup.cs | 86 ++++++++++++++++++++++++- 1 file changed, 83 insertions(+), 3 deletions(-) diff --git a/KustoSchemaTools/Model/WorkloadGroup.cs b/KustoSchemaTools/Model/WorkloadGroup.cs index f32cedf..97c5e86 100644 --- a/KustoSchemaTools/Model/WorkloadGroup.cs +++ b/KustoSchemaTools/Model/WorkloadGroup.cs @@ -1,12 +1,26 @@ using Newtonsoft.Json; +using System.Collections.Generic; +using System.Linq; namespace KustoSchemaTools.Model { + public enum RateLimitKind + { + ConcurrentRequests, + ResourceUtilization + } + + public enum RateLimitScope + { + WorkloadGroup, + Principal + } + public class WorkloadGroup : IEquatable { public required string WorkloadGroupName { get; set; } - public WorkloadGroupPolicy WorkloadGroupPolicy { get; set; } + public WorkloadGroupPolicy? WorkloadGroupPolicy { get; set; } public bool Equals(WorkloadGroup? other) { if (other is null) return false; @@ -27,6 +41,11 @@ public override int GetHashCode() public string ToUpdateScript() { + if (WorkloadGroupPolicy == null) + { + throw new InvalidOperationException("WorkloadGroupPolicy cannot be null when generating update script"); + } + var workloadGroupPolicyJson = WorkloadGroupPolicy.ToJson(); var script = $".alter-merge workload_group {WorkloadGroupName} ```{workloadGroupPolicyJson}```"; return script; @@ -38,18 +57,33 @@ public class WorkloadGroupPolicy : IEquatable [JsonProperty("RequestLimitsPolicy")] public RequestLimitsPolicy? RequestLimitsPolicy { get; set; } + [JsonProperty("RequestRateLimitPolicies")] + public List? RequestRateLimitPolicies { get; set; } + public bool Equals(WorkloadGroupPolicy? other) { if (other is null) return false; if (ReferenceEquals(this, other)) return true; - return EqualityComparer.Default.Equals(RequestLimitsPolicy, other.RequestLimitsPolicy); + return EqualityComparer.Default.Equals(RequestLimitsPolicy, other.RequestLimitsPolicy) && + (RequestRateLimitPolicies == null && other.RequestRateLimitPolicies == null || + RequestRateLimitPolicies != null && other.RequestRateLimitPolicies != null && + RequestRateLimitPolicies.SequenceEqual(other.RequestRateLimitPolicies)); } public override bool Equals(object? obj) => Equals(obj as WorkloadGroupPolicy); public override int GetHashCode() { - return RequestLimitsPolicy?.GetHashCode() ?? 0; + var hc = new HashCode(); + hc.Add(RequestLimitsPolicy); + if (RequestRateLimitPolicies != null) + { + foreach (var policy in RequestRateLimitPolicies) + { + hc.Add(policy); + } + } + return hc.ToHashCode(); } public string ToJson() @@ -174,4 +208,50 @@ public override int GetHashCode() return hc.ToHashCode(); } } + + public class RequestRateLimitPolicy : IEquatable + { + /// + /// Whether rate limiting is enabled for the workload group + /// + [JsonProperty("IsEnabled")] + public bool IsEnabled { get; set; } + + /// + /// The scope of rate limiting - can be "WorkloadGroup" or "Principal" + /// + [JsonProperty("Scope")] + [JsonConverter(typeof(Newtonsoft.Json.Converters.StringEnumConverter))] + public RateLimitScope Scope { get; set; } + + /// + /// Type of rate limiting - "ConcurrentRequests" or "ResourceUtilization" + /// + [JsonProperty("LimitKind")] + [JsonConverter(typeof(Newtonsoft.Json.Converters.StringEnumConverter))] + public RateLimitKind LimitKind { get; set; } + + /// + /// Rate limit properties containing specific limits + /// + [JsonProperty("Properties")] + public object Properties { get; set; } = new(); + + public bool Equals(RequestRateLimitPolicy? other) + { + if (other is null) return false; + if (ReferenceEquals(this, other)) return true; + return IsEnabled == other.IsEnabled && + Scope == other.Scope && + LimitKind == other.LimitKind && + EqualityComparer.Default.Equals(Properties, other.Properties); + } + + public override bool Equals(object? obj) => Equals(obj as RequestRateLimitPolicy); + + public override int GetHashCode() + { + return HashCode.Combine(IsEnabled, Scope, LimitKind, Properties); + } + } } \ No newline at end of file From fdc426abfad666c292b5dfb80caf145f1c8e5aea Mon Sep 17 00:00:00 2001 From: Ashley Van Spankeren <25673124+ashleyvansp@users.noreply.github.com> Date: Fri, 4 Jul 2025 18:48:00 +0000 Subject: [PATCH 03/23] Add RequestRateLimitsEnforcementPolicy to WorkloadGroup model --- KustoSchemaTools/Model/WorkloadGroup.cs | 91 ++++++++++++++++++++++++- 1 file changed, 90 insertions(+), 1 deletion(-) diff --git a/KustoSchemaTools/Model/WorkloadGroup.cs b/KustoSchemaTools/Model/WorkloadGroup.cs index 97c5e86..a4e79ac 100644 --- a/KustoSchemaTools/Model/WorkloadGroup.cs +++ b/KustoSchemaTools/Model/WorkloadGroup.cs @@ -4,18 +4,70 @@ namespace KustoSchemaTools.Model { + /// + /// Types of rate limiting available for workload groups + /// public enum RateLimitKind { + /// + /// Limits the number of concurrent requests + /// ConcurrentRequests, + + /// + /// Limits resource utilization (CPU, memory, etc.) + /// ResourceUtilization } + /// + /// Scope of rate limiting + /// public enum RateLimitScope { + /// + /// Rate limiting applies to the entire workload group + /// WorkloadGroup, + + /// + /// Rate limiting applies per principal (user/application) + /// Principal } + /// + /// Enforcement level for queries rate limits + /// + public enum QueriesEnforcementLevel + { + /// + /// Enforcement at query head level + /// + QueryHead, + + /// + /// Enforcement at cluster level + /// + Cluster + } + + /// + /// Enforcement level for commands rate limits + /// + public enum CommandsEnforcementLevel + { + /// + /// Enforcement at cluster level + /// + Cluster, + + /// + /// Enforcement at database level + /// + Database + } + public class WorkloadGroup : IEquatable { public required string WorkloadGroupName { get; set; } @@ -60,6 +112,9 @@ public class WorkloadGroupPolicy : IEquatable [JsonProperty("RequestRateLimitPolicies")] public List? RequestRateLimitPolicies { get; set; } + [JsonProperty("RequestRateLimitsEnforcementPolicy")] + public RequestRateLimitsEnforcementPolicy? RequestRateLimitsEnforcementPolicy { get; set; } + public bool Equals(WorkloadGroupPolicy? other) { if (other is null) return false; @@ -67,7 +122,8 @@ public bool Equals(WorkloadGroupPolicy? other) return EqualityComparer.Default.Equals(RequestLimitsPolicy, other.RequestLimitsPolicy) && (RequestRateLimitPolicies == null && other.RequestRateLimitPolicies == null || RequestRateLimitPolicies != null && other.RequestRateLimitPolicies != null && - RequestRateLimitPolicies.SequenceEqual(other.RequestRateLimitPolicies)); + RequestRateLimitPolicies.SequenceEqual(other.RequestRateLimitPolicies)) && + EqualityComparer.Default.Equals(RequestRateLimitsEnforcementPolicy, other.RequestRateLimitsEnforcementPolicy); } public override bool Equals(object? obj) => Equals(obj as WorkloadGroupPolicy); @@ -76,6 +132,7 @@ public override int GetHashCode() { var hc = new HashCode(); hc.Add(RequestLimitsPolicy); + hc.Add(RequestRateLimitsEnforcementPolicy); if (RequestRateLimitPolicies != null) { foreach (var policy in RequestRateLimitPolicies) @@ -254,4 +311,36 @@ public override int GetHashCode() return HashCode.Combine(IsEnabled, Scope, LimitKind, Properties); } } + + public class RequestRateLimitsEnforcementPolicy : IEquatable + { + /// + /// Enforcement level for queries + /// + [JsonProperty("QueriesEnforcementLevel")] + [JsonConverter(typeof(Newtonsoft.Json.Converters.StringEnumConverter))] + public QueriesEnforcementLevel QueriesEnforcementLevel { get; set; } + + /// + /// Enforcement level for commands + /// + [JsonProperty("CommandsEnforcementLevel")] + [JsonConverter(typeof(Newtonsoft.Json.Converters.StringEnumConverter))] + public CommandsEnforcementLevel CommandsEnforcementLevel { get; set; } + + public bool Equals(RequestRateLimitsEnforcementPolicy? other) + { + if (other is null) return false; + if (ReferenceEquals(this, other)) return true; + return QueriesEnforcementLevel == other.QueriesEnforcementLevel && + CommandsEnforcementLevel == other.CommandsEnforcementLevel; + } + + public override bool Equals(object? obj) => Equals(obj as RequestRateLimitsEnforcementPolicy); + + public override int GetHashCode() + { + return HashCode.Combine(QueriesEnforcementLevel, CommandsEnforcementLevel); + } + } } \ No newline at end of file From 84f4f3dcf5c1eaccd6cce671c6577d27a492c332 Mon Sep 17 00:00:00 2001 From: Ashley Van Spankeren <25673124+ashleyvansp@users.noreply.github.com> Date: Fri, 4 Jul 2025 18:59:18 +0000 Subject: [PATCH 04/23] Add QueryConsistencyPolicy to the WorkloadGroup model --- KustoSchemaTools/Model/WorkloadGroup.cs | 124 ++++++++---------------- 1 file changed, 38 insertions(+), 86 deletions(-) diff --git a/KustoSchemaTools/Model/WorkloadGroup.cs b/KustoSchemaTools/Model/WorkloadGroup.cs index a4e79ac..8c3f35f 100644 --- a/KustoSchemaTools/Model/WorkloadGroup.cs +++ b/KustoSchemaTools/Model/WorkloadGroup.cs @@ -4,70 +4,38 @@ namespace KustoSchemaTools.Model { - /// - /// Types of rate limiting available for workload groups - /// public enum RateLimitKind { - /// - /// Limits the number of concurrent requests - /// ConcurrentRequests, - - /// - /// Limits resource utilization (CPU, memory, etc.) - /// ResourceUtilization } - /// - /// Scope of rate limiting - /// public enum RateLimitScope { - /// - /// Rate limiting applies to the entire workload group - /// WorkloadGroup, - - /// - /// Rate limiting applies per principal (user/application) - /// Principal } - /// - /// Enforcement level for queries rate limits - /// public enum QueriesEnforcementLevel { - /// - /// Enforcement at query head level - /// QueryHead, - - /// - /// Enforcement at cluster level - /// Cluster } - /// - /// Enforcement level for commands rate limits - /// public enum CommandsEnforcementLevel { - /// - /// Enforcement at cluster level - /// Cluster, - - /// - /// Enforcement at database level - /// Database } + public enum QueryConsistency + { + Strong, + Weak, + WeakAffinitizedByQuery, + WeakAffinitizedByDatabase + } + public class WorkloadGroup : IEquatable { public required string WorkloadGroupName { get; set; } @@ -115,6 +83,9 @@ public class WorkloadGroupPolicy : IEquatable [JsonProperty("RequestRateLimitsEnforcementPolicy")] public RequestRateLimitsEnforcementPolicy? RequestRateLimitsEnforcementPolicy { get; set; } + [JsonProperty("QueryConsistencyPolicy")] + public QueryConsistencyPolicy? QueryConsistencyPolicy { get; set; } + public bool Equals(WorkloadGroupPolicy? other) { if (other is null) return false; @@ -123,7 +94,8 @@ public bool Equals(WorkloadGroupPolicy? other) (RequestRateLimitPolicies == null && other.RequestRateLimitPolicies == null || RequestRateLimitPolicies != null && other.RequestRateLimitPolicies != null && RequestRateLimitPolicies.SequenceEqual(other.RequestRateLimitPolicies)) && - EqualityComparer.Default.Equals(RequestRateLimitsEnforcementPolicy, other.RequestRateLimitsEnforcementPolicy); + EqualityComparer.Default.Equals(RequestRateLimitsEnforcementPolicy, other.RequestRateLimitsEnforcementPolicy) && + EqualityComparer.Default.Equals(QueryConsistencyPolicy, other.QueryConsistencyPolicy); } public override bool Equals(object? obj) => Equals(obj as WorkloadGroupPolicy); @@ -133,6 +105,7 @@ public override int GetHashCode() var hc = new HashCode(); hc.Add(RequestLimitsPolicy); hc.Add(RequestRateLimitsEnforcementPolicy); + hc.Add(QueryConsistencyPolicy); if (RequestRateLimitPolicies != null) { foreach (var policy in RequestRateLimitPolicies) @@ -179,57 +152,30 @@ public override int GetHashCode() public class RequestLimitsPolicy : IEquatable { - /// - /// Limits the data scope that the query is allowed to reference - /// [JsonProperty("DataScope")] public PolicyValue? DataScope { get; set; } - /// - /// Maximum amount of memory a single query operator may allocate per node (in bytes) - /// [JsonProperty("MaxMemoryPerQueryPerNode")] public PolicyValue? MaxMemoryPerQueryPerNode { get; set; } - /// - /// Maximum amount of memory a single query operator iterator can allocate (in bytes) - /// [JsonProperty("MaxMemoryPerIterator")] public PolicyValue? MaxMemoryPerIterator { get; set; } - /// - /// Maximum percentage of total fanout threads in the cluster that a query can utilize - /// [JsonProperty("MaxFanoutThreadsPercentage")] public PolicyValue? MaxFanoutThreadsPercentage { get; set; } - /// - /// Maximum percentage of nodes in the cluster that a query can fanout to - /// [JsonProperty("MaxFanoutNodesPercentage")] public PolicyValue? MaxFanoutNodesPercentage { get; set; } - /// - /// Maximum number of records a query is allowed to return to the caller - /// [JsonProperty("MaxResultRecords")] public PolicyValue? MaxResultRecords { get; set; } - /// - /// Maximum amount of data a query is allowed to return to the caller (in bytes) - /// [JsonProperty("MaxResultBytes")] public PolicyValue? MaxResultBytes { get; set; } - /// - /// Maximum amount of time a request may execute - /// [JsonProperty("MaxExecutionTime")] public PolicyValue? MaxExecutionTime { get; set; } - /// - /// How frequently progress of query results is reported (only takes effect if query results are progressive) - /// [JsonProperty("QueryResultsProgressiveUpdatePeriod")] public PolicyValue? QueryResultsProgressiveUpdatePeriod { get; set; } @@ -268,29 +214,17 @@ public override int GetHashCode() public class RequestRateLimitPolicy : IEquatable { - /// - /// Whether rate limiting is enabled for the workload group - /// [JsonProperty("IsEnabled")] public bool IsEnabled { get; set; } - /// - /// The scope of rate limiting - can be "WorkloadGroup" or "Principal" - /// [JsonProperty("Scope")] [JsonConverter(typeof(Newtonsoft.Json.Converters.StringEnumConverter))] public RateLimitScope Scope { get; set; } - /// - /// Type of rate limiting - "ConcurrentRequests" or "ResourceUtilization" - /// [JsonProperty("LimitKind")] [JsonConverter(typeof(Newtonsoft.Json.Converters.StringEnumConverter))] public RateLimitKind LimitKind { get; set; } - /// - /// Rate limit properties containing specific limits - /// [JsonProperty("Properties")] public object Properties { get; set; } = new(); @@ -314,16 +248,10 @@ public override int GetHashCode() public class RequestRateLimitsEnforcementPolicy : IEquatable { - /// - /// Enforcement level for queries - /// [JsonProperty("QueriesEnforcementLevel")] [JsonConverter(typeof(Newtonsoft.Json.Converters.StringEnumConverter))] public QueriesEnforcementLevel QueriesEnforcementLevel { get; set; } - /// - /// Enforcement level for commands - /// [JsonProperty("CommandsEnforcementLevel")] [JsonConverter(typeof(Newtonsoft.Json.Converters.StringEnumConverter))] public CommandsEnforcementLevel CommandsEnforcementLevel { get; set; } @@ -343,4 +271,28 @@ public override int GetHashCode() return HashCode.Combine(QueriesEnforcementLevel, CommandsEnforcementLevel); } } + + public class QueryConsistencyPolicy : IEquatable + { + [JsonProperty("QueryConsistency")] + public PolicyValue? QueryConsistency { get; set; } + + [JsonProperty("CachedResultsMaxAge")] + public PolicyValue? CachedResultsMaxAge { get; set; } + + public bool Equals(QueryConsistencyPolicy? other) + { + if (other is null) return false; + if (ReferenceEquals(this, other)) return true; + return EqualityComparer?>.Default.Equals(QueryConsistency, other.QueryConsistency) && + EqualityComparer?>.Default.Equals(CachedResultsMaxAge, other.CachedResultsMaxAge); + } + + public override bool Equals(object? obj) => Equals(obj as QueryConsistencyPolicy); + + public override int GetHashCode() + { + return HashCode.Combine(QueryConsistency, CachedResultsMaxAge); + } + } } \ No newline at end of file From 31a3fed45f9fe014a40842d39b54c7108780c687 Mon Sep 17 00:00:00 2001 From: Ashley Van Spankeren <25673124+ashleyvansp@users.noreply.github.com> Date: Fri, 4 Jul 2025 19:01:33 +0000 Subject: [PATCH 05/23] remove unused imports --- KustoSchemaTools/Model/WorkloadGroup.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/KustoSchemaTools/Model/WorkloadGroup.cs b/KustoSchemaTools/Model/WorkloadGroup.cs index 8c3f35f..b380c61 100644 --- a/KustoSchemaTools/Model/WorkloadGroup.cs +++ b/KustoSchemaTools/Model/WorkloadGroup.cs @@ -1,6 +1,4 @@ using Newtonsoft.Json; -using System.Collections.Generic; -using System.Linq; namespace KustoSchemaTools.Model { From af75cfc35b798a2862e33fec30b99a9a62c97696 Mon Sep 17 00:00:00 2001 From: Ashley Van Spankeren <25673124+ashleyvansp@users.noreply.github.com> Date: Fri, 4 Jul 2025 19:06:14 +0000 Subject: [PATCH 06/23] Add create script to workload group --- KustoSchemaTools/Model/WorkloadGroup.cs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/KustoSchemaTools/Model/WorkloadGroup.cs b/KustoSchemaTools/Model/WorkloadGroup.cs index b380c61..2373fc2 100644 --- a/KustoSchemaTools/Model/WorkloadGroup.cs +++ b/KustoSchemaTools/Model/WorkloadGroup.cs @@ -57,13 +57,25 @@ public override int GetHashCode() return hc.ToHashCode(); } + public string ToCreateScript() + { + if (WorkloadGroupPolicy == null) + { + throw new InvalidOperationException("WorkloadGroupPolicy cannot be null when generating create script"); + } + + var workloadGroupPolicyJson = WorkloadGroupPolicy.ToJson(); + var script = $".create-or-alter workload_group {WorkloadGroupName} ```{workloadGroupPolicyJson}```"; + return script; + } + public string ToUpdateScript() { if (WorkloadGroupPolicy == null) { throw new InvalidOperationException("WorkloadGroupPolicy cannot be null when generating update script"); } - + var workloadGroupPolicyJson = WorkloadGroupPolicy.ToJson(); var script = $".alter-merge workload_group {WorkloadGroupName} ```{workloadGroupPolicyJson}```"; return script; From 865447ec2b268623a39bbb1a3ad1f8e7b88de917 Mon Sep 17 00:00:00 2001 From: Ashley Van Spankeren <25673124+ashleyvansp@users.noreply.github.com> Date: Fri, 4 Jul 2025 19:15:56 +0000 Subject: [PATCH 07/23] Add workload groups to the Cluster model --- KustoSchemaTools/Model/Cluster.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/KustoSchemaTools/Model/Cluster.cs b/KustoSchemaTools/Model/Cluster.cs index 832b648..e8bb2fb 100644 --- a/KustoSchemaTools/Model/Cluster.cs +++ b/KustoSchemaTools/Model/Cluster.cs @@ -6,6 +6,7 @@ public class Cluster public string Url { get; set; } public List Scripts { get; set; } = new List(); public ClusterCapacityPolicy? CapacityPolicy { get; set; } + public List WorkloadGroups { get; set; } = new List(); } } From eb80585a49211de3831d66d45db7e2a9b6334dd9 Mon Sep 17 00:00:00 2001 From: Ashley Van Spankeren <25673124+ashleyvansp@users.noreply.github.com> Date: Fri, 4 Jul 2025 19:24:39 +0000 Subject: [PATCH 08/23] Add deletion script for workload groups --- KustoSchemaTools/Model/WorkloadGroup.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/KustoSchemaTools/Model/WorkloadGroup.cs b/KustoSchemaTools/Model/WorkloadGroup.cs index 2373fc2..4ff0e84 100644 --- a/KustoSchemaTools/Model/WorkloadGroup.cs +++ b/KustoSchemaTools/Model/WorkloadGroup.cs @@ -63,7 +63,7 @@ public string ToCreateScript() { throw new InvalidOperationException("WorkloadGroupPolicy cannot be null when generating create script"); } - + var workloadGroupPolicyJson = WorkloadGroupPolicy.ToJson(); var script = $".create-or-alter workload_group {WorkloadGroupName} ```{workloadGroupPolicyJson}```"; return script; @@ -80,6 +80,12 @@ public string ToUpdateScript() var script = $".alter-merge workload_group {WorkloadGroupName} ```{workloadGroupPolicyJson}```"; return script; } + + public string ToDeleteScript() + { + var script = $".drop workload_group {WorkloadGroupName}"; + return script; + } } public class WorkloadGroupPolicy : IEquatable From e5454dcc23e88a23e08d73b54bc43844cec5e970 Mon Sep 17 00:00:00 2001 From: Ashley Van Spankeren <25673124+ashleyvansp@users.noreply.github.com> Date: Fri, 4 Jul 2025 21:26:14 +0000 Subject: [PATCH 09/23] Load workload groups from live cluster --- .../Parser/KustoClusterHandlerTests.cs | 191 ++++++++++++++++-- .../Parser/KustoClusterHandler.cs | 22 ++ 2 files changed, 196 insertions(+), 17 deletions(-) diff --git a/KustoSchemaTools.Tests/Parser/KustoClusterHandlerTests.cs b/KustoSchemaTools.Tests/Parser/KustoClusterHandlerTests.cs index a16866a..0f52905 100644 --- a/KustoSchemaTools.Tests/Parser/KustoClusterHandlerTests.cs +++ b/KustoSchemaTools.Tests/Parser/KustoClusterHandlerTests.cs @@ -248,15 +248,22 @@ public async Task LoadAsync_WithCapacityPolicy_ReturnsClusterWithPolicy() } """; - var mockReader = new Mock(); - mockReader.SetupSequence(x => x.Read()) + var mockCapacityReader = new Mock(); + mockCapacityReader.SetupSequence(x => x.Read()) .Returns(true) // First call returns true (data available) .Returns(false); // Second call returns false (no more data) - mockReader.Setup(x => x["Policy"]).Returns(policyJson); + mockCapacityReader.Setup(x => x["Policy"]).Returns(policyJson); + + var mockWorkloadGroupsReader = new Mock(); + mockWorkloadGroupsReader.Setup(x => x.Read()).Returns(false); // No workload groups _adminClientMock .Setup(x => x.ExecuteControlCommandAsync("", ".show cluster policy capacity", It.IsAny())) - .ReturnsAsync(mockReader.Object); + .ReturnsAsync(mockCapacityReader.Object); + + _adminClientMock + .Setup(x => x.ExecuteControlCommandAsync("", ".show workload_groups", It.IsAny())) + .ReturnsAsync(mockWorkloadGroupsReader.Object); // Act var result = await _handler.LoadAsync(); @@ -278,23 +285,34 @@ public async Task LoadAsync_WithCapacityPolicy_ReturnsClusterWithPolicy() Assert.NotNull(result.CapacityPolicy.MirroringCapacity); Assert.Equal(50, result.CapacityPolicy.MirroringCapacity.ClusterMaximumConcurrentOperations); - // Verify the correct command was executed + // Verify the correct commands were executed _adminClientMock.Verify(x => x.ExecuteControlCommandAsync( "", ".show cluster policy capacity", It.IsAny()), Times.Once); + _adminClientMock.Verify(x => x.ExecuteControlCommandAsync( + "", + ".show workload_groups", + It.IsAny()), Times.Once); } [Fact] public async Task LoadAsync_WithNoPolicyData_ReturnsClusterWithoutPolicy() { // Arrange - var mockReader = new Mock(); - mockReader.Setup(x => x.Read()).Returns(false); // No data + var mockCapacityReader = new Mock(); + mockCapacityReader.Setup(x => x.Read()).Returns(false); // No data + + var mockWorkloadGroupsReader = new Mock(); + mockWorkloadGroupsReader.Setup(x => x.Read()).Returns(false); // No workload groups _adminClientMock .Setup(x => x.ExecuteControlCommandAsync("", ".show cluster policy capacity", It.IsAny())) - .ReturnsAsync(mockReader.Object); + .ReturnsAsync(mockCapacityReader.Object); + + _adminClientMock + .Setup(x => x.ExecuteControlCommandAsync("", ".show workload_groups", It.IsAny())) + .ReturnsAsync(mockWorkloadGroupsReader.Object); // Act var result = await _handler.LoadAsync(); @@ -305,26 +323,37 @@ public async Task LoadAsync_WithNoPolicyData_ReturnsClusterWithoutPolicy() Assert.Equal("test.eastus", result.Url); Assert.Null(result.CapacityPolicy); - // Verify the correct command was executed + // Verify the correct commands were executed _adminClientMock.Verify(x => x.ExecuteControlCommandAsync( "", ".show cluster policy capacity", It.IsAny()), Times.Once); + _adminClientMock.Verify(x => x.ExecuteControlCommandAsync( + "", + ".show workload_groups", + It.IsAny()), Times.Once); } [Fact] public async Task LoadAsync_WithEmptyPolicyJson_ReturnsClusterWithoutPolicy() { // Arrange - var mockReader = new Mock(); - mockReader.SetupSequence(x => x.Read()) + var mockCapacityReader = new Mock(); + mockCapacityReader.SetupSequence(x => x.Read()) .Returns(true) // First call returns true (data available) .Returns(false); // Second call returns false (no more data) - mockReader.Setup(x => x["Policy"]).Returns(""); // Empty policy + mockCapacityReader.Setup(x => x["Policy"]).Returns(""); // Empty policy + + var mockWorkloadGroupsReader = new Mock(); + mockWorkloadGroupsReader.Setup(x => x.Read()).Returns(false); // No workload groups _adminClientMock .Setup(x => x.ExecuteControlCommandAsync("", ".show cluster policy capacity", It.IsAny())) - .ReturnsAsync(mockReader.Object); + .ReturnsAsync(mockCapacityReader.Object); + + _adminClientMock + .Setup(x => x.ExecuteControlCommandAsync("", ".show workload_groups", It.IsAny())) + .ReturnsAsync(mockWorkloadGroupsReader.Object); // Act var result = await _handler.LoadAsync(); @@ -334,21 +363,38 @@ public async Task LoadAsync_WithEmptyPolicyJson_ReturnsClusterWithoutPolicy() Assert.Equal("test-cluster", result.Name); Assert.Equal("test.eastus", result.Url); Assert.Null(result.CapacityPolicy); + + // Verify the correct commands were executed + _adminClientMock.Verify(x => x.ExecuteControlCommandAsync( + "", + ".show cluster policy capacity", + It.IsAny()), Times.Once); + _adminClientMock.Verify(x => x.ExecuteControlCommandAsync( + "", + ".show workload_groups", + It.IsAny()), Times.Once); } [Fact] public async Task LoadAsync_WithNullPolicyJson_ReturnsClusterWithoutPolicy() { // Arrange - var mockReader = new Mock(); - mockReader.SetupSequence(x => x.Read()) + var mockCapacityReader = new Mock(); + mockCapacityReader.SetupSequence(x => x.Read()) .Returns(true) // First call returns true (data available) .Returns(false); // Second call returns false (no more data) - mockReader.Setup(x => x["Policy"]).Returns((object?)null); // Null policy + mockCapacityReader.Setup(x => x["Policy"]).Returns(null as object); // Null policy + + var mockWorkloadGroupsReader = new Mock(); + mockWorkloadGroupsReader.Setup(x => x.Read()).Returns(false); // No workload groups _adminClientMock .Setup(x => x.ExecuteControlCommandAsync("", ".show cluster policy capacity", It.IsAny())) - .ReturnsAsync(mockReader.Object); + .ReturnsAsync(mockCapacityReader.Object); + + _adminClientMock + .Setup(x => x.ExecuteControlCommandAsync("", ".show workload_groups", It.IsAny())) + .ReturnsAsync(mockWorkloadGroupsReader.Object); // Act var result = await _handler.LoadAsync(); @@ -358,6 +404,117 @@ public async Task LoadAsync_WithNullPolicyJson_ReturnsClusterWithoutPolicy() Assert.Equal("test-cluster", result.Name); Assert.Equal("test.eastus", result.Url); Assert.Null(result.CapacityPolicy); + + // Verify the correct commands were executed + _adminClientMock.Verify(x => x.ExecuteControlCommandAsync( + "", + ".show cluster policy capacity", + It.IsAny()), Times.Once); + _adminClientMock.Verify(x => x.ExecuteControlCommandAsync( + "", + ".show workload_groups", + It.IsAny()), Times.Once); + } + + [Fact] + public async Task LoadAsync_WithWorkloadGroups_ReturnsClusterWithWorkloadGroups() + { + // Arrange + var workloadGroupJson1 = """ + { + "RequestLimitsPolicy": { + "DataScope": { + "IsRelaxable": true, + "Value": "All" + }, + "MaxMemoryPerQueryPerNode": { + "IsRelaxable": true, + "Value": 8589346816 + }, + "MaxExecutionTime": { + "IsRelaxable": true, + "Value": "00:04:00" + } + }, + "RequestRateLimitPolicies": [ + { + "IsEnabled": true, + "Scope": "WorkloadGroup", + "LimitKind": "ConcurrentRequests", + "Properties": { + "MaxConcurrentRequests": 20 + } + } + ] + } + """; + + var workloadGroupJson2 = """ + { + "RequestRateLimitPolicies": [] + } + """; + + var mockCapacityReader = new Mock(); + mockCapacityReader.Setup(x => x.Read()).Returns(false); // No capacity policy + + var mockWorkloadGroupsReader = new Mock(); + mockWorkloadGroupsReader.SetupSequence(x => x.Read()) + .Returns(true) // First workload group + .Returns(true) // Second workload group + .Returns(false); // No more data + mockWorkloadGroupsReader.SetupSequence(x => x["WorkloadGroupName"]) + .Returns("default") + .Returns("custom"); + mockWorkloadGroupsReader.SetupSequence(x => x["WorkloadGroup"]) + .Returns(workloadGroupJson1) + .Returns(workloadGroupJson2); + + _adminClientMock + .Setup(x => x.ExecuteControlCommandAsync("", ".show cluster policy capacity", It.IsAny())) + .ReturnsAsync(mockCapacityReader.Object); + + _adminClientMock + .Setup(x => x.ExecuteControlCommandAsync("", ".show workload_groups", It.IsAny())) + .ReturnsAsync(mockWorkloadGroupsReader.Object); + + // Act + var result = await _handler.LoadAsync(); + + // Assert + Assert.NotNull(result); + Assert.Equal("test-cluster", result.Name); + Assert.Equal("test.eastus", result.Url); + Assert.Null(result.CapacityPolicy); + + // Verify workload groups were loaded + Assert.NotNull(result.WorkloadGroups); + Assert.Equal(2, result.WorkloadGroups.Count); + + var defaultGroup = result.WorkloadGroups.FirstOrDefault(wg => wg.WorkloadGroupName == "default"); + Assert.NotNull(defaultGroup); + Assert.NotNull(defaultGroup.WorkloadGroupPolicy); + Assert.NotNull(defaultGroup.WorkloadGroupPolicy.RequestLimitsPolicy); + Assert.Equal("All", defaultGroup.WorkloadGroupPolicy.RequestLimitsPolicy.DataScope?.Value); + Assert.Equal(8589346816, defaultGroup.WorkloadGroupPolicy.RequestLimitsPolicy.MaxMemoryPerQueryPerNode?.Value); + Assert.NotNull(defaultGroup.WorkloadGroupPolicy.RequestRateLimitPolicies); + Assert.Single(defaultGroup.WorkloadGroupPolicy.RequestRateLimitPolicies); + + var customGroup = result.WorkloadGroups.FirstOrDefault(wg => wg.WorkloadGroupName == "custom"); + Assert.NotNull(customGroup); + Assert.NotNull(customGroup.WorkloadGroupPolicy); + Assert.NotNull(customGroup.WorkloadGroupPolicy.RequestRateLimitPolicies); + Assert.Empty(customGroup.WorkloadGroupPolicy.RequestRateLimitPolicies); + + // Verify the correct commands were executed + _adminClientMock.Verify(x => x.ExecuteControlCommandAsync( + "", + ".show cluster policy capacity", + It.IsAny()), Times.Once); + _adminClientMock.Verify(x => x.ExecuteControlCommandAsync( + "", + ".show workload_groups", + It.IsAny()), Times.Once); } #region Helper Methods diff --git a/KustoSchemaTools/Parser/KustoClusterHandler.cs b/KustoSchemaTools/Parser/KustoClusterHandler.cs index ea2db1b..19e056f 100644 --- a/KustoSchemaTools/Parser/KustoClusterHandler.cs +++ b/KustoSchemaTools/Parser/KustoClusterHandler.cs @@ -42,6 +42,28 @@ public virtual async Task LoadAsync() } } + _logger.LogInformation("Loading workload groups..."); + using (var reader = await _adminClient.ExecuteControlCommandAsync("", ".show workload_groups", new ClientRequestProperties())) + { + while (reader.Read()) + { + var workloadGroupName = reader["WorkloadGroupName"]?.ToString(); + var workloadGroupJson = reader["WorkloadGroup"]?.ToString(); + + if (!string.IsNullOrEmpty(workloadGroupName) && !string.IsNullOrEmpty(workloadGroupJson)) + { + var workloadGroupPolicy = JsonConvert.DeserializeObject(workloadGroupJson); + var workloadGroup = new WorkloadGroup + { + WorkloadGroupName = workloadGroupName, + WorkloadGroupPolicy = workloadGroupPolicy + }; + cluster.WorkloadGroups.Add(workloadGroup); + } + } + } + + return cluster; } From 40fd40c806c013e4e95bfc1c8d93bbd07f67d5b4 Mon Sep 17 00:00:00 2001 From: Ashley Van Spankeren <25673124+ashleyvansp@users.noreply.github.com> Date: Fri, 4 Jul 2025 21:47:19 +0000 Subject: [PATCH 10/23] Setup for deleting workload groups --- KustoSchemaTools/Model/Cluster.cs | 2 +- KustoSchemaTools/Model/ClusterDeletions.cs | 7 +++++++ KustoSchemaTools/Parser/KustoClusterHandler.cs | 6 +++--- 3 files changed, 11 insertions(+), 4 deletions(-) create mode 100644 KustoSchemaTools/Model/ClusterDeletions.cs diff --git a/KustoSchemaTools/Model/Cluster.cs b/KustoSchemaTools/Model/Cluster.cs index e8bb2fb..cd6658c 100644 --- a/KustoSchemaTools/Model/Cluster.cs +++ b/KustoSchemaTools/Model/Cluster.cs @@ -7,6 +7,6 @@ public class Cluster public List Scripts { get; set; } = new List(); public ClusterCapacityPolicy? CapacityPolicy { get; set; } public List WorkloadGroups { get; set; } = new List(); + public ClusterDeletions? Deletions { get; set; } } - } diff --git a/KustoSchemaTools/Model/ClusterDeletions.cs b/KustoSchemaTools/Model/ClusterDeletions.cs new file mode 100644 index 0000000..cc7ff90 --- /dev/null +++ b/KustoSchemaTools/Model/ClusterDeletions.cs @@ -0,0 +1,7 @@ +namespace KustoSchemaTools.Model +{ + public class ClusterDeletions + { + public List WorkloadGroups { get; set; } = new List(); + } +} diff --git a/KustoSchemaTools/Parser/KustoClusterHandler.cs b/KustoSchemaTools/Parser/KustoClusterHandler.cs index 19e056f..9355421 100644 --- a/KustoSchemaTools/Parser/KustoClusterHandler.cs +++ b/KustoSchemaTools/Parser/KustoClusterHandler.cs @@ -47,15 +47,15 @@ public virtual async Task LoadAsync() { while (reader.Read()) { - var workloadGroupName = reader["WorkloadGroupName"]?.ToString(); + var workloadGroupName = reader["WorkloadGroupName"].ToString(); var workloadGroupJson = reader["WorkloadGroup"]?.ToString(); - if (!string.IsNullOrEmpty(workloadGroupName) && !string.IsNullOrEmpty(workloadGroupJson)) + if (!string.IsNullOrEmpty(workloadGroupJson)) { var workloadGroupPolicy = JsonConvert.DeserializeObject(workloadGroupJson); var workloadGroup = new WorkloadGroup { - WorkloadGroupName = workloadGroupName, + WorkloadGroupName = workloadGroupName!, WorkloadGroupPolicy = workloadGroupPolicy }; cluster.WorkloadGroups.Add(workloadGroup); From 8ba74850a02cdf850ae8f915f9b9926c346b264b Mon Sep 17 00:00:00 2001 From: Ashley Van Spankeren <25673124+ashleyvansp@users.noreply.github.com> Date: Fri, 4 Jul 2025 22:10:42 +0000 Subject: [PATCH 11/23] Generate changes for workload groups --- KustoSchemaTools/Changes/ClusterChanges.cs | 49 ++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/KustoSchemaTools/Changes/ClusterChanges.cs b/KustoSchemaTools/Changes/ClusterChanges.cs index bc5aa64..40c6109 100644 --- a/KustoSchemaTools/Changes/ClusterChanges.cs +++ b/KustoSchemaTools/Changes/ClusterChanges.cs @@ -48,6 +48,55 @@ public static ClusterChangeSet GenerateChanges(Cluster oldCluster, Cluster newCl } } + log.LogInformation($"Analyzing workload group changes for cluster {clusterName}..."); + + // Handle workload group deletions first + var workloadGroupsToDelete = newCluster.Deletions?.WorkloadGroups ?? new List(); + foreach (var workloadGroupName in workloadGroupsToDelete) + { + var existingWorkloadGroup = oldCluster.WorkloadGroups.FirstOrDefault(wg => wg.WorkloadGroupName == workloadGroupName); + if (existingWorkloadGroup != null) + { + log.LogInformation($"Marking workload group '{workloadGroupName}' for deletion."); + var deletionChange = new DeletionChange(workloadGroupName, "workload_group"); + changeSet.Changes.Add(deletionChange); + } + else + { + log.LogWarning($"Workload group '{workloadGroupName}' marked for deletion but does not exist in the live cluster."); + } + } + + // Handle workload group creations and updates + foreach (var newWorkloadGroup in newCluster.WorkloadGroups) + { + // Skip if this workload group is marked for deletion + if (workloadGroupsToDelete.Contains(newWorkloadGroup.WorkloadGroupName)) + { + log.LogInformation($"Skipping update to workload group {newWorkloadGroup.WorkloadGroupName} as it is marked for deletion."); + continue; + } + + var existingWorkloadGroup = oldCluster.WorkloadGroups.FirstOrDefault(wg => wg.WorkloadGroupName == newWorkloadGroup.WorkloadGroupName); + + var workloadGroupChange = ComparePolicy( + "Workload Group", + newWorkloadGroup.WorkloadGroupName, + existingWorkloadGroup, + newWorkloadGroup, + wg => new List { + new DatabaseScriptContainer( + existingWorkloadGroup == null ? "CreateWorkloadGroup" : "UpdateWorkloadGroup", + 5, + existingWorkloadGroup == null ? wg.ToCreateScript() : wg.ToUpdateScript()) + }); + + if (workloadGroupChange != null) + { + changeSet.Changes.Add(workloadGroupChange); + } + } + changeSet.Scripts.AddRange(changeSet.Changes.SelectMany(c => c.Scripts)); // Run Kusto code diagnostics From 24555c2b9913b0cf39fc3070bdf9e80b662dafae Mon Sep 17 00:00:00 2001 From: Ashley Van Spankeren <25673124+ashleyvansp@users.noreply.github.com> Date: Sun, 6 Jul 2025 15:40:55 +0000 Subject: [PATCH 12/23] Extract capacity policy changes into its own function --- KustoSchemaTools/Changes/ClusterChanges.cs | 54 ++++++++++++++-------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/KustoSchemaTools/Changes/ClusterChanges.cs b/KustoSchemaTools/Changes/ClusterChanges.cs index 40c6109..a15649a 100644 --- a/KustoSchemaTools/Changes/ClusterChanges.cs +++ b/KustoSchemaTools/Changes/ClusterChanges.cs @@ -30,26 +30,10 @@ public static ClusterChangeSet GenerateChanges(Cluster oldCluster, Cluster newCl var changeSet = new ClusterChangeSet(clusterName, oldCluster, newCluster); log.LogInformation($"Analyzing capacity policy changes for cluster {clusterName}..."); - if (newCluster.CapacityPolicy == null) { - log.LogInformation("No capacity policy defined in the new cluster configuration."); - } else { - var capacityPolicyChange = ComparePolicy( - "Cluster Capacity Policy", - "default", - oldCluster.CapacityPolicy, - newCluster.CapacityPolicy, - policy => new List { - new DatabaseScriptContainer("AlterMergeClusterCapacityPolicy", 10, policy.ToUpdateScript()) - }); - - if (capacityPolicyChange != null) - { - changeSet.Changes.Add(capacityPolicyChange); - } - } + HandleCapacityPolicyChanges(oldCluster, newCluster, changeSet, log); log.LogInformation($"Analyzing workload group changes for cluster {clusterName}..."); - + // Handle workload group deletions first var workloadGroupsToDelete = newCluster.Deletions?.WorkloadGroups ?? new List(); foreach (var workloadGroupName in workloadGroupsToDelete) @@ -135,7 +119,7 @@ public static ClusterChangeSet GenerateChanges(Cluster oldCluster, Cluster newCl var changedProperties = new List(); var properties = typeof(T).GetProperties().Where(p => p.CanRead && p.CanWrite); - + foreach (var prop in properties) { var oldValue = oldPolicy != null ? prop.GetValue(oldPolicy) : null; @@ -158,5 +142,37 @@ public static ClusterChangeSet GenerateChanges(Cluster oldCluster, Cluster newCl return null; } + + /// + /// Handles capacity policy changes between old and new cluster configurations. + /// Compares the capacity policies and adds any necessary changes to the change set. + /// + /// The current/existing cluster configuration. + /// The desired cluster configuration. + /// The change set to add capacity policy changes to. + /// Logger instance for recording the comparison process. + private static void HandleCapacityPolicyChanges(Cluster oldCluster, Cluster newCluster, ClusterChangeSet changeSet, ILogger log) + { + if (newCluster.CapacityPolicy == null) + { + log.LogInformation("No capacity policy defined in the new cluster configuration."); + } + else + { + var capacityPolicyChange = ComparePolicy( + "Cluster Capacity Policy", + "default", + oldCluster.CapacityPolicy, + newCluster.CapacityPolicy, + policy => new List { + new DatabaseScriptContainer("AlterMergeClusterCapacityPolicy", 10, policy.ToUpdateScript()) + }); + + if (capacityPolicyChange != null) + { + changeSet.Changes.Add(capacityPolicyChange); + } + } + } } } \ No newline at end of file From 044f8e34e847c602bfb97ccd7092979ce6faeffd Mon Sep 17 00:00:00 2001 From: Ashley Van Spankeren <25673124+ashleyvansp@users.noreply.github.com> Date: Sun, 6 Jul 2025 15:44:03 +0000 Subject: [PATCH 13/23] Extract workload group changes into its own function --- KustoSchemaTools/Changes/ClusterChanges.cs | 106 ++++++++++++--------- 1 file changed, 59 insertions(+), 47 deletions(-) diff --git a/KustoSchemaTools/Changes/ClusterChanges.cs b/KustoSchemaTools/Changes/ClusterChanges.cs index a15649a..2f4bd1a 100644 --- a/KustoSchemaTools/Changes/ClusterChanges.cs +++ b/KustoSchemaTools/Changes/ClusterChanges.cs @@ -33,53 +33,7 @@ public static ClusterChangeSet GenerateChanges(Cluster oldCluster, Cluster newCl HandleCapacityPolicyChanges(oldCluster, newCluster, changeSet, log); log.LogInformation($"Analyzing workload group changes for cluster {clusterName}..."); - - // Handle workload group deletions first - var workloadGroupsToDelete = newCluster.Deletions?.WorkloadGroups ?? new List(); - foreach (var workloadGroupName in workloadGroupsToDelete) - { - var existingWorkloadGroup = oldCluster.WorkloadGroups.FirstOrDefault(wg => wg.WorkloadGroupName == workloadGroupName); - if (existingWorkloadGroup != null) - { - log.LogInformation($"Marking workload group '{workloadGroupName}' for deletion."); - var deletionChange = new DeletionChange(workloadGroupName, "workload_group"); - changeSet.Changes.Add(deletionChange); - } - else - { - log.LogWarning($"Workload group '{workloadGroupName}' marked for deletion but does not exist in the live cluster."); - } - } - - // Handle workload group creations and updates - foreach (var newWorkloadGroup in newCluster.WorkloadGroups) - { - // Skip if this workload group is marked for deletion - if (workloadGroupsToDelete.Contains(newWorkloadGroup.WorkloadGroupName)) - { - log.LogInformation($"Skipping update to workload group {newWorkloadGroup.WorkloadGroupName} as it is marked for deletion."); - continue; - } - - var existingWorkloadGroup = oldCluster.WorkloadGroups.FirstOrDefault(wg => wg.WorkloadGroupName == newWorkloadGroup.WorkloadGroupName); - - var workloadGroupChange = ComparePolicy( - "Workload Group", - newWorkloadGroup.WorkloadGroupName, - existingWorkloadGroup, - newWorkloadGroup, - wg => new List { - new DatabaseScriptContainer( - existingWorkloadGroup == null ? "CreateWorkloadGroup" : "UpdateWorkloadGroup", - 5, - existingWorkloadGroup == null ? wg.ToCreateScript() : wg.ToUpdateScript()) - }); - - if (workloadGroupChange != null) - { - changeSet.Changes.Add(workloadGroupChange); - } - } + HandleWorkloadGroupChanges(oldCluster, newCluster, changeSet, log); changeSet.Scripts.AddRange(changeSet.Changes.SelectMany(c => c.Scripts)); @@ -174,5 +128,63 @@ private static void HandleCapacityPolicyChanges(Cluster oldCluster, Cluster newC } } } + + /// + /// Handles workload group changes between old and new cluster configurations. + /// Processes both deletions and creations/updates of workload groups. + /// + /// The current/existing cluster configuration. + /// The desired cluster configuration. + /// The change set to add workload group changes to. + /// Logger instance for recording the comparison process. + private static void HandleWorkloadGroupChanges(Cluster oldCluster, Cluster newCluster, ClusterChangeSet changeSet, ILogger log) + { + // Handle workload group deletions first + var workloadGroupsToDelete = newCluster.Deletions?.WorkloadGroups ?? new List(); + foreach (var workloadGroupName in workloadGroupsToDelete) + { + var existingWorkloadGroup = oldCluster.WorkloadGroups.FirstOrDefault(wg => wg.WorkloadGroupName == workloadGroupName); + if (existingWorkloadGroup != null) + { + log.LogInformation($"Marking workload group '{workloadGroupName}' for deletion."); + var deletionChange = new DeletionChange(workloadGroupName, "workload_group"); + changeSet.Changes.Add(deletionChange); + } + else + { + log.LogWarning($"Workload group '{workloadGroupName}' marked for deletion but does not exist in the live cluster."); + } + } + + // Handle workload group creations and updates + foreach (var newWorkloadGroup in newCluster.WorkloadGroups) + { + // Skip if this workload group is marked for deletion + if (workloadGroupsToDelete.Contains(newWorkloadGroup.WorkloadGroupName)) + { + log.LogInformation($"Skipping update to workload group {newWorkloadGroup.WorkloadGroupName} as it is marked for deletion."); + continue; + } + + var existingWorkloadGroup = oldCluster.WorkloadGroups.FirstOrDefault(wg => wg.WorkloadGroupName == newWorkloadGroup.WorkloadGroupName); + + var workloadGroupChange = ComparePolicy( + "Workload Group", + newWorkloadGroup.WorkloadGroupName, + existingWorkloadGroup, + newWorkloadGroup, + wg => new List { + new DatabaseScriptContainer( + existingWorkloadGroup == null ? "CreateWorkloadGroup" : "UpdateWorkloadGroup", + 5, + existingWorkloadGroup == null ? wg.ToCreateScript() : wg.ToUpdateScript()) + }); + + if (workloadGroupChange != null) + { + changeSet.Changes.Add(workloadGroupChange); + } + } + } } } \ No newline at end of file From ee60410de00178b6d9a408db523a17d990f6c03a Mon Sep 17 00:00:00 2001 From: Ashley Van Spankeren <25673124+ashleyvansp@users.noreply.github.com> Date: Sun, 6 Jul 2025 15:50:23 +0000 Subject: [PATCH 14/23] Add toString methods for workload groups --- KustoSchemaTools/Model/WorkloadGroup.cs | 45 +++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/KustoSchemaTools/Model/WorkloadGroup.cs b/KustoSchemaTools/Model/WorkloadGroup.cs index 4ff0e84..be203f3 100644 --- a/KustoSchemaTools/Model/WorkloadGroup.cs +++ b/KustoSchemaTools/Model/WorkloadGroup.cs @@ -164,6 +164,15 @@ public override int GetHashCode() { return HashCode.Combine(IsRelaxable, Value); } + + public override string ToString() + { + return JsonConvert.SerializeObject(this, new JsonSerializerSettings + { + NullValueHandling = NullValueHandling.Ignore, + Formatting = Formatting.None + }); + } } public class RequestLimitsPolicy : IEquatable @@ -226,6 +235,15 @@ public override int GetHashCode() hc.Add(QueryResultsProgressiveUpdatePeriod); return hc.ToHashCode(); } + + public override string ToString() + { + return JsonConvert.SerializeObject(this, new JsonSerializerSettings + { + NullValueHandling = NullValueHandling.Ignore, + Formatting = Formatting.None + }); + } } public class RequestRateLimitPolicy : IEquatable @@ -260,6 +278,15 @@ public override int GetHashCode() { return HashCode.Combine(IsEnabled, Scope, LimitKind, Properties); } + + public override string ToString() + { + return JsonConvert.SerializeObject(this, new JsonSerializerSettings + { + NullValueHandling = NullValueHandling.Ignore, + Formatting = Formatting.None + }); + } } public class RequestRateLimitsEnforcementPolicy : IEquatable @@ -286,6 +313,15 @@ public override int GetHashCode() { return HashCode.Combine(QueriesEnforcementLevel, CommandsEnforcementLevel); } + + public override string ToString() + { + return JsonConvert.SerializeObject(this, new JsonSerializerSettings + { + NullValueHandling = NullValueHandling.Ignore, + Formatting = Formatting.None + }); + } } public class QueryConsistencyPolicy : IEquatable @@ -310,5 +346,14 @@ public override int GetHashCode() { return HashCode.Combine(QueryConsistency, CachedResultsMaxAge); } + + public override string ToString() + { + return JsonConvert.SerializeObject(this, new JsonSerializerSettings + { + NullValueHandling = NullValueHandling.Ignore, + Formatting = Formatting.None + }); + } } } \ No newline at end of file From 139c08837f1e129ac5cbaa63c701f6e0afb4cbc1 Mon Sep 17 00:00:00 2001 From: Ashley Van Spankeren <25673124+ashleyvansp@users.noreply.github.com> Date: Sun, 6 Jul 2025 15:55:46 +0000 Subject: [PATCH 15/23] Update HandleWorkloadGroupChanges --- KustoSchemaTools/Changes/ClusterChanges.cs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/KustoSchemaTools/Changes/ClusterChanges.cs b/KustoSchemaTools/Changes/ClusterChanges.cs index 2f4bd1a..58b0943 100644 --- a/KustoSchemaTools/Changes/ClusterChanges.cs +++ b/KustoSchemaTools/Changes/ClusterChanges.cs @@ -167,17 +167,19 @@ private static void HandleWorkloadGroupChanges(Cluster oldCluster, Cluster newCl } var existingWorkloadGroup = oldCluster.WorkloadGroups.FirstOrDefault(wg => wg.WorkloadGroupName == newWorkloadGroup.WorkloadGroupName); - + var scriptType = existingWorkloadGroup == null ? "ClusterWorkloadGroupCreateOrAlterCommand" : "ClusterWorkloadGroupAlterMergeCommand"; + var scriptText = existingWorkloadGroup == null ? newWorkloadGroup.ToCreateScript() : newWorkloadGroup.ToUpdateScript(); var workloadGroupChange = ComparePolicy( "Workload Group", newWorkloadGroup.WorkloadGroupName, - existingWorkloadGroup, - newWorkloadGroup, + existingWorkloadGroup?.WorkloadGroupPolicy, + newWorkloadGroup.WorkloadGroupPolicy, wg => new List { new DatabaseScriptContainer( - existingWorkloadGroup == null ? "CreateWorkloadGroup" : "UpdateWorkloadGroup", + scriptType, 5, - existingWorkloadGroup == null ? wg.ToCreateScript() : wg.ToUpdateScript()) + scriptText + ) }); if (workloadGroupChange != null) From c63d44c3b076996ef0ad7c38aa5d447abc1ed197 Mon Sep 17 00:00:00 2001 From: Ashley Van Spankeren <25673124+ashleyvansp@users.noreply.github.com> Date: Sun, 6 Jul 2025 17:15:45 +0000 Subject: [PATCH 16/23] Update markdown for a cluster change --- KustoSchemaTools/Changes/ClusterChanges.cs | 93 ++++++++++++++++++---- KustoSchemaTools/Changes/DeletionChange.cs | 12 ++- 2 files changed, 88 insertions(+), 17 deletions(-) diff --git a/KustoSchemaTools/Changes/ClusterChanges.cs b/KustoSchemaTools/Changes/ClusterChanges.cs index 58b0943..8aa755d 100644 --- a/KustoSchemaTools/Changes/ClusterChanges.cs +++ b/KustoSchemaTools/Changes/ClusterChanges.cs @@ -1,6 +1,7 @@ using KustoSchemaTools.Model; using Microsoft.Extensions.Logging; using Kusto.Language; +using System.Text; namespace KustoSchemaTools.Changes { @@ -44,6 +45,9 @@ public static ClusterChangeSet GenerateChanges(Cluster oldCluster, Cluster newCl var diagnostics = code.GetDiagnostics(); script.IsValid = !diagnostics.Any(); } + + changeSet.Markdown = GenerateClusterMarkdown(changeSet); + return changeSet; } @@ -89,7 +93,8 @@ public static ClusterChangeSet GenerateChanges(Cluster oldCluster, Cluster newCl if (changedProperties.Any()) { - policyChange.Markdown = $"## {entityType} Changes\n\n{string.Join("\n", changedProperties)}"; + var action = oldPolicy == null ? "Create" : "Update"; + policyChange.Markdown = $"### {action} {entityType} `{entityName}`\n\n{string.Join("\n", changedProperties)}"; policyChange.Scripts.AddRange(scriptGenerator(newPolicy)); return policyChange; } @@ -114,7 +119,7 @@ private static void HandleCapacityPolicyChanges(Cluster oldCluster, Cluster newC else { var capacityPolicyChange = ComparePolicy( - "Cluster Capacity Policy", + "Capacity Policy", "default", oldCluster.CapacityPolicy, newCluster.CapacityPolicy, @@ -148,6 +153,12 @@ private static void HandleWorkloadGroupChanges(Cluster oldCluster, Cluster newCl { log.LogInformation($"Marking workload group '{workloadGroupName}' for deletion."); var deletionChange = new DeletionChange(workloadGroupName, "workload_group"); + + // Replace the header in the deletion markdown + var originalMarkdown = deletionChange.Markdown; + var modifiedMarkdown = originalMarkdown.Replace($"## {workloadGroupName}", $"#### :heavy_minus_sign: Drop {workloadGroupName}"); + deletionChange.Markdown = modifiedMarkdown; + changeSet.Changes.Add(deletionChange); } else @@ -169,24 +180,74 @@ private static void HandleWorkloadGroupChanges(Cluster oldCluster, Cluster newCl var existingWorkloadGroup = oldCluster.WorkloadGroups.FirstOrDefault(wg => wg.WorkloadGroupName == newWorkloadGroup.WorkloadGroupName); var scriptType = existingWorkloadGroup == null ? "ClusterWorkloadGroupCreateOrAlterCommand" : "ClusterWorkloadGroupAlterMergeCommand"; var scriptText = existingWorkloadGroup == null ? newWorkloadGroup.ToCreateScript() : newWorkloadGroup.ToUpdateScript(); - var workloadGroupChange = ComparePolicy( - "Workload Group", - newWorkloadGroup.WorkloadGroupName, - existingWorkloadGroup?.WorkloadGroupPolicy, - newWorkloadGroup.WorkloadGroupPolicy, - wg => new List { - new DatabaseScriptContainer( - scriptType, - 5, - scriptText - ) - }); + + // Only create a change if the workload group policy is not null + if (newWorkloadGroup.WorkloadGroupPolicy != null) + { + var workloadGroupChange = ComparePolicy( + "Workload Group", + newWorkloadGroup.WorkloadGroupName, + existingWorkloadGroup?.WorkloadGroupPolicy, + newWorkloadGroup.WorkloadGroupPolicy, + wg => new List { + new DatabaseScriptContainer( + scriptType, + 5, + scriptText + ) + }); + + if (workloadGroupChange != null) + { + changeSet.Changes.Add(workloadGroupChange); + } + } + } + } - if (workloadGroupChange != null) + /// + /// Generates a markdown representation of all cluster changes. + /// + /// The cluster change set containing all detected changes. + /// A formatted markdown string documenting all changes and scripts. + private static string GenerateClusterMarkdown(ClusterChangeSet changeSet) + { + var sb = new StringBuilder(); + + sb.AppendLine($"## Cluster: {changeSet.Entity}"); + sb.AppendLine(); + + if (changeSet.Changes.Count == 0) + { + sb.AppendLine("No changes detected for this cluster."); + return sb.ToString(); + } + else + { + foreach (var change in changeSet.Changes) { - changeSet.Changes.Add(workloadGroupChange); + if (!string.IsNullOrEmpty(change.Markdown)) + { + sb.AppendLine(change.Markdown); + } + sb.AppendLine(); } } + + // Add scripts section + if (changeSet.Scripts.Count != 0) + { + sb.AppendLine("## Scripts to be executed:"); + sb.AppendLine("```kql"); + foreach (var script in changeSet.Scripts) + { + sb.AppendLine(script.Text); + sb.AppendLine(); + } + sb.AppendLine("```"); + } + + return sb.ToString(); } } } \ No newline at end of file diff --git a/KustoSchemaTools/Changes/DeletionChange.cs b/KustoSchemaTools/Changes/DeletionChange.cs index 55b39be..6328729 100644 --- a/KustoSchemaTools/Changes/DeletionChange.cs +++ b/KustoSchemaTools/Changes/DeletionChange.cs @@ -29,10 +29,17 @@ public List Scripts } } + private string? _customMarkdown; + public string Markdown { get { + if (!string.IsNullOrEmpty(_customMarkdown)) + { + return _customMarkdown; + } + var sb = new StringBuilder(); sb.AppendLine($"## {Entity}"); sb.AppendLine(); @@ -46,7 +53,10 @@ public string Markdown sb.AppendLine(""); return sb.ToString(); - + } + set + { + _customMarkdown = value; } } From 4ce11483eea241fc13b8099f04204735357fc4d8 Mon Sep 17 00:00:00 2001 From: Ashley Van Spankeren <25673124+ashleyvansp@users.noreply.github.com> Date: Sun, 6 Jul 2025 18:33:03 +0000 Subject: [PATCH 17/23] Add stricter types for RequestRateLimitPolicy properties --- KustoSchemaTools/Model/WorkloadGroup.cs | 102 ++++++++++++++++++++++-- 1 file changed, 94 insertions(+), 8 deletions(-) diff --git a/KustoSchemaTools/Model/WorkloadGroup.cs b/KustoSchemaTools/Model/WorkloadGroup.cs index 4ff0e84..980976e 100644 --- a/KustoSchemaTools/Model/WorkloadGroup.cs +++ b/KustoSchemaTools/Model/WorkloadGroup.cs @@ -34,6 +34,12 @@ public enum QueryConsistency WeakAffinitizedByDatabase } + public enum RateLimitResourceKind + { + RequestCount, + TotalCpuSeconds + } + public class WorkloadGroup : IEquatable { public required string WorkloadGroupName { get; set; } @@ -80,12 +86,6 @@ public string ToUpdateScript() var script = $".alter-merge workload_group {WorkloadGroupName} ```{workloadGroupPolicyJson}```"; return script; } - - public string ToDeleteScript() - { - var script = $".drop workload_group {WorkloadGroupName}"; - return script; - } } public class WorkloadGroupPolicy : IEquatable @@ -164,6 +164,15 @@ public override int GetHashCode() { return HashCode.Combine(IsRelaxable, Value); } + + public override string ToString() + { + return JsonConvert.SerializeObject(this, new JsonSerializerSettings + { + NullValueHandling = NullValueHandling.Ignore, + Formatting = Formatting.None + }); + } } public class RequestLimitsPolicy : IEquatable @@ -226,6 +235,56 @@ public override int GetHashCode() hc.Add(QueryResultsProgressiveUpdatePeriod); return hc.ToHashCode(); } + + public override string ToString() + { + return JsonConvert.SerializeObject(this, new JsonSerializerSettings + { + NullValueHandling = NullValueHandling.Ignore, + Formatting = Formatting.None + }); + } + } + + public class RateLimitProperties : IEquatable + { + [JsonProperty("MaxConcurrentRequests")] + public int? MaxConcurrentRequests { get; set; } + + [JsonProperty("ResourceKind")] + public RateLimitResourceKind? ResourceKind { get; set; } + + [JsonProperty("MaxUtilization")] + public double? MaxUtilization { get; set; } + + [JsonProperty("TimeWindow")] + public TimeSpan? TimeWindow { get; set; } + + public bool Equals(RateLimitProperties? other) + { + if (other is null) return false; + if (ReferenceEquals(this, other)) return true; + return MaxConcurrentRequests == other.MaxConcurrentRequests && + ResourceKind == other.ResourceKind && + MaxUtilization == other.MaxUtilization && + TimeWindow == other.TimeWindow; + } + + public override bool Equals(object? obj) => Equals(obj as RateLimitProperties); + + public override int GetHashCode() + { + return HashCode.Combine(MaxConcurrentRequests, ResourceKind, MaxUtilization, TimeWindow); + } + + public override string ToString() + { + return JsonConvert.SerializeObject(this, new JsonSerializerSettings + { + NullValueHandling = NullValueHandling.Ignore, + Formatting = Formatting.None + }); + } } public class RequestRateLimitPolicy : IEquatable @@ -242,7 +301,7 @@ public class RequestRateLimitPolicy : IEquatable public RateLimitKind LimitKind { get; set; } [JsonProperty("Properties")] - public object Properties { get; set; } = new(); + public RateLimitProperties? Properties { get; set; } public bool Equals(RequestRateLimitPolicy? other) { @@ -251,7 +310,7 @@ public bool Equals(RequestRateLimitPolicy? other) return IsEnabled == other.IsEnabled && Scope == other.Scope && LimitKind == other.LimitKind && - EqualityComparer.Default.Equals(Properties, other.Properties); + EqualityComparer.Default.Equals(Properties, other.Properties); } public override bool Equals(object? obj) => Equals(obj as RequestRateLimitPolicy); @@ -260,6 +319,15 @@ public override int GetHashCode() { return HashCode.Combine(IsEnabled, Scope, LimitKind, Properties); } + + public override string ToString() + { + return JsonConvert.SerializeObject(this, new JsonSerializerSettings + { + NullValueHandling = NullValueHandling.Ignore, + Formatting = Formatting.None + }); + } } public class RequestRateLimitsEnforcementPolicy : IEquatable @@ -286,6 +354,15 @@ public override int GetHashCode() { return HashCode.Combine(QueriesEnforcementLevel, CommandsEnforcementLevel); } + + public override string ToString() + { + return JsonConvert.SerializeObject(this, new JsonSerializerSettings + { + NullValueHandling = NullValueHandling.Ignore, + Formatting = Formatting.None + }); + } } public class QueryConsistencyPolicy : IEquatable @@ -310,5 +387,14 @@ public override int GetHashCode() { return HashCode.Combine(QueryConsistency, CachedResultsMaxAge); } + + public override string ToString() + { + return JsonConvert.SerializeObject(this, new JsonSerializerSettings + { + NullValueHandling = NullValueHandling.Ignore, + Formatting = Formatting.None + }); + } } } \ No newline at end of file From 8a2bcb123c2f2bec27f05cb18c06da06512a56e5 Mon Sep 17 00:00:00 2001 From: Ashley Van Spankeren <25673124+ashleyvansp@users.noreply.github.com> Date: Sun, 6 Jul 2025 19:01:39 +0000 Subject: [PATCH 18/23] Add tests for detecting workload group changes --- .../Changes/ClusterChangesTest.cs | 368 ++++++++++++++++++ KustoSchemaTools/Changes/ClusterChanges.cs | 5 +- 2 files changed, 371 insertions(+), 2 deletions(-) diff --git a/KustoSchemaTools.Tests/Changes/ClusterChangesTest.cs b/KustoSchemaTools.Tests/Changes/ClusterChangesTest.cs index 991f8d1..253982a 100644 --- a/KustoSchemaTools.Tests/Changes/ClusterChangesTest.cs +++ b/KustoSchemaTools.Tests/Changes/ClusterChangesTest.cs @@ -89,6 +89,315 @@ public void GenerateChanges_WithNullNewCapacityPolicy_ShouldNotGenerateChanges() Assert.Empty(changeSet.Scripts); } + #region Workload Group Tests + + [Fact] + public void GenerateChanges_WithNewWorkloadGroup_ShouldDetectCreation() + { + // Arrange + var oldCluster = new Cluster { Name = "TestCluster", WorkloadGroups = new List() }; + var newCluster = new Cluster + { + Name = "TestCluster", + WorkloadGroups = new List + { + CreateWorkloadGroup("test-group", maxMemoryPerQueryPerNode: 1024) + } + }; + + // Act + var changeSet = ClusterChanges.GenerateChanges(oldCluster, newCluster, _loggerMock.Object); + + // Assert + Assert.NotNull(changeSet); + Assert.NotEmpty(changeSet.Changes); + Assert.NotEmpty(changeSet.Scripts); + + var policyChange = Assert.Single(changeSet.Changes) as PolicyChange; + Assert.NotNull(policyChange); + Assert.Equal("test-group", policyChange.Entity); + + var scriptContainer = Assert.Single(changeSet.Scripts); + Assert.Contains(".create-or-alter workload_group test-group", scriptContainer.Script.Text); + } + + [Fact] + public void GenerateChanges_WithUpdatedWorkloadGroup_ShouldDetectUpdate() + { + // Arrange + var oldCluster = new Cluster + { + Name = "TestCluster", + WorkloadGroups = new List + { + CreateWorkloadGroup("test-group", maxMemoryPerQueryPerNode: 1024) + } + }; + var newCluster = new Cluster + { + Name = "TestCluster", + WorkloadGroups = new List + { + CreateWorkloadGroup("test-group", maxMemoryPerQueryPerNode: 2048) + } + }; + + // Act + var changeSet = ClusterChanges.GenerateChanges(oldCluster, newCluster, _loggerMock.Object); + + // Assert + Assert.NotNull(changeSet); + Assert.NotEmpty(changeSet.Changes); + Assert.NotEmpty(changeSet.Scripts); + + var policyChange = Assert.Single(changeSet.Changes) as PolicyChange; + Assert.NotNull(policyChange); + Assert.Equal("test-group", policyChange.Entity); + + var scriptContainer = Assert.Single(changeSet.Scripts); + Assert.Contains(".alter-merge workload_group test-group", scriptContainer.Script.Text); + } + + [Fact] + public void GenerateChanges_WithIdenticalWorkloadGroups_ShouldDetectNoChanges() + { + // Arrange + var workloadGroup = CreateWorkloadGroup("test-group", maxMemoryPerQueryPerNode: 1024); + var oldCluster = new Cluster + { + Name = "TestCluster", + WorkloadGroups = new List { workloadGroup } + }; + var newCluster = new Cluster + { + Name = "TestCluster", + WorkloadGroups = new List + { + CreateWorkloadGroup("test-group", maxMemoryPerQueryPerNode: 1024) + } + }; + + // Act + var changeSet = ClusterChanges.GenerateChanges(oldCluster, newCluster, _loggerMock.Object); + + // Assert + Assert.NotNull(changeSet); + Assert.Empty(changeSet.Changes); + Assert.Empty(changeSet.Scripts); + } + + [Fact] + public void GenerateChanges_WithWorkloadGroupDeletion_ShouldDetectDeletion() + { + // Arrange + var oldCluster = new Cluster + { + Name = "TestCluster", + WorkloadGroups = new List + { + CreateWorkloadGroup("test-group", maxMemoryPerQueryPerNode: 1024) + } + }; + var newCluster = new Cluster + { + Name = "TestCluster", + WorkloadGroups = new List(), + Deletions = new ClusterDeletions + { + WorkloadGroups = new List { "test-group" } + } + }; + + // Act + var changeSet = ClusterChanges.GenerateChanges(oldCluster, newCluster, _loggerMock.Object); + + // Assert + Assert.NotNull(changeSet); + Assert.NotEmpty(changeSet.Changes); + + var deletionChange = Assert.Single(changeSet.Changes) as DeletionChange; + Assert.NotNull(deletionChange); + Assert.Equal("test-group", deletionChange.Entity); + Assert.Equal("workload_group", deletionChange.EntityType); + Assert.Contains("Drop test-group", deletionChange.Markdown); + } + + [Fact] + public void GenerateChanges_WithWorkloadGroupDeletionOfNonExistentGroup_ShouldNotCreateChange() + { + // Arrange + var oldCluster = new Cluster { Name = "TestCluster", WorkloadGroups = new List() }; + var newCluster = new Cluster + { + Name = "TestCluster", + WorkloadGroups = new List(), + Deletions = new ClusterDeletions + { + WorkloadGroups = new List { "non-existent-group" } + } + }; + + // Act + var changeSet = ClusterChanges.GenerateChanges(oldCluster, newCluster, _loggerMock.Object); + + // Assert + Assert.NotNull(changeSet); + Assert.Empty(changeSet.Changes); + Assert.Empty(changeSet.Scripts); + } + + [Fact] + public void GenerateChanges_WithWorkloadGroupMarkedForDeletionButAlsoInNewList_ShouldOnlyProcessDeletion() + { + // Arrange + var oldCluster = new Cluster + { + Name = "TestCluster", + WorkloadGroups = new List + { + CreateWorkloadGroup("test-group", maxMemoryPerQueryPerNode: 1024) + } + }; + var newCluster = new Cluster + { + Name = "TestCluster", + WorkloadGroups = new List + { + CreateWorkloadGroup("test-group", maxMemoryPerQueryPerNode: 2048) + }, + Deletions = new ClusterDeletions + { + WorkloadGroups = new List { "test-group" } + } + }; + + // Act + var changeSet = ClusterChanges.GenerateChanges(oldCluster, newCluster, _loggerMock.Object); + + // Assert + Assert.NotNull(changeSet); + Assert.NotEmpty(changeSet.Changes); + + var deletionChange = Assert.Single(changeSet.Changes) as DeletionChange; + Assert.NotNull(deletionChange); + Assert.Equal("test-group", deletionChange.Entity); + Assert.Equal("workload_group", deletionChange.EntityType); + } + + [Fact] + public void GenerateChanges_WithMultipleWorkloadGroupChanges_ShouldDetectAllChanges() + { + // Arrange + var oldCluster = new Cluster + { + Name = "TestCluster", + WorkloadGroups = new List + { + CreateWorkloadGroup("group1", maxMemoryPerQueryPerNode: 1024), + CreateWorkloadGroup("group2", maxMemoryPerQueryPerNode: 2048), + CreateWorkloadGroup("group3", maxMemoryPerQueryPerNode: 512) + } + }; + var newCluster = new Cluster + { + Name = "TestCluster", + WorkloadGroups = new List + { + CreateWorkloadGroup("group1", maxMemoryPerQueryPerNode: 1024), // No change + CreateWorkloadGroup("group2", maxMemoryPerQueryPerNode: 4096), // Update + CreateWorkloadGroup("group4", maxMemoryPerQueryPerNode: 256) // New + }, + Deletions = new ClusterDeletions + { + WorkloadGroups = new List { "group3" } // Delete + } + }; + + // Act + var changeSet = ClusterChanges.GenerateChanges(oldCluster, newCluster, _loggerMock.Object); + + // Assert + Assert.NotNull(changeSet); + Assert.Equal(3, changeSet.Changes.Count); // 1 deletion + 1 update + 1 creation + + // Check deletion + var deletionChange = changeSet.Changes.OfType().Single(); + Assert.Equal("group3", deletionChange.Entity); + + // Check updates/creations + var policyChanges = changeSet.Changes.OfType>().ToList(); + Assert.Equal(2, policyChanges.Count); + + var group2Change = policyChanges.First(c => c.Entity == "group2"); + var group4Change = policyChanges.First(c => c.Entity == "group4"); + + Assert.NotNull(group2Change); + Assert.NotNull(group4Change); + + // Verify scripts + Assert.Equal(3, changeSet.Scripts.Count); + } + + [Fact] + public void GenerateChanges_WithWorkloadGroupHavingComplexPolicy_ShouldDetectChanges() + { + // Arrange + var oldCluster = new Cluster + { + Name = "TestCluster", + WorkloadGroups = new List + { + CreateComplexWorkloadGroup("complex-group", 1024, TimeSpan.FromMinutes(5)) + } + }; + var newCluster = new Cluster + { + Name = "TestCluster", + WorkloadGroups = new List + { + CreateComplexWorkloadGroup("complex-group", 2048, TimeSpan.FromMinutes(10)) + } + }; + + // Act + var changeSet = ClusterChanges.GenerateChanges(oldCluster, newCluster, _loggerMock.Object); + + // Assert + Assert.NotNull(changeSet); + Assert.NotEmpty(changeSet.Changes); + + var policyChange = Assert.Single(changeSet.Changes) as PolicyChange; + Assert.NotNull(policyChange); + Assert.Equal("complex-group", policyChange.Entity); + Assert.Contains("MaxMemoryPerQueryPerNode", policyChange.Markdown); + Assert.Contains("MaxExecutionTime", policyChange.Markdown); + } + + [Fact] + public void GenerateChanges_WithWorkloadGroupHavingNullPolicy_ShouldNotCreateChange() + { + // Arrange + var oldCluster = new Cluster { Name = "TestCluster", WorkloadGroups = new List() }; + var newCluster = new Cluster + { + Name = "TestCluster", + WorkloadGroups = new List + { + new WorkloadGroup { WorkloadGroupName = "null-policy-group", WorkloadGroupPolicy = null } + } + }; + + // Act + var changeSet = ClusterChanges.GenerateChanges(oldCluster, newCluster, _loggerMock.Object); + + // Assert + Assert.NotNull(changeSet); + Assert.Empty(changeSet.Changes); + Assert.Empty(changeSet.Scripts); + } + + #endregion + #region Helper Methods private Cluster CreateClusterWithPolicy( double? ingestionCapacityCoreUtilizationCoefficient = null, @@ -112,6 +421,65 @@ private Cluster CreateClusterWithPolicy( } }; } + + private WorkloadGroup CreateWorkloadGroup(string name, long? maxMemoryPerQueryPerNode = null) + { + return new WorkloadGroup + { + WorkloadGroupName = name, + WorkloadGroupPolicy = new WorkloadGroupPolicy + { + RequestLimitsPolicy = new RequestLimitsPolicy + { + MaxMemoryPerQueryPerNode = maxMemoryPerQueryPerNode.HasValue + ? new PolicyValue { Value = maxMemoryPerQueryPerNode.Value, IsRelaxable = false } + : null + } + } + }; + } + + private WorkloadGroup CreateComplexWorkloadGroup(string name, long maxMemoryPerQueryPerNode, TimeSpan maxExecutionTime) + { + return new WorkloadGroup + { + WorkloadGroupName = name, + WorkloadGroupPolicy = new WorkloadGroupPolicy + { + RequestLimitsPolicy = new RequestLimitsPolicy + { + MaxMemoryPerQueryPerNode = new PolicyValue { Value = maxMemoryPerQueryPerNode, IsRelaxable = false }, + MaxExecutionTime = new PolicyValue { Value = maxExecutionTime, IsRelaxable = true }, + MaxResultRecords = new PolicyValue { Value = 10000, IsRelaxable = false } + }, + RequestRateLimitPolicies = new List + { + new RequestRateLimitPolicy + { + IsEnabled = true, + Scope = RateLimitScope.WorkloadGroup, + LimitKind = RateLimitKind.ConcurrentRequests, + Properties = new RateLimitProperties + { + MaxConcurrentRequests = 100 + } + }, + new RequestRateLimitPolicy + { + IsEnabled = true, + Scope = RateLimitScope.Principal, + LimitKind = RateLimitKind.ResourceUtilization, + Properties = new RateLimitProperties + { + ResourceKind = RateLimitResourceKind.TotalCpuSeconds, + MaxUtilization = 0.8, + TimeWindow = TimeSpan.FromMinutes(5) + } + } + } + } + }; + } #endregion } } \ No newline at end of file diff --git a/KustoSchemaTools/Changes/ClusterChanges.cs b/KustoSchemaTools/Changes/ClusterChanges.cs index 8aa755d..7bbeb9e 100644 --- a/KustoSchemaTools/Changes/ClusterChanges.cs +++ b/KustoSchemaTools/Changes/ClusterChanges.cs @@ -178,12 +178,13 @@ private static void HandleWorkloadGroupChanges(Cluster oldCluster, Cluster newCl } var existingWorkloadGroup = oldCluster.WorkloadGroups.FirstOrDefault(wg => wg.WorkloadGroupName == newWorkloadGroup.WorkloadGroupName); - var scriptType = existingWorkloadGroup == null ? "ClusterWorkloadGroupCreateOrAlterCommand" : "ClusterWorkloadGroupAlterMergeCommand"; - var scriptText = existingWorkloadGroup == null ? newWorkloadGroup.ToCreateScript() : newWorkloadGroup.ToUpdateScript(); // Only create a change if the workload group policy is not null if (newWorkloadGroup.WorkloadGroupPolicy != null) { + var scriptType = existingWorkloadGroup == null ? "ClusterWorkloadGroupCreateOrAlterCommand" : "ClusterWorkloadGroupAlterMergeCommand"; + var scriptText = existingWorkloadGroup == null ? newWorkloadGroup.ToCreateScript() : newWorkloadGroup.ToUpdateScript(); + var workloadGroupChange = ComparePolicy( "Workload Group", newWorkloadGroup.WorkloadGroupName, From ea09f2b1783cf3d97b1af9f35116e12fa3718f73 Mon Sep 17 00:00:00 2001 From: Ashley Van Spankeren <25673124+ashleyvansp@users.noreply.github.com> Date: Sun, 6 Jul 2025 19:10:00 +0000 Subject: [PATCH 19/23] Add warning to consult support before modifying cluster config --- KustoSchemaTools/Changes/ClusterChanges.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/KustoSchemaTools/Changes/ClusterChanges.cs b/KustoSchemaTools/Changes/ClusterChanges.cs index 7bbeb9e..1bf2bb4 100644 --- a/KustoSchemaTools/Changes/ClusterChanges.cs +++ b/KustoSchemaTools/Changes/ClusterChanges.cs @@ -225,6 +225,9 @@ private static string GenerateClusterMarkdown(ClusterChangeSet changeSet) } else { + sb.AppendLine(":warning: **Warning**: It is strongly recommended to consult with Microsoft Support before making cluster configuration changes."); + sb.AppendLine(); + foreach (var change in changeSet.Changes) { if (!string.IsNullOrEmpty(change.Markdown)) From 8d42dd7bcb9d19afe333be000e073dd11080e757 Mon Sep 17 00:00:00 2001 From: Ashley Van Spankeren <25673124+ashleyvansp@users.noreply.github.com> Date: Sun, 6 Jul 2025 19:27:54 +0000 Subject: [PATCH 20/23] Update README with workload group management --- README.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b98f142..c2a4138 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ The `KustoSchemaHandler` is the central place for synching schemas between yaml ### Cluster configuration management -Cluster configuration changes are handled by the `KustoClusterOrchestrator`. Currently, the only supported feature is [`Capacity Policies`](https://learn.microsoft.com/en-us/kusto/management/capacity-policy?view=azure-data-explorer). The orchestrator expects a file path to a configuration file. A key design principle is that you only need to specify the properties you wish to set or change. Any property omitted in your policy file will be ignored, preserving its current value on the cluster. +Cluster configuration changes are handled by the `KustoClusterOrchestrator`. Currently supported features include [`Capacity Policies`](https://learn.microsoft.com/en-us/kusto/management/capacity-policy?view=azure-data-explorer) and [`Workload Groups`](https://learn.microsoft.com/en-us/kusto/management/workload-groups?view=azure-data-explorer). The orchestrator expects a file path to a configuration file. A key design principle is that you only need to specify the properties you wish to set or change. Any property omitted in your policy file will be ignored, preserving its current value on the cluster. A sample file could look like this: ```yaml @@ -54,14 +54,19 @@ connections: maximumConcurrentOperationsPerNode: 3 extentsPurgeRebuildCapacity: maximumConcurrentOperationsPerNode: 1 + workloadGroups: + - workloadGroupName: DataScience + workloadGroupPolicy: + requestRateLimitsEnforcementPolicy: + commandsEnforcementLevel: Cluster ``` The `KustoClusterOrchestrator` coordinates between cluster handlers to manage cluster configuration changes: 1. **Loading Configuration**: Uses `YamlClusterHandler` to parse the YAML configuration file and load the desired cluster state -2. **Reading Current State**: Uses `KustoClusterHandler` to connect to each live cluster and retrieve the current capacity policy settings +2. **Reading Current State**: Uses `KustoClusterHandler` to connect to each live cluster and retrieve the current capacity policy and workload group settings 3. **Generating Changes**: Compares the desired state (from YAML) with the current state (from Kusto) to identify differences -4. **Creating Scripts**: Generates the necessary Kusto control commands (like `.alter-merge cluster policy capacity`) to apply the changes +4. **Creating Scripts**: Generates the necessary Kusto control commands (like `.alter-merge cluster policy capacity` and `.create-or-alter workload_group`) to apply the changes 5. **Applying Updates**: Executes the generated scripts against the live clusters to synchronize them with the desired configuration Currently no plugins are supported. The orchestrator expects all cluster configuration in a central file. @@ -70,6 +75,9 @@ Currently no plugins are supported. The orchestrator expects all cluster configu Currently following features are supported: +* Cluster + * Capacity Policies + * Workload Groups * Database * Permissions * Default Retention From 0b44245a71ef447d4116e34ba6ef8bd60dc028a0 Mon Sep 17 00:00:00 2001 From: Ashley Van Spankeren <25673124+ashleyvansp@users.noreply.github.com> Date: Wed, 16 Jul 2025 11:03:04 -0700 Subject: [PATCH 21/23] Update KustoSchemaTools.Tests/Parser/KustoClusterHandlerTests.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- KustoSchemaTools.Tests/Parser/KustoClusterHandlerTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KustoSchemaTools.Tests/Parser/KustoClusterHandlerTests.cs b/KustoSchemaTools.Tests/Parser/KustoClusterHandlerTests.cs index 0f52905..b82c7a4 100644 --- a/KustoSchemaTools.Tests/Parser/KustoClusterHandlerTests.cs +++ b/KustoSchemaTools.Tests/Parser/KustoClusterHandlerTests.cs @@ -383,7 +383,7 @@ public async Task LoadAsync_WithNullPolicyJson_ReturnsClusterWithoutPolicy() mockCapacityReader.SetupSequence(x => x.Read()) .Returns(true) // First call returns true (data available) .Returns(false); // Second call returns false (no more data) - mockCapacityReader.Setup(x => x["Policy"]).Returns(null as object); // Null policy + mockCapacityReader.Setup(x => x["Policy"]).Returns(null); // Null policy var mockWorkloadGroupsReader = new Mock(); mockWorkloadGroupsReader.Setup(x => x.Read()).Returns(false); // No workload groups From 90b7f7e40c4dd5664be2c92f894f9e00cf9c57ed Mon Sep 17 00:00:00 2001 From: Ashley Van Spankeren <25673124+ashleyvansp@users.noreply.github.com> Date: Wed, 16 Jul 2025 12:25:42 -0700 Subject: [PATCH 22/23] Update KustoSchemaTools/Parser/KustoClusterHandler.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- KustoSchemaTools/Parser/KustoClusterHandler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KustoSchemaTools/Parser/KustoClusterHandler.cs b/KustoSchemaTools/Parser/KustoClusterHandler.cs index 9355421..9d2c76b 100644 --- a/KustoSchemaTools/Parser/KustoClusterHandler.cs +++ b/KustoSchemaTools/Parser/KustoClusterHandler.cs @@ -55,7 +55,7 @@ public virtual async Task LoadAsync() var workloadGroupPolicy = JsonConvert.DeserializeObject(workloadGroupJson); var workloadGroup = new WorkloadGroup { - WorkloadGroupName = workloadGroupName!, + WorkloadGroupName = !string.IsNullOrEmpty(workloadGroupName) ? workloadGroupName : throw new InvalidOperationException("WorkloadGroupName cannot be null or empty."), WorkloadGroupPolicy = workloadGroupPolicy }; cluster.WorkloadGroups.Add(workloadGroup); From 2eeb5c353e074117405fbdce8e0c7afe3e4f0015 Mon Sep 17 00:00:00 2001 From: Ashley Van Spankeren <25673124+ashleyvansp@users.noreply.github.com> Date: Wed, 16 Jul 2025 23:00:49 +0000 Subject: [PATCH 23/23] Add validations for reading cluster config from YAML --- .../Parser/YamlClusterHandlerTests.cs | 227 ++++++++++++++++++ KustoSchemaTools/Parser/YamlClusterHandler.cs | 40 +++ 2 files changed, 267 insertions(+) diff --git a/KustoSchemaTools.Tests/Parser/YamlClusterHandlerTests.cs b/KustoSchemaTools.Tests/Parser/YamlClusterHandlerTests.cs index 307cb41..1fa2d6c 100644 --- a/KustoSchemaTools.Tests/Parser/YamlClusterHandlerTests.cs +++ b/KustoSchemaTools.Tests/Parser/YamlClusterHandlerTests.cs @@ -116,5 +116,232 @@ public async Task LoadAsync_InvalidClustersProperties_ThrowsInvalidOperationExce File.Delete(tempFilePath); } } + + [Fact] + public async Task LoadAsync_ClusterMissingName_ThrowsInvalidOperationException() + { + // Arrange + var tempFilePath = Path.GetTempFileName(); + try + { + var yamlContent = @" +connections: +- url: test.eastus + workloadGroups: [] +"; + await File.WriteAllTextAsync(tempFilePath, yamlContent); + var handler = new YamlClusterHandler(tempFilePath); + + // Act & Assert + var exception = await Assert.ThrowsAsync(() => handler.LoadAsync()); + Assert.Contains("Cluster at index 0 is missing a required 'name' property", exception.Message); + } + finally + { + if (File.Exists(tempFilePath)) + File.Delete(tempFilePath); + } + } + + [Fact] + public async Task LoadAsync_ClusterMissingUrl_ThrowsInvalidOperationException() + { + // Arrange + var tempFilePath = Path.GetTempFileName(); + try + { + var yamlContent = @" +connections: +- name: testcluster + workloadGroups: [] +"; + await File.WriteAllTextAsync(tempFilePath, yamlContent); + var handler = new YamlClusterHandler(tempFilePath); + + // Act & Assert + var exception = await Assert.ThrowsAsync(() => handler.LoadAsync()); + Assert.Contains("Cluster 'testcluster' is missing a required 'url' property", exception.Message); + } + finally + { + if (File.Exists(tempFilePath)) + File.Delete(tempFilePath); + } + } + + [Fact] + public async Task LoadAsync_WorkloadGroupMissingName_ThrowsInvalidOperationException() + { + // Arrange + var tempFilePath = Path.GetTempFileName(); + try + { + var yamlContent = @" +connections: +- name: testcluster + url: testcluster.eastus + workloadGroups: + - workloadGroupPolicy: + requestRateLimitsEnforcementPolicy: + commandsEnforcementLevel: Cluster + - workloadGroupName: validgroup + workloadGroupPolicy: + requestRateLimitPolicies: [] +"; + await File.WriteAllTextAsync(tempFilePath, yamlContent); + var handler = new YamlClusterHandler(tempFilePath); + + // Act & Assert + var exception = await Assert.ThrowsAsync(() => handler.LoadAsync()); + Assert.Contains("Cluster 'testcluster' has a workload group at index 0 that is missing a required 'workloadGroupName' property", exception.Message); + Assert.Contains("All workload groups must have a non-empty name", exception.Message); + } + finally + { + if (File.Exists(tempFilePath)) + File.Delete(tempFilePath); + } + } + + [Fact] + public async Task LoadAsync_WorkloadGroupEmptyName_ThrowsInvalidOperationException() + { + // Arrange + var tempFilePath = Path.GetTempFileName(); + try + { + var yamlContent = @" +connections: +- name: testcluster + url: testcluster.eastus + workloadGroups: + - workloadGroupName: '' + workloadGroupPolicy: + requestRateLimitPolicies: [] +"; + await File.WriteAllTextAsync(tempFilePath, yamlContent); + var handler = new YamlClusterHandler(tempFilePath); + + // Act & Assert + var exception = await Assert.ThrowsAsync(() => handler.LoadAsync()); + Assert.Contains("Cluster 'testcluster' has a workload group at index 0 that is missing a required 'workloadGroupName' property", exception.Message); + } + finally + { + if (File.Exists(tempFilePath)) + File.Delete(tempFilePath); + } + } + + [Fact] + public async Task LoadAsync_WorkloadGroupWhitespaceOnlyName_ThrowsInvalidOperationException() + { + // Arrange + var tempFilePath = Path.GetTempFileName(); + try + { + var yamlContent = @" +connections: +- name: testcluster + url: testcluster.eastus + workloadGroups: + - workloadGroupName: ' ' + workloadGroupPolicy: + requestRateLimitPolicies: [] +"; + await File.WriteAllTextAsync(tempFilePath, yamlContent); + var handler = new YamlClusterHandler(tempFilePath); + + // Act & Assert + var exception = await Assert.ThrowsAsync(() => handler.LoadAsync()); + Assert.Contains("Cluster 'testcluster' has a workload group at index 0 that is missing a required 'workloadGroupName' property", exception.Message); + } + finally + { + if (File.Exists(tempFilePath)) + File.Delete(tempFilePath); + } + } + + [Fact] + public async Task LoadAsync_ValidWorkloadGroups_DoesNotThrow() + { + // Arrange + var tempFilePath = Path.GetTempFileName(); + try + { + var yamlContent = @" +connections: +- name: testcluster + url: testcluster.eastus + workloadGroups: + - workloadGroupName: group1 + workloadGroupPolicy: + requestRateLimitPolicies: + - limitKind: ConcurrentRequests + scope: WorkloadGroup + isEnabled: true + properties: + maxConcurrentRequests: 100 + - workloadGroupName: group2 + workloadGroupPolicy: + requestRateLimitsEnforcementPolicy: + commandsEnforcementLevel: Cluster +"; + await File.WriteAllTextAsync(tempFilePath, yamlContent); + var handler = new YamlClusterHandler(tempFilePath); + + // Act + var result = await handler.LoadAsync(); + + // Assert + Assert.NotNull(result); + Assert.Single(result); + var cluster = result[0]; + Assert.Equal("testcluster", cluster.Name); + Assert.Equal("testcluster.eastus", cluster.Url); + Assert.Equal(2, cluster.WorkloadGroups.Count); + Assert.Equal("group1", cluster.WorkloadGroups[0].WorkloadGroupName); + Assert.Equal("group2", cluster.WorkloadGroups[1].WorkloadGroupName); + } + finally + { + if (File.Exists(tempFilePath)) + File.Delete(tempFilePath); + } + } + + [Fact] + public async Task LoadAsync_ClusterWithoutWorkloadGroups_DoesNotThrow() + { + // Arrange + var tempFilePath = Path.GetTempFileName(); + try + { + var yamlContent = @" +connections: +- name: testcluster + url: testcluster.eastus +"; + await File.WriteAllTextAsync(tempFilePath, yamlContent); + var handler = new YamlClusterHandler(tempFilePath); + + // Act + var result = await handler.LoadAsync(); + + // Assert + Assert.NotNull(result); + Assert.Single(result); + var cluster = result[0]; + Assert.Equal("testcluster", cluster.Name); + Assert.Equal("testcluster.eastus", cluster.Url); + Assert.Empty(cluster.WorkloadGroups); + } + finally + { + if (File.Exists(tempFilePath)) + File.Delete(tempFilePath); + } + } } } diff --git a/KustoSchemaTools/Parser/YamlClusterHandler.cs b/KustoSchemaTools/Parser/YamlClusterHandler.cs index a069249..8d0bcda 100644 --- a/KustoSchemaTools/Parser/YamlClusterHandler.cs +++ b/KustoSchemaTools/Parser/YamlClusterHandler.cs @@ -30,6 +30,8 @@ public async Task> LoadAsync() var clusters = Serialization.YamlPascalCaseDeserializer.Deserialize(clustersFileContent); + ValidateClusters(clusters); + return clusters.Connections.ToList(); } catch (Exception ex) when (!(ex is FileNotFoundException || ex is InvalidOperationException)) @@ -37,5 +39,43 @@ public async Task> LoadAsync() throw new InvalidOperationException($"Failed to parse clusters file '{_filePath}': {ex.Message}", ex); } } + + private static void ValidateClusters(Clusters clusters) + { + if (clusters?.Connections == null) + return; + + for (int clusterIndex = 0; clusterIndex < clusters.Connections.Count; clusterIndex++) + { + var cluster = clusters.Connections[clusterIndex]; + + // Validate cluster basic properties + if (string.IsNullOrWhiteSpace(cluster.Name)) + { + throw new InvalidOperationException($"Cluster at index {clusterIndex} is missing a required 'name' property."); + } + + if (string.IsNullOrWhiteSpace(cluster.Url)) + { + throw new InvalidOperationException($"Cluster '{cluster.Name}' is missing a required 'url' property."); + } + + // Validate workload groups + if (cluster.WorkloadGroups?.Count > 0) + { + for (int wgIndex = 0; wgIndex < cluster.WorkloadGroups.Count; wgIndex++) + { + var workloadGroup = cluster.WorkloadGroups[wgIndex]; + + if (string.IsNullOrWhiteSpace(workloadGroup.WorkloadGroupName)) + { + throw new InvalidOperationException( + $"Cluster '{cluster.Name}' has a workload group at index {wgIndex} that is missing a required 'workloadGroupName' property. " + + "All workload groups must have a non-empty name."); + } + } + } + } + } } } \ No newline at end of file