Skip to content

Implement apply() for cluster configuration changes #110

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ashleyvansp merged 9 commits into from
Jul 4, 2025
Merged

Implement apply() for cluster configuration changes #110

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
8f3a437
Fix YamlClusterHandlerFactory
ashleyvansp Jul 2, 2025
8e40779
Add Apply method to KustoClusterOrchestrator
ashleyvansp Jul 2, 2025
6f472e4
Use `.execute cluster script` for running updates on the cluster
ashleyvansp Jul 3, 2025
98cedea
Parity between ClusterCapacityModel and the model returned by Kusto
ashleyvansp Jul 3, 2025
ad721ad
Refactor KustoClusterHandler for better testability
ashleyvansp Jul 3, 2025
15676a9
too much filtering
ashleyvansp Jul 3, 2025
74ba8f4
better handling of change sets with 0 changes
ashleyvansp Jul 3, 2025
b6dd611
Fix init
ashleyvansp Jul 3, 2025
44f5f58
Update KustoSchemaTools/Parser/KustoClusterHandler.cs
ashleyvansp Jul 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 6 additions & 11 deletions KustoSchemaTools.Tests/Changes/ClusterChangesTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ public ClusterChangesTests()
public void GenerateChanges_WithIdenticalPolicies_ShouldDetectNoChanges()
{
// Arrange
var oldCluster = CreateClusterWithPolicy(0.2, 1, 2, 3);
var newCluster = CreateClusterWithPolicy(0.2, 1, 2, 3);
var oldCluster = CreateClusterWithPolicy(0.2, 1, 2);
var newCluster = CreateClusterWithPolicy(0.2, 1, 2);

// Act
var changeSet = ClusterChanges.GenerateChanges(oldCluster, newCluster, _loggerMock.Object);
Expand All @@ -32,8 +32,8 @@ public void GenerateChanges_WithIdenticalPolicies_ShouldDetectNoChanges()
public void GenerateChanges_WithSingleChange_ShouldDetectChangeAndCreateScript()
{
// Arrange
var oldCluster = CreateClusterWithPolicy(0.2, 1, 2, 3);
var newCluster = CreateClusterWithPolicy(0.2, 1, 2, 5);
var oldCluster = CreateClusterWithPolicy(0.2, 1, 2);
var newCluster = CreateClusterWithPolicy(0.2, 1, 5);

// Act
var changeSet = ClusterChanges.GenerateChanges(oldCluster, newCluster, _loggerMock.Object);
Expand Down Expand Up @@ -93,8 +93,7 @@ public void GenerateChanges_WithNullNewCapacityPolicy_ShouldNotGenerateChanges()
private Cluster CreateClusterWithPolicy(
double? ingestionCapacityCoreUtilizationCoefficient = null,
int? materializedViewsCapacityClusterMaximumConcurrentOperations = null,
int? extentsRebuildClusterMaximumConcurrentOperations = null,
int? extentsRebuildMaximumConcurrentOperationsPerNode = null
int? materializedViewsCapacityClusterMinimumConcurrentOperations = null
)
{
return new Cluster
Expand All @@ -104,11 +103,7 @@ private Cluster CreateClusterWithPolicy(
MaterializedViewsCapacity = new MaterializedViewsCapacity
{
ClusterMaximumConcurrentOperations = materializedViewsCapacityClusterMaximumConcurrentOperations,
ExtentsRebuildCapacity = (extentsRebuildClusterMaximumConcurrentOperations != null || extentsRebuildMaximumConcurrentOperationsPerNode != null) ? new ExtentsRebuildCapacity
{
ClusterMaximumConcurrentOperations = extentsRebuildClusterMaximumConcurrentOperations,
MaximumConcurrentOperationsPerNode = extentsRebuildMaximumConcurrentOperationsPerNode
} : null
ClusterMinimumConcurrentOperations = materializedViewsCapacityClusterMinimumConcurrentOperations
},
IngestionCapacity = new IngestionCapacity
{
Expand Down
30 changes: 9 additions & 21 deletions KustoSchemaTools.Tests/KustoClusterOrchestratorTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,6 @@
using KustoSchemaTools.Parser;
using Microsoft.Extensions.Logging;
using Moq;
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;
using Xunit;
using System.Data;
using System;
using System.Linq;
using Kusto.Data.Common;

namespace KustoSchemaTools.Tests
Expand All @@ -28,10 +21,9 @@ public KustoClusterOrchestratorTests()
kustoClusterHandlerFactoryMock = new Mock<IKustoClusterHandlerFactory>();
yamlClusterHandlerFactoryMock = new Mock<IYamlClusterHandlerFactory>();

// Create mock for KustoClusterHandler
var kustoClientMock = new Mock<KustoClient>("test.eastus");
var adminClientMock = new Mock<ICslAdminProvider>();
var kustoLoggerMock = new Mock<ILogger<KustoClusterHandler>>();
kustoHandlerMock = new Mock<KustoClusterHandler>(kustoClientMock.Object, kustoLoggerMock.Object, "test", "test.eastus");
kustoHandlerMock = new Mock<KustoClusterHandler>(adminClientMock.Object, kustoLoggerMock.Object, "test", "test.eastus");

orchestrator = new KustoClusterOrchestrator(
loggerMock.Object,
Expand All @@ -57,12 +49,10 @@ private Clusters CreateClustersWithCapacityPolicy(ClusterCapacityPolicy? capacit

private void SetupMockHandler(Cluster kustoCluster)
{
// Configure the handler factory to return our mock handler
kustoClusterHandlerFactoryMock
.Setup(f => f.Create("test", "test.eastus"))
.Returns(kustoHandlerMock.Object);

// Set up the mock handler to return our test cluster
kustoHandlerMock
.Setup(h => h.LoadAsync())
.ReturnsAsync(kustoCluster);
Expand Down Expand Up @@ -106,8 +96,7 @@ private Clusters CreateMultipleClusters()

private void SetupMultipleClusterMocks()
{
// Mock for cluster1
var kustoHandler1Mock = new Mock<KustoClusterHandler>(new Mock<KustoClient>("cluster1.eastus").Object, new Mock<ILogger<KustoClusterHandler>>().Object, "cluster1", "cluster1.eastus");
var kustoHandler1Mock = new Mock<KustoClusterHandler>(new Mock<ICslAdminProvider>().Object, new Mock<ILogger<KustoClusterHandler>>().Object, "cluster1", "cluster1.eastus");
var kustoCluster1 = new Cluster
{
Name = "cluster1",
Expand All @@ -128,8 +117,7 @@ private void SetupMultipleClusterMocks()
.Setup(h => h.LoadAsync())
.ReturnsAsync(kustoCluster1);

// Mock for cluster2 - same as config, no changes
var kustoHandler2Mock = new Mock<KustoClusterHandler>(new Mock<KustoClient>("cluster2.westus").Object, new Mock<ILogger<KustoClusterHandler>>().Object, "cluster2", "cluster2.westus");
var kustoHandler2Mock = new Mock<KustoClusterHandler>(new Mock<ICslAdminProvider>().Object, new Mock<ILogger<KustoClusterHandler>>().Object, "cluster2", "cluster2.westus");
var kustoCluster2 = new Cluster
{
Name = "cluster2",
Expand Down Expand Up @@ -404,7 +392,7 @@ public async Task GenerateChangesFromFileAsync_ValidYamlFile_ReturnsChanges()
.Returns(new YamlClusterHandler(yamlFilePath));

// Set up mocks for the clusters defined in the YAML file
var kustoHandler1Mock = new Mock<KustoClusterHandler>(new Mock<KustoClient>("test1.eastus").Object, new Mock<ILogger<KustoClusterHandler>>().Object, "test1", "test1.eastus");
var kustoHandler1Mock = new Mock<KustoClusterHandler>(new Mock<ICslAdminProvider>().Object, new Mock<ILogger<KustoClusterHandler>>().Object, "test1", "test1.eastus");
var kustoCluster1 = new Cluster
{
Name = "test1",
Expand All @@ -425,7 +413,7 @@ public async Task GenerateChangesFromFileAsync_ValidYamlFile_ReturnsChanges()
.Setup(h => h.LoadAsync())
.ReturnsAsync(kustoCluster1);

var kustoHandler2Mock = new Mock<KustoClusterHandler>(new Mock<KustoClient>("test2.eastus").Object, new Mock<ILogger<KustoClusterHandler>>().Object, "test2", "test2.eastus");
var kustoHandler2Mock = new Mock<KustoClusterHandler>(new Mock<ICslAdminProvider>().Object, new Mock<ILogger<KustoClusterHandler>>().Object, "test2", "test2.eastus");
var kustoCluster2 = new Cluster
{
Name = "test2",
Expand Down Expand Up @@ -568,8 +556,8 @@ public async Task GenerateChangesFromFileAsync_VerifyLoggingCalled()
.Returns(new YamlClusterHandler(yamlFilePath));

// Set up a simple mock for the clusters
var kustoHandler1Mock = new Mock<KustoClusterHandler>(new Mock<KustoClient>("test1.eastus").Object, new Mock<ILogger<KustoClusterHandler>>().Object, "test1", "test1.eastus");
var kustoHandler2Mock = new Mock<KustoClusterHandler>(new Mock<KustoClient>("test2.eastus").Object, new Mock<ILogger<KustoClusterHandler>>().Object, "test2", "test2.eastus");
var kustoHandler1Mock = new Mock<KustoClusterHandler>(new Mock<ICslAdminProvider>().Object, new Mock<ILogger<KustoClusterHandler>>().Object, "test1", "test1.eastus");
var kustoHandler2Mock = new Mock<KustoClusterHandler>(new Mock<ICslAdminProvider>().Object, new Mock<ILogger<KustoClusterHandler>>().Object, "test2", "test2.eastus");

kustoClusterHandlerFactoryMock
.Setup(f => f.Create("test1", "test1.eastus"))
Expand Down Expand Up @@ -639,7 +627,7 @@ public async Task GenerateChangesFromFileAsync_LoadAsyncThrowsException_Propagat
.Setup(f => f.Create(yamlFilePath))
.Returns(new YamlClusterHandler(yamlFilePath));

var kustoHandler1Mock = new Mock<KustoClusterHandler>(new Mock<KustoClient>("test1.eastus").Object, new Mock<ILogger<KustoClusterHandler>>().Object, "test1", "test1.eastus");
var kustoHandler1Mock = new Mock<KustoClusterHandler>(new Mock<ICslAdminProvider>().Object, new Mock<ILogger<KustoClusterHandler>>().Object, "test1", "test1.eastus");

kustoClusterHandlerFactoryMock
.Setup(f => f.Create("test1", "test1.eastus"))
Expand Down
Loading
Loading