diff --git a/KustoSchemaTools.Tests/DemoData/DemoDeployment/DemoDatabase/functions/COMMENT_END.yml b/KustoSchemaTools.Tests/DemoData/DemoDeployment/DemoDatabase/functions/COMMENT_END.yml
new file mode 100644
index 0000000..489d424
--- /dev/null
+++ b/KustoSchemaTools.Tests/DemoData/DemoDeployment/DemoDatabase/functions/COMMENT_END.yml
@@ -0,0 +1,6 @@
+folder: test
+docString: Function with comment at end
+body: |-
+ sourceTable
+ | limit 100
+ | where IsNotEmpty(EventId) // this is a comment at the end
\ No newline at end of file
diff --git a/KustoSchemaTools.Tests/DemoData/DemoDeployment/DemoDatabase/functions/complicated.yml b/KustoSchemaTools.Tests/DemoData/DemoDeployment/DemoDatabase/functions/complicated.yml
new file mode 100644
index 0000000..51f70f5
--- /dev/null
+++ b/KustoSchemaTools.Tests/DemoData/DemoDeployment/DemoDatabase/functions/complicated.yml
@@ -0,0 +1,18 @@
+folder: test
+docString: issues for relevant services, filtered
+preformatted: false
+body: |-
+ sourceTable
+ | where t between(startofday(_startTime)..endofday(_endTime)) or classifier == "somevalue"
+ // comments
+ | where repository_id in (table_function(_aaaaaaaaa,_bbbbbbbb,_cccccccc,_eeeeeeeee,_fffffffff) | distinct id) // prefer `in` over `join` for short right columns
+ | project id
+ , type
+ , t
+ | summarize arg_max(t, *) by id
+ | lookup (table_function(_aaaaaaaaa,_bbbbbbbb,_cccccccc,_eeeeeeeee,_fffffffff) | distinct id, classifier) on id
+ | extend type = case(
+ id == 1, "a",
+ id == 2, "b",
+ "other") // comments
+ | project id, type, classifier
diff --git a/KustoSchemaTools.Tests/DemoData/DemoDeployment/DemoDatabase/functions/complicated_preformatted.yml b/KustoSchemaTools.Tests/DemoData/DemoDeployment/DemoDatabase/functions/complicated_preformatted.yml
new file mode 100644
index 0000000..9e3270d
--- /dev/null
+++ b/KustoSchemaTools.Tests/DemoData/DemoDeployment/DemoDatabase/functions/complicated_preformatted.yml
@@ -0,0 +1,18 @@
+folder: test
+docString: issues for relevant services, filtered
+preformatted: true
+body: |-
+ sourceTable
+ | where t between(startofday(_startTime)..endofday(_endTime)) or classifier == "somevalue"
+ // comments
+ | where repository_id in (table_function(_aaaaaaaaa,_bbbbbbbb,_cccccccc,_eeeeeeeee,_fffffffff) | distinct id) // prefer `in` over `join` for short right columns
+ | project id
+ , type
+ , t
+ | summarize arg_max(t, *) by id
+ | lookup (table_function(_aaaaaaaaa,_bbbbbbbb,_cccccccc,_eeeeeeeee,_fffffffff) | distinct id, classifier) on id
+ | extend type = case(
+ id == 1, "a",
+ id == 2, "b",
+ "other") // comments
+ | project id, type, classifier
diff --git a/KustoSchemaTools.Tests/DemoData/DemoDeployment/DemoDatabase/functions/needs_formatting.yml b/KustoSchemaTools.Tests/DemoData/DemoDeployment/DemoDatabase/functions/needs_formatting.yml
new file mode 100644
index 0000000..1f169d7
--- /dev/null
+++ b/KustoSchemaTools.Tests/DemoData/DemoDeployment/DemoDatabase/functions/needs_formatting.yml
@@ -0,0 +1,4 @@
+folder: test
+docString: test function that would change with formatting
+body: |-
+ sourceTable | limit 100
diff --git a/KustoSchemaTools.Tests/DemoData/DemoDeployment/DemoDatabase/materialized-views/mv.yml b/KustoSchemaTools.Tests/DemoData/DemoDeployment/DemoDatabase/materialized-views/mv.yml
new file mode 100644
index 0000000..b992b11
--- /dev/null
+++ b/KustoSchemaTools.Tests/DemoData/DemoDeployment/DemoDatabase/materialized-views/mv.yml
@@ -0,0 +1,12 @@
+source: sourceTable
+kind: table
+folder: test
+retentionAndCachePolicy:
+ retention: 720d
+query: |-
+ sourceTable
+ | where type == "a"
+ | summarize hint.strategy=shuffle active=countif(is_active != true),
+ archived=countif(is_archived)
+ by id
+ , day
diff --git a/KustoSchemaTools.Tests/DemoData/DemoDeployment/DemoDatabase/materialized-views/mv_preformatted.yml b/KustoSchemaTools.Tests/DemoData/DemoDeployment/DemoDatabase/materialized-views/mv_preformatted.yml
new file mode 100644
index 0000000..2d408a8
--- /dev/null
+++ b/KustoSchemaTools.Tests/DemoData/DemoDeployment/DemoDatabase/materialized-views/mv_preformatted.yml
@@ -0,0 +1,13 @@
+source: sourceTable
+kind: table
+folder: test
+preformatted: true
+retentionAndCachePolicy:
+ retention: 720d
+query: |-
+ sourceTable
+ | where type == "a"
+ | summarize hint.strategy=shuffle active=countif(is_active != true),
+ archived=countif(is_archived)
+ by id
+ , day
diff --git a/KustoSchemaTools.Tests/KustoSchemaTools.Tests.csproj b/KustoSchemaTools.Tests/KustoSchemaTools.Tests.csproj
index f240916..e493112 100644
--- a/KustoSchemaTools.Tests/KustoSchemaTools.Tests.csproj
+++ b/KustoSchemaTools.Tests/KustoSchemaTools.Tests.csproj
@@ -27,20 +27,9 @@
+
-
- PreserveNewest
-
-
- PreserveNewest
-
-
- PreserveNewest
-
-
- PreserveNewest
-
-
+
PreserveNewest
diff --git a/KustoSchemaTools.Tests/YamlDatabaseParserTests.cs b/KustoSchemaTools.Tests/YamlDatabaseParserTests.cs
index df99733..b7ed477 100644
--- a/KustoSchemaTools.Tests/YamlDatabaseParserTests.cs
+++ b/KustoSchemaTools.Tests/YamlDatabaseParserTests.cs
@@ -2,6 +2,9 @@
using KustoSchemaTools.Parser;
using KustoSchemaTools.Plugins;
using KustoSchemaTools.Model;
+using KustoSchemaTools.Changes;
+using Kusto.Data;
+using System.IO;
namespace KustoSchemaTools.Tests.Parser
{
@@ -17,7 +20,6 @@ public async Task GetDatabase()
var factory = new YamlDatabaseHandlerFactory()
.WithPlugin(new TablePlugin())
.WithPlugin(new FunctionPlugin())
- .WithPlugin(new MaterializedViewsPlugin())
.WithPlugin(new DatabaseCleanup());
var loader = factory.Create(Path.Combine(BasePath, Deployment), Database);
@@ -25,19 +27,151 @@ public async Task GetDatabase()
Assert.NotNull(db);
Assert.Equal(2, db.Tables.Count);
- Assert.Single(db.Functions);
Assert.Equal(6, db.Functions["UP"].Body.RowLength());
Assert.Equal("DemoDatabase", db.Name);
- var policies = db.Tables["sourceTable"].Policies;
- Assert.NotNull(policies);
- Assert.Equal("120d", policies.Retention);
- Assert.Equal("120d", policies.HotCache);
- Assert.Equal("Test team", db.Team);
- Assert.True(db.Tables["sourceTable"].RestrictedViewAccess);
-
- // these tests do not compile! to be removed in a future PR.
- // Assert.Equal("120d", db.Tables["tableWithUp"].RetentionAndCachePolicy.Retention);
- // Assert.Equal("120d", db.Tables["sourceTable"].RetentionAndCachePolicy.HotCache);
+
+ var st = db.Tables["sourceTable"];
+ Assert.NotNull(st);
+ Assert.NotNull(st.Policies);
+ Assert.True(st.Policies!.RestrictedViewAccess);
+ Assert.Equal("120d", st.Policies?.HotCache);
+
+ var tt = db.Tables["tableWithUp"];
+ Assert.NotNull(tt);
+ Assert.NotNull(tt.Policies);
+ Assert.False(tt.Policies!.RestrictedViewAccess);
+ Assert.Equal("120d", tt.Policies?.Retention);
+ }
+
+ [Fact]
+ public async Task VerifyFunctionPreformatted()
+ {
+ // WITHOUT the DatabaseCleanup plugin
+ var factoryWithoutCleanup = new YamlDatabaseHandlerFactory()
+ .WithPlugin(new TablePlugin())
+ .WithPlugin(new FunctionPlugin());
+ // DatabaseCleanup intentionally omitted
+ var loaderWithoutCleanup = factoryWithoutCleanup.Create(Path.Combine(BasePath, Deployment), Database);
+ var dbWithoutCleanup = await loaderWithoutCleanup.LoadAsync();
+
+ // with the DatabaseCleanup plugin
+ var factoryWithCleanup = new YamlDatabaseHandlerFactory()
+ .WithPlugin(new TablePlugin())
+ .WithPlugin(new FunctionPlugin())
+ .WithPlugin(new MaterializedViewsPlugin())
+ .WithPlugin(new DatabaseCleanup());
+ var loaderWithCleanup = factoryWithCleanup.Create(Path.Combine(BasePath, Deployment), Database);
+ var dbWithCleanup = await loaderWithCleanup.LoadAsync();
+
+ // Assert
+ Assert.NotNull(dbWithCleanup);
+ Assert.NotNull(dbWithoutCleanup);
+ Assert.Equal(dbWithCleanup.Functions.Count, dbWithoutCleanup.Functions.Count);
+
+ // Verify the UP function has preformatted set to false (default)
+ var up_withCleanup = dbWithCleanup.Functions["UP"];
+ var up_withoutCleanup = dbWithoutCleanup.Functions["UP"];
+ Assert.NotNull(up_withCleanup);
+ Assert.NotNull(up_withoutCleanup);
+ Assert.False(up_withCleanup.Preformatted);
+ Assert.False(up_withoutCleanup.Preformatted);
+
+ // this case is simple and formatting has no impact.
+ Assert.Equal(up_withoutCleanup.Body.RowLength(), up_withCleanup.Body.RowLength());
+
+ // Verify the needs_formatting query changed when formatting.
+ var f_withCleanup = dbWithCleanup.Functions["needs_formatting"];
+ var f_withoutCleanup = dbWithoutCleanup.Functions["needs_formatting"];
+ Assert.NotNull(f_withCleanup);
+ Assert.NotNull(f_withoutCleanup);
+ Assert.False(f_withCleanup.Preformatted);
+ Assert.False(f_withoutCleanup.Preformatted);
+
+ // preformatted function should have been formatted by DatabaseCleanup
+ Assert.NotEqual(f_withCleanup.Body, f_withoutCleanup.Body);
+
+ // much more complicated function where formatting breaks the query
+ var complicated_with_cleanup = dbWithCleanup.Functions["complicated"].Body;
+ var complicated_without_cleanup = dbWithoutCleanup.Functions["complicated"].Body;
+ Assert.NotEqual(complicated_with_cleanup, complicated_without_cleanup);
+
+ var complicated_pf_with_cleanup = dbWithCleanup.Functions["complicated_preformatted"].Body;
+ var complicated_pf_without_cleanup = dbWithoutCleanup.Functions["complicated_preformatted"].Body;
+
+ // preformatted option makes query match non-formatted version
+ Assert.Equal(complicated_pf_without_cleanup, complicated_pf_with_cleanup);
+
+ // preformatted option makes query match non-formatted version
+ Assert.Equal(complicated_without_cleanup, complicated_pf_with_cleanup);
+ }
+
+ [Fact]
+ public async Task VerifyMaterializedView()
+ {
+ // WITHOUT the DatabaseCleanup plugin
+ var factoryWithoutCleanup = new YamlDatabaseHandlerFactory()
+ .WithPlugin(new TablePlugin())
+ .WithPlugin(new MaterializedViewsPlugin());
+ // DatabaseCleanup intentionally omitted
+ var loaderWithoutCleanup = factoryWithoutCleanup.Create(Path.Combine(BasePath, Deployment), Database);
+ var dbWithoutCleanup = await loaderWithoutCleanup.LoadAsync();
+
+ // with the DatabaseCleanup plugin
+ var factoryWithCleanup = new YamlDatabaseHandlerFactory()
+ .WithPlugin(new TablePlugin())
+ .WithPlugin(new MaterializedViewsPlugin())
+ .WithPlugin(new DatabaseCleanup());
+ var loaderWithCleanup = factoryWithCleanup.Create(Path.Combine(BasePath, Deployment), Database);
+ var dbWithCleanup = await loaderWithCleanup.LoadAsync();
+
+ // Assert
+ Assert.NotNull(dbWithCleanup);
+ Assert.NotNull(dbWithoutCleanup);
+ Assert.Equal(dbWithCleanup.MaterializedViews.Count, dbWithoutCleanup.MaterializedViews.Count);
+
+ // basic materialized view tests
+ void AssertMaterializedView(
+ string file_name,
+ bool should_match)
+ {
+ var mv_with_cleanup = dbWithCleanup.MaterializedViews[file_name];
+ var mv_without_cleanup = dbWithoutCleanup.MaterializedViews[file_name];
+ Assert.NotNull(mv_with_cleanup);
+ Assert.NotNull(mv_without_cleanup);
+ Assert.Equal(should_match, mv_without_cleanup.Query == mv_with_cleanup.Query);
+
+ Assert.DoesNotContain("Preformatted", mv_with_cleanup.Query);
+ Assert.DoesNotContain("Preformatted", mv_without_cleanup.Query);
+ }
+ AssertMaterializedView("mv", false);
+ AssertMaterializedView("mv_preformatted", true);
+ }
+
+ [Fact]
+ public async Task VerifyFunctionWithCommentAtEnd()
+ {
+ // This test verifies that functions with comments at the end without a newline
+ // are handled correctly when scripts are generated
+
+ // Arrange - First load the database
+ var factory = new YamlDatabaseHandlerFactory()
+ .WithPlugin(new TablePlugin())
+ .WithPlugin(new FunctionPlugin())
+ .WithPlugin(new DatabaseCleanup());
+ var loader = factory.Create(Path.Combine(BasePath, Deployment), Database);
+
+ // Act - Load the database
+ var db = await loader.LoadAsync();
+ var commentEndFunction = db.Functions["COMMENT_END"];
+ Assert.NotNull(commentEndFunction);
+
+ // Generate the script container for the function
+ var scriptContainers = commentEndFunction.CreateScripts("COMMENT_END", false);
+ Assert.Single(scriptContainers);
+
+ var script = scriptContainers[0].Script.Text;
+ var expected = ".create-or-alter function with(SkipValidation=```False```, View=```False```, Folder=```test```, DocString=```Function with comment at end```) COMMENT_END () { sourceTable\n| limit 100\n| where IsNotEmpty(EventId) // this is a comment at the end\n }";
+ Assert.Equal(expected, script);
}
}
-}
+}
\ No newline at end of file
diff --git a/KustoSchemaTools/Model/Function.cs b/KustoSchemaTools/Model/Function.cs
index 6b617e5..8de795b 100644
--- a/KustoSchemaTools/Model/Function.cs
+++ b/KustoSchemaTools/Model/Function.cs
@@ -4,6 +4,7 @@
using KustoSchemaTools.Parser;
using System.Text;
using YamlDotNet.Serialization;
+using System.Xml.Schema;
namespace KustoSchemaTools.Model
{
@@ -15,43 +16,97 @@ public class Function : IKustoBaseEntity
public string DocString { get; set; } = "";
public string Parameters { get; set; } = "";
[YamlMember(ScalarStyle = YamlDotNet.Core.ScalarStyle.Literal)]
-
+ public bool Preformatted { get; set; } = false;
public string Body { get; set; }
public List CreateScripts(string name, bool isNew)
{
+ // load the non-query parts of the yaml model
+ var excludedProperties = new HashSet(["Body", "Parameters", "Preformatted"]);
var properties = GetType().GetProperties()
- .Where(p => p.GetValue(this) != null && p.Name != "Body" && p.Name != "Parameters")
+ .Where(p => p.GetValue(this) != null && !excludedProperties.Contains(p.Name))
.Select(p => $"{p.Name}=```{p.GetValue(this)}```");
var propertiesString = string.Join(", ", properties);
+ // Process function parameters to ensure proper syntax when creating Kusto function
var parameters = Parameters;
if (!string.IsNullOrWhiteSpace(Parameters))
{
+ // PARAMETER PROCESSING WORKFLOW:
+ // 1. Create a dummy Kusto function that uses our parameters to leverage Kusto parser
+ // 2. Parse the function to extract parameter declarations AST
+ // 3. For each parameter name, apply bracketing if needed (for identifiers with special chars)
+ // 4. Reconstruct the parameter string with properly formatted parameter names
+
+ // Create a simple dummy function to parse, embedding our parameters
var dummyFunction = $"let x = ({parameters}) {{print \"abc\"}}";
var parsed = KustoCode.Parse(dummyFunction);
+ // Extract all parameter name declarations from the parsed syntax tree
var descs = parsed.Syntax
.GetDescendants()
.First()
.GetDescendants()
.ToList();
+ // Rebuild the parameters string with proper bracketing for each parameter name
var sb = new StringBuilder();
int lastPos = 0;
foreach (var desc in descs)
{
+ // Apply bracketing to parameter name if needed (for identifiers with spaces or special chars)
var bracketified = desc.Name.ToString().Trim().BracketIfIdentifier();
+
+ // Append everything from the last position up to the current parameter name
sb.Append(dummyFunction[lastPos..desc.TextStart]);
+
+ // Append the properly bracketed parameter name
sb.Append(bracketified);
+
+ // Update position tracker to end of this parameter name
lastPos = desc.End;
}
+
+ // Append any remaining text after the last parameter
sb.Append(dummyFunction.Substring(lastPos));
var replacedFunction = sb.ToString();
+
+ // Extract just the parameter portion from the reconstructed dummy function
+ // The slice removes "let x = (" from the start and "){print "abc"}" from the end
parameters = replacedFunction[9..^15];
}
- return new List { new DatabaseScriptContainer("CreateOrAlterFunction", 40, $".create-or-alter function with({propertiesString}) {name} ({parameters}) {{ {Body} }}") };
+ // Normalize the body to ensure it ends with exactly one newline character
+ // and remove trailing whitespace from each line
+ string normalizedBody = Body;
+
+ if (string.IsNullOrEmpty(normalizedBody))
+ {
+ // Empty body case
+ normalizedBody = string.Empty;
+ }
+ else
+ {
+ // Split the body into lines, trim each line, and rejoin
+ string[] lines = normalizedBody.Replace("\r\n", "\n").Replace("\r", "\n").Split('\n');
+
+ // Process all lines except the last one
+ for (int i = 0; i < lines.Length - 1; i++)
+ {
+ lines[i] = lines[i].TrimEnd();
+ }
+
+ // Handle the last line separately - no need to trim trailing newlines since we split on them
+ if (lines.Length > 0)
+ {
+ lines[lines.Length - 1] = lines[lines.Length - 1].TrimEnd();
+ }
+
+ // Rejoin the lines and add exactly one newline character at the end
+ normalizedBody = string.Join(Environment.NewLine, lines) + Environment.NewLine;
+ }
+
+ return new List { new DatabaseScriptContainer("CreateOrAlterFunction", 40, $".create-or-alter function with({propertiesString}) {name} ({parameters}) {{ {normalizedBody} }}") };
}
}
diff --git a/KustoSchemaTools/Model/MaterializedView.cs b/KustoSchemaTools/Model/MaterializedView.cs
index ad24687..c197f93 100644
--- a/KustoSchemaTools/Model/MaterializedView.cs
+++ b/KustoSchemaTools/Model/MaterializedView.cs
@@ -24,13 +24,21 @@ public class MaterializedView : IKustoBaseEntity
[Obsolete("Use policies instead")]
public string? RowLevelSecurity { get; set; }
public Policy? Policies { get; set; }
-
+ public bool Preformatted { get; set; } = false;
public List CreateScripts(string name, bool isNew)
{
var asyncSetup = isNew && Backfill == true;
-
- var excludedProperties = new HashSet(["Query", "Source", "Kind", "RetentionAndCachePolicy", "RowLevelSecurity", "Policies"]);
+ var excludedProperties = new HashSet([
+ "Query",
+ "Source",
+ "Kind",
+ "RetentionAndCachePolicy",
+ "RowLevelSecurity",
+ "Policies",
+ "Preformatted"
+ ]);
+
if (!asyncSetup)
{
excludedProperties.Add("EffectiveDateTime");
@@ -40,11 +48,10 @@ public List CreateScripts(string name, bool isNew)
var scripts = new List();
var properties = string.Join(", ", GetType().GetProperties()
.Where(p => p.GetValue(this) != null && excludedProperties.Contains(p.Name) == false)
- .Select(p => new {Name = p.Name, Value = p.GetValue(this) })
+ .Select(p => new { Name = p.Name, Value = p.GetValue(this) })
.Where(p => !string.IsNullOrWhiteSpace(p.Value?.ToString()))
.Select(p => $"{p.Name}=```{p.Value}```"));
-
if (asyncSetup)
{
scripts.Add(new DatabaseScriptContainer("CreateMaterializedViewAsync", Kind == "table" ? 40 : 41, $".create async ifnotexists materialized-view with ({properties}) {name} on {Kind} {Source} {{ {Query} }}", true));
@@ -57,9 +64,7 @@ public List CreateScripts(string name, bool isNew)
{
scripts.AddRange(Policies.CreateScripts(name, "materialized-view"));
}
-
return scripts;
}
}
-
}
diff --git a/KustoSchemaTools/Parser/DatabaseCleanup.cs b/KustoSchemaTools/Parser/DatabaseCleanup.cs
index ef91308..9ce4c62 100644
--- a/KustoSchemaTools/Parser/DatabaseCleanup.cs
+++ b/KustoSchemaTools/Parser/DatabaseCleanup.cs
@@ -122,7 +122,12 @@ public void CleanUp(Database database)
{
policy.HotCache = null;
}
- entity.Value.Query = entity.Value.Query.PrettifyKql();
+
+ if (entity.Value.Preformatted == false)
+ {
+ // format the query unless the materialized view opts out
+ entity.Value.Query = entity.Value.Query.PrettifyKql();
+ }
}
foreach(var entity in database.MaterializedViews)
@@ -135,7 +140,12 @@ public void CleanUp(Database database)
foreach (var entity in database.Functions)
{
- entity.Value.Body = entity.Value.Body.PrettifyKql();
+ // format unless the function opts out
+ // there are known issues with PrettifyKql function.
+ if (!entity.Value.Preformatted)
+ {
+ entity.Value.Body = entity.Value.Body.PrettifyKql();
+ }
}
foreach (var up in database.Tables.Values.Where(itm => itm.Policies?.UpdatePolicies != null).SelectMany(itm => itm.Policies.UpdatePolicies))
{
diff --git a/README.md b/README.md
index 3443df8..073dc82 100644
--- a/README.md
+++ b/README.md
@@ -53,12 +53,14 @@ Currently following features are supported:
* Body
* Docstring
* Folder
+ * Preformatted
* Materialized Views
* Query
* Retention
* HotCache
* Docstring
* Folder
+ * Preformatted
* External Tables (managed identity/impersonation only)
* Storage / Delta / SQL
* Folder
@@ -73,4 +75,5 @@ Currently following features are supported:
* Extenal Tables
* Continuous Exports
-The `DatabaseCleanup` will remove redundant retention and hotcache definitions. It will also pretty print KQL queries in functions, update policies, materialized views and continuous exports.
+The `DatabaseCleanup` will remove redundant retention and hotcache definitions.
+It will also pretty print KQL queries in functions (unless the `preformatted` feature is used) , update policies, materialized views and continuous exports.