Skip to content

Commit 3796366

Browse files
zachariahcoxzachariahcox
committed
feature: preformatting option on functions and materialized-views
1 parent f9aa451 commit 3796366

10 files changed

Lines changed: 220 additions & 25 deletions

File tree

‎KustoSchemaTools.Tests/DemoData/DemoDeployment/DemoDatabase/functions/complicated.yml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
folder: test
2+
docString: issues for relevant services, filtered
3+
preformatted: false
4+
body: |-
5+
sourceTable
6+
| where t betwee(5..10)
7+
| where repository_id in (table_function(_a,_b,_c) | distinct id)
8+
| project id, type
9+
| summarize arg_max(t, *) by id
10+
| lookup (table_function(_a,_b,_c) | distinct id, classifier) on id
11+
| extend type = case(
12+
id = 1, "a",
13+
id = 2, "b",
14+
"other")

‎KustoSchemaTools.Tests/DemoData/DemoDeployment/DemoDatabase/functions/complicated_preformatted.yml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
folder: test
2+
docString: issues for relevant services, filtered
3+
preformatted: true
4+
body: |-
5+
sourceTable
6+
| where t betwee(5..10)
7+
| where repository_id in (table_function(_a,_b,_c) | distinct id)
8+
| project id, type
9+
| summarize arg_max(t, *) by id
10+
| lookup (table_function(_a,_b,_c) | distinct id, classifier) on id
11+
| extend type = case(
12+
id = 1, "a",
13+
id = 2, "b",
14+
"other")

‎KustoSchemaTools.Tests/DemoData/DemoDeployment/DemoDatabase/functions/needs_formatting.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
folder: test
2+
docString: test function that would change with formatting
3+
body: |-
4+
sourceTable | limit 100
5+

‎KustoSchemaTools.Tests/DemoData/DemoDeployment/DemoDatabase/materialized-views/mv.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
source: sourceTable
2+
kind: table
3+
folder: test
4+
retentionAndCachePolicy:
5+
retention: 720d
6+
query: |-
7+
sourceTable
8+
| where type == "a"
9+
| summarize hint.strategy=shuffle active=countif(is_active != true),
10+
archived=countif(is_archived)
11+
by id
12+
, day

‎KustoSchemaTools.Tests/DemoData/DemoDeployment/DemoDatabase/materialized-views/mv_preformatted.yml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
source: sourceTable
2+
kind: table
3+
folder: test
4+
preformatted: true
5+
retentionAndCachePolicy:
6+
retention: 720d
7+
query: |-
8+
sourceTable
9+
| where type == "a"
10+
| summarize hint.strategy=shuffle active=countif(is_active != true),
11+
archived=countif(is_archived)
12+
by id
13+
, day

‎KustoSchemaTools.Tests/KustoSchemaTools.Tests.csproj‎

Lines changed: 3 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
<Project Sdk="Microsoft.NET.Sdk">
22

33
<PropertyGroup>
4-
<TargetFramework>net6.0</TargetFramework>
4+
<TargetFramework>net8.0</TargetFramework>
55
<ImplicitUsings>enable</ImplicitUsings>
66
<Nullable>enable</Nullable>
77

@@ -27,20 +27,9 @@
2727
<ProjectReference Include="..\KustoSchemaTools\KustoSchemaTools.csproj" />
2828
</ItemGroup>
2929

30+
<!-- Automatically include all files in the DemoData directory -->
3031
<ItemGroup>
31-
<None Update="DemoData\DemoDeployment\clusters.yml">
32-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
33-
</None>
34-
<None Update="DemoData\DemoDeployment\DemoDatabase\database.yml">
35-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
36-
</None>
37-
<None Update="DemoData\DemoDeployment\DemoDatabase\functions\UP.yml">
38-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
39-
</None>
40-
<None Update="DemoData\DemoDeployment\DemoDatabase\tables\sourceTable.yml">
41-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
42-
</None>
43-
<None Update="DemoData\DemoDeployment\DemoDatabase\tables\tableWithUp.yml">
32+
<None Include="DemoData\**\*.*">
4433
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
4534
</None>
4635
</ItemGroup>

‎KustoSchemaTools.Tests/YamlDatabaseParserTests.cs‎

Lines changed: 118 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
using KustoSchemaTools.Helpers;
22
using KustoSchemaTools.Parser;
33
using KustoSchemaTools.Plugins;
4+
using KustoSchemaTools.Model;
5+
using KustoSchemaTools.Changes;
6+
using Kusto.Data;
7+
using System.IO;
48

59
namespace KustoSchemaTools.Tests.Parser
610
{
@@ -14,24 +18,132 @@ public class YamlDatabaseParserTests
1418
[Fact]
1519
public async Task GetDatabase()
1620
{
17-
var factory = new YamlDatabaseHandlerFactory()
21+
var factory = new YamlDatabaseHandlerFactory<Model.Database>()
1822
.WithPlugin(new TablePlugin())
1923
.WithPlugin(new FunctionPlugin())
20-
.WithPlugin(new MaterializedViewsPlugin())
2124
.WithPlugin(new DatabaseCleanup());
2225
var loader = factory.Create(Path.Combine(BasePath, Deployment), Database);
2326

2427
var db = await loader.LoadAsync();
2528

2629
Assert.NotNull(db);
2730
Assert.Equal(2, db.Tables.Count);
28-
Assert.Equal(1, db.Functions.Count);
31+
Assert.Equal(4, db.Functions.Count);
2932
Assert.Equal(6, db.Functions["UP"].Body.RowLength());
3033
Assert.Equal("DemoDatabase", db.Name);
31-
Assert.True(db.Tables["sourceTable"].RestrictedViewAccess);
32-
Assert.Equal("120d", db.Tables["tableWithUp"].RetentionAndCachePolicy.Retention);
33-
Assert.Equal("120d", db.Tables["sourceTable"].RetentionAndCachePolicy.HotCache);
34+
35+
var st = db.Tables["sourceTable"];
36+
Assert.NotNull(st);
37+
Assert.NotNull(st.Policies);
38+
Assert.True(st.Policies!.RestrictedViewAccess);
39+
Assert.Equal("120d", st.Policies?.HotCache);
40+
41+
var tt = db.Tables["tableWithUp"];
42+
Assert.NotNull(tt);
43+
Assert.NotNull(tt.Policies);
44+
Assert.False(tt.Policies!.RestrictedViewAccess);
45+
Assert.Equal("120d", tt.Policies?.Retention);
46+
}
47+
48+
[Fact]
49+
public async Task VerifyFunctionPreformatted()
50+
{
51+
// WITHOUT the DatabaseCleanup plugin
52+
var factoryWithoutCleanup = new YamlDatabaseHandlerFactory<Model.Database>()
53+
.WithPlugin(new TablePlugin())
54+
.WithPlugin(new FunctionPlugin());
55+
// DatabaseCleanup intentionally omitted
56+
var loaderWithoutCleanup = factoryWithoutCleanup.Create(Path.Combine(BasePath, Deployment), Database);
57+
var dbWithoutCleanup = await loaderWithoutCleanup.LoadAsync();
58+
59+
// with the DatabaseCleanup plugin
60+
var factoryWithCleanup = new YamlDatabaseHandlerFactory<Model.Database>()
61+
.WithPlugin(new TablePlugin())
62+
.WithPlugin(new FunctionPlugin())
63+
.WithPlugin(new MaterializedViewsPlugin())
64+
.WithPlugin(new DatabaseCleanup());
65+
var loaderWithCleanup = factoryWithCleanup.Create(Path.Combine(BasePath, Deployment), Database);
66+
var dbWithCleanup = await loaderWithCleanup.LoadAsync();
67+
68+
// Assert
69+
Assert.NotNull(dbWithCleanup);
70+
Assert.NotNull(dbWithoutCleanup);
71+
Assert.Equal(dbWithCleanup.Functions.Count, dbWithoutCleanup.Functions.Count);
72+
73+
// Verify the UP function has preformatted set to false (default)
74+
var up_withCleanup = dbWithCleanup.Functions["UP"];
75+
var up_withoutCleanup = dbWithoutCleanup.Functions["UP"];
76+
Assert.NotNull(up_withCleanup);
77+
Assert.NotNull(up_withoutCleanup);
78+
Assert.False(up_withCleanup.Preformatted);
79+
Assert.False(up_withoutCleanup.Preformatted);
80+
81+
// this case is simple and formatting has no impact.
82+
Assert.Equal(up_withoutCleanup.Body.RowLength(), up_withCleanup.Body.RowLength());
83+
84+
// Verify the needs_formatting query changed when formatting.
85+
var f_withCleanup = dbWithCleanup.Functions["needs_formatting"];
86+
var f_withoutCleanup = dbWithoutCleanup.Functions["needs_formatting"];
87+
Assert.NotNull(f_withCleanup);
88+
Assert.NotNull(f_withoutCleanup);
89+
Assert.False(f_withCleanup.Preformatted);
90+
Assert.False(f_withoutCleanup.Preformatted);
91+
92+
// preformatted function should have been formatted by DatabaseCleanup
93+
Assert.NotEqual(f_withCleanup.Body, f_withoutCleanup.Body);
94+
95+
// much more complicated function where formatting breaks the query
96+
var complicated_with_cleanup = dbWithCleanup.Functions["complicated"].Body;
97+
var complicated_without_cleanup = dbWithoutCleanup.Functions["complicated"].Body;
98+
Assert.NotEqual(complicated_with_cleanup, complicated_without_cleanup);
99+
100+
var complicated_pf_with_cleanup = dbWithCleanup.Functions["complicated_preformatted"].Body;
101+
var complicated_pf_without_cleanup = dbWithoutCleanup.Functions["complicated_preformatted"].Body;
102+
103+
// preformatted option makes query match non-formatted version
104+
Assert.Equal(complicated_pf_without_cleanup, complicated_pf_with_cleanup);
105+
106+
// preformatted option makes query match non-formatted version
107+
Assert.Equal(complicated_without_cleanup, complicated_pf_with_cleanup);
34108
}
35109

110+
[Fact]
111+
public async Task VerifyMaterializedView()
112+
{
113+
// WITHOUT the DatabaseCleanup plugin
114+
var factoryWithoutCleanup = new YamlDatabaseHandlerFactory<Model.Database>()
115+
.WithPlugin(new TablePlugin())
116+
.WithPlugin(new MaterializedViewsPlugin());
117+
// DatabaseCleanup intentionally omitted
118+
var loaderWithoutCleanup = factoryWithoutCleanup.Create(Path.Combine(BasePath, Deployment), Database);
119+
var dbWithoutCleanup = await loaderWithoutCleanup.LoadAsync();
120+
121+
// with the DatabaseCleanup plugin
122+
var factoryWithCleanup = new YamlDatabaseHandlerFactory<Model.Database>()
123+
.WithPlugin(new TablePlugin())
124+
.WithPlugin(new MaterializedViewsPlugin())
125+
.WithPlugin(new DatabaseCleanup());
126+
var loaderWithCleanup = factoryWithCleanup.Create(Path.Combine(BasePath, Deployment), Database);
127+
var dbWithCleanup = await loaderWithCleanup.LoadAsync();
128+
129+
// Assert
130+
Assert.NotNull(dbWithCleanup);
131+
Assert.NotNull(dbWithoutCleanup);
132+
Assert.Equal(dbWithCleanup.MaterializedViews.Count, dbWithoutCleanup.MaterializedViews.Count);
133+
134+
// basic materialized view tests
135+
void AssertMaterializedView(
136+
string file_name,
137+
bool should_match)
138+
{
139+
var mv_with_cleanup = dbWithCleanup.MaterializedViews[file_name];
140+
var mv_without_cleanup = dbWithoutCleanup.MaterializedViews[file_name];
141+
Assert.NotNull(mv_with_cleanup);
142+
Assert.NotNull(mv_without_cleanup);
143+
Assert.Equal(should_match, mv_without_cleanup.Query == mv_with_cleanup.Query);
144+
}
145+
AssertMaterializedView("mv", false);
146+
AssertMaterializedView("mv_preformatted", true);
147+
}
36148
}
37149
}

‎KustoSchemaTools/Model/Function.cs‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,39 +15,63 @@ public class Function : IKustoBaseEntity
1515
public string DocString { get; set; } = "";
1616
public string Parameters { get; set; } = "";
1717
[YamlMember(ScalarStyle = YamlDotNet.Core.ScalarStyle.Literal)]
18+
public bool Preformatted { get; set; } = false;
1819

1920
public string Body { get; set; }
2021

2122
public List<DatabaseScriptContainer> CreateScripts(string name, bool isNew)
2223
{
24+
// load the non-query parts of the yaml model
2325
var properties = GetType().GetProperties()
2426
.Where(p => p.GetValue(this) != null && p.Name != "Body" && p.Name != "Parameters")
2527
.Select(p => $"{p.Name}=```{p.GetValue(this)}```");
2628
var propertiesString = string.Join(", ", properties);
2729

30+
// Process function parameters to ensure proper syntax when creating Kusto function
2831
var parameters = Parameters;
2932
if (!string.IsNullOrWhiteSpace(Parameters))
3033
{
34+
// PARAMETER PROCESSING WORKFLOW:
35+
// 1. Create a dummy Kusto function that uses our parameters to leverage Kusto parser
36+
// 2. Parse the function to extract parameter declarations AST
37+
// 3. For each parameter name, apply bracketing if needed (for identifiers with special chars)
38+
// 4. Reconstruct the parameter string with properly formatted parameter names
39+
40+
// Create a simple dummy function to parse, embedding our parameters
3141
var dummyFunction = $"let x = ({parameters}) {{print \"abc\"}}";
3242
var parsed = KustoCode.Parse(dummyFunction);
3343

44+
// Extract all parameter name declarations from the parsed syntax tree
3445
var descs = parsed.Syntax
3546
.GetDescendants<FunctionParameters>()
3647
.First()
3748
.GetDescendants<NameDeclaration>()
3849
.ToList();
3950

51+
// Rebuild the parameters string with proper bracketing for each parameter name
4052
var sb = new StringBuilder();
4153
int lastPos = 0;
4254
foreach (var desc in descs)
4355
{
56+
// Apply bracketing to parameter name if needed (for identifiers with spaces or special chars)
4457
var bracketified = desc.Name.ToString().Trim().BracketIfIdentifier();
58+
59+
// Append everything from the last position up to the current parameter name
4560
sb.Append(dummyFunction[lastPos..desc.TextStart]);
61+
62+
// Append the properly bracketed parameter name
4663
sb.Append(bracketified);
64+
65+
// Update position tracker to end of this parameter name
4766
lastPos = desc.End;
4867
}
68+
69+
// Append any remaining text after the last parameter
4970
sb.Append(dummyFunction.Substring(lastPos));
5071
var replacedFunction = sb.ToString();
72+
73+
// Extract just the parameter portion from the reconstructed dummy function
74+
// The slice removes "let x = (" from the start and "){print "abc"}" from the end
5175
parameters = replacedFunction[9..^15];
5276
}
5377

‎KustoSchemaTools/Model/MaterializedView.cs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ public class MaterializedView : IKustoBaseEntity
2424
[Obsolete("Use policies instead")]
2525
public string? RowLevelSecurity { get; set; }
2626
public Policy? Policies { get; set; }
27+
28+
public bool Preformatted { get; set; } = false;
2729

2830
public List<DatabaseScriptContainer> CreateScripts(string name, bool isNew)
2931
{
@@ -40,11 +42,11 @@ public List<DatabaseScriptContainer> CreateScripts(string name, bool isNew)
4042
var scripts = new List<DatabaseScriptContainer>();
4143
var properties = string.Join(", ", GetType().GetProperties()
4244
.Where(p => p.GetValue(this) != null && excludedProperties.Contains(p.Name) == false)
43-
.Select(p => new {Name = p.Name, Value = p.GetValue(this) })
45+
.Select(p => new { Name = p.Name, Value = p.GetValue(this) })
4446
.Where(p => !string.IsNullOrWhiteSpace(p.Value?.ToString()))
4547
.Select(p => $"{p.Name}=```{p.Value}```"));
4648

47-
49+
4850
if (asyncSetup)
4951
{
5052
scripts.Add(new DatabaseScriptContainer("CreateMaterializedViewAsync", Kind == "table" ? 40 : 41, $".create async ifnotexists materialized-view with ({properties}) {name} on {Kind} {Source} {{ {Query} }}", true));
@@ -57,7 +59,7 @@ public List<DatabaseScriptContainer> CreateScripts(string name, bool isNew)
5759
{
5860
scripts.AddRange(Policies.CreateScripts(name, "materialized-view"));
5961
}
60-
62+
6163
return scripts;
6264
}
6365
}

‎KustoSchemaTools/Parser/DatabaseCleanup.cs‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,12 @@ public void CleanUp(Database database)
122122
{
123123
policy.HotCache = null;
124124
}
125-
entity.Value.Query = entity.Value.Query.PrettifyKql();
125+
126+
if (entity.Value.Preformatted == false)
127+
{
128+
// format the query unless the materialized view opts out
129+
entity.Value.Query = entity.Value.Query.PrettifyKql();
130+
}
126131
}
127132

128133
foreach(var entity in database.MaterializedViews)
@@ -135,7 +140,12 @@ public void CleanUp(Database database)
135140

136141
foreach (var entity in database.Functions)
137142
{
138-
entity.Value.Body = entity.Value.Body.PrettifyKql();
143+
// format unless the function opts out
144+
// there are known issues with PrettifyKql function.
145+
if (!entity.Value.Preformatted)
146+
{
147+
entity.Value.Body = entity.Value.Body.PrettifyKql();
148+
}
139149
}
140150
foreach (var up in database.Tables.Values.Where(itm => itm.Policies?.UpdatePolicies != null).SelectMany(itm => itm.Policies.UpdatePolicies))
141151
{

0 commit comments

Comments
 (0)