Skip to content

Commit e380546

Browse files
zachariahcoxzachariahcox
committed
resolve merge conflicts
1 parent 8df7793 commit e380546

11 files changed

Lines changed: 229 additions & 31 deletions

File tree

‎KustoSchemaTools.Tests/DemoData/DemoDeployment/DemoDatabase/functions/complicated.yml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
folder: test
2+
docString: issues for relevant services, filtered
3+
preformatted: false
4+
body: |-
5+
sourceTable
6+
| where t between(startofday(_startTime)..endofday(_endTime)) or classifier == "somevalue"
7+
// comments
8+
| where repository_id in (table_function(_aaaaaaaaa,_bbbbbbbb,_cccccccc,_eeeeeeeee,_fffffffff) | distinct id) // prefer `in` over `join` for short right columns
9+
| project id
10+
, type
11+
, t
12+
| summarize arg_max(t, *) by id
13+
| lookup (table_function(_aaaaaaaaa,_bbbbbbbb,_cccccccc,_eeeeeeeee,_fffffffff) | distinct id, classifier) on id
14+
| extend type = case(
15+
id == 1, "a",
16+
id == 2, "b",
17+
"other") // comments
18+
| project id, type, classifier

‎KustoSchemaTools.Tests/DemoData/DemoDeployment/DemoDatabase/functions/complicated_preformatted.yml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
folder: test
2+
docString: issues for relevant services, filtered
3+
preformatted: true
4+
body: |-
5+
sourceTable
6+
| where t between(startofday(_startTime)..endofday(_endTime)) or classifier == "somevalue"
7+
// comments
8+
| where repository_id in (table_function(_aaaaaaaaa,_bbbbbbbb,_cccccccc,_eeeeeeeee,_fffffffff) | distinct id) // prefer `in` over `join` for short right columns
9+
| project id
10+
, type
11+
, t
12+
| summarize arg_max(t, *) by id
13+
| lookup (table_function(_aaaaaaaaa,_bbbbbbbb,_cccccccc,_eeeeeeeee,_fffffffff) | distinct id, classifier) on id
14+
| extend type = case(
15+
id == 1, "a",
16+
id == 2, "b",
17+
"other") // comments
18+
| project id, type, classifier

‎KustoSchemaTools.Tests/DemoData/DemoDeployment/DemoDatabase/functions/needs_formatting.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
folder: test
2+
docString: test function that would change with formatting
3+
body: |-
4+
sourceTable | limit 100

‎KustoSchemaTools.Tests/DemoData/DemoDeployment/DemoDatabase/materialized-views/mv.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
source: sourceTable
2+
kind: table
3+
folder: test
4+
retentionAndCachePolicy:
5+
retention: 720d
6+
query: |-
7+
sourceTable
8+
| where type == "a"
9+
| summarize hint.strategy=shuffle active=countif(is_active != true),
10+
archived=countif(is_archived)
11+
by id
12+
, day

‎KustoSchemaTools.Tests/DemoData/DemoDeployment/DemoDatabase/materialized-views/mv_preformatted.yml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
source: sourceTable
2+
kind: table
3+
folder: test
4+
preformatted: true
5+
retentionAndCachePolicy:
6+
retention: 720d
7+
query: |-
8+
sourceTable
9+
| where type == "a"
10+
| summarize hint.strategy=shuffle active=countif(is_active != true),
11+
archived=countif(is_archived)
12+
by id
13+
, day

‎KustoSchemaTools.Tests/KustoSchemaTools.Tests.csproj‎

Lines changed: 2 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -27,20 +27,9 @@
2727
<ProjectReference Include="..\KustoSchemaTools\KustoSchemaTools.csproj" />
2828
</ItemGroup>
2929

30+
<!-- Automatically include all files in the DemoData directory -->
3031
<ItemGroup>
31-
<None Update="DemoData\DemoDeployment\clusters.yml">
32-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
33-
</None>
34-
<None Update="DemoData\DemoDeployment\DemoDatabase\database.yml">
35-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
36-
</None>
37-
<None Update="DemoData\DemoDeployment\DemoDatabase\functions\UP.yml">
38-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
39-
</None>
40-
<None Update="DemoData\DemoDeployment\DemoDatabase\tables\sourceTable.yml">
41-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
42-
</None>
43-
<None Update="DemoData\DemoDeployment\DemoDatabase\tables\tableWithUp.yml">
32+
<None Include="DemoData\**\*.*">
4433
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
4534
</None>
4635
</ItemGroup>

‎KustoSchemaTools.Tests/YamlDatabaseParserTests.cs‎

Lines changed: 117 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22
using KustoSchemaTools.Parser;
33
using KustoSchemaTools.Plugins;
44
using KustoSchemaTools.Model;
5+
using KustoSchemaTools.Changes;
6+
using Kusto.Data;
7+
using System.IO;
58

69
namespace KustoSchemaTools.Tests.Parser
710
{
@@ -17,27 +20,129 @@ public async Task GetDatabase()
1720
var factory = new YamlDatabaseHandlerFactory<Model.Database>()
1821
.WithPlugin(new TablePlugin())
1922
.WithPlugin(new FunctionPlugin())
20-
.WithPlugin(new MaterializedViewsPlugin())
2123
.WithPlugin(new DatabaseCleanup());
2224
var loader = factory.Create(Path.Combine(BasePath, Deployment), Database);
2325

2426
var db = await loader.LoadAsync();
2527

2628
Assert.NotNull(db);
2729
Assert.Equal(2, db.Tables.Count);
28-
Assert.Single(db.Functions);
30+
Assert.Equal(4, db.Functions.Count);
2931
Assert.Equal(6, db.Functions["UP"].Body.RowLength());
3032
Assert.Equal("DemoDatabase", db.Name);
31-
var policies = db.Tables["sourceTable"].Policies;
32-
Assert.NotNull(policies);
33-
Assert.Equal("120d", policies.Retention);
34-
Assert.Equal("120d", policies.HotCache);
35-
Assert.Equal("Test team", db.Team);
36-
Assert.True(db.Tables["sourceTable"].RestrictedViewAccess);
37-
38-
// these tests do not compile! to be removed in a future PR.
39-
// Assert.Equal("120d", db.Tables["tableWithUp"].RetentionAndCachePolicy.Retention);
40-
// Assert.Equal("120d", db.Tables["sourceTable"].RetentionAndCachePolicy.HotCache);
33+
34+
var st = db.Tables["sourceTable"];
35+
Assert.NotNull(st);
36+
Assert.NotNull(st.Policies);
37+
Assert.True(st.Policies!.RestrictedViewAccess);
38+
Assert.Equal("120d", st.Policies?.HotCache);
39+
40+
var tt = db.Tables["tableWithUp"];
41+
Assert.NotNull(tt);
42+
Assert.NotNull(tt.Policies);
43+
Assert.False(tt.Policies!.RestrictedViewAccess);
44+
Assert.Equal("120d", tt.Policies?.Retention);
45+
}
46+
47+
[Fact]
48+
public async Task VerifyFunctionPreformatted()
49+
{
50+
// WITHOUT the DatabaseCleanup plugin
51+
var factoryWithoutCleanup = new YamlDatabaseHandlerFactory<Model.Database>()
52+
.WithPlugin(new TablePlugin())
53+
.WithPlugin(new FunctionPlugin());
54+
// DatabaseCleanup intentionally omitted
55+
var loaderWithoutCleanup = factoryWithoutCleanup.Create(Path.Combine(BasePath, Deployment), Database);
56+
var dbWithoutCleanup = await loaderWithoutCleanup.LoadAsync();
57+
58+
// with the DatabaseCleanup plugin
59+
var factoryWithCleanup = new YamlDatabaseHandlerFactory<Model.Database>()
60+
.WithPlugin(new TablePlugin())
61+
.WithPlugin(new FunctionPlugin())
62+
.WithPlugin(new MaterializedViewsPlugin())
63+
.WithPlugin(new DatabaseCleanup());
64+
var loaderWithCleanup = factoryWithCleanup.Create(Path.Combine(BasePath, Deployment), Database);
65+
var dbWithCleanup = await loaderWithCleanup.LoadAsync();
66+
67+
// Assert
68+
Assert.NotNull(dbWithCleanup);
69+
Assert.NotNull(dbWithoutCleanup);
70+
Assert.Equal(dbWithCleanup.Functions.Count, dbWithoutCleanup.Functions.Count);
71+
72+
// Verify the UP function has preformatted set to false (default)
73+
var up_withCleanup = dbWithCleanup.Functions["UP"];
74+
var up_withoutCleanup = dbWithoutCleanup.Functions["UP"];
75+
Assert.NotNull(up_withCleanup);
76+
Assert.NotNull(up_withoutCleanup);
77+
Assert.False(up_withCleanup.Preformatted);
78+
Assert.False(up_withoutCleanup.Preformatted);
79+
80+
// this case is simple and formatting has no impact.
81+
Assert.Equal(up_withoutCleanup.Body.RowLength(), up_withCleanup.Body.RowLength());
82+
83+
// Verify the needs_formatting query changed when formatting.
84+
var f_withCleanup = dbWithCleanup.Functions["needs_formatting"];
85+
var f_withoutCleanup = dbWithoutCleanup.Functions["needs_formatting"];
86+
Assert.NotNull(f_withCleanup);
87+
Assert.NotNull(f_withoutCleanup);
88+
Assert.False(f_withCleanup.Preformatted);
89+
Assert.False(f_withoutCleanup.Preformatted);
90+
91+
// preformatted function should have been formatted by DatabaseCleanup
92+
Assert.NotEqual(f_withCleanup.Body, f_withoutCleanup.Body);
93+
94+
// much more complicated function where formatting breaks the query
95+
var complicated_with_cleanup = dbWithCleanup.Functions["complicated"].Body;
96+
var complicated_without_cleanup = dbWithoutCleanup.Functions["complicated"].Body;
97+
Assert.NotEqual(complicated_with_cleanup, complicated_without_cleanup);
98+
99+
var complicated_pf_with_cleanup = dbWithCleanup.Functions["complicated_preformatted"].Body;
100+
var complicated_pf_without_cleanup = dbWithoutCleanup.Functions["complicated_preformatted"].Body;
101+
102+
// preformatted option makes query match non-formatted version
103+
Assert.Equal(complicated_pf_without_cleanup, complicated_pf_with_cleanup);
104+
105+
// preformatted option makes query match non-formatted version
106+
Assert.Equal(complicated_without_cleanup, complicated_pf_with_cleanup);
107+
}
108+
109+
[Fact]
110+
public async Task VerifyMaterializedView()
111+
{
112+
// WITHOUT the DatabaseCleanup plugin
113+
var factoryWithoutCleanup = new YamlDatabaseHandlerFactory<Model.Database>()
114+
.WithPlugin(new TablePlugin())
115+
.WithPlugin(new MaterializedViewsPlugin());
116+
// DatabaseCleanup intentionally omitted
117+
var loaderWithoutCleanup = factoryWithoutCleanup.Create(Path.Combine(BasePath, Deployment), Database);
118+
var dbWithoutCleanup = await loaderWithoutCleanup.LoadAsync();
119+
120+
// with the DatabaseCleanup plugin
121+
var factoryWithCleanup = new YamlDatabaseHandlerFactory<Model.Database>()
122+
.WithPlugin(new TablePlugin())
123+
.WithPlugin(new MaterializedViewsPlugin())
124+
.WithPlugin(new DatabaseCleanup());
125+
var loaderWithCleanup = factoryWithCleanup.Create(Path.Combine(BasePath, Deployment), Database);
126+
var dbWithCleanup = await loaderWithCleanup.LoadAsync();
127+
128+
// Assert
129+
Assert.NotNull(dbWithCleanup);
130+
Assert.NotNull(dbWithoutCleanup);
131+
Assert.Equal(dbWithCleanup.MaterializedViews.Count, dbWithoutCleanup.MaterializedViews.Count);
132+
133+
// basic materialized view tests
134+
void AssertMaterializedView(
135+
string file_name,
136+
bool should_match)
137+
{
138+
var mv_with_cleanup = dbWithCleanup.MaterializedViews[file_name];
139+
var mv_without_cleanup = dbWithoutCleanup.MaterializedViews[file_name];
140+
Assert.NotNull(mv_with_cleanup);
141+
Assert.NotNull(mv_without_cleanup);
142+
Assert.Equal(should_match, mv_without_cleanup.Query == mv_with_cleanup.Query);
143+
}
144+
AssertMaterializedView("mv", false);
145+
AssertMaterializedView("mv_preformatted", true);
41146
}
42147
}
43148
}

‎KustoSchemaTools/Model/Function.cs‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,39 +15,63 @@ public class Function : IKustoBaseEntity
1515
public string DocString { get; set; } = "";
1616
public string Parameters { get; set; } = "";
1717
[YamlMember(ScalarStyle = YamlDotNet.Core.ScalarStyle.Literal)]
18+
public bool Preformatted { get; set; } = false;
1819

1920
public string Body { get; set; }
2021

2122
public List<DatabaseScriptContainer> CreateScripts(string name, bool isNew)
2223
{
24+
// load the non-query parts of the yaml model
2325
var properties = GetType().GetProperties()
2426
.Where(p => p.GetValue(this) != null && p.Name != "Body" && p.Name != "Parameters")
2527
.Select(p => $"{p.Name}=```{p.GetValue(this)}```");
2628
var propertiesString = string.Join(", ", properties);
2729

30+
// Process function parameters to ensure proper syntax when creating Kusto function
2831
var parameters = Parameters;
2932
if (!string.IsNullOrWhiteSpace(Parameters))
3033
{
34+
// PARAMETER PROCESSING WORKFLOW:
35+
// 1. Create a dummy Kusto function that uses our parameters to leverage Kusto parser
36+
// 2. Parse the function to extract parameter declarations AST
37+
// 3. For each parameter name, apply bracketing if needed (for identifiers with special chars)
38+
// 4. Reconstruct the parameter string with properly formatted parameter names
39+
40+
// Create a simple dummy function to parse, embedding our parameters
3141
var dummyFunction = $"let x = ({parameters}) {{print \"abc\"}}";
3242
var parsed = KustoCode.Parse(dummyFunction);
3343

44+
// Extract all parameter name declarations from the parsed syntax tree
3445
var descs = parsed.Syntax
3546
.GetDescendants<FunctionParameters>()
3647
.First()
3748
.GetDescendants<NameDeclaration>()
3849
.ToList();
3950

51+
// Rebuild the parameters string with proper bracketing for each parameter name
4052
var sb = new StringBuilder();
4153
int lastPos = 0;
4254
foreach (var desc in descs)
4355
{
56+
// Apply bracketing to parameter name if needed (for identifiers with spaces or special chars)
4457
var bracketified = desc.Name.ToString().Trim().BracketIfIdentifier();
58+
59+
// Append everything from the last position up to the current parameter name
4560
sb.Append(dummyFunction[lastPos..desc.TextStart]);
61+
62+
// Append the properly bracketed parameter name
4663
sb.Append(bracketified);
64+
65+
// Update position tracker to end of this parameter name
4766
lastPos = desc.End;
4867
}
68+
69+
// Append any remaining text after the last parameter
4970
sb.Append(dummyFunction.Substring(lastPos));
5071
var replacedFunction = sb.ToString();
72+
73+
// Extract just the parameter portion from the reconstructed dummy function
74+
// The slice removes "let x = (" from the start and "){print "abc"}" from the end
5175
parameters = replacedFunction[9..^15];
5276
}
5377

‎KustoSchemaTools/Model/MaterializedView.cs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ public class MaterializedView : IKustoBaseEntity
2424
[Obsolete("Use policies instead")]
2525
public string? RowLevelSecurity { get; set; }
2626
public Policy? Policies { get; set; }
27+
28+
public bool Preformatted { get; set; } = false;
2729

2830
public List<DatabaseScriptContainer> CreateScripts(string name, bool isNew)
2931
{
@@ -40,11 +42,11 @@ public List<DatabaseScriptContainer> CreateScripts(string name, bool isNew)
4042
var scripts = new List<DatabaseScriptContainer>();
4143
var properties = string.Join(", ", GetType().GetProperties()
4244
.Where(p => p.GetValue(this) != null && excludedProperties.Contains(p.Name) == false)
43-
.Select(p => new {Name = p.Name, Value = p.GetValue(this) })
45+
.Select(p => new { Name = p.Name, Value = p.GetValue(this) })
4446
.Where(p => !string.IsNullOrWhiteSpace(p.Value?.ToString()))
4547
.Select(p => $"{p.Name}=```{p.Value}```"));
4648

47-
49+
4850
if (asyncSetup)
4951
{
5052
scripts.Add(new DatabaseScriptContainer("CreateMaterializedViewAsync", Kind == "table" ? 40 : 41, $".create async ifnotexists materialized-view with ({properties}) {name} on {Kind} {Source} {{ {Query} }}", true));
@@ -57,7 +59,7 @@ public List<DatabaseScriptContainer> CreateScripts(string name, bool isNew)
5759
{
5860
scripts.AddRange(Policies.CreateScripts(name, "materialized-view"));
5961
}
60-
62+
6163
return scripts;
6264
}
6365
}

‎KustoSchemaTools/Parser/DatabaseCleanup.cs‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,12 @@ public void CleanUp(Database database)
122122
{
123123
policy.HotCache = null;
124124
}
125-
entity.Value.Query = entity.Value.Query.PrettifyKql();
125+
126+
if (entity.Value.Preformatted == false)
127+
{
128+
// format the query unless the materialized view opts out
129+
entity.Value.Query = entity.Value.Query.PrettifyKql();
130+
}
126131
}
127132

128133
foreach(var entity in database.MaterializedViews)
@@ -135,7 +140,12 @@ public void CleanUp(Database database)
135140

136141
foreach (var entity in database.Functions)
137142
{
138-
entity.Value.Body = entity.Value.Body.PrettifyKql();
143+
// format unless the function opts out
144+
// there are known issues with PrettifyKql function.
145+
if (!entity.Value.Preformatted)
146+
{
147+
entity.Value.Body = entity.Value.Body.PrettifyKql();
148+
}
139149
}
140150
foreach (var up in database.Tables.Values.Where(itm => itm.Policies?.UpdatePolicies != null).SelectMany(itm => itm.Policies.UpdatePolicies))
141151
{

0 commit comments

Comments
 (0)