cqthyy/params integration unit tests (Azure#7352)

* Bicep parameter file array & object unit tests

* created bicepparam sample file for integration test, unit test fixes

* add integration tests first iteration

* comment cleanup

* Bicep param file integration test fixes

* changed SyntaxCollectorVisitor to public helper class, added more test cases for integration test

* error message fix

* one more error message fix

* semantic token test fixes

* integration test baseline update

* additional integration test baseline update

* additional integration test baseline update

* additional integration test baseline update

* additional integration test baseline update

Co-authored-by: Cathy Cao <[email protected]>

.gitattributes

@@ -1,6 +1,6 @@
-*.bicep -text
-*.ts text eol=lf
-*.cs text eol=lf
-*.sh text eol=lf
-/src/Bicep.Core.Samples/Files/**/Assets/**/*.txt -text
-
+*.bicep -text
+*.bicepparam -text
+*.ts text eol=lf
+*.cs text eol=lf
+*.sh text eol=lf
+/src/Bicep.Core.Samples/Files/**/Assets/**/*.txt -text

src/Bicep.Core.IntegrationTests/ParamsParserTests.cs

@@ -0,0 +1,77 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.IO;
+using Bicep.Core.UnitTests.Assertions;
+using Bicep.Core.Parsing;
+using Bicep.Core.Samples;
+using Bicep.Core.UnitTests.Utils;
+using FluentAssertions;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using System.Linq;
+using System;
+
+namespace Bicep.Core.IntegrationTests
+{
+ [TestClass]
+ public class ParamsParserTests
+ {
+ [NotNull]
+ public TestContext? TestContext { get; set; }
+
+ [DataTestMethod]
+ [DynamicData(nameof(GetParamData), DynamicDataSourceType.Method, DynamicDataDisplayNameDeclaringType = typeof(DataSet), DynamicDataDisplayName = nameof(DataSet.GetDisplayName))]
+ [TestCategory(BaselineHelper.BaselineTestCategory)]
+ public void Params_Parser_should_produce_expected_syntax(DataSet dataSet)
+ {
+ if (dataSet.BicepParam is null)
+ {
+ throw new InvalidOperationException($"Expected {nameof(dataSet.BicepParam)} to be non-null");
+ }
+
+ var program = ParamsParserHelper.ParamsParse(dataSet.BicepParam);
+ var syntaxList = SyntaxCollectorVisitorHelper.SyntaxCollectorVisitor.Build(program);
+ var syntaxByParent = syntaxList.ToLookup(x => x.Parent);
+
+ string getLoggingString(SyntaxCollectorVisitorHelper.SyntaxCollectorVisitor.SyntaxItem data)
+ {
+ // Build a visual graph with lines to help understand the syntax hierarchy
+ var graphPrefix = "";
+ if (data.Depth > 0)
+ {
+ var lastSibling = syntaxByParent[data.Parent].Last();
+ var isLast = data.Syntax == lastSibling.Syntax;
+
+ graphPrefix = string.Concat(Enumerable.Repeat("| ", data.Depth - 1));
+ graphPrefix += isLast switch {
+ true => "└─",
+ _ => "├─",
+ };
+ }
+
+ return data.Syntax switch {
+ Token token => $"{graphPrefix}Token({token.Type}) |{OutputHelper.EscapeWhitespace(token.Text)}|",
+ _ => $"{graphPrefix}{data.Syntax.GetType().Name}",
+ };
+ }
+
+ TextSpan getSpan(SyntaxCollectorVisitorHelper.SyntaxCollectorVisitor.SyntaxItem data) => data.Syntax.Span;
+
+ var sourceTextWithDiags = DataSet.AddDiagsToParamSourceText(dataSet, syntaxList, getSpan, getLoggingString);
+ var resultsFile = FileHelper.SaveResultFile(this.TestContext, Path.Combine(dataSet.Name, DataSet.TestFileMainParamSyntax), sourceTextWithDiags);
+
+ sourceTextWithDiags.Should().EqualWithLineByLineDiffOutput(
+ TestContext,
+ dataSet.ParamSyntax ?? throw new InvalidOperationException($"Expected {nameof(dataSet.ParamSyntax)} to be non-null."),
+ expectedLocation: DataSet.GetBaselineUpdatePath(dataSet, DataSet.TestFileMainParamSyntax),
+ actualLocation: resultsFile);
+ }
+
+ private static IEnumerable<object[]> GetParamData()
+ {
+ return DataSets.ParamDataSets.ToDynamicTestData();
+ }
+
+ }
+}

src/Bicep.Core.IntegrationTests/ParserTests.cs

@@ -1,7 +1,6 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 using System.Collections.Generic;
-using System.Collections.Immutable;
 using System.Diagnostics.CodeAnalysis;
 using System.IO;
 using Bicep.Core.Extensions;
@@ -20,39 +19,6 @@ namespace Bicep.Core.IntegrationTests
  [TestClass]
  public class ParserTests
  {
- private class SyntaxCollectorVisitor : SyntaxVisitor
- {
- public record SyntaxItem(SyntaxBase Syntax, SyntaxBase? Parent, int Depth);
-
- private readonly IList<SyntaxItem> syntaxList = new List<SyntaxItem>();
- private SyntaxBase? parent = null;
- private int depth = 0;
-
- private SyntaxCollectorVisitor()
- {
- }
-
- public static ImmutableArray<SyntaxItem> Build(ProgramSyntax syntax)
- {
- var visitor = new SyntaxCollectorVisitor();
- visitor.Visit(syntax);
-
- return visitor.syntaxList.ToImmutableArray();
- }
-
- protected override void VisitInternal(SyntaxBase syntax)
- {
- syntaxList.Add(new(Syntax: syntax, Parent: parent, Depth: depth));
-
- var prevParent = parent;
- parent = syntax;
- depth++;
- base.VisitInternal(syntax);
- depth--;
- parent = prevParent;
- }
- }
-
  [NotNull]
  public TestContext? TestContext { get; set; }
 
@@ -91,10 +57,10 @@ public void Oneliners_ShouldRoundTripSuccessfully(string contents)
  public void Parser_should_produce_expected_syntax(DataSet dataSet)
  {
  var program = ParserHelper.Parse(dataSet.Bicep);
- var syntaxList = SyntaxCollectorVisitor.Build(program);
+ var syntaxList = SyntaxCollectorVisitorHelper.SyntaxCollectorVisitor.Build(program);
  var syntaxByParent = syntaxList.ToLookup(x => x.Parent);
 
- string getLoggingString(SyntaxCollectorVisitor.SyntaxItem data)
+ string getLoggingString(SyntaxCollectorVisitorHelper.SyntaxCollectorVisitor.SyntaxItem data)
  {
  // Build a visual graph with lines to help understand the syntax hierarchy
  var graphPrefix = "";
@@ -116,7 +82,7 @@ string getLoggingString(SyntaxCollectorVisitor.SyntaxItem data)
  };
  }
 
- TextSpan getSpan(SyntaxCollectorVisitor.SyntaxItem data) => data.Syntax.Span;
+ TextSpan getSpan(SyntaxCollectorVisitorHelper.SyntaxCollectorVisitor.SyntaxItem data) => data.Syntax.Span;
 
  var sourceTextWithDiags = DataSet.AddDiagsToSourceText(dataSet, syntaxList, getSpan, getLoggingString);
  var resultsFile = FileHelper.SaveResultFile(this.TestContext, Path.Combine(dataSet.Name, DataSet.TestFileMainSyntax), sourceTextWithDiags);

src/Bicep.Core.IntegrationTests/SyntaxCollectorVisitorHelper.cs

@@ -0,0 +1,44 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using Bicep.Core.Syntax;
+
+namespace Bicep.Core.IntegrationTests
+{
+ public class SyntaxCollectorVisitorHelper
+ {
+ public class SyntaxCollectorVisitor : SyntaxVisitor
+ {
+ public record SyntaxItem(SyntaxBase Syntax, SyntaxBase? Parent, int Depth);
+
+ private readonly IList<SyntaxItem> syntaxList = new List<SyntaxItem>();
+ private SyntaxBase? parent = null;
+ private int depth = 0;
+
+ private SyntaxCollectorVisitor()
+ {
+ }
+
+ public static ImmutableArray<SyntaxItem> Build(ProgramSyntax syntax)
+ {
+ var visitor = new SyntaxCollectorVisitor();
+ visitor.Visit(syntax);
+
+ return visitor.syntaxList.ToImmutableArray();
+ }
+
+ protected override void VisitInternal(SyntaxBase syntax)
+ {
+ syntaxList.Add(new(Syntax: syntax, Parent: parent, Depth: depth));
+
+ var prevParent = parent;
+ parent = syntax;
+ depth++;
+ base.VisitInternal(syntax);
+ depth--;
+ parent = prevParent;
+ }
+ }
+ }
+}

src/Bicep.Core.Samples/Bicep.Core.Samples.csproj

@@ -8,6 +8,7 @@
  <!-- Bicep.Core.Samples -->
  <EmbeddedResource Include="./Files/**/*.json" LogicalName="$([System.String]::new('Bicep.Core.Samples/%(RecursiveDir)%(Filename)%(Extension)').Replace('\', '/'))" />
  <EmbeddedResource Include="./Files/**/*.bicep" LogicalName="$([System.String]::new('Bicep.Core.Samples/%(RecursiveDir)%(Filename)%(Extension)').Replace('\', '/'))" />
+ <EmbeddedResource Include="./Files/**/*.bicepparam" LogicalName="$([System.String]::new('Bicep.Core.Samples/%(RecursiveDir)%(Filename)%(Extension)').Replace('\', '/'))" />
  <EmbeddedResource Include="./Files/**/Assets/**/*" LogicalName="$([System.String]::new('Bicep.Core.Samples/%(RecursiveDir)%(Filename)%(Extension)').Replace('\', '/'))" />
 
  <!-- Path prefix: docs/examples -->

src/Bicep.Core.Samples/DataSet.cs

@@ -20,10 +20,12 @@ namespace Bicep.Core.Samples
  public class DataSet
  {
  public const string TestFileMain = "main.bicep";
+ public const string TestFileMainParam = "main.bicepparam";
  public const string TestFileMainDiagnostics = "main.diagnostics.bicep";
  public const string TestFileMainTokens = "main.tokens.bicep";
  public const string TestFileMainSymbols = "main.symbols.bicep";
  public const string TestFileMainSyntax = "main.syntax.bicep";
+ public const string TestFileMainParamSyntax = "main.syntax.bicepparam";
  public const string TestFileMainFormatted = "main.formatted.bicep";
  public const string TestFileMainCompiled = "main.json";
  public const string TestFileMainCompiledWithSymbolicNames = "main.symbolicnames.json";
@@ -44,6 +46,8 @@ public record ExternalModuleMetadata(string Target);
 
  private readonly Lazy<string> lazyBicep;
 
+ private readonly Lazy<string>? lazyBicepParam;
+
  private readonly Lazy<string> lazyTokens;
 
  private readonly Lazy<string> lazyDiagnostics;
@@ -54,6 +58,8 @@ public record ExternalModuleMetadata(string Target);
 
  private readonly Lazy<string> lazySyntax;
 
+ private readonly Lazy<string>? lazyParamSyntax;
+
  private readonly Lazy<string> lazySymbols;
 
  private readonly Lazy<string> lazyFormatted;
@@ -69,12 +75,14 @@ public DataSet(string name)
  this.Name = name;
 
  this.lazyBicep = this.CreateRequired(TestFileMain);
+ this.lazyBicepParam = this.CreateOptional(TestFileMainParam);
  this.lazyTokens = this.CreateRequired(TestFileMainTokens);
  this.lazyDiagnostics = this.CreateRequired(TestFileMainDiagnostics);
  this.lazyCompiled = this.CreateIffValid(TestFileMainCompiled);
  this.lazyCompiledWithSymbolicNames = this.CreateIffValid(TestFileMainCompiledWithSymbolicNames);
  this.lazySymbols = this.CreateRequired(TestFileMainSymbols);
  this.lazySyntax = this.CreateRequired(TestFileMainSyntax);
+ this.lazyParamSyntax = this.CreateOptional(TestFileMainParamSyntax);
  this.lazyFormatted = this.CreateRequired(TestFileMainFormatted);
  this.lazyCompletions = new(() => ReadDataSetDictionary(GetStreamName(TestCompletionsPrefix)), LazyThreadSafetyMode.PublicationOnly);
  this.lazyModulesToPublish = new(() => ReadPublishData(GetStreamName(TestPublishPrefix)), LazyThreadSafetyMode.PublicationOnly);
@@ -87,6 +95,8 @@ public DataSet(string name)
 
  public string Bicep => this.lazyBicep.Value;
 
+ public string? BicepParam => this.lazyBicepParam?.Value;
+
  public string Tokens => this.lazyTokens.Value;
 
  public string Diagnostics => this.lazyDiagnostics.Value;
@@ -99,6 +109,8 @@ public DataSet(string name)
 
  public string Syntax => this.lazySyntax.Value;
 
+ public string? ParamSyntax => this.lazyParamSyntax?.Value;
+
  public string Formatted => this.lazyFormatted.Value;
 
  public ImmutableDictionary<string, string> Completions => this.lazyCompletions.Value;
@@ -119,13 +131,17 @@ public DataSet(string name)
 
  public bool IsStress => this.Name.Contains("Stress", StringComparison.Ordinal);
 
+ public bool HasParamFile => this.BicepParam is not null;
+
  private Lazy<string> CreateRequired(string fileName)
  {
  return new Lazy<string>(() => this.ReadDataSetFile(fileName), LazyThreadSafetyMode.PublicationOnly);
  }
 
  private Lazy<string>? CreateIffValid(string fileName) => this.IsValid ? this.CreateRequired(fileName) : null;
 
+ private Lazy<string>? CreateOptional(string fileName) => ExistsFile(GetStreamName(fileName)) ? this.CreateRequired(fileName) : null;
+
  public static string GetDisplayName(MethodInfo info, object[] data) => $"{info.Name}_{((DataSet)data[0]).Name}";
 
  private string ReadDataSetFile(string fileName) => ReadFile(GetStreamName(fileName));
@@ -144,6 +160,13 @@ public static string ReadFile(string streamName)
  return reader.ReadToEnd();
  }
 
+ public static bool ExistsFile(string streamName)
+ {
+ using Stream? stream = Assembly.GetExecutingAssembly().GetManifestResourceStream(streamName);
+
+ return stream is not null;
+ }
+
  public static ImmutableDictionary<string, string> ReadDataSetDictionary(string streamNamePrefix)
  {
  var matches = Assembly.GetExecutingAssembly()
@@ -161,6 +184,13 @@ public static string ReadFile(string streamName)
  return builder.ToImmutable();
  }
 
+ public static string AddDiagsToParamSourceText<T>(DataSet dataSet, IEnumerable<T> items, Func<T, TextSpan> getSpanFunc, Func<T, string> diagsFunc)
+ => OutputHelper.AddDiagsToSourceText(dataSet.BicepParam ?? throw new ArgumentException($"{nameof(dataSet.BicepParam)} is null."), dataSet.HasCrLfNewlines() ? "\r\n" : "\n", items, getSpanFunc, diagsFunc);
+
+ public static string AddDiagsToParamSourceText<TPositionable>(DataSet dataSet, IEnumerable<TPositionable> items, Func<TPositionable, string> diagsFunc)
+ where TPositionable : IPositionable
+ => OutputHelper.AddDiagsToSourceText(dataSet.BicepParam ?? throw new ArgumentException($"{nameof(dataSet.BicepParam)} is null."), dataSet.HasCrLfNewlines() ? "\r\n" : "\n", items, item => item.Span, diagsFunc);
+
  public static string AddDiagsToSourceText<T>(DataSet dataSet, IEnumerable<T> items, Func<T, TextSpan> getSpanFunc, Func<T, string> diagsFunc)
  => OutputHelper.AddDiagsToSourceText<T>(dataSet.Bicep, dataSet.HasCrLfNewlines() ? "\r\n" : "\n", items, getSpanFunc, diagsFunc);
 

src/Bicep.Core.Samples/DataSets.cs

@@ -104,6 +104,8 @@ public static class DataSets
 
  public static IEnumerable<DataSet> NonStressDataSets => AllDataSets.Where(ds => !ds.IsStress);
 
+ public static IEnumerable<DataSet> ParamDataSets => AllDataSets.Where(ds => ds.HasParamFile);
+
  public static ImmutableDictionary<string, string> Completions => DataSet.ReadDataSetDictionary($"{DataSet.Prefix}{DataSet.TestCompletionsPrefix}");
 
  public static ImmutableDictionary<string, string> Functions => DataSet.ReadDataSetDictionary($"{DataSet.Prefix}{DataSet.TestFunctionsPrefix}");