Task 26201472: Adding negative test cases for dynamic type loading (A…

…zure#12903)

## Overview

Adding test cases to cover the following scenarios:

- Legacy declaration for `az` provider should yield a diagnostic and is
no longer valid
- Specifying a module artifact address in a provider declaration syntax
should yield a diagnostic
- A diagnostic should be emitted when one of the following is true for
the OCI registry:
  - The registry host is unreachable 
  - The repository is not found
  - The version for the artifact is not found
###### Microsoft Reviewers: [Open in
CodeFlow](https://microsoft.github.io/open-pr/?codeflow=https://github.com/Azure/bicep/pull/12903)

---------

Co-authored-by: asilverman <[email protected]>
Co-authored-by: Anthony Martin <[email protected]>

src/Bicep.Core.IntegrationTests/DynamicAzTypesTests.cs

@@ -1,27 +1,23 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 using System;
-using System.Diagnostics.CodeAnalysis;
+using System.Collections.Generic;
 using System.IO;
-using System.IO.Abstractions.TestingHelpers;
-using System.Text.Json;
+using System.Linq;
+using System.Threading;
 using System.Threading.Tasks;
-using Azure.Bicep.Types.Az;
+using Azure;
 using Bicep.Core.Configuration;
 using Bicep.Core.Diagnostics;
-using Bicep.Core.IntegrationTests.Extensibility;
-using Bicep.Core.Registry;
 using Bicep.Core.Samples;
-using Bicep.Core.Semantics.Namespaces;
-using Bicep.Core.TypeSystem.Providers;
-using Bicep.Core.TypeSystem.Providers.Az;
 using Bicep.Core.UnitTests;
 using Bicep.Core.UnitTests.Assertions;
-using Bicep.Core.UnitTests.FileSystem;
+using Bicep.Core.UnitTests.Mock;
+using Bicep.Core.UnitTests.Registry;
 using Bicep.Core.UnitTests.Utils;
 using FluentAssertions;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
-using Newtonsoft.Json.Linq;
+using Moq;
 
 namespace Bicep.Core.IntegrationTests
 {
@@ -49,8 +45,8 @@ public async Task Az_namespace_can_be_used_without_configuration()
  {
  var services = await GetServices();
  var result = await CompilationHelper.RestoreAndCompile(services, ("main.bicep", @$"
-provider 'br/public:az@{BicepTestConstants.BuiltinAzProviderVersion}'
-"));
+ provider 'br/public:az@{BicepTestConstants.BuiltinAzProviderVersion}'
+ "));
 
  result.Should().GenerateATemplate();
  result.ExcludingLinterDiagnostics().Should().NotHaveAnyDiagnostics();
@@ -61,8 +57,8 @@ public async Task Az_namespace_errors_with_configuration()
  {
  var services = await GetServices();
  var result = await CompilationHelper.RestoreAndCompile(services, @$"
-provider 'br/public:az@{BicepTestConstants.BuiltinAzProviderVersion}' with {{}}
-");
+ provider 'br/public:az@{BicepTestConstants.BuiltinAzProviderVersion}' with {{}}
+ ");
 
  result.Should().NotGenerateATemplate();
  result.ExcludingLinterDiagnostics().Should().HaveDiagnostics(new[] {
@@ -87,14 +83,16 @@ public async Task Az_namespace_can_be_used_with_bicepconfig_provider_alias()
  {
  var services = await GetServices();
 
- services = services.WithConfigurationPatch(c => c.WithProviderAlias(@"{
-  ""br"": {
-  ""customAlias"": {
-  ""registry"": ""mcr.microsoft.com"",
-  ""providerPath"": ""bicep/providers""
- }
+ services = services.WithConfigurationPatch(c => c.WithProviderAlias($$"""
+ {
+ "br": {
+ "customAlias": {
+ "registry": "{{LanguageConstants.BicepPublicMcrRegistry}}",
+ "providerPath": "bicep/providers"
  }
- }"));
+ }
+ }
+ """));
 
  var result = await CompilationHelper.RestoreAndCompile(services, @$"
  provider 'br/customAlias:az@{BicepTestConstants.BuiltinAzProviderVersion}'
@@ -104,5 +102,124 @@ public async Task Az_namespace_can_be_used_with_bicepconfig_provider_alias()
  result.Compilation.GetEntrypointSemanticModel().Root.ProviderDeclarations.Should().Contain(x => x.Name.Equals("az"));
  }
 
+ [TestMethod]
+ public async Task Az_namespace_specified_using_legacy_declaration_syntax_yields_diagnostic()
+ {
+ var services = new ServiceBuilder()
+ .WithFeatureOverrides(new(ExtensibilityEnabled: true, DynamicTypeLoadingEnabled: true));
+
+ var result = await CompilationHelper.RestoreAndCompile(services, @"
+ provider '[email protected]'
+ ");
+
+ result.Should().NotGenerateATemplate();
+ result.Should().HaveDiagnostics(
+ new[] {
+ ("BCP304", DiagnosticLevel.Error, "Invalid provider specifier string. Specify a valid provider of format \"<providerName>@<providerVersion>\"."),
+ });
+
+ }
+
+ [TestMethod]
+ public async Task Bicep_module_artifact_specified_in_provider_declaration_syntax_yields_diagnostic()
+ {
+ var testArtifact = new ArtifactRegistryAddress(LanguageConstants.BicepPublicMcrRegistry, "bicep/providers/az", "0.2.661");
+ var clientFactory = DataSetsExtensions.CreateMockRegistryClients(
+ (new Uri($"https://{testArtifact.RegistryAddress}"), testArtifact.RepositoryPath)).factoryMock;
+ await DataSetsExtensions.PublishModuleToRegistryAsync(
+ clientFactory,
+ moduleName: "az",
+ target: testArtifact.ToSpecificationString(':'),
+ moduleSource: "",
+ publishSource: false,
+ documentationUri: "mydocs.org/abc");
+
+ var services = new ServiceBuilder()
+ .WithFeatureOverrides(new(ExtensibilityEnabled: true, DynamicTypeLoadingEnabled: true))
+ .WithContainerRegistryClientFactory(clientFactory);
+
+ // ACT
+ var result = await CompilationHelper.RestoreAndCompile(services, @$"
+ provider '{testArtifact.ToSpecificationString('@')}'
+ ");
+
+ // ASSERT
+ result.Should().NotGenerateATemplate();
+ result.Should().HaveDiagnostics(
+ new[] {
+ ("BCP190", DiagnosticLevel.Error, @$"The artifact with reference ""{testArtifact.ToSpecificationString(':')}"" has not been restored."),
+ ("BCP084", DiagnosticLevel.Error, "The symbolic name \"az\" is reserved. Please use a different symbolic name. Reserved namespaces are \"az\", \"sys\".")
+ });
+ }
+
+ public record ArtifactRegistryAddress(string RegistryAddress, string RepositoryPath, string ProviderVersion)
+ {
+ public string ToSpecificationString(char delim) => $"br:{RegistryAddress}/{RepositoryPath}{delim}{ProviderVersion}";
+ }
+
+ [TestMethod]
+ [DynamicData(nameof(ArtifactRegistryAddressNegativeTestScenarios), DynamicDataSourceType.Method)]
+ public async Task Repository_not_found_in_registry(
+ ArtifactRegistryAddress artifactRegistryAddress,
+ Exception exceptionToThrow,
+ IEnumerable<(string, DiagnosticLevel, string)> expectedDiagnostics)
+ {
+ // ARRANGE
+ // mock the blob client to throw the expected exception
+ var mockBlobClient = StrictMock.Of<MockRegistryBlobClient>();
+ mockBlobClient.Setup(m => m.GetManifestAsync(It.IsAny<string>(), It.IsAny<CancellationToken>())).ThrowsAsync(exceptionToThrow);
+
+ // mock the registry client to return the mock blob client
+ var containerRegistryFactoryBuilder = new TestContainerRegistryClientFactoryBuilder();
+ containerRegistryFactoryBuilder.RegisterMockRepositoryBlobClient(
+ new Uri($"https://{artifactRegistryAddress.RegistryAddress}"), artifactRegistryAddress.RepositoryPath, mockBlobClient.Object);
+
+ var (clientFactory, _) = containerRegistryFactoryBuilder.Build();
+
+ var services = new ServiceBuilder()
+ .WithFeatureOverrides(new(ExtensibilityEnabled: true, DynamicTypeLoadingEnabled: true))
+ .WithContainerRegistryClientFactory(clientFactory);
+
+ // ACT
+ var result = await CompilationHelper.RestoreAndCompile(services, @$"
+ provider '{artifactRegistryAddress.ToSpecificationString('@')}'
+ ");
+
+ // ASSERT
+ result.Should().NotGenerateATemplate();
+ result.Should().HaveDiagnostics(expectedDiagnostics);
+ }
+
+ public static IEnumerable<object[]> ArtifactRegistryAddressNegativeTestScenarios()
+ {
+ // constants
+ const string placeholderProviderVersion = "0.0.0-placeholder";
+
+ // unresolvable host registry. For example if DNS is down or unresponsive
+ const string unreachableRegistryAddress = "unknown.registry.azurecr.io";
+ const string NoSuchHostMessage = $" (No such host is known. ({unreachableRegistryAddress}:443))";
+ var AggregateExceptionMessage = $"Retry failed after 4 tries. Retry settings can be adjusted in ClientOptions.Retry or by configuring a custom retry policy in ClientOptions.RetryPolicy.{string.Concat(Enumerable.Repeat(NoSuchHostMessage, 4))}";
+ var unreacheable = new ArtifactRegistryAddress(unreachableRegistryAddress, "bicep/providers/az", placeholderProviderVersion);
+ yield return new object[] {
+ unreacheable,
+ new AggregateException(AggregateExceptionMessage),
+ new (string, DiagnosticLevel, string)[]{
+ new ("BCP192", DiagnosticLevel.Error, @$"Unable to restore the artifact with reference ""{unreacheable.ToSpecificationString(':')}"": {AggregateExceptionMessage}"),
+ ("BCP084", DiagnosticLevel.Error, "The symbolic name \"az\" is reserved. Please use a different symbolic name. Reserved namespaces are \"az\", \"sys\".")
+ },
+ };
+
+ // manifest not found is thrown when the repository address is not registered and/or the version doesn't exist in the registry
+ const string NotFoundMessage = "The artifact does not exist in the registry.";
+ var withoutRepo = new ArtifactRegistryAddress(LanguageConstants.BicepPublicMcrRegistry, "unknown/path/az", placeholderProviderVersion);
+ yield return new object[] {
+ withoutRepo,
+ new RequestFailedException(404, NotFoundMessage),
+ new (string, DiagnosticLevel, string)[]{
+ new ("BCP192", DiagnosticLevel.Error, $@"Unable to restore the artifact with reference ""{withoutRepo.ToSpecificationString(':')}"": {NotFoundMessage}"),
+ ("BCP084", DiagnosticLevel.Error, "The symbolic name \"az\" is reserved. Please use a different symbolic name. Reserved namespaces are \"az\", \"sys\".")
+ },
+ };
+ }
  }
 }

src/Bicep.Core.IntegrationTests/RegistryTests.cs

@@ -159,6 +159,7 @@ public async Task InvalidRootCachePathShouldProduceReasonableErrors()
  }
 
  [TestMethod]
+ [DoNotParallelize()]
  public async Task ModuleRestoreContentionShouldProduceConsistentState()
  {
  var dataSet = DataSets.Registry_LF;

src/Bicep.Core.UnitTests/Utils/TestContainerRegistryClientFactoryBuilder.cs

@@ -22,6 +22,11 @@ public void RegisterMockRepositoryBlobClient(Uri registryUri, string repository)
  clientsBuilder.TryAdd((registryUri, repository), new MockRegistryBlobClient());
  }
 
+ public void RegisterMockRepositoryBlobClient(Uri registryUri, string repository, MockRegistryBlobClient client)
+ {
+ clientsBuilder.TryAdd((registryUri, repository), client);
+ }
+
  public (IContainerRegistryClientFactory mockContrainerRegistryClientFactory, ImmutableDictionary<(Uri, string), MockRegistryBlobClient> blobClientMocks) Build()
  {
  var repoToClient = clientsBuilder.ToImmutable();