Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix dynamic loading feature to load providers from custom bicepconfig.json aliases #12267

Merged
merged 61 commits into from
Nov 14, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
61 commits
Select commit Hold shift + click to select a range
fea7164
initial commit
Oct 12, 2023
0e2742d
refactor for simplicity
Oct 12, 2023
d587067
Fix Bicep.Cli.IntegrationTests
Oct 12, 2023
490b331
fix tests
Oct 12, 2023
755cd6d
simplify code
Oct 13, 2023
db01b75
fix conditional
Oct 13, 2023
e0f3582
adressed pr comments
Oct 13, 2023
d65557a
adress pr comments
Oct 13, 2023
49f9c83
adress pr comments
Oct 13, 2023
5374f96
fix tests
Oct 15, 2023
ce64dc7
rename moduleReference to reference
Oct 16, 2023
7b6c1ff
move ArtifactType to its own file
Oct 16, 2023
115b453
fix diagnostic to not use the class name
Oct 16, 2023
ee2dd62
Various renames to address pr comments
Oct 16, 2023
7a03b80
add GetArtifactType() fn
Oct 16, 2023
e190cb2
refactor functions to get type as first argument
Oct 16, 2023
89c2997
refactor functions to get type as first argument
Oct 16, 2023
27fd979
rename matcher
Oct 16, 2023
fa45b1a
rename
Oct 16, 2023
3011cd3
address pr comments
Oct 20, 2023
fd93bda
addressing pr comments
Oct 22, 2023
9ca749e
refactor TryGetNamespace signature
Oct 23, 2023
26e1148
more test cases
Oct 23, 2023
3b680cf
Merge remote-tracking branch 'origin/main' into asilverman/refactor/T…
Oct 23, 2023
f2470a0
refaactor T
Oct 23, 2023
4351706
various changes
Oct 30, 2023
a3f4135
various changes
Oct 30, 2023
0dcc38b
merge main
Oct 30, 2023
d8f2288
fix test setup
Oct 30, 2023
4004fea
fix tests
Oct 30, 2023
5b41e8e
fix tests
Nov 1, 2023
751aa77
fix tests
Nov 1, 2023
1aa2838
fix tests
Nov 1, 2023
ffd7fd9
fix tests
Nov 1, 2023
1a11767
revert unintentional rename
Nov 1, 2023
21ab650
fix refactors
Nov 1, 2023
fa2b074
dotnet format
Nov 1, 2023
c053734
revert unintentional rename
Nov 1, 2023
cafd535
fix compilation error
Nov 1, 2023
a04e43a
revert unintended changes
Nov 1, 2023
8f94298
various trims
Nov 1, 2023
52ec607
final touches
Nov 2, 2023
85b3b12
more finishing touches
Nov 2, 2023
f84d465
merge main
Nov 2, 2023
4712de5
fix diagnostic id
Nov 2, 2023
595bff7
make test os agnostic
Nov 2, 2023
eae634c
fix path resolution
Nov 2, 2023
e7d24ce
adress pr comments
Nov 3, 2023
7fa9ffa
fix test
Nov 3, 2023
769dfe6
use a test path
asilverman Nov 6, 2023
d63ed24
move file location to correct ns
Nov 6, 2023
e56322f
remove comment
Nov 6, 2023
524e774
move prodiver descriptor namespace
Nov 7, 2023
e5803cb
move files to correct namespace
Nov 7, 2023
0d0d2f1
rename to address pr comment
Nov 7, 2023
0e22a6e
address pr comments
Nov 7, 2023
18e4ea2
address PR comments
Nov 8, 2023
6a09d29
fixes
Nov 9, 2023
14e083e
adress pr comment
Nov 9, 2023
e1531cf
address pr comments
Nov 9, 2023
24f610b
address PR comments
Nov 14, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
adress pr comments
  • Loading branch information
asilverman committed Nov 3, 2023
commit e7d24ce7e33c2fcbb599f4a73cdc301d793224b6
47 changes: 24 additions & 23 deletions src/Bicep.Core.UnitTests/BicepTestConstants.cs
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,8 @@ public static class BicepTestConstants
public static readonly ITemplateSpecRepositoryFactory TemplateSpecRepositoryFactory = StrictMock.Of<ITemplateSpecRepositoryFactory>().Object;

// Linter rules added to this list will be automtically disabled for most tests.
public static readonly string[] AnalyzerRulesToDisableInTests = Array.Empty<string>();
// use-recent-api-versions is problematic for tests but it's off by default so doesn't need to appear here
public static readonly string[] AnalyzerRulesToDisableInTests = Array.Empty<string>();

public static readonly RootConfiguration BuiltInConfigurationWithAllAnalyzersEnabled = IConfigurationManager.GetBuiltInConfiguration();
public static readonly RootConfiguration BuiltInConfigurationWithAllAnalyzersDisabled = IConfigurationManager.GetBuiltInConfiguration().WithAllAnalyzersDisabled();
Expand Down Expand Up @@ -135,27 +136,27 @@ public static Mock<ITelemetryProvider> CreateMockTelemetryProvider()
}

public static BinaryData BicepProviderManifestWithEmptyTypesLayer = BinaryData.FromString($$"""
{
"schemaVersion": 2,
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"artifactType": "{{BicepMediaTypes.BicepProviderArtifactType}}",
"config": {
"mediaType": "{{BicepMediaTypes.BicepProviderConfigV1}}",
"digest": "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a",
"size": 2
},
"layers": [
{
"mediaType": "{{BicepMediaTypes.BicepProviderArtifactLayerV1TarGzip}}",
"digest": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"size": 0
}
],
"annotations": {
"bicep.serialization.format": "v1",
"org.opencontainers.image.created": "2023-05-04T16:40:05Z"
}
}
""");
{
"schemaVersion": 2,
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"artifactType": "{{BicepMediaTypes.BicepProviderArtifactType}}",
"config": {
"mediaType": "{{BicepMediaTypes.BicepProviderConfigV1}}",
"digest": "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a",
"size": 2
},
"layers": [
{
"mediaType": "{{BicepMediaTypes.BicepProviderArtifactLayerV1TarGzip}}",
"digest": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"size": 0
}
],
"annotations": {
"bicep.serialization.format": "v1",
"org.opencontainers.image.created": "2023-05-04T16:40:05Z"
}
}
""");
}
}
4 changes: 2 additions & 2 deletions src/Bicep.Core/Diagnostics/DiagnosticBuilder.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2110,7 +2110,7 @@ public ErrorDiagnostic RuntimeValueNotAllowedInFunctionDeclaration(string? acces
"BCP380",
$"Artifacts of type: \"{artifactType}\" are not supported."
);

public FixableDiagnostic ProviderDeclarationViaImportKeywordIsDeprecated(ProviderDeclarationSyntax syntax)
{
var codeFix = new CodeFix(
Expand All @@ -2132,7 +2132,7 @@ public FixableDiagnostic ProviderDeclarationViaImportKeywordIsDeprecated(Provide
public ErrorDiagnostic ArtifactFilePathCouldNotBeResolved(string ociManifestPath) => new(
asilverman marked this conversation as resolved.
Show resolved Hide resolved
TextSpan,
"BCP382",
$"The artifact file path could not be resolved from the OCI manifest file: \"{ociManifestPath}\"."
$"The provider package is malformed and could not be loaded from \"{ociManifestPath}\"."
);
}

Expand Down
9 changes: 7 additions & 2 deletions src/Bicep.Core/Registry/OciArtifactRegistry.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
Expand Down Expand Up @@ -88,8 +89,12 @@ public override bool IsArtifactRestoreRequired(OciArtifactReference reference)
{
ArtifactType.Module => !this.FileResolver.FileExists(this.GetArtifactFileUri(reference, ArtifactFileType.ModuleMain)),
ArtifactType.Provider => !this.FileResolver.FileExists(this.GetArtifactFileUri(reference, ArtifactFileType.Provider)),
_ => default // should never happen
_ => throw new UnreachableException()
};

// TODO: Provider artifacts don't write a ModuleMain file, so this code is incorrect.
asilverman marked this conversation as resolved.
Show resolved Hide resolved
// That can be solved by only writing layer data files only (see code under features.PublishSourceEnabled)
// and not main.json directly (https://github.com/Azure/bicep/issues/11900)
return artifactFilesNotFound ||
!this.FileResolver.FileExists(this.GetArtifactFileUri(reference, ArtifactFileType.Manifest)) ||
!this.FileResolver.FileExists(this.GetArtifactFileUri(reference, ArtifactFileType.Metadata));
Expand Down Expand Up @@ -138,7 +143,7 @@ public override ResultWithDiagnostic<Uri> TryGetLocalArtifactEntryPointUri(OciAr
{
ArtifactType.Module => ArtifactFileType.ModuleMain,
ArtifactType.Provider => ArtifactFileType.Provider,
_ => default // should never happen
_ => throw new UnreachableException()
};

var localUri = this.GetArtifactFileUri(reference, artifactFileType);
Expand Down
80 changes: 40 additions & 40 deletions src/Bicep.Core/Semantics/DeclarationVisitor.cs
Original file line number Diff line number Diff line change
Expand Up @@ -162,55 +162,55 @@ public override void VisitAssertDeclarationSyntax(AssertDeclarationSyntax syntax
public override void VisitProviderDeclarationSyntax(ProviderDeclarationSyntax syntax)
{
base.VisitProviderDeclarationSyntax(syntax);
DeclareSymbol(new ProviderNamespaceSymbol(this.context, syntax, ResolveProviderSymbol(syntax)));
}

TypeSymbol resolveProviderSymbol()
private TypeSymbol ResolveProviderSymbol(ProviderDeclarationSyntax syntax)
{
if (!features.ExtensibilityEnabled)
{
if (!features.ExtensibilityEnabled)
{
return ErrorType.Create(DiagnosticBuilder.ForPosition(syntax).ProvidersAreDisabled());
}

// Check for interpolated specification strings
if (syntax.SpecificationString is StringSyntax specificationString && specificationString.IsInterpolated())
{
return ErrorType.Create(DiagnosticBuilder.ForPosition(syntax.SpecificationString).ProviderSpecificationInterpolationUnsupported());
}
return ErrorType.Create(DiagnosticBuilder.ForPosition(syntax).ProvidersAreDisabled());
}

if (!syntax.Specification.IsValid)
{
return (syntax.SpecificationString is StringSyntax)
? ErrorType.Create(DiagnosticBuilder.ForPosition(syntax.Specification).InvalidProviderSpecification())
: ErrorType.Empty();
}
// Check for interpolated specification strings
if (syntax.SpecificationString is StringSyntax specificationString && specificationString.IsInterpolated())
{
return ErrorType.Create(DiagnosticBuilder.ForPosition(syntax.SpecificationString).ProviderSpecificationInterpolationUnsupported());
}

// Check if the MSGraph provider is recognized and enabled
if (syntax.Specification.Name == MicrosoftGraphNamespaceType.BuiltInName && !features.MicrosoftGraphPreviewEnabled)
{
return ErrorType.Create(DiagnosticBuilder.ForPosition(syntax).UnrecognizedProvider(syntax.Specification.Name));
}
if (!syntax.Specification.IsValid)
{
return (syntax.SpecificationString is StringSyntax)
? ErrorType.Create(DiagnosticBuilder.ForPosition(syntax.Specification).InvalidProviderSpecification())
: ErrorType.Empty();
}

Uri? providerUri = null;
if (syntax.Specification.Name == AzNamespaceType.BuiltInName &&
features.DynamicTypeLoadingEnabled &&
!this.artifactFileLookup.TryGetProviderFileUri(syntax).IsSuccess(out providerUri, out var providerUriLookupErrorBuilder))
{
return ErrorType.Create(providerUriLookupErrorBuilder(DiagnosticBuilder.ForPosition(syntax)));
}
// Check if the MSGraph provider is recognized and enabled
if (syntax.Specification.Name == MicrosoftGraphNamespaceType.BuiltInName && !features.MicrosoftGraphPreviewEnabled)
{
return ErrorType.Create(DiagnosticBuilder.ForPosition(syntax).UnrecognizedProvider(syntax.Specification.Name));
}

ResourceTypesProviderDescriptor providerDescriptor = new(
syntax.Specification.Name,
syntax.Alias?.IdentifierName,
providerUri,
syntax.Specification.Version);
Uri? providerUri = null;
if (syntax.Specification.Name == AzNamespaceType.BuiltInName &&
features.DynamicTypeLoadingEnabled &&
!this.artifactFileLookup.TryGetProviderFileUri(syntax).IsSuccess(out providerUri, out var providerUriLookupErrorBuilder))
{
return ErrorType.Create(providerUriLookupErrorBuilder(DiagnosticBuilder.ForPosition(syntax)));
}

if (namespaceProvider.TryGetNamespace(providerDescriptor, targetScope, features, sourceFileKind) is not { } namespaceType)
{
return ErrorType.Create(DiagnosticBuilder.ForPosition(syntax).UnrecognizedProvider(syntax.Specification!.Name));
}
ResourceTypesProviderDescriptor providerDescriptor = new(
syntax.Specification.Name,
syntax.Alias?.IdentifierName,
providerUri,
syntax.Specification.Version);

return namespaceType;
if (namespaceProvider.TryGetNamespace(providerDescriptor, targetScope, features, sourceFileKind) is not { } namespaceType)
{
return ErrorType.Create(DiagnosticBuilder.ForPosition(syntax).UnrecognizedProvider(syntax.Specification!.Name));
}
DeclareSymbol(new ProviderNamespaceSymbol(this.context, syntax, resolveProviderSymbol()));

return namespaceType;
}

public override void VisitParameterAssignmentSyntax(ParameterAssignmentSyntax syntax)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,14 @@ public DefaultNamespaceProvider(IResourceTypeProviderFactory resourceTypeLoaderF
this.resourceTypeLoaderFactory = resourceTypeLoaderFactory;
this.providerLookup = new Dictionary<string, GetNamespaceDelegate>
{
[AzNamespaceType.BuiltInName] = CreateAzNamespace,
[AzNamespaceType.BuiltInName] = TryCreateAzNamespace,
[SystemNamespaceType.BuiltInName] = (providerDescriptor, resourceScope, features, sourceFileKind) => SystemNamespaceType.Create(providerDescriptor.Alias, features, sourceFileKind),
[K8sNamespaceType.BuiltInName] = (providerDescriptor, resourceScope, features, sourceFileKind) => K8sNamespaceType.Create(providerDescriptor.Alias),
[MicrosoftGraphNamespaceType.BuiltInName] = (providerDescriptor, resourceScope, features, sourceFileKind) => MicrosoftGraphNamespaceType.Create(providerDescriptor.Alias),
}.ToImmutableDictionary();
}

private NamespaceType? CreateAzNamespace(ResourceTypesProviderDescriptor providerDescriptor, ResourceScope scope, IFeatureProvider features, BicepSourceFileKind sourceFileKind)
private NamespaceType? TryCreateAzNamespace(ResourceTypesProviderDescriptor providerDescriptor, ResourceScope scope, IFeatureProvider features, BicepSourceFileKind sourceFileKind)
{
if (!features.DynamicTypeLoadingEnabled)
{
Expand Down
6 changes: 3 additions & 3 deletions src/Bicep.Core/TypeSystem/Az/OciTypeLoader.cs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ public class OciTypeLoader : TypeLoader
{
private readonly ImmutableDictionary<string, byte[]> typesCache;
private const string typesArtifactFilename = "types.tgz";
public OciTypeLoader(ImmutableDictionary<string, byte[]> typesCache)
private OciTypeLoader(ImmutableDictionary<string, byte[]> typesCache)
{
this.typesCache = typesCache;
}
Expand All @@ -33,7 +33,7 @@ public static OciTypeLoader FromTgz(string typesFileDir)
if (entry.DataStream is null)
{
var errorMessage = $"Failed to restore {entry.Name} from OCI provider data";
throw new ArgumentException(errorMessage, entry.Name);
throw new InvalidOperationException(errorMessage);
}

using var memoryStream = new MemoryStream();
Expand All @@ -53,7 +53,7 @@ public static OciTypeLoader FromTgz(string typesFileDir)

protected override Stream GetContentStreamAtPath(string path)
{
if (typesCache.TryGetValue($"{path}", out var bytes))
if (typesCache.TryGetValue(path, out var bytes))
{
return new MemoryStream(bytes);
}
Expand Down
25 changes: 14 additions & 11 deletions src/Bicep.Core/TypeSystem/Az/ResourceTypeProviderFactory.cs
Original file line number Diff line number Diff line change
Expand Up @@ -16,29 +16,33 @@

namespace Bicep.Core.TypeSystem
asilverman marked this conversation as resolved.
Show resolved Hide resolved
{

asilverman marked this conversation as resolved.
Show resolved Hide resolved

public class ResourceTypeProviderFactory : IResourceTypeProviderFactory
{
private record Key(string Name, string Version);
asilverman marked this conversation as resolved.
Show resolved Hide resolved

private readonly (string, string) BuiltInAzLoaderKey = ("az", IResourceTypeProvider.BuiltInVersion);
private readonly Dictionary<(string, string), Lazy<IResourceTypeProvider>> resourceTypeLoaders;
private readonly Key BuiltInAzLoaderKey = new("az", IResourceTypeProvider.BuiltInVersion);
private readonly Dictionary<Key, IResourceTypeProvider> resourceTypeLoaders;

public ResourceTypeProviderFactory()
{
resourceTypeLoaders = new() {
{BuiltInAzLoaderKey, new(new AzResourceTypeProvider(new AzResourceTypeLoader(new AzTypeLoader())))},
{BuiltInAzLoaderKey, new AzResourceTypeProvider(new AzResourceTypeLoader(new AzTypeLoader()))},
asilverman marked this conversation as resolved.
Show resolved Hide resolved
};
}

public ResultWithDiagnostic<IResourceTypeProvider> GetResourceTypeProvider(ResourceTypesProviderDescriptor providerDescriptor, IFeatureProvider features)
{
if (!features.DynamicTypeLoadingEnabled)
{
return new(resourceTypeLoaders[BuiltInAzLoaderKey].Value);
return new(resourceTypeLoaders[BuiltInAzLoaderKey]);
}
var key = (providerDescriptor.Alias, providerDescriptor.Version);
var key = new Key(providerDescriptor.Alias, providerDescriptor.Version);

if (resourceTypeLoaders.ContainsKey(key))
{
return new(resourceTypeLoaders[key].Value);
return new(resourceTypeLoaders[key]);
}

// compose the path to the OCI manifest based on the cache root directory and provider version
Expand All @@ -58,20 +62,19 @@ public ResultWithDiagnostic<IResourceTypeProvider> GetResourceTypeProvider(Resou
}

// Register a new types loader
Lazy<IResourceTypeProvider> newResourceTypeLoader = providerDescriptor.Alias switch
IResourceTypeProvider newResourceTypeLoader = providerDescriptor.Alias switch
{
AzNamespaceType.BuiltInName => new(new AzResourceTypeProvider(new AzResourceTypeLoader(OciTypeLoader.FromTgz(providerDescriptor.Path)), providerDescriptor.Version)),
AzNamespaceType.BuiltInName => new AzResourceTypeProvider(new AzResourceTypeLoader(OciTypeLoader.FromTgz(providerDescriptor.Path)), providerDescriptor.Version),
_ => throw new NotImplementedException($"The provider {providerDescriptor.Alias} is not supported."),
};

resourceTypeLoaders[key] = newResourceTypeLoader;
return new(newResourceTypeLoader.Value);
return new(resourceTypeLoaders[key] = newResourceTypeLoader);

}

public IResourceTypeProvider GetBuiltInAzResourceTypesProvider()
asilverman marked this conversation as resolved.
Show resolved Hide resolved
{
return resourceTypeLoaders[BuiltInAzLoaderKey].Value;
return resourceTypeLoaders[BuiltInAzLoaderKey];
}
}
}
1 change: 0 additions & 1 deletion src/Bicep.LangServer/IServiceCollectionExtensions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ public static class IServiceCollectionExtensions
{
public static IServiceCollection AddBicepCore(this IServiceCollection services) => services
.AddSingleton<INamespaceProvider, DefaultNamespaceProvider>()
.AddSingleton<IResourceTypeLoader, AzResourceTypeLoader>()
.AddSingleton<IResourceTypeProviderFactory, ResourceTypeProviderFactory>()
.AddSingleton<IContainerRegistryClientFactory, ContainerRegistryClientFactory>()
.AddSingleton<ITemplateSpecRepositoryFactory, TemplateSpecRepositoryFactory>()
Expand Down
Loading