From 2e6938dc3315f64eee9737b241ee583a83cedc3a Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 30 Apr 2024 08:07:16 -0700 Subject: [PATCH] feat: Translation records (#5346) --- .../ClientApp/GroceryListPrinter.dfy.expect | 1 + Source/DafnyCore/AST/CompilationData.cs | 3 + Source/DafnyCore/AST/Members/Method.cs | 3 +- .../DafnyCore/AST/Members/MethodOrFunction.cs | 3 +- .../AST/Statements/Methods/PrintStmt.cs | 5 +- .../Backends/DafnyExecutableBackend.cs | 4 +- .../DafnyCore/Backends/ExecutableBackend.cs | 68 ++++++- .../Backends/Library/LibraryBackend.cs | 2 +- Source/DafnyCore/DooFile.cs | 102 +---------- Source/DafnyCore/Options/BoogieOptionBag.cs | 7 +- Source/DafnyCore/Options/CommonOptionBag.cs | 17 +- Source/DafnyCore/Options/DafnyCommands.cs | 8 +- .../DafnyCore/Options/OptionCompatibility.cs | 87 +++++++++ Source/DafnyCore/Options/TranslationRecord.cs | 172 ++++++++++++++++++ .../DafnyCore/Pipeline/NoExecutableBackend.cs | 2 +- .../DafnyCore/Plugins/IExecutableBackend.cs | 26 ++- .../DafnyDriver/Commands/TranslateCommand.cs | 3 +- Source/DafnyDriver/Commands/VerifyCommand.cs | 2 + .../Legacy/SynchronousCliCompilation.cs | 14 +- .../CompileAndThenRun.dfy.expect | 157 ++++++++-------- .../LitTest/comp/compile3/JustRun.dfy.expect | 156 ++++++++-------- .../manualcompile/ManualCompile.dfy.expect | 158 ++++++++-------- .../translation-records/Bar.dtr | 4 + .../translation-records/DuplicateModules.dfy | 11 ++ .../DuplicateModules.dfy.expect | 10 + .../translation-records/DuplicateModules.dtr | 3 + .../translation-records/Foo.dtr | 4 + .../translation-records/InvalidFormat.dfy | 7 + .../InvalidFormat.dfy.expect | 8 + .../translation-records/NoGood.dtr | 1 + .../translation-records/WrongDafnyVersion.dtr | 2 + .../separate-compilation/usesTimesTwo.dfy | 8 +- docs/DafnyRef/UserGuide.md | 29 ++- docs/dev/news/5346.feat | 4 + 34 files changed, 721 insertions(+), 370 deletions(-) create mode 100644 Source/DafnyCore/Options/OptionCompatibility.cs create mode 100644 Source/DafnyCore/Options/TranslationRecord.cs create mode 100644 Source/IntegrationTests/TestFiles/LitTests/LitTest/comp/separate-compilation/translation-records/Bar.dtr create mode 100644 Source/IntegrationTests/TestFiles/LitTests/LitTest/comp/separate-compilation/translation-records/DuplicateModules.dfy create mode 100644 Source/IntegrationTests/TestFiles/LitTests/LitTest/comp/separate-compilation/translation-records/DuplicateModules.dfy.expect create mode 100644 Source/IntegrationTests/TestFiles/LitTests/LitTest/comp/separate-compilation/translation-records/DuplicateModules.dtr create mode 100644 Source/IntegrationTests/TestFiles/LitTests/LitTest/comp/separate-compilation/translation-records/Foo.dtr create mode 100644 Source/IntegrationTests/TestFiles/LitTests/LitTest/comp/separate-compilation/translation-records/InvalidFormat.dfy create mode 100644 Source/IntegrationTests/TestFiles/LitTests/LitTest/comp/separate-compilation/translation-records/InvalidFormat.dfy.expect create mode 100644 Source/IntegrationTests/TestFiles/LitTests/LitTest/comp/separate-compilation/translation-records/NoGood.dtr create mode 100644 Source/IntegrationTests/TestFiles/LitTests/LitTest/comp/separate-compilation/translation-records/WrongDafnyVersion.dtr create mode 100644 docs/dev/news/5346.feat diff --git a/Source/AutoExtern.Test/Tutorial/ClientApp/GroceryListPrinter.dfy.expect b/Source/AutoExtern.Test/Tutorial/ClientApp/GroceryListPrinter.dfy.expect index 03a358a4c2b..44366de2ee0 100644 --- a/Source/AutoExtern.Test/Tutorial/ClientApp/GroceryListPrinter.dfy.expect +++ b/Source/AutoExtern.Test/Tutorial/ClientApp/GroceryListPrinter.dfy.expect @@ -2,6 +2,7 @@ Warning: this way of using the CLI is deprecated. Use 'dafny --help' to see help Dafny program verifier did not attempt verification Wrote textual form of target program to GroceryListPrinter.cs +Additional output written to GroceryListPrinter-cs.dtr # Printing a grocery list (2 items in the list) ## Apple: 3 @ 1.00 diff --git a/Source/DafnyCore/AST/CompilationData.cs b/Source/DafnyCore/AST/CompilationData.cs index 45f9c30a283..ce827a7c0d9 100644 --- a/Source/DafnyCore/AST/CompilationData.cs +++ b/Source/DafnyCore/AST/CompilationData.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using DafnyCore.Options; namespace Microsoft.Dafny; @@ -28,4 +29,6 @@ public class CompilationData { // TODO move to DocumentAfterParsing once that's used by the CLI [FilledInDuringResolution] public ISet UrisToCompile; + + public TranslationRecord AlreadyTranslatedRecord { get; set; } } \ No newline at end of file diff --git a/Source/DafnyCore/AST/Members/Method.cs b/Source/DafnyCore/AST/Members/Method.cs index f99df5613f6..1d615e908a7 100644 --- a/Source/DafnyCore/AST/Members/Method.cs +++ b/Source/DafnyCore/AST/Members/Method.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Numerics; using DafnyCore; +using DafnyCore.Options; using Microsoft.Dafny.Auditor; using OmniSharp.Extensions.LanguageServer.Protocol.Models; @@ -20,7 +21,7 @@ public class Method : MethodOrFunction, TypeParameter.ParentType, DafnyOptions.RegisterLegacyUi(ReadsClausesOnMethods, DafnyOptions.ParseBoolean, "Language feature selection", "readsClausesOnMethods", @" 0 (default) - Reads clauses on methods are forbidden. 1 - Reads clauses on methods are permitted (with a default of 'reads *').".TrimStart(), defaultValue: false); - DooFile.RegisterLibraryCheck(ReadsClausesOnMethods, DooFile.CheckOptionLocalImpliesLibrary); + DooFile.RegisterLibraryCheck(ReadsClausesOnMethods, OptionCompatibility.CheckOptionLocalImpliesLibrary); } public override IEnumerable Children => new Node[] { Body, Decreases }.Where(x => x != null). diff --git a/Source/DafnyCore/AST/Members/MethodOrFunction.cs b/Source/DafnyCore/AST/Members/MethodOrFunction.cs index 74d375af2b3..76e791c6b8b 100644 --- a/Source/DafnyCore/AST/Members/MethodOrFunction.cs +++ b/Source/DafnyCore/AST/Members/MethodOrFunction.cs @@ -2,6 +2,7 @@ using System.CommandLine; using System.Linq; using DafnyCore; +using DafnyCore.Options; namespace Microsoft.Dafny; @@ -10,7 +11,7 @@ public abstract class MethodOrFunction : MemberDecl { "Allow exporting callables with preconditions, and importing callables with postconditions"); static MethodOrFunction() { - DooFile.RegisterLibraryCheck(AllowExternalContracts, DooFile.CheckOptionLibraryImpliesLocal); + DooFile.RegisterLibraryCheck(AllowExternalContracts, OptionCompatibility.CheckOptionLibraryImpliesLocal); } public readonly List TypeArgs; diff --git a/Source/DafnyCore/AST/Statements/Methods/PrintStmt.cs b/Source/DafnyCore/AST/Statements/Methods/PrintStmt.cs index a12311e6521..cf08c54bfa6 100644 --- a/Source/DafnyCore/AST/Statements/Methods/PrintStmt.cs +++ b/Source/DafnyCore/AST/Statements/Methods/PrintStmt.cs @@ -3,6 +3,7 @@ using System.Diagnostics.Contracts; using System.Linq; using DafnyCore; +using DafnyCore.Options; namespace Microsoft.Dafny; @@ -17,8 +18,8 @@ public class PrintStmt : Statement, ICloneable, ICanFormat { }); DooFile.RegisterLibraryChecks( - checks: new Dictionary { - { TrackPrintEffectsOption, DooFile.CheckOptionLocalImpliesLibrary } + checks: new Dictionary { + { TrackPrintEffectsOption, OptionCompatibility.CheckOptionLocalImpliesLibrary } }); } diff --git a/Source/DafnyCore/Backends/DafnyExecutableBackend.cs b/Source/DafnyCore/Backends/DafnyExecutableBackend.cs index 18c955848cd..baa2446c162 100644 --- a/Source/DafnyCore/Backends/DafnyExecutableBackend.cs +++ b/Source/DafnyCore/Backends/DafnyExecutableBackend.cs @@ -31,9 +31,11 @@ public abstract class DafnyExecutableBackend : ExecutableBackend { DafnyCodeGenerator = CreateDafnyWrittenCompiler(); } - public override void Compile(Program dafnyProgram, ConcreteSyntaxTree output) { + public override void Compile(Program dafnyProgram, string dafnyProgramName, ConcreteSyntaxTree output) { + ProcessTranslationRecords(dafnyProgram, dafnyProgramName, output); CheckInstantiationReplaceableModules(dafnyProgram); ProcessOuterModules(dafnyProgram); + ((DafnyCodeGenerator)codeGenerator).Start(); codeGenerator.Compile(dafnyProgram, new ConcreteSyntaxTree()); var dast = ((DafnyCodeGenerator)codeGenerator).Build(); diff --git a/Source/DafnyCore/Backends/ExecutableBackend.cs b/Source/DafnyCore/Backends/ExecutableBackend.cs index cd4c50a8a1a..0c6ce2785ba 100644 --- a/Source/DafnyCore/Backends/ExecutableBackend.cs +++ b/Source/DafnyCore/Backends/ExecutableBackend.cs @@ -6,6 +6,7 @@ using System.IO; using System.Linq; using System.Threading.Tasks; +using DafnyCore.Options; using Microsoft.Dafny.Compilers; using Microsoft.Dafny.Plugins; @@ -25,12 +26,33 @@ public abstract class ExecutableBackend : IExecutableBackend { public override string ModuleSeparator => CodeGenerator.ModuleSeparator; - public override void Compile(Program dafnyProgram, ConcreteSyntaxTree output) { + public override void Compile(Program dafnyProgram, string dafnyProgramName, ConcreteSyntaxTree output) { + ProcessTranslationRecords(dafnyProgram, dafnyProgramName, output); CheckInstantiationReplaceableModules(dafnyProgram); ProcessOuterModules(dafnyProgram); + CodeGenerator.Compile(dafnyProgram, output); } + protected void ProcessTranslationRecords(Program dafnyProgram, string dafnyProgramName, ConcreteSyntaxTree output) { + // Process --translation-record options, since translation may need that data to translate correctly. + dafnyProgram.Compilation.AlreadyTranslatedRecord = TranslationRecord.Empty(dafnyProgram); + var records = dafnyProgram.Options.Get(TranslationRecords); + if (records != null) { + foreach (var path in records) { + TranslationRecord.ReadValidateAndMerge(dafnyProgram, path.FullName, Token.Cli); + } + } + + // Write out the translation record for THIS translation before compiling, + // in case the compilation process mutates the program. + var translationRecord = new TranslationRecord(dafnyProgram); + var baseName = Path.GetFileNameWithoutExtension(dafnyProgramName); + var dtrFilePath = dafnyProgram.Options.Get(TranslationRecordOutput)?.FullName ?? $"{baseName}-{TargetId}.dtr"; + var dtrWriter = output.NewFile(dtrFilePath); + translationRecord.Write(dtrWriter); + } + protected void CheckInstantiationReplaceableModules(Program dafnyProgram) { foreach (var compiledModule in dafnyProgram.Modules()) { if (compiledModule.Implements is { Kind: ImplementationKind.Replacement }) { @@ -50,17 +72,27 @@ public abstract class ExecutableBackend : IExecutableBackend { } protected void ProcessOuterModules(Program dafnyProgram) { - var outerModules = GetOuterModules(); - ModuleDefinition rootUserModule = null; - foreach (var outerModule in outerModules) { - var newRoot = new ModuleDefinition(RangeToken.NoToken, new Name(outerModule), new List(), - ModuleKindEnum.Concrete, false, - null, null, null); - newRoot.EnclosingModule = rootUserModule; - rootUserModule = newRoot; + // Apply the --outer-module option from any translation records for libraries, + // but only to top-level modules. + var outerModules = new Dictionary(); + foreach (var module in dafnyProgram.CompileModules) { + if (module.EnclosingModule is not DefaultModuleDefinition) { + continue; + } + + var recordedOuterModuleName = (string)dafnyProgram.Compilation.AlreadyTranslatedRecord.Get(null, module.FullDafnyName, OuterModule); + if (recordedOuterModuleName == null) { + continue; + } + + var outerModule = outerModules.GetOrCreate(recordedOuterModuleName, () => CreateOuterModule(recordedOuterModuleName)); + module.EnclosingModule = outerModule; } - if (rootUserModule != null) { + // Apply the local --output-module option if there is one + var outerModuleName = Options.Get(OuterModule); + if (outerModuleName != null) { + var rootUserModule = outerModules.GetOrCreate(outerModuleName, () => CreateOuterModule(outerModuleName)); dafnyProgram.DefaultModuleDef.NameNode = rootUserModule.NameNode; dafnyProgram.DefaultModuleDef.EnclosingModule = rootUserModule.EnclosingModule; } @@ -70,6 +102,22 @@ public abstract class ExecutableBackend : IExecutableBackend { } } + private static ModuleDefinition CreateOuterModule(string moduleName) { + var outerModules = moduleName.Split("."); + + ModuleDefinition module = null; + foreach (var outerModule in outerModules) { + var thisModule = new ModuleDefinition(RangeToken.NoToken, new Name(outerModule), new List(), + ModuleKindEnum.Concrete, false, + null, null, null) { + EnclosingModule = module + }; + module = thisModule; + } + + return module; + } + public override void OnPreCompile(ErrorReporter reporter, ReadOnlyCollection otherFileNames) { base.OnPreCompile(reporter, otherFileNames); codeGenerator = CreateCodeGenerator(); diff --git a/Source/DafnyCore/Backends/Library/LibraryBackend.cs b/Source/DafnyCore/Backends/Library/LibraryBackend.cs index 5f125bab805..185a7f0678e 100644 --- a/Source/DafnyCore/Backends/Library/LibraryBackend.cs +++ b/Source/DafnyCore/Backends/Library/LibraryBackend.cs @@ -52,7 +52,7 @@ public class LibraryBackend : ExecutableBackend { throw new NotSupportedException(); } - public override void Compile(Program dafnyProgram, ConcreteSyntaxTree output) { + public override void Compile(Program dafnyProgram, string dafnyProgramName, ConcreteSyntaxTree output) { if (!Options.UsingNewCli) { throw new UnsupportedFeatureException(dafnyProgram.GetStartOfFirstFileToken(), Feature.LegacyCLI); } diff --git a/Source/DafnyCore/DooFile.cs b/Source/DafnyCore/DooFile.cs index cc49b4ed32d..9d7bddf4df0 100644 --- a/Source/DafnyCore/DooFile.cs +++ b/Source/DafnyCore/DooFile.cs @@ -8,6 +8,7 @@ using System.Text; using System.Threading.Tasks; using DafnyCore.Generic; +using DafnyCore.Options; using Microsoft.Dafny; using Microsoft.Dafny.LanguageServer.IntegrationTest.Util; using Tomlyn; @@ -48,7 +49,7 @@ public class ManifestData { Options = new Dictionary(); foreach (var (option, _) in OptionChecks) { - var optionValue = GetOptionValue(options, option); + var optionValue = options.Get((dynamic)option); Options.Add(option.Name, optionValue); } } @@ -173,7 +174,8 @@ public class ManifestData { } result.Options.OptionArguments[option] = libraryValue; - success = success && check(reporter, origin, option, localValue, options.GetPrintPath(libraryFile), libraryValue); + var prefix = $"cannot load {options.GetPrintPath(libraryFile)}"; + success = success && check(reporter, origin, prefix, option, localValue, libraryValue); } if (!success) { @@ -183,25 +185,6 @@ public class ManifestData { return result; } - private static object GetOptionValue(DafnyOptions options, Option option) { - // This is annoyingly necessary because only DafnyOptions.Get(Option option) - // handles falling back to the configured default option value, - // whereas the non-generic DafnyOptions.Get(Option option) doesn't. - // TODO: Move somewhere more generic if this is useful in other cases? - var optionType = option.ValueType; - if (optionType == typeof(bool)) { - return options.Get((Option)option); - } - if (optionType == typeof(string)) { - return options.Get((Option)option); - } - if (optionType == typeof(IEnumerable)) { - return options.Get((Option>)option); - } - - throw new ArgumentException(); - } - public void Write(ConcreteSyntaxTree wr) { var manifestWr = wr.NewFile(ManifestFileEntry); using var manifestWriter = new StringWriter(); @@ -237,88 +220,17 @@ public class ManifestData { // more difficult to completely categorize, which is the main reason the LibraryBackend // is restricted to only the new CLI. - public delegate bool OptionCheck(ErrorReporter reporter, IToken origin, Option option, object localValue, string libraryFile, object libraryValue); - private static readonly Dictionary OptionChecks = new(); + private static readonly Dictionary OptionChecks = new(); private static readonly HashSet