[UWP] Remove image comparison from unit tests

source/uwp/UWPTestLibrary/TestResultViewModel.cs

@@ -199,6 +199,8 @@ public TestStatus Status
  }
  }
 
+ answer.Status.Error = answer.TestResult.Error;
+
  // If both had error, compare via the error
  if (answer.ExpectedError != null && answer.TestResult.Error != null)
  {
@@ -463,6 +465,8 @@ public class TestStatus
  /// <summary> This is a new card</summary>
  public bool NewCard { get; set; } = false;
 
+ public string Error;
+
  /// <summary> Set the status to a passing result</summary>
  public void SetToPassingStatus(bool matchedViaError)
  {
@@ -488,7 +492,14 @@ public override string ToString()
  }
  else if (NewCard == true)
  {
- return "New Card Added";
+ if (Error == null)
+ {
+ return "New Card Added";
+ }
+ else
+ {
+ return "Error in new card";
+ }
  }
  else if (!OriginalMatched && ((!ImageMatched || !JsonRoundTripMatched) && !MatchedViaError))
  {

source/uwp/UWPUnitTests/UnitTest.cs

@@ -113,72 +113,58 @@ public async Task TestCardInDispatcher(FileViewModel hostConfig, FileViewModel c
 
  async public Task TestCard(FileViewModel hostConfigFile, FileViewModel cardFile)
  {
- uint reruns = 0;
- TestResultViewModel result = null;
- bool retryImage = true;
- bool testPass = false;
+ var renderResult = await UWPTestLibrary.RenderTestHelpers.RenderCard(cardFile, hostConfigFile);
 
- while (retryImage)
+ if (renderResult.Tree != null)
  {
- var renderResult = await UWPTestLibrary.RenderTestHelpers.RenderCard(cardFile, hostConfigFile);
+ UWPTestLibrary.ImageWaiter imageWaiter = new ImageWaiter(renderResult.Tree);
 
- if (renderResult.Tree != null)
- {
- UWPTestLibrary.ImageWaiter imageWaiter = new ImageWaiter(renderResult.Tree);
-
- StackPanel stackPanel = new StackPanel();
- stackPanel.Children.Add(renderResult.Tree);
-
- Border border = new Border();
- border.Width = renderResult.CardWidth;
- border.Child = stackPanel;
-
- ScrollViewer scrollViewer = new ScrollViewer();
- scrollViewer.Content = border;
+ StackPanel stackPanel = new StackPanel();
+ stackPanel.Children.Add(renderResult.Tree);
 
- (Window.Current.Content as Frame).Content = scrollViewer;
+ Border border = new Border();
+ border.Width = renderResult.CardWidth;
+ border.Child = stackPanel;
 
- await imageWaiter.WaitOnAllImagesAsync();
-
- }
-
- StorageFile imageResultFile = null;
- StorageFile jsonResultFile = null;
- if (renderResult.Error == null)
- {
- imageResultFile = await _tempResultsFolder.CreateFileAsync("Result.png", CreationCollisionOption.GenerateUniqueName);
- jsonResultFile = await _tempResultsFolder.CreateFileAsync("Result.json", CreationCollisionOption.GenerateUniqueName);
+ ScrollViewer scrollViewer = new ScrollViewer();
+ scrollViewer.Content = border;
 
- await UWPTestLibrary.RenderTestHelpers.ResultsToFile(imageResultFile, jsonResultFile, renderResult.RoundTrippedJSON, renderResult.Tree);
- }
+ (Window.Current.Content as Frame).Content = scrollViewer;
 
- await Task.Delay(10);
+ await imageWaiter.WaitOnAllImagesAsync();
 
- result = await TestResultViewModel.CreateAsync(
- cardFile: cardFile,
- hostConfigFile: hostConfigFile,
- renderedTestResult: renderResult,
- actualImageFile: imageResultFile,
- actualJsonFile: jsonResultFile,
- expectedFolder: _expectedFolder,
- sourceHostConfigsFolder: _sourceHostConfigsFolder,
- sourceCardsFolder: _sourceCardsFolder);
+ }
 
- testPass = result.Status.IsPassingStatus() && result.Status.OriginalMatched;
+ StorageFile imageResultFile = null;
+ StorageFile jsonResultFile = null;
+ if (renderResult.Error == null)
+ {
+ imageResultFile = await _tempResultsFolder.CreateFileAsync("Result.png", CreationCollisionOption.GenerateUniqueName);
+ jsonResultFile = await _tempResultsFolder.CreateFileAsync("Result.json", CreationCollisionOption.GenerateUniqueName);
 
- if(!testPass)
- {
- // Retry if we failed on image matching for an unchanged card to allow for
- // occasional differences in image rendering
- retryImage = result.Status.OriginalMatched && !result.Status.ImageMatched && (reruns < 3);
- reruns++;
- }
- else
- {
- retryImage = false;
- }
+ await UWPTestLibrary.RenderTestHelpers.ResultsToFile(imageResultFile, jsonResultFile, renderResult.RoundTrippedJSON, renderResult.Tree);
  }
 
+ await Task.Delay(10);
+
+ TestResultViewModel result = await TestResultViewModel.CreateAsync(
+ cardFile: cardFile,
+ hostConfigFile: hostConfigFile,
+ renderedTestResult: renderResult,
+ actualImageFile: imageResultFile,
+ actualJsonFile: jsonResultFile,
+ expectedFolder: _expectedFolder,
+ sourceHostConfigsFolder: _sourceHostConfigsFolder,
+ sourceCardsFolder: _sourceCardsFolder);
+
+ // For the unit test, only check that the renderer succeeded (or not) as expected, and that the round tripped json is unchanged.
+ // Image comparison for the time being is not quite stable enough to run as a unit test.
+ // Allow new cards to pass as long as they don't have an error
+ bool testPass =
+ result.Status.MatchedViaError ||
+ result.Status.JsonRoundTripMatched ||
+ (result.Status.NewCard && (result.Status.Error == null));
+
  if (!testPass)
  {
  throw new Exception(result.Status.ToString() + ": " + result.HostConfigName + "\\" + result.CardName);