refactor(Indexer): Tentative fix for database locking issue

This should cleanly fix AdGoBye#62, the only database operations that could still fail as-is would be Live.
We can't easily batch Live without some additional logic since it's event based.

AdGoBye/Indexer.cs

@@ -15,13 +15,16 @@ public class Indexer
  private static readonly ILogger Logger = Log.ForContext(typeof(Indexer));
  public static readonly string WorkingDirectory = GetWorkingDirectory();
 
+
  public static void ManageIndex()
  {
  using var db = new State.IndexContext();
- if (db.Content.Any()) VerifyDbTruth();
+ var container = new DatabaseOperationsContainer();
+ if (db.Content.Any()) VerifyDbTruth(ref container);
+ CommitToDatabase(container);
+
  var contentFolders = new DirectoryInfo(GetCacheDir()).GetDirectories();
  if (contentFolders.Length == db.Content.Count() - SafeAllowlistCount()) return;
-
  var content = contentFolders
  .ExceptBy(db.Content.Select(content => content.StableContentName), info => info.Name)
  .Select(newContent => GetLatestFileVersion(newContent).HighestVersionDirectory)
@@ -41,15 +44,15 @@ static int SafeAllowlistCount()
  }
  }
 
- private static void VerifyDbTruth()
+ private static void VerifyDbTruth(ref DatabaseOperationsContainer container)
  {
  using var db = new State.IndexContext();
  foreach (var content in db.Content.Include(content => content.VersionMeta))
  {
  var directoryMeta = new DirectoryInfo(content.VersionMeta.Path);
  if (!directoryMeta.Parent!.Exists) // This content doesn't have a StableContentId folder anymore
  {
- db.Remove(content);
+ container.RemoveContent.Add(content);
  continue;
  }
 
@@ -67,7 +70,7 @@ private static void VerifyDbTruth()
 
  if (!File.Exists(highestVersionDir.FullName + "/__data"))
  {
- db.Remove(content);
+ container.RemoveContent.Add(content);
  Logger.Warning(
  "{directory} is highest version but doesn't have __data, hell might have frozen over. Removed from Index",
  highestVersionDir.FullName);
@@ -78,49 +81,41 @@ private static void VerifyDbTruth()
  content.VersionMeta.Version = highestVersion;
  content.VersionMeta.Path = highestVersionDir.FullName;
  content.VersionMeta.PatchedBy = [];
+ container.EditContent.Add(content);
  }
-
- db.SaveChanges();
  }
 
  public static void AddToIndex(string path)
  {
- using var db = new State.IndexContext();
- if (AddToIndexPart1(path, out var content) && content != null)
- {
- AddToIndexPart2(content, db);
- }
-
- db.SaveChanges();
+ var container = new DatabaseOperationsContainer();
+ AddToIndexPart1(path, ref container);
+ if (!container.AddContent.IsEmpty) AddToIndexPart2(container.AddContent.First(), ref container);
+ CommitToDatabase(container);
  }
 
  public static void AddToIndex(IEnumerable<DirectoryInfo?> paths)
  {
- ConcurrentBag<Content> contents = [];
+ var dbActionsContainer = new DatabaseOperationsContainer();
  Parallel.ForEach(paths, path =>
  {
- if (path != null && AddToIndexPart1(path.FullName, out var content) && content != null)
- {
- contents.Add(content);
- }
+ if (path != null) AddToIndexPart1(path.FullName, ref dbActionsContainer);
  });
 
- var groupedById = contents.GroupBy(content => content.Id);
- using var writeDbContext = new State.IndexContext();
- foreach (var group in groupedById)
+ var groupedById = dbActionsContainer.AddContent.GroupBy(content => content.Id);
+ Parallel.ForEach(groupedById, group =>
  {
  {
  foreach (var content in group)
  {
- AddToIndexPart2(content, writeDbContext);
+ AddToIndexPart2(content, ref dbActionsContainer);
  }
  }
- }
+ });
 
- writeDbContext.SaveChanges();
+ CommitToDatabase(dbActionsContainer);
  }
 
- public static bool AddToIndexPart1(string path, out Content? content)
+ public static void AddToIndexPart1(string path, ref DatabaseOperationsContainer container)
  {
  using var db = new State.IndexContext();
  // - Folder (StableContentName) [singleton, we want this]
@@ -132,7 +127,7 @@ public static bool AddToIndexPart1(string path, out Content? content)
  if (directory.Name == "__data") directory = directory.Parent;
 
  // If we already have an item in Index that has the same StableContentName, then this is a newer version of something Indexed
- content = db.Content.Include(content => content.VersionMeta)
+ var content = db.Content.Include(content => content.VersionMeta)
  .FirstOrDefault(content => content.StableContentName == directory!.Parent!.Name);
  if (content is not null)
  {
@@ -142,24 +137,24 @@ public static bool AddToIndexPart1(string path, out Content? content)
  Logger.Verbose(
  "Skipped Indexation of {directory} since it isn't an upgrade (Index: {greaterVersion}, Parsed: {lesserVersion})",
  directory.FullName, content.VersionMeta.Version, version);
- return false;
+ return;
  }
 
  content.VersionMeta.Version = version;
  content.VersionMeta.Path = directory.FullName;
  content.VersionMeta.PatchedBy = [];
-
- db.SaveChanges();
- return false;
+ container.EditContent.Add(content);
+ return;
  }
 
- if (!File.Exists(directory!.FullName + "/__data")) return false;
+ if (!File.Exists(directory!.FullName + "/__data")) return;
  content = FileToContent(directory);
- return true;
+ if (content is not null) container.AddContent.Add(content);
  }
 
- private static void AddToIndexPart2(Content content, State.IndexContext db)
+ private static void AddToIndexPart2(Content content, ref DatabaseOperationsContainer container)
  {
+ using var db = new State.IndexContext();
  var indexCopy = db.Content.Include(existingFile => existingFile.VersionMeta)
  .FirstOrDefault(existingFile => existingFile.Id == content.Id);
  if (indexCopy is null)
@@ -170,7 +165,7 @@ private static void AddToIndexPart2(Content content, State.IndexContext db)
  return;
  }
 
- db.Content.Add(content);
+ container.AddContent.Add(content);
  Logger.Information("Added {id} [{type}] to Index", content.Id, content.Type);
  return;
  }
@@ -186,6 +181,7 @@ private static void AddToIndexPart2(Content content, State.IndexContext db)
  indexCopy.VersionMeta.Version = content.VersionMeta.Version;
  indexCopy.VersionMeta.Path = content.VersionMeta.Path;
  indexCopy.VersionMeta.PatchedBy = [];
+ container.EditContent.Add(indexCopy);
  return;
  // The second is an Imposter avatar, which we don't want to index.
  case ContentType.Avatar:
@@ -248,6 +244,15 @@ static string ResolveUnityVersion(string path)
  }
  }
 
+ private static void CommitToDatabase(DatabaseOperationsContainer container)
+ {
+ using var writeDbContext = new State.IndexContext();
+ writeDbContext.Content.AddRange(container.AddContent);
+ writeDbContext.Content.UpdateRange(container.EditContent);
+ writeDbContext.Content.RemoveRange(container.RemoveContent);
+ writeDbContext.SaveChanges();
+ }
+
 
  public static Content? GetFromIndex(string path)
  {
@@ -534,4 +539,11 @@ private static string GetCacheDir()
  {
  return WorkingDirectory + "/Cache-WindowsPlayer/";
  }
+
+ public record DatabaseOperationsContainer
+ {
+ public ConcurrentBag<Content> AddContent = [];
+ public ConcurrentBag<Content> EditContent = [];
+ public ConcurrentBag<Content> RemoveContent = [];
+ }
 }