Code refactoring + perf optimizations

Assets/Scripts/Pal3.Core/DataReader/Cpk/CpkArchive.cs

@@ -6,6 +6,7 @@
 namespace Pal3.Core.DataReader.Cpk
 {
  using System;
+ using System.Buffers;
  using System.Collections.Generic;
  using System.IO;
  using System.Runtime.InteropServices;
@@ -29,7 +30,7 @@ public sealed class CpkArchive
  private readonly int _codepage;
  private Dictionary<uint, CpkTableEntity> _tableEntities;
 
- private readonly Dictionary<uint, byte[]> _fileNameMap = new ();
+ private readonly Dictionary<uint, string> _crcToFileNameMap = new ();
  private readonly Dictionary<uint, uint> _crcToTableIndexMap = new ();
  private readonly Dictionary<uint, HashSet<uint>> _fatherCrcToChildCrcTableIndexMap = new ();
 
@@ -55,7 +56,7 @@ public void Init()
  bufferSize: 8196,
  FileOptions.SequentialScan); // Use SequentialScan option to increase reading speed
 
- var header = CoreUtility.ReadStruct<CpkHeader>(stream);
+ CpkHeader header = CoreUtility.ReadStruct<CpkHeader>(stream);
 
  if (!IsValidCpkHeader(header))
  {
@@ -111,7 +112,8 @@ private void ExtractToInternal(string outputFolder, IEnumerable<CpkEntry> nodes)
  }
  else
  {
- File.WriteAllBytes(outputFolder + relativePath, ReadAllBytes(node.VirtualPath));
+ uint crcHash = _crcHash.Compute(node.VirtualPath.ToLower(), _codepage);
+ File.WriteAllBytes(outputFolder + relativePath, ReadAllBytes(crcHash));
  }
  }
  }
@@ -120,45 +122,29 @@ private void ExtractToInternal(string outputFolder, IEnumerable<CpkEntry> nodes)
  /// Check if file exists inside the archive.
  /// </summary>
  /// <param name="fileVirtualPath">Virtualized file path inside CPK archive</param>
+ /// <param name="filePathCrcHash">CRC hash of the file path</param>
  /// <returns>True if file exists</returns>
- public bool FileExists(string fileVirtualPath)
+ public bool FileExists(string fileVirtualPath, out uint filePathCrcHash)
  {
- var crc = _crcHash.Compute(fileVirtualPath.ToLower(), _codepage);
- return _crcToTableIndexMap.ContainsKey(crc);
+ filePathCrcHash = _crcHash.Compute(fileVirtualPath.ToLower(), _codepage);
+ return _crcToTableIndexMap.ContainsKey(filePathCrcHash);
  }
 
  /// <summary>
  /// Read all bytes of the give file stored inside the archive.
  /// </summary>
- /// <param name="fileVirtualPath">Virtualized file path inside CPK archive</param>
+ /// <param name="fileVirtualPathCrcHash">CRC32 hash of the virtualized file path inside CPK archive</param>
  /// <returns>File content in byte array</returns>
- public byte[] ReadAllBytes(string fileVirtualPath)
- {
- return _archiveInMemory ?
- ReadAllBytesUsingInMemoryCache(fileVirtualPath) :
- GetFileContent(fileVirtualPath);
- }
-
- private byte[] ReadAllBytesUsingInMemoryCache(string fileVirtualPath)
+ public byte[] ReadAllBytes(uint fileVirtualPathCrcHash)
  {
- CpkTableEntity entity = ValidateAndGetTableEntity(fileVirtualPath);
-
- ReadOnlySpan<byte> dataInArchive = new ReadOnlySpan<byte>(_archiveData)
- .Slice((int)entity.StartPos, (int)entity.PackedSize);
+ CpkTableEntity entity = GetTableEntity(fileVirtualPathCrcHash);
 
- return entity.IsCompressed() ?
- DecompressDataInArchive(dataInArchive, entity.OriginSize) :
- dataInArchive.ToArray();
- }
-
- private byte[] GetFileContent(string fileVirtualPath)
- {
- CpkTableEntity entity = ValidateAndGetTableEntity(fileVirtualPath);
- return GetFileContentInternal(entity);
- }
+ if (entity.IsDirectory())
+ {
+ throw new InvalidOperationException(
+ $"Cannot read file <{fileVirtualPathCrcHash}> since it is a directory");
+ }
 
- private byte[] GetFileContentInternal(CpkTableEntity entity)
- {
  if (_archiveInMemory)
  {
  ReadOnlySpan<byte> rawData = new ReadOnlySpan<byte>(_archiveData)
@@ -186,22 +172,14 @@ private byte[] DecompressDataInArchive(ReadOnlySpan<byte> compressedData, uint o
  return decompressedData;
  }
 
- private CpkTableEntity ValidateAndGetTableEntity(string fileVirtualPath)
+ private CpkTableEntity GetTableEntity(uint fileVirtualPathCrcHash)
  {
- var crc = _crcHash.Compute(fileVirtualPath.ToLower(), _codepage);
- if (!_crcToTableIndexMap.ContainsKey(crc))
+ if (!_crcToTableIndexMap.ContainsKey(fileVirtualPathCrcHash))
  {
- throw new ArgumentException($"<{fileVirtualPath}> does not exists in the archive");
+ throw new ArgumentException($"File <{fileVirtualPathCrcHash}> does not exists in the archive");
  }
 
- CpkTableEntity entity = _tableEntities[_crcToTableIndexMap[crc]];
-
- if (entity.IsDirectory())
- {
- throw new InvalidOperationException($"Cannot open <{fileVirtualPath}> since it is a directory");
- }
-
- return entity;
+ return _tableEntities[_crcToTableIndexMap[fileVirtualPathCrcHash]];
  }
 
  /// <summary>
@@ -258,8 +236,8 @@ private void BuildCrcIndexMap()
  /// <returns>Root level CpkEntry nodes</returns>
  public IEnumerable<CpkEntry> GetRootEntries()
  {
- if (_fileNameMap.Count == 0) BuildFileNameMap();
- return GetChildren(0);
+ if (_crcToFileNameMap.Count == 0) BuildFileNameMap();
+ return GetChildren(0); // 0 is the CRC of the root directory
  }
 
  private void BuildFileNameMap()
@@ -281,13 +259,21 @@ private void BuildFileNameMap()
 
  foreach (CpkTableEntity entity in _tableEntities.Values)
  {
- long extraInfoOffset = entity.StartPos + entity.PackedSize;
- var extraInfo = new byte[entity.ExtraInfoSize];
- stream.Seek(extraInfoOffset, SeekOrigin.Begin);
- _ = stream.Read(extraInfo);
+ byte[] extraInfo = ArrayPool<byte>.Shared.Rent((int)entity.ExtraInfoSize);
 
- var fileName = CoreUtility.TrimEnd(extraInfo, new byte[] { 0x00, 0x00 });
- _fileNameMap[entity.CRC] = fileName;
+ try
+ {
+ long extraInfoOffset = entity.StartPos + entity.PackedSize;
+ stream.Seek(extraInfoOffset, SeekOrigin.Begin);
+ _ = stream.Read(extraInfo);
+ string fileName = Encoding.GetEncoding(_codepage)
+ .GetString(extraInfo, 0, Array.IndexOf(extraInfo, (byte)0));
+ _crcToFileNameMap[entity.CRC] = fileName;
+ }
+ finally
+ {
+ ArrayPool<byte>.Shared.Return(extraInfo);
+ }
  }
 
  stream.Close();
@@ -306,13 +292,13 @@ private IEnumerable<CpkEntry> GetChildren(uint fatherCrc, string rootPath = "")
  rootPath += CpkConstants.DirectorySeparatorChar;
  }
 
- foreach (var childCrc in _fatherCrcToChildCrcTableIndexMap[fatherCrc])
+ foreach (uint childCrc in _fatherCrcToChildCrcTableIndexMap[fatherCrc])
  {
- var index = _crcToTableIndexMap[childCrc];
+ uint index = _crcToTableIndexMap[childCrc];
  CpkTableEntity child = _tableEntities[index];
- var fileName = Encoding.GetEncoding(_codepage).GetString(_fileNameMap[child.CRC]);
+ string fileName = _crcToFileNameMap[child.CRC];
 
- var virtualPath = rootPath + fileName;
+ string virtualPath = rootPath + fileName;
 
  if (child.IsDirectory())
  {

Assets/Scripts/Pal3.Core/DataReader/Cpk/CrcHash.cs

@@ -6,7 +6,7 @@
 namespace Pal3.Core.DataReader.Cpk
 {
  using System;
- using System.Collections.Generic;
+ using System.Buffers;
  using System.Text;
 
  /// <summary>
@@ -20,9 +20,6 @@ public sealed class Crc32Hash
  private static readonly uint[] CrcTable = new uint[CRC_TABLE_MAX + 1];
  private bool _initialized;
 
- // Cache the crc32 hash result for string since it's expensive to compute.
- private readonly Dictionary<string, uint> _crcResultCache = new();
-
  public void Init()
  {
  // generate the table of CRC remainders for all possible bytes
@@ -41,31 +38,34 @@ public void Init()
  _initialized = true;
  }
 
- public uint Compute(string str, int codepage, bool useCache = true)
+ public uint Compute(string str, int codepage)
  {
- if (!useCache)
- {
- return ComputeInternal(Encoding.GetEncoding(codepage).GetBytes(str));
- }
+ if (!_initialized) throw new InvalidOperationException($"Crc32 hash table is not initialized yet");
 
- var cacheKey = $"{str}-{codepage}";
+ Encoding encoding = Encoding.GetEncoding(codepage);
+ int byteCount = encoding.GetByteCount(str);
 
- if (_crcResultCache.TryGetValue(cacheKey, out var hash))
+ byte[] rentedArray = null;
+ Span<byte> buffer = byteCount <= 512 // To prevent stack overflow when string is too long
+ ? stackalloc byte[byteCount]
+ : (Span<byte>)(rentedArray = ArrayPool<byte>.Shared.Rent(byteCount));
+
+ try
  {
- return hash;
+ encoding.GetBytes(str, buffer);
+ return ComputeInternal(buffer[..byteCount]);
+ }
+ finally
+ {
+ if (rentedArray != null)
+ {
+ ArrayPool<byte>.Shared.Return(rentedArray);
+ }
  }
-
- uint result = ComputeInternal(Encoding.GetEncoding(codepage).GetBytes(str));
-
- _crcResultCache[cacheKey] = result;
-
- return result;
  }
 
- private unsafe uint ComputeInternal(byte[] data)
+ private unsafe uint ComputeInternal(ReadOnlySpan<byte> data)
  {
- if (!_initialized) throw new InvalidOperationException($"Crc32 hash table is not initialized yet");
-
  if (data == null || data.Length == 0 || data[0] == 0)
  {
  return 0;

Assets/Scripts/Pal3.Core/DataReader/Cvd/CvdFileReader.cs

@@ -301,9 +301,9 @@ private static CvdMesh ReadMesh(IBinaryReader reader, float version, int codepag
  var vertices = new CvdVertex[numberOfVertices];
  for (var j = 0; j < numberOfVertices; j++)
  {
- GameBoxVector2 uv = reader.ReadVector2();
- GameBoxVector3 normal = reader.ReadVector3();
- GameBoxVector3 position = reader.ReadVector3();
+ GameBoxVector2 uv = reader.ReadGameBoxVector2();
+ GameBoxVector3 normal = reader.ReadGameBoxVector3();
+ GameBoxVector3 position = reader.ReadGameBoxVector3();
 
  // Quick fix for the missing/wrong normals
  if (normal == GameBoxVector3.Zero)
@@ -350,10 +350,10 @@ private static CvdMesh ReadMesh(IBinaryReader reader, float version, int codepag
 
  GameBoxMaterial material = new ()
  {
- Diffuse = CoreUtility.ToColor32(reader.ReadBytes(4)),
- Ambient = CoreUtility.ToColor32(reader.ReadBytes(4)),
- Specular = CoreUtility.ToColor32(reader.ReadBytes(4)),
- Emissive = CoreUtility.ToColor32(reader.ReadBytes(4)),
+ Diffuse = (Color)reader.ReadColor32(),
+ Ambient = (Color)reader.ReadColor32(),
+ Specular = (Color)reader.ReadColor32(),
+ Emissive = (Color)reader.ReadColor32(),
  SpecularPower = reader.ReadSingle(),
  TextureFileNames = new [] { reader.ReadString(64, codepage) }
  };
@@ -386,10 +386,10 @@ private static CvdMesh ReadMesh(IBinaryReader reader, float version, int codepag
  {
  animationMaterials[i] = new GameBoxMaterial()
  {
- Diffuse = CoreUtility.ToColor(reader.ReadSingles(4)),
- Ambient = CoreUtility.ToColor(reader.ReadSingles(4)),
- Specular = CoreUtility.ToColor(reader.ReadSingles(4)),
- Emissive = CoreUtility.ToColor(reader.ReadSingles(4)),
+ Diffuse = reader.ReadColor(),
+ Ambient = reader.ReadColor(),
+ Specular = reader.ReadColor(),
+ Emissive = reader.ReadColor(),
  SpecularPower = reader.ReadSingle()
  };
  }

Assets/Scripts/Pal3.Core/DataReader/Dat/EffectLinkerFileReader.cs

@@ -48,7 +48,7 @@ private static EffectGroupInfo ReadEffectGroupInfo(IBinaryReader reader)
  IsCenteredAroundReceiver = reader.ReadBoolean(),
  IsCasterActor = reader.ReadBoolean(),
  IsReceiverActor = reader.ReadBoolean(),
- EffectGameBoxPosition = reader.ReadVector3(),
+ EffectGameBoxPosition = reader.ReadGameBoxVector3(),
  WaitDurationInSeconds = reader.ReadSingle(),
  EffectDurationInSeconds = reader.ReadSingle(),
  };

Assets/Scripts/Pal3.Core/DataReader/Dxt/Dxt1Decoder.cs

@@ -32,7 +32,7 @@ public static unsafe void ToRgba32(byte[] data, int width, int height, byte[] bu
  throw new ArgumentException("buffer is null or too small");
  }
 
- fixed (byte* srcStart = &data[DDS_FILE_HEADER_SIZE], dstStart = &buffer[0])
+ fixed (byte* srcStart = &data[DDS_FILE_HEADER_SIZE], dstStart = buffer)
  {
  var src = srcStart;
  var dst = dstStart;

Assets/Scripts/Pal3.Core/DataReader/Dxt/Dxt3Decoder.cs

@@ -32,7 +32,7 @@ public static unsafe void ToRgba32(byte[] data, int width, int height, byte[] bu
  throw new ArgumentException("buffer is null or too small");
  }
 
- fixed (byte* srcStart = &data[DDS_FILE_HEADER_SIZE], dstStart = &buffer[0])
+ fixed (byte* srcStart = &data[DDS_FILE_HEADER_SIZE], dstStart = buffer)
  {
  var src = srcStart;
  var dst = dstStart;