Compress entire mapblock

.github/workflows/build.yml

@@ -216,7 +216,7 @@ jobs:
  env:
  VCPKG_VERSION: 0bf3923f9fab4001c00f0f429682a0853b5749e0
 # 2020.11
- vcpkg_packages: irrlicht zlib curl[winssl] openal-soft libvorbis libogg sqlite3 freetype luajit
+ vcpkg_packages: irrlicht zlib zstd curl[winssl] openal-soft libvorbis libogg sqlite3 freetype luajit
  strategy:
  fail-fast: false
  matrix:

src/CMakeLists.txt

@@ -262,9 +262,13 @@ if(WIN32)
  find_path(ZLIB_INCLUDE_DIR "zlib.h" DOC "Zlib include directory")
  find_library(ZLIB_LIBRARIES "zlib" DOC "Path to zlib library")
 
+ find_path(ZSTD_INCLUDE_DIR "zstd.h" DOC "Zstd include directory")
+ find_library(ZSTD_LIBRARIES "zstd" DOC "Path to zstd library")
+
  # Dll's are automatically copied to the output directory by vcpkg when VCPKG_APPLOCAL_DEPS=ON
  if(NOT VCPKG_APPLOCAL_DEPS)
  find_file(ZLIB_DLL NAMES "zlib.dll" "zlib1.dll" DOC "Path to zlib.dll for installation (optional)")
+ find_file(ZSTD_DLL NAMES "zstd.dll" DOC "Path to zstd.dll for installation (optional)")
  if(ENABLE_SOUND)
  set(OPENAL_DLL "" CACHE FILEPATH "Path to OpenAL32.dll for installation (optional)")
  set(OGG_DLL "" CACHE FILEPATH "Path to libogg.dll for installation (optional)")
@@ -816,6 +820,9 @@ if(WIN32)
  if(ZLIB_DLL)
  install(FILES ${ZLIB_DLL} DESTINATION ${BINDIR})
  endif()
+ if(ZSTD_DLL)
+ install(FILES ${ZSTD_DLL} DESTINATION ${BINDIR})
+ endif()
  if(FREETYPE_DLL)
  install(FILES ${FREETYPE_DLL} DESTINATION ${BINDIR})
  endif()

src/map.h

@@ -31,6 +31,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "voxel.h"
 #include "modifiedstate.h"
 #include "util/container.h"
+#include "util/pointer.h"
 #include "util/metricsbackend.h"
 #include "nodetimer.h"
 #include "map_settings_manager.h"

src/mapblock.cpp

@@ -355,7 +355,7 @@ static void correctBlockNodeIds(const NameIdMapping *nimap, MapNode *nodes,
  }
 }
 
-void MapBlock::serialize(std::ostream &os, u8 version, bool disk, int compression_level)
+void MapBlock::serialize(std::ostream &out, u8 version, bool disk, int compression_level)
 {
  if(!ser_ver_supported(version))
  throw VersionMismatchException("ERROR: MapBlock format not supported");
@@ -365,6 +365,8 @@ void MapBlock::serialize(std::ostream &os, u8 version, bool disk, int compressio
 
  FATAL_ERROR_IF(version < SER_FMT_VER_LOWEST_WRITE, "Serialisation version error");
 
+ std::ostringstream oss(std::ios_base::binary);
+ std::ostream &os = version >= 29 ? oss : out;
  // First byte
  u8 flags = 0;
  if(is_underground)
@@ -381,38 +383,46 @@ void MapBlock::serialize(std::ostream &os, u8 version, bool disk, int compressio
  /*
  Bulk node data
  */
+ u8 content_width = 2;
+ u8 params_width = 2;
+ writeU8(os, content_width);
+ writeU8(os, params_width);
  NameIdMapping nimap;
+ SharedBuffer<u8> buf;
  if(disk)
  {
  MapNode *tmp_nodes = new MapNode[nodecount];
  for(u32 i=0; i<nodecount; i++)
  tmp_nodes[i] = data[i];
  getBlockNodeIdMapping(&nimap, tmp_nodes, m_gamedef->ndef());
 
- u8 content_width = 2;
- u8 params_width = 2;
- writeU8(os, content_width);
- writeU8(os, params_width);
- MapNode::serializeBulk(os, version, tmp_nodes, nodecount,
- content_width, params_width, compression_level);
+ buf = MapNode::serializeBulk(version, tmp_nodes, nodecount,
+ content_width, params_width);
  delete[] tmp_nodes;
  }
  else
  {
- u8 content_width = 2;
- u8 params_width = 2;
- writeU8(os, content_width);
- writeU8(os, params_width);
- MapNode::serializeBulk(os, version, data, nodecount,
- content_width, params_width, compression_level);
+ buf = MapNode::serializeBulk(version, data, nodecount,
+ content_width, params_width);
+ }
+ if(version >= 29) {
+ os.write((const char*) &buf[0], buf.getSize());
+ } else {
+ // prior to 29 node data was compressed individually
+ compress(buf, os, version, compression_level);
  }
 
  /*
  Node metadata
  */
- std::ostringstream oss(std::ios_base::binary);
- m_node_metadata.serialize(oss, version, disk);
- compress(oss.str(), os, version, compression_level);
+ if (version >= 29) {
+ m_node_metadata.serialize(os, version, disk);
+ } else {
+ // use oss from above to avoid allocating another stream object
+ m_node_metadata.serialize(oss, version, disk);
+ // prior to 29 node data was compressed individually
+ compress(oss.str(), os, version, compression_level);
+ }
 
  /*
  Data that goes to disk, but not the network
@@ -438,6 +448,11 @@ void MapBlock::serialize(std::ostream &os, u8 version, bool disk, int compressio
  m_node_timers.serialize(os, version);
  }
  }
+
+ if(version >= 29) {
+ // now compress the whole thing
+ compress(oss.str(), out, version, compression_level);
+ }
 }
 
 void MapBlock::serializeNetworkSpecific(std::ostream &os)
@@ -449,7 +464,7 @@ void MapBlock::serializeNetworkSpecific(std::ostream &os)
  writeU8(os, 2); // version
 }
 
-void MapBlock::deSerialize(std::istream &is, u8 version, bool disk)
+void MapBlock::deSerialize(std::istream &in, u8 version, bool disk)
 {
  if(!ser_ver_supported(version))
  throw VersionMismatchException("ERROR: MapBlock format not supported");
@@ -460,10 +475,17 @@ void MapBlock::deSerialize(std::istream &is, u8 version, bool disk)
 
  if(version <= 21)
  {
- deSerialize_pre22(is, version, disk);
+ deSerialize_pre22(in, version, disk);
  return;
  }
 
+ std::stringstream iss(std::ios_base::binary | std::ios_base::in | std::ios_base::out);
+ if (version >= 29) {
+ // decompress the whole thing
+ decompress(in, iss, version);
+ }
+ std::istream &is = version >= 29 ? iss : in;
+
  u8 flags = readU8(is);
  is_underground = (flags & 0x01) != 0;
  m_day_night_differs = (flags & 0x02) != 0;
@@ -484,8 +506,16 @@ void MapBlock::deSerialize(std::istream &is, u8 version, bool disk)
  throw SerializationError("MapBlock::deSerialize(): invalid content_width");
  if(params_width != 2)
  throw SerializationError("MapBlock::deSerialize(): invalid params_width");
- MapNode::deSerializeBulk(is, version, data, nodecount,
+
+ if (version >= 29) {
+ MapNode::deSerializeBulk(is, version, data, nodecount,
+ content_width, params_width);
+ } else {
+ // use iss from above to avoid allocating another stream object
+ decompress(is, iss, version);
+ MapNode::deSerializeBulk(iss, version, data, nodecount,
  content_width, params_width);
+ }
 
  /*
  NodeMetadata
@@ -494,15 +524,20 @@ void MapBlock::deSerialize(std::istream &is, u8 version, bool disk)
  <<": Node metadata"<<std::endl);
  // Ignore errors
  try {
- std::ostringstream oss(std::ios_base::binary);
- decompress(is, oss, version);
- std::istringstream iss(oss.str(), std::ios_base::binary);
- if (version >= 23)
- m_node_metadata.deSerialize(iss, m_gamedef->idef());
- else
- content_nodemeta_deserialize_legacy(iss,
- &m_node_metadata, &m_node_timers,
- m_gamedef->idef());
+ if (version >= 29) {
+ m_node_metadata.deSerialize(is, m_gamedef->idef());
+ } else {
+ // resuse iss
+ iss.str(std::string());
+ iss.clear();
+ decompress(is, iss, version);
+ if (version >= 23)
+ m_node_metadata.deSerialize(iss, m_gamedef->idef());
+ else
+ content_nodemeta_deserialize_legacy(iss,
+ &m_node_metadata, &m_node_timers,
+ m_gamedef->idef());
+ }
  } catch(SerializationError &e) {
  warningstream<<"MapBlock::deSerialize(): Ignoring an error"
  <<" while deserializing node metadata at ("

src/mapblock.h

@@ -482,7 +482,7 @@ class MapBlock
  // These don't write or read version by itself
  // Set disk to true for on-disk format, false for over-the-network format
  // Precondition: version >= SER_FMT_VER_LOWEST_WRITE
- void serialize(std::ostream &os, u8 version, bool disk, int compression_level);
+ void serialize(std::ostream &result, u8 version, bool disk, int compression_level);
  // If disk == true: In addition to doing other things, will add
  // unknown blocks from id-name mapping to wndef
  void deSerialize(std::istream &is, u8 version, bool disk);

src/mapgen/mg_schematic.cpp

@@ -333,7 +333,9 @@ bool Schematic::deserializeFromMts(std::istream *is,
  delete []schemdata;
  schemdata = new MapNode[nodecount];
 
- MapNode::deSerializeBulk(ss, MTSCHEM_MAPNODE_SER_FMT_VER, schemdata,
+ std::stringstream d_ss(std::ios_base::binary | std::ios_base::in | std::ios_base::out);
+ decompress(ss, d_ss, MTSCHEM_MAPNODE_SER_FMT_VER);
+ MapNode::deSerializeBulk(d_ss, MTSCHEM_MAPNODE_SER_FMT_VER, schemdata,
  nodecount, 2, 2);
 
  // Fix probability values for nodes that were ignore; removed in v2
@@ -375,8 +377,9 @@ bool Schematic::serializeToMts(std::ostream *os,
  ss << serializeString16(names[i]); // node names
 
  // compressed bulk node data
- MapNode::serializeBulk(ss, MTSCHEM_MAPNODE_SER_FMT_VER,
- schemdata, size.X * size.Y * size.Z, 2, 2, -1);
+ SharedBuffer<u8> buf = MapNode::serializeBulk(MTSCHEM_MAPNODE_SER_FMT_VER,
+ schemdata, size.X * size.Y * size.Z, 2, 2);
+ compress(buf, ss, MTSCHEM_MAPNODE_SER_FMT_VER);
 
  return true;
 }

src/mapnode.cpp

@@ -704,9 +704,10 @@ void MapNode::deSerialize(u8 *source, u8 version)
  }
  }
 }
-void MapNode::serializeBulk(std::ostream &os, int version,
+
+SharedBuffer<u8> MapNode::serializeBulk(int version,
  const MapNode *nodes, u32 nodecount,
- u8 content_width, u8 params_width, int compression_level)
+ u8 content_width, u8 params_width)
 {
  if (!ser_ver_supported(version))
  throw VersionMismatchException("ERROR: MapNode format not supported");
@@ -731,12 +732,7 @@ void MapNode::serializeBulk(std::ostream &os, int version,
  writeU8(&databuf[start1 + i], nodes[i].param1);
  writeU8(&databuf[start2 + i], nodes[i].param2);
  }
-
- /*
- Compress data to output stream
- */
-
- compress(databuf, os, version, compression_level);
+ return databuf;
 }
 
 // Deserialize bulk node data
@@ -752,15 +748,10 @@ void MapNode::deSerializeBulk(std::istream &is, int version,
  || params_width != 2)
  FATAL_ERROR("Deserialize bulk node data error");
 
- // Uncompress or read data
+ // read data
  u32 len = nodecount * (content_width + params_width);
- std::ostringstream os(std::ios_base::binary);
- decompress(is, os, version);
- std::string s = os.str();
- if(s.size() != len)
- throw SerializationError("deSerializeBulkNodes: "
- "decompress resulted in invalid size");
- const u8 *databuf = reinterpret_cast<const u8*>(s.c_str());
+ u8 databuf[len];
+ is.read(reinterpret_cast<char*>(&databuf[0]), len);
 
  // Deserialize content
  if(content_width == 1)

src/mapnode.h

@@ -21,6 +21,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 
 #include "irrlichttypes_bloated.h"
 #include "light.h"
+#include "util/pointer.h"
 #include <string>
 #include <vector>
 
@@ -290,9 +291,9 @@ struct MapNode
  // content_width = the number of bytes of content per node
  // params_width = the number of bytes of params per node
  // compressed = true to zlib-compress output
- static void serializeBulk(std::ostream &os, int version,
+ static SharedBuffer<u8> serializeBulk(int version,
  const MapNode *nodes, u32 nodecount,
- u8 content_width, u8 params_width, int compression_level);
+ u8 content_width, u8 params_width);
  static void deSerializeBulk(std::istream &is, int version,
  MapNode *nodes, u32 nodecount,
  u8 content_width, u8 params_width);