Add zstd support.

builtin/settingtypes.txt

@@ -1051,11 +1051,10 @@ full_block_send_enable_min_time_from_building (Delay in sending blocks after bui
 # client number.
 max_packets_per_iteration (Max. packets per iteration) int 1024
 
-# ZLib compression level to use when sending mapblocks to the client.
-# -1 - Zlib's default compression level
-# 0 - no compresson, fastest
+# compression level to use when sending mapblocks to the client.
+# -1 - use default compression level
+# 0 - least compresson, fastest
 # 9 - best compression, slowest
-# (levels 1-3 use Zlib's "fast" method, 4-9 use the normal method)
 map_compression_level_net (Map Compression Level for Network Transfer) int -1 -1 9
 
 [*Game]
@@ -1250,12 +1249,11 @@ max_objects_per_block (Maximum objects per block) int 64
 # See https://www.sqlite.org/pragma.html#pragma_synchronous
 sqlite_synchronous (Synchronous SQLite) enum 2 0,1,2
 
-# ZLib compression level to use when saving mapblocks to disk.
-# -1 - Zlib's default compression level
-# 0 - no compresson, fastest
+# compression level to use when sending mapblocks to the client.
+# -1 - use default compression level
+# 0 - least compresson, fastest
 # 9 - best compression, slowest
-# (levels 1-3 use Zlib's "fast" method, 4-9 use the normal method)
-map_compression_level_disk (Map Compression Level for Disk Storage) int 3 -1 9
+map_compression_level_disk (Map Compression Level for Disk Storage) int -1 -1 9
 
 # Length of a server tick and the interval at which objects are generally updated over
 # network.

cmake/Modules/FindZstd.cmake

@@ -0,0 +1,9 @@
+mark_as_advanced(ZSTD_LIBRARY ZSTD_INCLUDE_DIR)
+
+find_path(ZSTD_INCLUDE_DIR NAMES zstd.h)
+
+find_library(ZSTD_LIBRARY NAMES zstd)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Zstd DEFAULT_MSG ZSTD_LIBRARY ZSTD_INCLUDE_DIR)
+

src/CMakeLists.txt

@@ -312,6 +312,7 @@ else()
  endif(BUILD_CLIENT)
 
  find_package(ZLIB REQUIRED)
+ find_package(Zstd REQUIRED)
  set(PLATFORM_LIBS -lpthread ${CMAKE_DL_LIBS})
  if(APPLE)
  set(PLATFORM_LIBS "-framework CoreFoundation" ${PLATFORM_LIBS})
@@ -503,6 +504,7 @@ include_directories(
  ${PROJECT_SOURCE_DIR}
  ${IRRLICHT_INCLUDE_DIR}
  ${ZLIB_INCLUDE_DIR}
+ ${ZSTD_INCLUDE_DIR}
  ${PNG_INCLUDE_DIR}
  ${SOUND_INCLUDE_DIRS}
  ${SQLITE3_INCLUDE_DIR}
@@ -535,6 +537,7 @@ if(BUILD_CLIENT)
  set(client_LIBS
  ${PROJECT_NAME}
  ${ZLIB_LIBRARIES}
+ ${ZSTD_LIBRARY}
  ${IRRLICHT_LIBRARY}
  ${JPEG_LIBRARIES}
  ${BZIP2_LIBRARIES}

src/clientiface.cpp

@@ -212,7 +212,7 @@ void RemoteClient::GetNextBlocks (
  s16 d_max = full_d_max;
 
  // Don't loop very much at a time
- s16 max_d_increment_at_time = 2;
+ s16 max_d_increment_at_time = 1;
  if (d_max > d_start + max_d_increment_at_time)
  d_max = d_start + max_d_increment_at_time;
 

src/constants.h

@@ -74,10 +74,10 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #define BS 10.0f
 
 // Dimension of a MapBlock
-#define MAP_BLOCKSIZE 16
+//#define MAP_BLOCKSIZE 16
 // This makes mesh updates too slow, as many meshes are updated during
 // the main loop (related to TempMods and day/night)
-//#define MAP_BLOCKSIZE 32
+#define MAP_BLOCKSIZE 32
 
 // Player step height in nodes
 #define PLAYER_DEFAULT_STEPHEIGHT 0.6f

src/defaultsettings.cpp

@@ -386,7 +386,7 @@ void set_default_settings(Settings *settings)
  settings->setDefault("chat_message_limit_per_10sec", "8.0");
  settings->setDefault("chat_message_limit_trigger_kick", "50");
  settings->setDefault("sqlite_synchronous", "2");
- settings->setDefault("map_compression_level_disk", "3");
+ settings->setDefault("map_compression_level_disk", "-1");
  settings->setDefault("map_compression_level_net", "-1");
  settings->setDefault("full_block_send_enable_min_time_from_building", "2.0");
  settings->setDefault("dedicated_server_step", "0.09");
@@ -474,7 +474,7 @@ void set_default_settings(Settings *settings)
  settings->setDefault("max_objects_per_block", "20");
  settings->setDefault("sqlite_synchronous", "1");
  settings->setDefault("map_compression_level_disk", "-1");
- settings->setDefault("map_compression_level_net", "3");
+ settings->setDefault("map_compression_level_net", "-1");
  settings->setDefault("server_map_save_interval", "15");
  settings->setDefault("client_mapblock_limit", "1000");
  settings->setDefault("active_block_range", "2");

src/mapblock.cpp

@@ -412,7 +412,7 @@ void MapBlock::serialize(std::ostream &os, u8 version, bool disk, int compressio
  */
  std::ostringstream oss(std::ios_base::binary);
  m_node_metadata.serialize(oss, version, disk);
- compressZlib(oss.str(), os, compression_level);
+ compress(oss.str(), os, version, compression_level);
 
  /*
  Data that goes to disk, but not the network
@@ -495,7 +495,7 @@ void MapBlock::deSerialize(std::istream &is, u8 version, bool disk)
  // Ignore errors
  try {
  std::ostringstream oss(std::ios_base::binary);
- decompressZlib(is, oss);
+ decompress(is, oss, version);
  std::istringstream iss(oss.str(), std::ios_base::binary);
  if (version >= 23)
  m_node_metadata.deSerialize(iss, m_gamedef->idef());

src/mapgen/mg_schematic.cpp

@@ -333,7 +333,7 @@ bool Schematic::deserializeFromMts(std::istream *is,
  delete []schemdata;
  schemdata = new MapNode[nodecount];
 
- MapNode::deSerializeBulk(ss, SER_FMT_VER_HIGHEST_READ, schemdata,
+ MapNode::deSerializeBulk(ss, MTSCHEM_MAPNODE_SER_FMT_VER, schemdata,
  nodecount, 2, 2);
 
  // Fix probability values for nodes that were ignore; removed in v2
@@ -375,7 +375,7 @@ bool Schematic::serializeToMts(std::ostream *os,
  ss << serializeString16(names[i]); // node names
 
  // compressed bulk node data
- MapNode::serializeBulk(ss, SER_FMT_VER_HIGHEST_WRITE,
+ MapNode::serializeBulk(ss, MTSCHEM_MAPNODE_SER_FMT_VER,
  schemdata, size.X * size.Y * size.Z, 2, 2, -1);
 
  return true;

src/mapgen/mg_schematic.h

@@ -70,6 +70,7 @@ class Server;
 #define MTSCHEM_FILE_SIGNATURE 0x4d54534d // 'MTSM'
 #define MTSCHEM_FILE_VER_HIGHEST_READ 4
 #define MTSCHEM_FILE_VER_HIGHEST_WRITE 4
+#define MTSCHEM_MAPNODE_SER_FMT_VER 28 // remember to up versions above when increasing this
 
 #define MTSCHEM_PROB_MASK 0x7F
 

src/mapnode.cpp

@@ -720,8 +720,9 @@ void MapNode::serializeBulk(std::ostream &os, int version,
  throw SerializationError("MapNode::serializeBulk: serialization to "
  "version < 24 not possible");
 
- size_t databuf_size = nodecount * (content_width + params_width);
- u8 *databuf = new u8[databuf_size];
+ SharedBuffer<u8> databuf(nodecount * (content_width + params_width));
+ //size_t databuf_size = nodecount * (content_width + params_width);
+ //u8 *databuf = new u8[databuf_size];
 
  u32 start1 = content_width * nodecount;
  u32 start2 = (content_width + 1) * nodecount;
@@ -737,9 +738,9 @@ void MapNode::serializeBulk(std::ostream &os, int version,
  Compress data to output stream
  */
 
- compressZlib(databuf, databuf_size, os, compression_level);
+ compress(databuf, os, version, compression_level);
 
- delete [] databuf;
+ //delete [] databuf;
 }
 
 // Deserialize bulk node data
@@ -758,11 +759,13 @@ void MapNode::deSerializeBulk(std::istream &is, int version,
  // Uncompress or read data
  u32 len = nodecount * (content_width + params_width);
  std::ostringstream os(std::ios_base::binary);
- decompressZlib(is, os);
+ decompress(is, os, version);
  std::string s = os.str();
- if(s.size() != len)
+ if(s.size() != len) {
+ warningstream << "Len: " << s.size() << " expected " << len << std::endl;
  throw SerializationError("deSerializeBulkNodes: "
  "decompress resulted in invalid size");
+ }
  const u8 *databuf = reinterpret_cast<const u8*>(s.c_str());
 
  // Deserialize content

src/serialization.cpp

@@ -22,6 +22,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "util/serialize.h"
 
 #include "zlib.h"
+#include "zstd.h"
 
 /* report a zlib or i/o error */
 void zerr(int ret)
@@ -197,11 +198,98 @@ void decompressZlib(std::istream &is, std::ostream &os, size_t limit)
  inflateEnd(&z);
 }
 
-void compress(const SharedBuffer<u8> &data, std::ostream &os, u8 version)
+struct ZSTD_Deleter {
+ void operator() (ZSTD_CCtx* cctx) {
+ ZSTD_freeCCtx(cctx);
+ }
+
+ void operator() (ZSTD_DCtx* dctx) {
+ ZSTD_freeDCtx(dctx);
+ }
+};
+
+void compressZstd(const u8 *data, size_t data_size, std::ostream &os, int level)
 {
+ // reusing the context is recommended for performance
+ // it will destroyed when the thread ends
+ thread_local std::unique_ptr<ZSTD_CCtx, ZSTD_Deleter> cctx(ZSTD_createCCtx());
+
+ const size_t bufsize = ZSTD_CStreamOutSize();
+ char output_buffer[bufsize];
+
+ ZSTD_CCtx_setParameter(cctx.get(), ZSTD_c_compressionLevel, level);
+
+ ZSTD_inBuffer input = { data, data_size, 0 };
+ ZSTD_outBuffer output = { output_buffer, bufsize, 0 };
+ size_t ret;
+ do
+ {
+ ret = ZSTD_compressStream2(cctx.get(), &output , &input, ZSTD_e_end);
+ if (ZSTD_isError(ret)) {
+ dstream << ZSTD_getErrorName(ret) << std::endl;
+ throw SerializationError("compressZstd: failed");
+ }
+ os.write(output_buffer, output.pos);
+ output.pos = 0;
+ } while (ret != 0);
+}
+
+void compressZstd(const std::string &data, std::ostream &os, int level)
+{
+ compressZstd((u8*)data.c_str(), data.size(), os, level);
+}
+
+void decompressZstd(std::istream &is, std::ostream &os)
+{
+ // reusing the context is recommended for performance
+ // it will destroyed when the thread ends
+ thread_local std::unique_ptr<ZSTD_DCtx, ZSTD_Deleter> dctx(ZSTD_createDCtx());
+
+ const size_t bufsize_out = ZSTD_CStreamOutSize();
+ const size_t bufsize_in = ZSTD_CStreamInSize();
+ char output_buffer[bufsize_out];
+ char input_buffer[bufsize_in];
+
+ ZSTD_outBuffer output = { output_buffer, bufsize_out, 0 };
+ ZSTD_inBuffer input = { input_buffer, 0, 0 };
+ size_t ret;
+ do
+ {
+ is.read(input_buffer, bufsize_in);
+ input.size = is.gcount();
+ input.pos = 0;
+
+ ret = ZSTD_decompressStream(dctx.get(), &output, &input);
+ if (ZSTD_isError(ret)) {
+ dstream << ZSTD_getErrorName(ret) << std::endl;
+ throw SerializationError("decompressZstd: failed");
+ }
+ os.write(output_buffer, output.pos);
+ output.pos = 0;
+ } while(ret != 0);
+
+ // Unget all the data that inflate didn't take
+ is.clear(); // Just in case EOF is set
+ for (u32 i = 0; i < input.size - input.pos; i++) {
+ is.unget();
+ if (is.fail() || is.bad())
+ throw SerializationError("decompressZlib: unget failed");
+ }
+}
+
+void compress(const SharedBuffer<u8> &data, std::ostream &os, u8 version, int level)
+{
+ if(version >= 29)
+ {
+ // map the zlib levels [-1,9]
+ // the range 1-10 is good enough, 0 is the default (currently 3)
+ compressZstd(*data, data.getSize(), os, level + 1);
+ return;
+ }
+
  if(version >= 11)
  {
- compressZlib(*data ,data.getSize(), os);
+ compressZlib(*data, data.getSize(), os, level);
  return;
  }
 
@@ -240,8 +328,20 @@ void compress(const SharedBuffer<u8> &data, std::ostream &os, u8 version)
  os.write((char*)&current_byte, 1);
 }
 
+void compress(const std::string &data, std::ostream &os, u8 version, int level)
+{
+ SharedBuffer<u8> databuf((u8*)data.c_str(), data.size());
+ compress(databuf, os, version, level);
+}
+
 void decompress(std::istream &is, std::ostream &os, u8 version)
 {
+ if(version >= 29)
+ {
+ decompressZstd(is, os);
+ return;
+ }
+
  if(version >= 11)
  {
  decompressZlib(is, os);

src/serialization.h

@@ -67,9 +67,9 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 // This represents an uninitialized or invalid format
 #define SER_FMT_VER_INVALID 255
 // Highest supported serialization version
-#define SER_FMT_VER_HIGHEST_READ 28
+#define SER_FMT_VER_HIGHEST_READ 29
 // Saved on disk version
-#define SER_FMT_VER_HIGHEST_WRITE 28
+#define SER_FMT_VER_HIGHEST_WRITE 29
 // Lowest supported serialization version
 #define SER_FMT_VER_LOWEST_READ 0
 // Lowest serialization version for writing
@@ -89,7 +89,11 @@ void compressZlib(const u8 *data, size_t data_size, std::ostream &os, int level
 void compressZlib(const std::string &data, std::ostream &os, int level = -1);
 void decompressZlib(std::istream &is, std::ostream &os, size_t limit = 0);
 
+void compressZstd(const u8 *data, size_t data_size, std::ostream &os, int level = 0);
+void compressZstd(const std::string &data, std::ostream &os, int level = 0);
+void decompressZstd(std::istream &is, std::ostream &os);
+
 // These choose between zlib and a self-made one according to version
-void compress(const SharedBuffer<u8> &data, std::ostream &os, u8 version);
-//void compress(const std::string &data, std::ostream &os, u8 version);
+void compress(const SharedBuffer<u8> &data, std::ostream &os, u8 version, int level = -1);
+void compress(const std::string &data, std::ostream &os, u8 version, int level = -1);
 void decompress(std::istream &is, std::ostream &os, u8 version);