Switch MapBlock compression to zstd

.github/workflows/build.yml

@@ -227,7 +227,7 @@ jobs:
  env:
  VCPKG_VERSION: 0bf3923f9fab4001c00f0f429682a0853b5749e0
 # 2020.11
- vcpkg_packages: irrlicht zlib curl[winssl] openal-soft libvorbis libogg sqlite3 freetype luajit
+ vcpkg_packages: irrlicht zlib zstd curl[winssl] openal-soft libvorbis libogg sqlite3 freetype luajit
  strategy:
  fail-fast: false
  matrix:

.github/workflows/macos.yml

@@ -34,7 +34,7 @@ jobs:
  - uses: actions/checkout@v2
  - name: Install deps
  run: |
- pkgs=(cmake freetype gettext gmp hiredis jpeg jsoncpp leveldb libogg libpng libvorbis luajit)
+ pkgs=(cmake freetype gettext gmp hiredis jpeg jsoncpp leveldb libogg libpng libvorbis luajit zstd)
  brew update
  brew install ${pkgs[@]}
  brew unlink $(brew ls --formula)

Dockerfile

@@ -20,7 +20,7 @@ COPY textures /usr/src/minetest/textures
 
 WORKDIR /usr/src/minetest
 
-RUN apk add --no-cache git build-base cmake sqlite-dev curl-dev zlib-dev \
+RUN apk add --no-cache git build-base cmake sqlite-dev curl-dev zlib-dev zstd-dev \
  gmp-dev jsoncpp-dev postgresql-dev ninja luajit-dev ca-certificates && \
  git clone --depth=1 -b ${MINETEST_GAME_VERSION} https://github.com/minetest/minetest_game.git ./games/minetest_game && \
  rm -fr ./games/minetest_game/.git

android/native/jni/Android.mk

@@ -57,6 +57,11 @@ LOCAL_MODULE := Vorbis
 LOCAL_SRC_FILES := deps/Android/Vorbis/${NDK_TOOLCHAIN_VERSION}/$(APP_ABI)/libvorbis.a
 include $(PREBUILT_STATIC_LIBRARY)
 
+include $(CLEAR_VARS)
+LOCAL_MODULE := Zstd
+LOCAL_SRC_FILES := deps/Android/Zstd/${NDK_TOOLCHAIN_VERSION}/$(APP_ABI)/libzstd.a
+include $(PREBUILT_STATIC_LIBRARY)
+
 include $(CLEAR_VARS)
 LOCAL_MODULE := Minetest
 
@@ -101,7 +106,8 @@ LOCAL_C_INCLUDES := \
  deps/Android/LuaJIT/src \
  deps/Android/OpenAL-Soft/include \
  deps/Android/sqlite \
- deps/Android/Vorbis/include
+ deps/Android/Vorbis/include \
+ deps/Android/Zstd/include
 
 LOCAL_SRC_FILES := \
  $(wildcard ../../src/client/*.cpp) \
@@ -201,7 +207,7 @@ LOCAL_SRC_FILES += \
 # SQLite3
 LOCAL_SRC_FILES += deps/Android/sqlite/sqlite3.c
 
-LOCAL_STATIC_LIBRARIES += Curl Freetype Irrlicht OpenAL mbedTLS mbedx509 mbedcrypto Vorbis LuaJIT GetText android_native_app_glue $(PROFILER_LIBS) #LevelDB
+LOCAL_STATIC_LIBRARIES += Curl Freetype Irrlicht OpenAL mbedTLS mbedx509 mbedcrypto Vorbis LuaJIT GetText Zstd android_native_app_glue $(PROFILER_LIBS) #LevelDB
 
 LOCAL_LDLIBS := -lEGL -lGLESv1_CM -lGLESv2 -landroid -lOpenSLES
 

builtin/settingtypes.txt

@@ -1100,11 +1100,10 @@ full_block_send_enable_min_time_from_building (Delay in sending blocks after bui
 # client number.
 max_packets_per_iteration (Max. packets per iteration) int 1024
 
-# ZLib compression level to use when sending mapblocks to the client.
-# -1 - Zlib's default compression level
-# 0 - no compresson, fastest
+# Compression level to use when sending mapblocks to the client.
+# -1 - use default compression level
+# 0 - least compresson, fastest
 # 9 - best compression, slowest
-# (levels 1-3 use Zlib's "fast" method, 4-9 use the normal method)
 map_compression_level_net (Map Compression Level for Network Transfer) int -1 -1 9
 
 [*Game]
@@ -1303,12 +1302,11 @@ max_objects_per_block (Maximum objects per block) int 64
 # See https://www.sqlite.org/pragma.html#pragma_synchronous
 sqlite_synchronous (Synchronous SQLite) enum 2 0,1,2
 
-# ZLib compression level to use when saving mapblocks to disk.
-# -1 - Zlib's default compression level
-# 0 - no compresson, fastest
+# Compression level to use when saving mapblocks to disk.
+# -1 - use default compression level
+# 0 - least compresson, fastest
 # 9 - best compression, slowest
-# (levels 1-3 use Zlib's "fast" method, 4-9 use the normal method)
-map_compression_level_disk (Map Compression Level for Disk Storage) int 3 -1 9
+map_compression_level_disk (Map Compression Level for Disk Storage) int -1 -1 9
 
 # Length of a server tick and the interval at which objects are generally updated over
 # network.

cmake/Modules/FindZstd.cmake

@@ -0,0 +1,9 @@
+mark_as_advanced(ZSTD_LIBRARY ZSTD_INCLUDE_DIR)
+
+find_path(ZSTD_INCLUDE_DIR NAMES zstd.h)
+
+find_library(ZSTD_LIBRARY NAMES zstd)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Zstd DEFAULT_MSG ZSTD_LIBRARY ZSTD_INCLUDE_DIR)
+

doc/world_format.txt

@@ -1,5 +1,5 @@
 =============================
-Minetest World Format 22...27
+Minetest World Format 22...29
 =============================
 
 This applies to a world format carrying the block serialization version
@@ -8,6 +8,7 @@ This applies to a world format carrying the block serialization version
 - 0.4.0 (23)
 - 24 was never released as stable and existed for ~2 days
 - 27 was added in 0.4.15-dev
+- 29 was added in 5.5.0-dev
 
 The block serialization version does not fully specify every aspect of this
 format; if compliance with this format is to be checked, it needs to be
@@ -281,6 +282,8 @@ MapBlock serialization format
 NOTE: Byte order is MSB first (big-endian).
 NOTE: Zlib data is in such a format that Python's zlib at least can
  directly decompress.
+NOTE: Since version 29 zstd is used instead of zlib. In addition the entire
+ block is first serialized and then compressed (except the version byte).
 
 u8 version
 - map format version number, see serialisation.h for the latest number
@@ -324,6 +327,21 @@ u16 lighting_complete
  then Minetest will correct lighting in the day light bank when
  the block at (1, 0, 0) is also loaded.
 
+if map format version >= 29:
+ u32 timestamp
+ - Timestamp when last saved, as seconds from starting the game.
+ - 0xffffffff = invalid/unknown timestamp, nothing should be done with the time
+ difference when loaded
+
+ u16 num_name_id_mappings
+
+ foreach num_name_id_mappings
+ u16 id
+ u16 name_len
+ u8[name_len] name
+if map format version < 29:
+ -- Nothing right here, timpstamp and node id mappings are serialized later
+
 u8 content_width
 - Number of bytes in the content (param0) fields of nodes
 if map format version <= 23:
@@ -335,7 +353,7 @@ u8 params_width
 - Number of bytes used for parameters per node
 - Always 2
 
-zlib-compressed node data:
+node data (zlib-compressed if version < 29):
 if content_width == 1:
  - content:
  u8[4096]: param0 fields
@@ -348,31 +366,34 @@ if content_width == 2:
  u8[4096]: param2 fields
 - The location of a node in each of those arrays is (z*16*16 + y*16 + x).
 
-zlib-compressed node metadata list
-- content:
-if map format version <= 22:
- u16 version (=1)
- u16 count of metadata
- foreach count:
- u16 position (p.Z*MAP_BLOCKSIZE*MAP_BLOCKSIZE + p.Y*MAP_BLOCKSIZE + p.X)
- u16 type_id
- u16 content_size
- u8[content_size] content of metadata. Format depends on type_id, see below.
-if map format version >= 23:
- u8 version -- Note: type was u16 for map format version <= 22
- -- = 1 for map format version < 28
- -- = 2 since map format version 28
- u16 count of metadata
- foreach count:
- u16 position (p.Z*MAP_BLOCKSIZE*MAP_BLOCKSIZE + p.Y*MAP_BLOCKSIZE + p.X)
- u32 num_vars
- foreach num_vars:
- u16 key_len
- u8[key_len] key
- u32 val_len
- u8[val_len] value
- u8 is_private -- only for version >= 2. 0 = not private, 1 = private
- serialized inventory
+if map format version < 29:
+ zlib-compressed node metadata list
+ - content:
+ if map format version <= 22:
+ u16 version (=1)
+ u16 count of metadata
+ foreach count:
+ u16 position (p.Z*MAP_BLOCKSIZE*MAP_BLOCKSIZE + p.Y*MAP_BLOCKSIZE + p.X)
+ u16 type_id
+ u16 content_size
+ u8[content_size] content of metadata. Format depends on type_id, see below.
+ if map format version >= 23:
+ u8 version -- Note: type was u16 for map format version <= 22
+ -- = 1 for map format version < 28
+ -- = 2 since map format version 28
+ u16 count of metadata
+ foreach count:
+ u16 position (p.Z*MAP_BLOCKSIZE*MAP_BLOCKSIZE + p.Y*MAP_BLOCKSIZE + p.X)
+ u32 num_vars
+ foreach num_vars:
+ u16 key_len
+ u8[key_len] key
+ u32 val_len
+ u8[val_len] value
+ u8 is_private -- only for version >= 2. 0 = not private, 1 = private
+ serialized inventory
+if map format version >= 29:
+ -- Nothing right here, node metadata is serialized later
 
 - Node timers
 if map format version == 23:
@@ -403,20 +424,22 @@ foreach static_object_count:
  u16 data_size
  u8[data_size] data
 
-u32 timestamp
-- Timestamp when last saved, as seconds from starting the game.
-- 0xffffffff = invalid/unknown timestamp, nothing should be done with the time
- difference when loaded
+if map format version < 29:
+ u32 timestamp
+ - Timestamp when last saved, as seconds from starting the game.
+ - 0xffffffff = invalid/unknown timestamp, nothing should be done with the time
+ difference when loaded
 
 u8 name-id-mapping version
 - Always 0
 
-u16 num_name_id_mappings
+if map format version < 29:
+ u16 num_name_id_mappings
 
-foreach num_name_id_mappings
- u16 id
- u16 name_len
- u8[name_len] name
+ foreach num_name_id_mappings
+  u16 id
+  u16 name_len
+  u8[name_len] name
 
 - Node timers
 if map format version == 25:
@@ -427,6 +450,22 @@ if map format version == 25:
  s32 timeout*1000
  s32 elapsed*1000
 
+if map format version >= 29:
+ node metadata list
+ - content:
+ u8 version (always = 2)
+ u16 count of metadata
+ foreach count:
+ u16 position (p.Z*MAP_BLOCKSIZE*MAP_BLOCKSIZE + p.Y*MAP_BLOCKSIZE + p.X)
+ u32 num_vars
+ foreach num_vars:
+ u16 key_len
+ u8[key_len] key
+ u32 val_len
+ u8[val_len] value
+ u8 is_private -- 0 = not private, 1 = private
+ serialized inventory
+
 EOF.
 
 Format of nodes

src/CMakeLists.txt

@@ -271,9 +271,13 @@ if(WIN32)
  find_path(ZLIB_INCLUDE_DIR "zlib.h" DOC "Zlib include directory")
  find_library(ZLIB_LIBRARIES "zlib" DOC "Path to zlib library")
 
+ find_path(ZSTD_INCLUDE_DIR "zstd.h" DOC "Zstd include directory")
+ find_library(ZSTD_LIBRARY "zstd" DOC "Path to zstd library")
+
  # Dll's are automatically copied to the output directory by vcpkg when VCPKG_APPLOCAL_DEPS=ON
  if(NOT VCPKG_APPLOCAL_DEPS)
  find_file(ZLIB_DLL NAMES "zlib.dll" "zlib1.dll" DOC "Path to zlib.dll for installation (optional)")
+ find_file(ZSTD_DLL NAMES "zstd.dll" DOC "Path to zstd.dll for installation (optional)")
  if(ENABLE_SOUND)
  set(OPENAL_DLL "" CACHE FILEPATH "Path to OpenAL32.dll for installation (optional)")
  set(OGG_DLL "" CACHE FILEPATH "Path to libogg.dll for installation (optional)")
@@ -296,6 +300,7 @@ else()
  endif()
 
  find_package(ZLIB REQUIRED)
+ find_package(Zstd REQUIRED)
  set(PLATFORM_LIBS -lpthread ${CMAKE_DL_LIBS})
  if(APPLE)
  set(PLATFORM_LIBS "-framework CoreFoundation" ${PLATFORM_LIBS})
@@ -486,6 +491,7 @@ include_directories(
  ${PROJECT_BINARY_DIR}
  ${PROJECT_SOURCE_DIR}
  ${ZLIB_INCLUDE_DIR}
+ ${ZSTD_INCLUDE_DIR}
  ${SOUND_INCLUDE_DIRS}
  ${SQLITE3_INCLUDE_DIR}
  ${LUA_INCLUDE_DIR}
@@ -521,6 +527,7 @@ if(BUILD_CLIENT)
  ${PROJECT_NAME}
  ${ZLIB_LIBRARIES}
  IrrlichtMt::IrrlichtMt
+ ${ZSTD_LIBRARY}
  ${X11_LIBRARIES}
  ${SOUND_LIBRARIES}
  ${SQLITE3_LIBRARY}
@@ -605,6 +612,7 @@ if(BUILD_SERVER)
  target_link_libraries(
  ${PROJECT_NAME}server
  ${ZLIB_LIBRARIES}
+ ${ZSTD_LIBRARY}
  ${SQLITE3_LIBRARY}
  ${JSON_LIBRARY}
  ${LUA_LIBRARY}
@@ -821,6 +829,9 @@ if(WIN32)
  if(ZLIB_DLL)
  install(FILES ${ZLIB_DLL} DESTINATION ${BINDIR})
  endif()
+ if(ZSTD_DLL)
+ install(FILES ${ZSTD_DLL} DESTINATION ${BINDIR})
+ endif()
  if(BUILD_CLIENT AND FREETYPE_DLL)
  install(FILES ${FREETYPE_DLL} DESTINATION ${BINDIR})
  endif()

src/defaultsettings.cpp

@@ -398,7 +398,7 @@ void set_default_settings()
  settings->setDefault("chat_message_limit_per_10sec", "8.0");
  settings->setDefault("chat_message_limit_trigger_kick", "50");
  settings->setDefault("sqlite_synchronous", "2");
- settings->setDefault("map_compression_level_disk", "3");
+ settings->setDefault("map_compression_level_disk", "-1");
  settings->setDefault("map_compression_level_net", "-1");
  settings->setDefault("full_block_send_enable_min_time_from_building", "2.0");
  settings->setDefault("dedicated_server_step", "0.09");
@@ -484,7 +484,7 @@ void set_default_settings()
  settings->setDefault("max_objects_per_block", "20");
  settings->setDefault("sqlite_synchronous", "1");
  settings->setDefault("map_compression_level_disk", "-1");
- settings->setDefault("map_compression_level_net", "3");
+ settings->setDefault("map_compression_level_net", "-1");
  settings->setDefault("server_map_save_interval", "15");
  settings->setDefault("client_mapblock_limit", "1000");
  settings->setDefault("active_block_range", "2");