Add zstd compression support (minetest#12515)

builtin/game/features.lua

@@ -26,6 +26,7 @@ core.features = {
  get_sky_as_table = true,
  get_light_data_buffer = true,
  mod_storage_on_disk = true,
+ compress_zstd = true,
 }
 
 function core.has_feature(arg)

doc/client_lua_api.txt

@@ -897,14 +897,19 @@ Call these functions only at load time!
  * Compress a string of data.
  * `method` is a string identifying the compression method to be used.
  * Supported compression methods:
- * Deflate (zlib): `"deflate"`
- * `...` indicates method-specific arguments. Currently defined arguments are:
- * Deflate: `level` - Compression level, `0`-`9` or `nil`.
+ * Deflate (zlib): `"deflate"`
+ * Zstandard: `"zstd"`
+ * `...` indicates method-specific arguments. Currently defined arguments
+ are:
+ * Deflate: `level` - Compression level, `0`-`9` or `nil`.
+ * Zstandard: `level` - Compression level. Integer or `nil`. Default `3`.
+ Note any supported Zstandard compression level could be used here,
+ but these are subject to change between Zstandard versions.
 * `minetest.decompress(compressed_data, method, ...)`: returns data
- * Decompress a string of data (using ZLib).
- * See documentation on `minetest.compress()` for supported compression methods.
- * currently supported.
- * `...` indicates method-specific arguments. Currently, no methods use this.
+ * Decompress a string of data using the algorithm specified by `method`.
+ * See documentation on `minetest.compress()` for supported compression
+  methods.
+ * `...` indicates method-specific arguments. Currently, no methods use this
 * `minetest.rgba(red, green, blue[, alpha])`: returns a string
  * Each argument is a 8 Bit unsigned integer
  * Returns the ColorString from rgb or rgba values

doc/lua_api.txt

@@ -4910,6 +4910,8 @@ Utilities
  -- the amount of data in mod storage is not constrained by
  -- the amount of RAM available. (5.7.0)
  mod_storage_on_disk = true,
+ -- "zstd" method for compress/decompress (5.7.0)
+ compress_zstd = true,
  }
 
 * `minetest.has_feature(arg)`: returns `boolean, missing_features`
@@ -6370,11 +6372,15 @@ Misc.
  * `method` is a string identifying the compression method to be used.
  * Supported compression methods:
  * Deflate (zlib): `"deflate"`
+ * Zstandard: `"zstd"`
  * `...` indicates method-specific arguments. Currently defined arguments
  are:
  * Deflate: `level` - Compression level, `0`-`9` or `nil`.
+ * Zstandard: `level` - Compression level. Integer or `nil`. Default `3`.
+ Note any supported Zstandard compression level could be used here,
+ but these are subject to change between Zstandard versions.
 * `minetest.decompress(compressed_data, method, ...)`: returns data
- * Decompress a string of data (using ZLib).
+ * Decompress a string of data using the algorithm specified by `method`.
  * See documentation on `minetest.compress()` for supported compression
  methods.
  * `...` indicates method-specific arguments. Currently, no methods use this

games/devtest/mods/unittests/misc.lua

@@ -80,3 +80,26 @@ unittests.register("test_punch_node", function(_, pos)
  minetest.remove_node(pos)
  -- currently failing: assert(on_punch_called)
 end, {map=true})
+
+local function test_compress()
+ -- This text should be compressible, to make sure the results are... normal
+ local text = "The\000 icey canoe couldn't move very well on the\128 lake. The\000 ice was too stiff and the icey canoe's paddles simply wouldn't punch through."
+ local methods = {
+ "deflate",
+ "zstd",
+ -- "noodle", -- for warning alarm test
+ }
+ local zstd_magic = string.char(0x28, 0xB5, 0x2F, 0xFD)
+ for _, method in ipairs(methods) do
+ local compressed = core.compress(text, method)
+ assert(core.decompress(compressed, method) == text, "input/output mismatch for compression method " .. method)
+ local has_zstd_magic = compressed:sub(1, 4) == zstd_magic
+ if method == "zstd" then
+ assert(has_zstd_magic, "zstd magic number not in zstd method")
+ else
+ assert(not has_zstd_magic, "zstd magic number in method " .. method .. " (which is not zstd)")
+ end
+ end
+end
+unittests.register("test_compress", test_compress)
+

src/script/lua_api/l_util.cpp

@@ -26,6 +26,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "cpp_api/s_async.h"
 #include "serialization.h"
 #include <json/json.h>
+#include <zstd.h>
 #include "cpp_api/s_security.h"
 #include "porting.h"
 #include "convert_json.h"
@@ -278,6 +279,34 @@ int ModApiUtil::l_get_user_path(lua_State *L)
  return 1;
 }
 
+enum LuaCompressMethod
+{
+ LUA_COMPRESS_METHOD_DEFLATE,
+ LUA_COMPRESS_METHOD_ZSTD,
+};
+
+static const struct EnumString es_LuaCompressMethod[] =
+{
+ {LUA_COMPRESS_METHOD_DEFLATE, "deflate"},
+ {LUA_COMPRESS_METHOD_ZSTD, "zstd"},
+ {0, nullptr},
+};
+
+static LuaCompressMethod get_compress_method(lua_State *L, int index)
+{
+ if (lua_isnoneornil(L, index))
+ return LUA_COMPRESS_METHOD_DEFLATE;
+ const char *name = luaL_checkstring(L, index);
+ int value;
+ if (!string_to_enum(es_LuaCompressMethod, value, name)) {
+ // Pretend it's deflate if we don't know, for compatibility reasons.
+ log_deprecated(L, "Unknown compression method \"" + std::string(name)
+ + "\", defaulting to \"deflate\". You should pass a valid value.");
+ return LUA_COMPRESS_METHOD_DEFLATE;
+ }
+ return (LuaCompressMethod) value;
+}
+
 // compress(data, method, level)
 int ModApiUtil::l_compress(lua_State *L)
 {
@@ -286,12 +315,23 @@ int ModApiUtil::l_compress(lua_State *L)
  size_t size;
  const char *data = luaL_checklstring(L, 1, &size);
 
- int level = -1;
- if (!lua_isnoneornil(L, 3))
- level = readParam<int>(L, 3);
+ LuaCompressMethod method = get_compress_method(L, 2);
 
  std::ostringstream os(std::ios_base::binary);
- compressZlib(reinterpret_cast<const u8 *>(data), size, os, level);
+
+ if (method == LUA_COMPRESS_METHOD_DEFLATE) {
+ int level = -1;
+ if (!lua_isnoneornil(L, 3))
+ level = readParam<int>(L, 3);
+
+ compressZlib(reinterpret_cast<const u8 *>(data), size, os, level);
+ } else if (method == LUA_COMPRESS_METHOD_ZSTD) {
+ int level = ZSTD_CLEVEL_DEFAULT;
+ if (!lua_isnoneornil(L, 3))
+ level = readParam<int>(L, 3);
+
+ compressZstd(reinterpret_cast<const u8 *>(data), size, os, level);
+ }
 
  std::string out = os.str();
 
@@ -307,9 +347,16 @@ int ModApiUtil::l_decompress(lua_State *L)
  size_t size;
  const char *data = luaL_checklstring(L, 1, &size);
 
+ LuaCompressMethod method = get_compress_method(L, 2);
+
  std::istringstream is(std::string(data, size), std::ios_base::binary);
  std::ostringstream os(std::ios_base::binary);
- decompressZlib(is, os);
+
+ if (method == LUA_COMPRESS_METHOD_DEFLATE) {
+ decompressZlib(is, os);
+ } else if (method == LUA_COMPRESS_METHOD_ZSTD) {
+ decompressZstd(is, os);
+ }
 
  std::string out = os.str();