Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Switch MapBlock compression to zstd #10788

Merged
merged 20 commits into from
Sep 1, 2021
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
move meta back
  • Loading branch information
sfan5 committed Aug 31, 2021
commit 21a94e3e89172c582f5e22c617972e5c3708e567
80 changes: 28 additions & 52 deletions doc/world_format.txt
Original file line number Diff line number Diff line change
Expand Up @@ -334,7 +334,6 @@ if map format version >= 29:
difference when loaded

u16 num_name_id_mappings

foreach num_name_id_mappings
u16 id
u16 name_len
Expand Down Expand Up @@ -366,34 +365,31 @@ if content_width == 2:
u8[4096]: param2 fields
- The location of a node in each of those arrays is (z*16*16 + y*16 + x).

if map format version < 29:
zlib-compressed node metadata list
- content:
if map format version <= 22:
u16 version (=1)
u16 count of metadata
foreach count:
u16 position (p.Z*MAP_BLOCKSIZE*MAP_BLOCKSIZE + p.Y*MAP_BLOCKSIZE + p.X)
u16 type_id
u16 content_size
u8[content_size] content of metadata. Format depends on type_id, see below.
if map format version >= 23:
u8 version -- Note: type was u16 for map format version <= 22
-- = 1 for map format version < 28
-- = 2 since map format version 28
u16 count of metadata
foreach count:
u16 position (p.Z*MAP_BLOCKSIZE*MAP_BLOCKSIZE + p.Y*MAP_BLOCKSIZE + p.X)
u32 num_vars
foreach num_vars:
u16 key_len
u8[key_len] key
u32 val_len
u8[val_len] value
u8 is_private -- only for version >= 2. 0 = not private, 1 = private
serialized inventory
if map format version >= 29:
-- Nothing right here, node metadata is serialized later
node metadata list (zlib-compressed if version < 29):
- content:
if map format version <= 22:
u16 version (=1)
u16 count of metadata
foreach count:
u16 position (p.Z*MAP_BLOCKSIZE*MAP_BLOCKSIZE + p.Y*MAP_BLOCKSIZE + p.X)
u16 type_id
u16 content_size
u8[content_size] content of metadata. Format depends on type_id, see below.
if map format version >= 23:
u8 version -- Note: type was u16 for map format version <= 22
-- = 1 for map format version < 28
-- = 2 since map format version 28
u16 count of metadata
foreach count:
u16 position (p.Z*MAP_BLOCKSIZE*MAP_BLOCKSIZE + p.Y*MAP_BLOCKSIZE + p.X)
u32 num_vars
foreach num_vars:
u16 key_len
u8[key_len] key
u32 val_len
u8[val_len] value
u8 is_private -- only for version >= 2. 0 = not private, 1 = private
serialized inventory

- Node timers
if map format version == 23:
Expand Down Expand Up @@ -426,16 +422,12 @@ foreach static_object_count:

if map format version < 29:
u32 timestamp
- Timestamp when last saved, as seconds from starting the game.
- 0xffffffff = invalid/unknown timestamp, nothing should be done with the time
difference when loaded
- Same meaning as the timestamp further up

u8 name-id-mapping version
- Always 0
u8 name-id-mapping version
- Always 0

if map format version < 29:
u16 num_name_id_mappings

foreach num_name_id_mappings
u16 id
u16 name_len
Expand All @@ -450,22 +442,6 @@ if map format version == 25:
s32 timeout*1000
s32 elapsed*1000

if map format version >= 29:
node metadata list
- content:
u8 version (always = 2)
u16 count of metadata
foreach count:
u16 position (p.Z*MAP_BLOCKSIZE*MAP_BLOCKSIZE + p.Y*MAP_BLOCKSIZE + p.X)
u32 num_vars
foreach num_vars:
u16 key_len
u8[key_len] key
u32 val_len
u8[val_len] value
u8 is_private -- 0 = not private, 1 = private
serialized inventory

EOF.

Format of nodes
Expand Down
62 changes: 23 additions & 39 deletions src/mapblock.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -367,6 +367,7 @@ void MapBlock::serialize(std::ostream &os_compressed, u8 version, bool disk, int

std::ostringstream os_raw(std::ios_base::binary);
std::ostream &os = version >= 29 ? os_raw : os_compressed;

// First byte
u8 flags = 0;
if(is_underground)
Expand All @@ -390,20 +391,17 @@ void MapBlock::serialize(std::ostream &os_compressed, u8 version, bool disk, int
if(disk)
{
MapNode *tmp_nodes = new MapNode[nodecount];
for(u32 i=0; i<nodecount; i++)
tmp_nodes[i] = data[i];
memcpy(tmp_nodes, data, nodecount * sizeof(MapNode));
getBlockNodeIdMapping(&nimap, tmp_nodes, m_gamedef->ndef());

buf = MapNode::serializeBulk(version, tmp_nodes, nodecount,
content_width, params_width);
delete[] tmp_nodes;

// write timestamp and node/id mapping first
if(version >= 29) {
// Timestamp
if (version >= 29) {
writeU32(os, getTimestamp());

// Write block-specific node definition id mapping
nimap.serialize(os);
}
}
Expand All @@ -415,17 +413,19 @@ void MapBlock::serialize(std::ostream &os_compressed, u8 version, bool disk, int

writeU8(os, content_width);
writeU8(os, params_width);
if(version >= 29) {
if (version >= 29) {
os.write(reinterpret_cast<char*>(*buf), buf.getSize());
} else {
// prior to 29 node data was compressed individually
compress(buf, os, version, compression_level);
}

/*
Node metadata (version < 29)
Node metadata
*/
if (version < 29) {
if (version >= 29) {
m_node_metadata.serialize(os, version, disk);
} else {
// use os_raw from above to avoid allocating another stream object
m_node_metadata.serialize(os_raw, version, disk);
// prior to 29 node data was compressed individually
Expand Down Expand Up @@ -459,12 +459,7 @@ void MapBlock::serialize(std::ostream &os_compressed, u8 version, bool disk, int
}
}

if(version >= 29) {
/*
Node metadata (version => 29)
*/
m_node_metadata.serialize(os, version, disk);

if (version >= 29) {
// now compress the whole thing
compress(os_raw.str(), os_compressed, version, compression_level);
}
Expand Down Expand Up @@ -494,11 +489,10 @@ void MapBlock::deSerialize(std::istream &in_compressed, u8 version, bool disk)
return;
}

// Decompress the whole block (version >= 29)
std::stringstream in_raw(std::ios_base::binary | std::ios_base::in | std::ios_base::out);
if (version >= 29) {
// decompress the whole thing
if (version >= 29)
decompress(in_compressed, in_raw, version);
}
std::istream &is = version >= 29 ? in_raw : in_compressed;

u8 flags = readU8(is);
Expand All @@ -511,7 +505,7 @@ void MapBlock::deSerialize(std::istream &in_compressed, u8 version, bool disk)
m_generated = (flags & 0x08) == 0;

NameIdMapping nimap;
if(disk && version >= 29) {
if (disk && version >= 29) {
// Timestamp
TRACESTREAM(<<"MapBlock::deSerialize "<<PP(getPos())
<<": Timestamp"<<std::endl);
Expand Down Expand Up @@ -544,15 +538,18 @@ void MapBlock::deSerialize(std::istream &in_compressed, u8 version, bool disk)
decompress(is, in_raw, version);
MapNode::deSerializeBulk(in_raw, version, data, nodecount,
content_width, params_width);
}

/*
NodeMetadata (version < 29)
*/
TRACESTREAM(<<"MapBlock::deSerialize "<<PP(getPos())
<<": Node metadata"<<std::endl);
// Ignore errors
/*
NodeMetadata
*/
TRACESTREAM(<<"MapBlock::deSerialize "<<PP(getPos())
<<": Node metadata"<<std::endl);
if (version >= 29) {
m_node_metadata.deSerialize(is, m_gamedef->idef());
} else {
try {
// resuse in_raw
// reuse in_raw
in_raw.str("");
in_raw.clear();
decompress(is, in_raw, version);
Expand Down Expand Up @@ -597,10 +594,9 @@ void MapBlock::deSerialize(std::istream &in_compressed, u8 version, bool disk)
setTimestampNoChangedFlag(readU32(is));
m_disk_timestamp = m_timestamp;

// Dynamically re-set ids based on node names
// Node/id mapping
TRACESTREAM(<<"MapBlock::deSerialize "<<PP(getPos())
<<": NameIdMapping"<<std::endl);
NameIdMapping nimap;
nimap.deSerialize(is);
}

Expand All @@ -613,18 +609,6 @@ void MapBlock::deSerialize(std::istream &in_compressed, u8 version, bool disk)
m_node_timers.deSerialize(is, version);
}
}
if (version >= 29) {
try {
TRACESTREAM(<<"MapBlock::deSerialize "<<PP(getPos())
<<": Node metadata"<<std::endl);

m_node_metadata.deSerialize(is, m_gamedef->idef());
} catch(SerializationError &e) {
warningstream<<"MapBlock::deSerialize(): Ignoring an error"
<<" while deserializing node metadata at ("
<<PP(getPos())<<": "<<e.what()<<std::endl;
}
}

TRACESTREAM(<<"MapBlock::deSerialize "<<PP(getPos())
<<": Done."<<std::endl);
Expand Down