add_inode: handle compressed files

CMakeLists.txt

@@ -16,6 +16,7 @@ find_package(fmt REQUIRED)
 
 set(SRC_FILES src/ntfs2btrfs.cpp
  src/ntfs.cpp
+ src/decomp.cpp
  src/crc32c.c)
 
 if(MSVC)

src/decomp.cpp

@@ -0,0 +1,146 @@
+/* Copyright (c) Mark Harmstone 2020
+ *
+ * This file is part of ntfs2btrfs.
+ *
+ * Ntfs2btrfs is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public Licence as published by
+ * the Free Software Foundation, either version 2 of the Licence, or
+ * (at your option) any later version.
+ *
+ * Ntfs2btrfs is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public Licence for more details.
+ *
+ * You should have received a copy of the GNU General Public Licence
+ * along with Ntfs2btrfs. If not, see <https://www.gnu.org/licenses/>. */
+
+#include "ntfs2btrfs.h"
+
+using namespace std;
+
+static string lznt1_decompress_chunk(string_view data) {
+ string s;
+
+ while (!data.empty()) {
+ auto fg = (uint8_t)data[0];
+
+ data = data.substr(1);
+
+ if (fg == 0) {
+ if (data.length() < 8) {
+ s.append(data);
+
+ return s;
+ } else {
+ s.append(data.substr(0, 8));
+ data = data.substr(8);
+ }
+ } else {
+ for (unsigned int i = 0; i < 8; i++) {
+ if (data.empty())
+ return s;
+
+ if (!(fg & 1)) {
+ s.append(data.substr(0, 1));
+ data = data.substr(1);
+ } else {
+ if (data.length() < sizeof(uint16_t))
+ throw formatted_error(FMT_STRING("Compressed chunk was {} bytes, expected at least 2."), data.length());
+
+ // See https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-xca/90fc6a28-f627-4ee5-82ce-445a6cf98b22
+
+ auto v = *(uint16_t*)data.data();
+
+ data = data.substr(2);
+
+ // Shamelessly stolen from https://github.com/you0708/lznt1 - thank you!
+
+ uint64_t u = s.length() - 1;
+ uint64_t lm = 0xfff;
+ uint64_t os = 12;
+
+ while (u >= 0x10) {
+ lm >>= 1;
+ os--;
+ u >>= 1;
+ }
+
+ auto l = (v & lm) + 3;
+ auto d = (v >> os) + 1;
+
+ s.reserve(s.length() + l);
+
+ while (l > 0) {
+ s.append(s.substr(s.length() - d, 1));
+ l--;
+ }
+ }
+
+ fg >>= 1;
+ }
+ }
+ }
+
+ return s;
+}
+
+string lznt1_decompress(string_view compdata, uint64_t size) {
+ string ret;
+ char* ptr;
+
+ ret.resize(size);
+ memset(ret.data(), 0, ret.size());
+
+ ptr = ret.data();
+
+ while (true) {
+ if (compdata.length() < sizeof(uint16_t))
+ throw formatted_error(FMT_STRING("compdata was {} bytes, expected at least 2."), compdata.length());
+
+ auto h = *(uint16_t*)compdata.data();
+
+ if (h == 0)
+ return ret;
+
+ compdata = compdata.substr(2);
+
+ auto sig = (h & 0x7000) >> 12;
+
+ if (sig != 3)
+ throw formatted_error(FMT_STRING("Compression signature was {}, expected 3."), sig);
+
+ auto len = (((uint64_t)h & 0xfff) + 1);
+
+ if (compdata.length() < len)
+ throw formatted_error(FMT_STRING("compdata was {} bytes, expected at least {}."), compdata.length(), len);
+
+ auto data = string_view(compdata.data(), len);
+
+ compdata = compdata.substr(len);
+
+ if (h & 0x8000) {
+ auto c = lznt1_decompress_chunk(data);
+
+ if (ptr + c.length() >= ret.data() + size) {
+ memcpy(ptr, c.data(), size - (ptr - ret.data()));
+
+ return ret;
+ } else {
+ memcpy(ptr, c.data(), c.length());
+ ptr += c.length();
+ }
+ } else {
+ if (ptr + data.length() >= ret.data() + size) {
+ memcpy(ptr, data.data(), size - (ptr - ret.data()));
+
+ return ret;
+ } else {
+ memcpy(ptr, data.data(), data.length());
+ ptr += data.length();
+ }
+ }
+ }
+
+ return ret;
+}

src/ntfs2btrfs.cpp

@@ -1687,29 +1687,35 @@ static void add_inode(root& r, uint64_t inode, uint64_t ntfs_inode, bool& is_dir
  return true;
  }
 
-
  if (att->FormCode == NTFS_ATTRIBUTE_FORM::RESIDENT_FORM) {
  file_size = att->Form.Resident.ValueLength;
 
  inline_data = res_data;
 
  // ATTRIBUTE_FLAG_COMPRESSION_MASK can be set in flags, but doesn't seem to do anything
  } else {
+ file_size = att->Form.Nonresident.FileSize;
+
  if (att->Flags & ATTRIBUTE_FLAG_COMPRESSION_MASK) {
- clear_line();
+ list<mapping> comp_mappings;
+ string compdata;
 
- if (filename.empty())
- filename = f.get_filename();
+ read_nonresident_mappings(att, comp_mappings, cluster_size);
 
- fmt::print(stderr, FMT_STRING("Skipping compressed inode {:x} ({})\n"), inode - inode_offset, filename.c_str());
- return true;
- }
+ compdata.resize(att->Form.Nonresident.TotalAllocated);
+ memset(compdata.data(), 0, compdata.length());
 
- file_size = att->Form.Nonresident.FileSize;
+ for (const auto& m : comp_mappings) {
+ dev.seek(m.lcn * cluster_size);
+ dev.read(compdata.data() + (m.vcn * cluster_size), m.length * cluster_size);
+ }
 
- // FIXME - if ValidDataLength < FileSize, will need to zero end
+ inline_data = lznt1_decompress(compdata, file_size);
+ } else {
+ // FIXME - if ValidDataLength < FileSize, will need to zero end
 
- read_nonresident_mappings(att, mappings, cluster_size);
+ read_nonresident_mappings(att, mappings, cluster_size);
+ }
  }
  } else { // ADS
  static const char xattr_prefix[] = "user.";

src/ntfs2btrfs.h

@@ -257,3 +257,6 @@ void read_nonresident_mappings(const ATTRIBUTE_RECORD_HEADER* att, std::list<map
 std::string_view find_sd(uint32_t id, ntfs_file& secure, ntfs& dev);
 void populate_skip_list(ntfs& dev, uint64_t inode, std::list<uint64_t>& skiplist);
 void process_fixups(MULTI_SECTOR_HEADER* header, uint64_t length, unsigned int sector_size);
+
+// decomp.cpp
+std::string lznt1_decompress(std::string_view compdata, uint64_t size);