lib: chunkio: sync dev changes (details below)

 8ce449f file: fix invalid metadata length
 8487554 crc32: endian.h is in /usr/include/sys on FreeBSD
 02a5eed file: fallocate exists only on Linux
 530063e meta: move the file writer into a separate function
 709e7d2 file_compat: include <chunkio_compat.h> explicitly
 d6edbdd meta: include <chunkio_compat.h> explicitly
 1e9947b scan: stub out cio_scan_streams() if file system backend is disabled
 4b56c09 compat: accumulate platform dependent headers
 7ef89bb os: include <sys/types.h> for mode_t
 1213306 compat: do not expose chunk_compat.h as part of library interface
 74d4651 compat: clean up the compatibility stuffs for Windows
 9146eae compat: fix the compat implementation of dirname()
 d1849a0 chunk: include <sys/types.h> explicitly
 b8ecd33 tests: memfs: avoid using mmap() for portability
 a9a4065 tests: memfs: do not touch file system while testing memfs
 496c2aa build: Use /W4 instead of /Wall on MSVC
 b75047a core: return early when given a unsupported backend
 90789c7 core: revert "CIO_HAVE_FILESYSTEM_BACKEND" ifdefs
 7378992 file_compat: implement a bare compat layer for cio_file_*
 7d0159c tools: cio: fix up the description of the command line options
 bc76f24 tests: fix SIGSEGV in 'cio-test-fs' and 'cio-test-memfs'
 26bd83c tools: cio: fix SIGSEGV due to incompatible argument types
 d11c50e tests: Add a missing ${UNIT_TESTS_FILES} if filesystem backend available
 b41a266 cio_meta: Add a missing CIO_HAVE_BACKEND_FILESYSTEM if branch
 22d4de5 compat: Add a missing ws2_32.lib pragma
 b7a83cc compat: Use strerror_s instead of strerror_r on Windows
 6e1fc85 compat: Implement getpagesize() on Windows
 8a00fd1 compat: Define alternative S_ISREG macro
 83f6a3c tests: Don't build filesystem backend testcase
 7f4546a Don't build filesystem backend related code
 f96d9bc build: Stop to build cio_file when filesystem backend is disabled
 d35623a build: Add switch for filesystem backend
 378d69e build: Adjust CFLAGS for MSVC
 0a14edb compat: new windows layer
 0bbb338 chunkio_compat: Use _access instead of access on Windows
 c00c404 Add compat layer
 7515cc0 tests: memfs: fix log callback function prototype

Signed-off-by: Eduardo Silva <[email protected]>

lib/chunkio/CMakeLists.txt

@@ -2,7 +2,12 @@ cmake_minimum_required(VERSION 2.8)
 project(chunk-io)
 
 # CFLAGS
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -fPIC ")
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+if(MSVC)
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /W4 ")
+else()
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall ")
+endif()
 
 # Set __FILENAME__
 if(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows")
@@ -17,6 +22,7 @@ option(CIO_LIB_STATIC "Enable static library build" On)
 option(CIO_LIB_SHARED "Enable shared library build" Off)
 option(CIO_SANITIZE_ADDRESS "Enable address sanitizer" Off)
 option(CIO_TESTS "Enable tests" Off)
+option(CIO_BACKEND_FILESYSTEM "Enable filesystem backend" On)
 
 # Force Option value
 macro(CIO_OPTION option value)
@@ -30,6 +36,13 @@ if(CIO_DEV)
  CIO_OPTION(CIO_LIB_STATIC On)
 endif()
 
+# Macro to set definitions
+macro(CIO_DEFINITION var)
+ add_definitions(-D${var})
+ set(CIO_BUILD_FLAGS "${CIO_BUILD_FLAGS}#ifndef ${var}\n#define ${var}\n#endif\n")
+ set(CIO_INFO_FLAGS "${CIO_INFO_FLAGS} ${var}")
+endmacro()
+
 # Check if Address Sanitizer is enabled
 if(CIO_SANITIZE_ADDRESS OR SANITIZE_ADDRESS)
  set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/sanitizers-cmake/cmake" ${CMAKE_MODULE_PATH})
@@ -47,6 +60,10 @@ if(CIO_SANITIZE_ADDRESS OR SANITIZE_ADDRESS)
  endif()
 endif()
 
+if(CIO_BACKEND_FILESYSTEM)
+ CIO_DEFINITION(CIO_HAVE_BACKEND_FILESYSTEM)
+endif()
+
 include_directories(
  include
  deps/

lib/chunkio/deps/crc32/crc32.c

@@ -29,6 +29,8 @@
 # define htole16(x) (x)
 # define be16toh(x) ntohs(x)
 # define le16toh(x) (x)
+#elif defined(__FreeBSD__)
+# include <sys/endian.h>
 #else
 # include <endian.h>
 #endif

lib/chunkio/include/chunkio/chunkio_compat.h

@@ -0,0 +1,72 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+
+/* Chunk I/O
+ * =========
+ * Copyright 2018 Eduardo Silva <[email protected]>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CHUNKIO_COMPAT_H
+#define CHUNKIO_COMPAT_H
+
+#ifdef _WIN32
+#include <winsock2.h>
+#include <windows.h>
+#include <io.h>
+#include <direct.h>
+#pragma comment(lib, "ws2_32.lib")
+
+/** mode flags for access() */
+#define R_OK 04
+#define W_OK 02
+#define X_OK 01
+#define F_OK 00
+
+#define PATH_MAX MAX_PATH
+#define S_ISREG(m) (((m) & S_IFMT) == S_IFREG)
+#define strerror_r(errno,buf,len) strerror_s(buf,len,errno)
+
+typedef SSIZE_T ssize_t;
+typedef unsigned mode_t;
+
+static inline char* dirname(const char *path)
+{
+ char drive[_MAX_DRIVE];
+ char dir[_MAX_DIR];
+ char fname[_MAX_FNAME];
+ char ext[_MAX_EXT];
+ static char buf[_MAX_PATH];
+
+ _splitpath_s(path, drive, _MAX_DRIVE, dir, _MAX_DIR,
+ fname, _MAX_FNAME, ext, _MAX_EXT);
+
+ _makepath_s(buf, _MAX_PATH, drive, dir, "", "");
+
+ return buf;
+}
+
+inline int getpagesize(void)
+{
+ SYSTEM_INFO system_info;
+ GetSystemInfo(&system_info);
+ return system_info.dwPageSize;
+}
+#else
+#include <unistd.h>
+#include <libgen.h>
+#include <dirent.h>
+#include <arpa/inet.h>
+#endif
+
+#endif

lib/chunkio/include/chunkio/cio_chunk.h

@@ -20,7 +20,7 @@
 #ifndef CIO_CHUNK_H
 #define CIO_CHUNK_H
 
-#include <unistd.h>
+#include <sys/types.h>
 #include <inttypes.h>
 
 struct cio_chunk {

lib/chunkio/include/chunkio/cio_file.h

@@ -48,6 +48,7 @@ struct cio_file *cio_file_open(struct cio_ctx *ctx,
  size_t size);
 void cio_file_close(struct cio_chunk *ch, int delete);
 int cio_file_write(struct cio_chunk *ch, const void *buf, size_t count);
+int cio_file_write_metadata(struct cio_chunk *ch, char *buf, size_t size);
 int cio_file_sync(struct cio_chunk *ch);
 int cio_file_fs_size_change(struct cio_file *cf, size_t new_size);
 int cio_file_close_stream(struct cio_stream *st);

lib/chunkio/include/chunkio/cio_file_st.h

@@ -69,7 +69,7 @@ static inline char *cio_file_st_get_hash(char *map)
 /* Return metadata length */
 static inline uint16_t cio_file_st_get_meta_len(char *map)
 {
- return (uint16_t) (map[22] << 8) | map[23];
+ return (uint16_t) ((uint8_t) map[22] << 8) | (uint8_t) map[23];
 }
 
 /* Set metadata length */

lib/chunkio/include/chunkio/cio_os.h

@@ -19,6 +19,7 @@
 
 #ifndef CIO_OS_H
 #define CIO_OS_H
+#include <sys/types.h>
 
 int cio_os_isdir(const char *dir);
 int cio_os_mkpath(const char *dir, mode_t mode);

lib/chunkio/src/CMakeLists.txt

@@ -2,7 +2,6 @@ set(src
  cio_os.c
  cio_log.c
  cio_memfs.c
- cio_file.c
  cio_chunk.c
  cio_meta.c
  cio_scan.c
@@ -11,6 +10,18 @@ set(src
  chunkio.c
  )
 
+if(CIO_BACKEND_FILESYSTEM)
+ set(src
+ ${src}
+ cio_file.c
+ )
+else()
+ set(src
+ ${src}
+ cio_file_compat.c
+ )
+endif()
+
 if(CIO_LIB_STATIC)
  add_library(chunkio-static STATIC ${src})
  target_link_libraries(chunkio-static cio-crc32)

lib/chunkio/src/chunkio.c

@@ -19,9 +19,9 @@
 
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
 
+#include <chunkio/chunkio_compat.h>
 #include <chunkio/chunkio.h>
 #include <chunkio/cio_os.h>
 #include <chunkio/cio_log.h>
@@ -63,6 +63,12 @@ struct cio_ctx *cio_create(const char *root_path,
  fprintf(stderr, "[cio] invalid log level, aborting");
  return NULL;
  }
+#ifndef CIO_HAVE_BACKEND_FILESYSTEM
+ if (root_path) {
+ fprintf(stderr, "[cio] file system backend not supported\n");
+ return NULL;
+ }
+#endif
 
  cio_page_size = getpagesize();
 

lib/chunkio/src/cio_chunk.c

@@ -17,6 +17,7 @@
  * limitations under the License.
  */
 
+#include <chunkio/chunkio_compat.h>
 #include <chunkio/chunkio.h>
 #include <chunkio/cio_file.h>
 #include <chunkio/cio_memfs.h>
@@ -46,6 +47,12 @@ struct cio_chunk *cio_chunk_open(struct cio_ctx *ctx, struct cio_stream *st,
  cio_log_error(ctx, "[cio chunk] invalid file name");
  return NULL;
  }
+#ifndef CIO_HAVE_BACKEND_FILESYSTEM
+ if (st->type == CIO_STORE_FS) {
+ cio_log_error(ctx, "[cio chunk] file system backend not supported");
+ return NULL;
+ }
+#endif
 
  /* allocate chunk context */
  ch = malloc(sizeof(struct cio_chunk));
@@ -88,12 +95,12 @@ void cio_chunk_close(struct cio_chunk *ch, int delete)
  int type;
 
  type = ch->st->type;
- if (type == CIO_STORE_FS) {
- cio_file_close(ch, delete);
- }
- else if (type == CIO_STORE_MEM) {
+ if (type == CIO_STORE_MEM) {
  cio_memfs_close(ch);
  }
+ else if (type == CIO_STORE_FS) {
+ cio_file_close(ch, delete);
+ }
 
  mk_list_del(&ch->_head);
  free(ch->name);
@@ -112,14 +119,14 @@ int cio_chunk_write_at(struct cio_chunk *ch, off_t offset,
  struct cio_file *cf;
 
  type = ch->st->type;
- if (type == CIO_STORE_FS) {
- cf = ch->backend;
- cf->data_size = offset;
- }
- else if (type == CIO_STORE_MEM) {
+ if (type == CIO_STORE_MEM) {
  mf = ch->backend;
  mf->buf_len = offset;
  }
+ else if (type == CIO_STORE_FS) {
+ cf = ch->backend;
+ cf->data_size = offset;
+ }
 
  /*
  * By default backends (fs, mem) appends data after the it last position,
@@ -130,16 +137,16 @@ int cio_chunk_write_at(struct cio_chunk *ch, off_t offset,
 
 int cio_chunk_write(struct cio_chunk *ch, const void *buf, size_t count)
 {
- int ret;
+ int ret = 0;
  int type;
 
  type = ch->st->type;
- if (type == CIO_STORE_FS) {
- ret = cio_file_write(ch, buf, count);
- }
- else if (type == CIO_STORE_MEM) {
+ if (type == CIO_STORE_MEM) {
  ret = cio_memfs_write(ch, buf, count);
  }
+ else if (type == CIO_STORE_FS) {
+ ret = cio_file_write(ch, buf, count);
+ }
 
  return ret;
 }
@@ -159,7 +166,7 @@ int cio_chunk_sync(struct cio_chunk *ch)
 
 int cio_chunk_get_content(struct cio_chunk *ch, char **buf, size_t *size)
 {
- int ret;
+ int ret = 0;
  int type;
  struct cio_memfs *mf;
  struct cio_file *cf;
@@ -169,7 +176,7 @@ int cio_chunk_get_content(struct cio_chunk *ch, char **buf, size_t *size)
  mf = ch->backend;
  *size = mf->buf_len;
  *buf = mf->buf_data;
- return 0;
+ return ret;
  }
  else if (type == CIO_STORE_FS) {
  cf = ch->backend;
@@ -179,7 +186,7 @@ int cio_chunk_get_content(struct cio_chunk *ch, char **buf, size_t *size)
  }
  *size = cf->data_size;
  *buf = cio_file_st_get_content(cf->map);
- return 0;
+ return ret;
  }
 
  return -1;