lib: chunkio: upgrade from v1.0.2 to v1.0.3

Signed-off-by: Eduardo Silva <[email protected]>

lib/chunkio/CMakeLists.txt

@@ -3,7 +3,7 @@ project(chunk-io)
 
 set(CIO_VERSION_MAJOR 1)
 set(CIO_VERSION_MINOR 0)
-set(CIO_VERSION_PATCH 2)
+set(CIO_VERSION_PATCH 3)
 set(CIO_VERSION_STR "${CIO_VERSION_MAJOR}.${CIO_VERSION_MINOR}.${CIO_VERSION_PATCH}")
 
 # CFLAGS

lib/chunkio/deps/crc32/crc32.c

@@ -2,8 +2,8 @@
  * \file
  * Functions and types for CRC checks.
  *
- * Generated on Tue Oct 30 22:09:40 2018
- * by pycrc v0.9.1, https://pycrc.org
+ * Generated on Wed Mar 18 12:41:20 2020
+ * by pycrc v0.9.2, https://pycrc.org
  * using the configuration:
  * - Width = 32
  * - Poly = 0x04c11db7
@@ -17,6 +17,8 @@
 #include "crc32.h" /* include the header file generated with pycrc */
 #include <stdlib.h>
 #include <stdint.h>
+
+/* Patch by Fluent Bit Authors */
 #ifdef __APPLE__
 # include <machine/endian.h>
 # include <libkern/OSByteOrder.h>
@@ -35,8 +37,6 @@
 # include <endian.h>
 #endif
 
-
-
 /**
  * Static table used for the table_driven implementation.
  */
@@ -321,6 +321,14 @@ crc_t crc_update(crc_t crc, const void *data, size_t data_len)
  const unsigned char *d = (const unsigned char *)data;
  unsigned int tbl_idx;
 
+ /* Align to a multiple of 8 bytes */
+ while (data_len && (((uintptr_t)(const void *)d) % 8 != 0)) {
+ tbl_idx = (crc ^ *d) & 0xff;
+ crc = (crc_table[0][tbl_idx] ^ (crc >> 8)) & 0xffffffff;
+ d++;
+ data_len--;
+ }
+
  const uint32_t *d32 = (const uint32_t *)d;
  while (data_len >= 8)
  {

lib/chunkio/deps/crc32/crc32.h

@@ -2,8 +2,8 @@
  * \file
  * Functions and types for CRC checks.
  *
- * Generated on Tue Oct 30 22:09:48 2018
- * by pycrc v0.9.1, https://pycrc.org
+ * Generated on Wed Mar 18 12:41:20 2020
+ * by pycrc v0.9.2, https://pycrc.org
  * using the configuration:
  * - Width = 32
  * - Poly = 0x04c11db7

lib/chunkio/deps/crc32/gen.sh

@@ -1,2 +1,2 @@
-./pycrc.py --generate c --algorithm table-driven --model crc-32 --slice-by 8 -o crc32.c
-./pycrc.py --generate h --algorithm table-driven --model crc-32 --slice-by 8 -o crc32.h
+pycrc-0.9.2/pycrc.py --generate c --algorithm table-driven --model crc-32 --slice-by 8 -o crc32.c
+pycrc-0.9.2/pycrc.py --generate h --algorithm table-driven --model crc-32 --slice-by 8 -o crc32.h

lib/chunkio/include/chunkio/cio_file_st.h

@@ -104,7 +104,7 @@ static inline ssize_t cio_file_st_get_content_size(char *map, size_t size)
  }
 
  meta_len = cio_file_st_get_meta_len(map);
- s = abs((size - CIO_FILE_HEADER_MIN) - meta_len);
+ s = (size - CIO_FILE_HEADER_MIN) - meta_len;
  if (s < size) {
  return s;
  }

lib/chunkio/src/cio_file.c

@@ -45,7 +45,7 @@ char cio_file_init_bytes[] = {
  /* crc32 (4 bytes) in network byte order */
  0xff, 0x12, 0xd9, 0x41,
 
- /* padding bytes (we have 16 extra bytes */
+ /* padding bytes (we have 16 extra bytes) */
  0x00, 0x00, 0x00, 0x00, 0x00,
  0x00, 0x00, 0x00, 0x00, 0x00,
  0x00, 0x00, 0x00, 0x00, 0x00,
@@ -124,9 +124,18 @@ static int adjust_layout(struct cio_chunk *ch,
  return 0;
 }
 
-static void write_init_header(struct cio_file *cf)
+/* Initialize Chunk header & structure */
+static void write_init_header(struct cio_chunk *ch, struct cio_file *cf)
 {
  memcpy(cf->map, cio_file_init_bytes, sizeof(cio_file_init_bytes));
+
+ /* If no checksum is enabled, reset the initial crc32 bytes */
+ if (!(ch->ctx->flags & CIO_CHECKSUM)) {
+ cf->map[2] = 0;
+ cf->map[3] = 0;
+ cf->map[4] = 0;
+ cf->map[5] = 0;
+ }
 }
 
 /* Return the available size in the file map to write data */
@@ -174,10 +183,12 @@ static int cio_file_format_check(struct cio_chunk *ch,
  }
 
  /* Initialize init bytes */
- write_init_header(cf);
+ write_init_header(ch, cf);
 
  /* Write checksum in context (note: crc32 not finalized) */
- cio_file_calculate_checksum(cf, &cf->crc_cur);
+ if (ch->ctx->flags & CIO_CHECKSUM) {
+ cio_file_calculate_checksum(cf, &cf->crc_cur);
+ }
  }
  else {
  /* Check first two bytes */
@@ -187,22 +198,23 @@ static int cio_file_format_check(struct cio_chunk *ch,
  return -1;
  }
 
- /* Initialize CRC variable */
- cf->crc_cur = cio_crc32_init();
+ /* Checksum */
+ if (ch->ctx->flags & CIO_CHECKSUM) {
+ /* Initialize CRC variable */
+ cf->crc_cur = cio_crc32_init();
 
- /* Get hash stored in the mmap */
- p = (unsigned char *) cio_file_st_get_hash(cf->map);
+  /* Get checksum stored in the mmap */
+  p = (unsigned char *) cio_file_st_get_hash(cf->map);
 
- /* Calculate data checksum in variable */
- cio_file_calculate_checksum(cf, &crc);
+  /* Calculate content checksum */
+  cio_file_calculate_checksum(cf, &crc);
 
- /* Compare checksum */
- if (ch->ctx->flags & CIO_CHECKSUM) {
+ /* Compare */
  crc_check = cio_crc32_finalize(crc);
  crc_check = htonl(crc_check);
  if (memcmp(p, &crc_check, sizeof(crc_check)) != 0) {
- cio_log_debug(ch->ctx, "[cio file] invalid crc32 at %s",
- ch->name);
+ cio_log_debug(ch->ctx, "[cio file] invalid crc32 at %s/%s",
+ ch->name, cf->path);
  return -1;
  }
  cf->crc_cur = crc;
@@ -272,15 +284,25 @@ static int mmap_file(struct cio_ctx *ctx, struct cio_chunk *ch, size_t size)
  return CIO_OK;
  }
 
- /* Check if some previous content exists */
- ret = fstat(cf->fd, &fst);
- if (ret == -1) {
- cio_errno();
- return CIO_ERROR;
+ /*
+ * 'size' value represents the value of a previous fstat(2) set by a previous
+ * caller. If the value is greater than zero, just use it, otherwise do a new
+ * fstat(2) of the file descriptor.
+ */
+
+ if (size > 0) {
+ fs_size = size;
  }
+ else {
+ ret = fstat(cf->fd, &fst);
+ if (ret == -1) {
+ cio_errno();
+ return CIO_ERROR;
+ }
 
- /* Get file size from the file system */
- fs_size = fst.st_size;
+ /* Get file size from the file system */
+ fs_size = fst.st_size;
+ }
 
  /* Mmap */
  if (cf->flags & CIO_OPEN) {
@@ -508,7 +530,7 @@ struct cio_file *cio_file_open(struct cio_ctx *ctx,
  }
 
  /* Map the file */
- ret = mmap_file(ctx, ch, size);
+ ret = mmap_file(ctx, ch, cf->fs_size);
  if (ret == CIO_ERROR || ret == CIO_CORRUPTED || ret == CIO_RETRY) {
  cio_file_close(ch, CIO_FALSE);
  *err = ret;
@@ -678,8 +700,12 @@ int cio_file_write(struct cio_chunk *ch, const void *buf, size_t count)
  return 0;
  }
 
+ if (!ch) {
+ return -1;
+ }
+
  if (cio_chunk_is_up(ch) == CIO_FALSE) {
- cio_log_error("[cio file] file is not mmaped: %s:%s",
+ cio_log_error("[cio file] file is not mmap()ed: %s:%s",
  ch->st->name, ch->name);
  return -1;
  }
@@ -689,7 +715,6 @@ int cio_file_write(struct cio_chunk *ch, const void *buf, size_t count)
 
  /* validate there is enough space, otherwise resize */
  if (av_size < count) {
-
  /* Set the pre-content size (chunk header + metadata) */
  pre_content = (CIO_FILE_HEADER_MIN + meta_len);
 
@@ -893,6 +918,7 @@ int cio_file_sync(struct cio_chunk *ch)
  }
  }
 
+
  /* If the mmap size changed, adjust mapping to the proper size */
  if (old_size != cf->alloc_size) {
 #ifndef MREMAP_MAYMOVE /* OSX */
@@ -942,17 +968,18 @@ int cio_file_sync(struct cio_chunk *ch)
  return 0;
 }
 
-/* Change the size of file in the file system (not memory map) */
+/*
+ * Change the size of 'file' in the file system (not memory map). This function
+ * MUST honor the required new size.
+ */
 int cio_file_fs_size_change(struct cio_file *cf, size_t new_size)
 {
  int ret;
 
- if (new_size == cf->alloc_size) {
- return 0;
- }
-
- /* macOS does not have fallocate().
- * So, we should use ftruncate always. */
+ /*
+ * fallocate() is not portable an Linux only. Since macOS does not have
+ * fallocate() we use ftruncate().
+ */
 #if defined(CIO_HAVE_FALLOCATE)
  if (new_size > cf->alloc_size) {
  /*

lib/chunkio/tests/fs.c

@@ -413,10 +413,50 @@ static void test_issue_51()
  cio_destroy(ctx);
 }
 
+/* ref: https://github.com/fluent/fluent-bit/2025 */
+static void test_issue_flb_2025()
+{
+ int i;
+ int ret;
+ int err;
+ int len;
+ char line[] = "this is a test line\n";
+ struct cio_ctx *ctx;
+ struct cio_chunk *chunk;
+ struct cio_stream *stream;
+
+ cio_utils_recursive_delete("tmp");
+
+ /* Create a temporal storage */
+ ctx = cio_create("tmp", log_cb, CIO_LOG_DEBUG, CIO_CHECKSUM);
+ stream = cio_stream_create(ctx, "test", CIO_STORE_FS);
+ chunk = cio_chunk_open(ctx, stream, "c", CIO_OPEN, 1000, &err);
+ TEST_CHECK(chunk != NULL);
+ if (!chunk) {
+ printf("cannot open chunk\n");
+ exit(1);
+ }
+
+ len = strlen(line);
+ for (i = 0; i < 1000; i++) {
+ ret = cio_chunk_write(chunk, line, len);
+ TEST_CHECK(ret == CIO_OK);
+
+ ret = cio_chunk_down(chunk);
+ TEST_CHECK(ret == CIO_OK);
+
+ ret = cio_chunk_up(chunk);
+ TEST_CHECK(ret == CIO_OK);
+ }
+
+ cio_destroy(ctx);
+}
+
 TEST_LIST = {
  {"fs_write", test_fs_write},
  {"fs_checksum", test_fs_checksum},
  {"fs_up_down", test_fs_up_down},
  {"issue_51", test_issue_51},
+ {"issue_flb_2025", test_issue_flb_2025},
  { 0 }
 };