diff --git a/lib/chunkio/CMakeLists.txt b/lib/chunkio/CMakeLists.txt index e3a85bf30ff..7351c2666ef 100644 --- a/lib/chunkio/CMakeLists.txt +++ b/lib/chunkio/CMakeLists.txt @@ -3,7 +3,7 @@ project(chunk-io) set(CIO_VERSION_MAJOR 1) set(CIO_VERSION_MINOR 0) -set(CIO_VERSION_PATCH 2) +set(CIO_VERSION_PATCH 3) set(CIO_VERSION_STR "${CIO_VERSION_MAJOR}.${CIO_VERSION_MINOR}.${CIO_VERSION_PATCH}") # CFLAGS diff --git a/lib/chunkio/deps/crc32/crc32.c b/lib/chunkio/deps/crc32/crc32.c index a9694d77f91..4e29afe6195 100644 --- a/lib/chunkio/deps/crc32/crc32.c +++ b/lib/chunkio/deps/crc32/crc32.c @@ -2,8 +2,8 @@ * \file * Functions and types for CRC checks. * - * Generated on Tue Oct 30 22:09:40 2018 - * by pycrc v0.9.1, https://pycrc.org + * Generated on Wed Mar 18 12:41:20 2020 + * by pycrc v0.9.2, https://pycrc.org * using the configuration: * - Width = 32 * - Poly = 0x04c11db7 @@ -17,6 +17,8 @@ #include "crc32.h" /* include the header file generated with pycrc */ #include #include + +/* Patch by Fluent Bit Authors */ #ifdef __APPLE__ # include # include @@ -35,8 +37,6 @@ # include #endif - - /** * Static table used for the table_driven implementation. */ @@ -321,6 +321,14 @@ crc_t crc_update(crc_t crc, const void *data, size_t data_len) const unsigned char *d = (const unsigned char *)data; unsigned int tbl_idx; + /* Align to a multiple of 8 bytes */ + while (data_len && (((uintptr_t)(const void *)d) % 8 != 0)) { + tbl_idx = (crc ^ *d) & 0xff; + crc = (crc_table[0][tbl_idx] ^ (crc >> 8)) & 0xffffffff; + d++; + data_len--; + } + const uint32_t *d32 = (const uint32_t *)d; while (data_len >= 8) { diff --git a/lib/chunkio/deps/crc32/crc32.h b/lib/chunkio/deps/crc32/crc32.h index 540323a3479..c3ec3fa30c8 100644 --- a/lib/chunkio/deps/crc32/crc32.h +++ b/lib/chunkio/deps/crc32/crc32.h @@ -2,8 +2,8 @@ * \file * Functions and types for CRC checks. * - * Generated on Tue Oct 30 22:09:48 2018 - * by pycrc v0.9.1, https://pycrc.org + * Generated on Wed Mar 18 12:41:20 2020 + * by pycrc v0.9.2, https://pycrc.org * using the configuration: * - Width = 32 * - Poly = 0x04c11db7 diff --git a/lib/chunkio/deps/crc32/gen.sh b/lib/chunkio/deps/crc32/gen.sh index a7955e5e8b2..8fc39597a4d 100755 --- a/lib/chunkio/deps/crc32/gen.sh +++ b/lib/chunkio/deps/crc32/gen.sh @@ -1,2 +1,2 @@ -./pycrc.py --generate c --algorithm table-driven --model crc-32 --slice-by 8 -o crc32.c -./pycrc.py --generate h --algorithm table-driven --model crc-32 --slice-by 8 -o crc32.h +pycrc-0.9.2/pycrc.py --generate c --algorithm table-driven --model crc-32 --slice-by 8 -o crc32.c +pycrc-0.9.2/pycrc.py --generate h --algorithm table-driven --model crc-32 --slice-by 8 -o crc32.h diff --git a/lib/chunkio/include/chunkio/cio_file_st.h b/lib/chunkio/include/chunkio/cio_file_st.h index 88a33211396..028922f3174 100644 --- a/lib/chunkio/include/chunkio/cio_file_st.h +++ b/lib/chunkio/include/chunkio/cio_file_st.h @@ -104,7 +104,7 @@ static inline ssize_t cio_file_st_get_content_size(char *map, size_t size) } meta_len = cio_file_st_get_meta_len(map); - s = abs((size - CIO_FILE_HEADER_MIN) - meta_len); + s = (size - CIO_FILE_HEADER_MIN) - meta_len; if (s < size) { return s; } diff --git a/lib/chunkio/src/cio_file.c b/lib/chunkio/src/cio_file.c index 54f7e593e41..bf44a7d3a50 100644 --- a/lib/chunkio/src/cio_file.c +++ b/lib/chunkio/src/cio_file.c @@ -45,7 +45,7 @@ char cio_file_init_bytes[] = { /* crc32 (4 bytes) in network byte order */ 0xff, 0x12, 0xd9, 0x41, - /* padding bytes (we have 16 extra bytes */ + /* padding bytes (we have 16 extra bytes) */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -124,9 +124,18 @@ static int adjust_layout(struct cio_chunk *ch, return 0; } -static void write_init_header(struct cio_file *cf) +/* Initialize Chunk header & structure */ +static void write_init_header(struct cio_chunk *ch, struct cio_file *cf) { memcpy(cf->map, cio_file_init_bytes, sizeof(cio_file_init_bytes)); + + /* If no checksum is enabled, reset the initial crc32 bytes */ + if (!(ch->ctx->flags & CIO_CHECKSUM)) { + cf->map[2] = 0; + cf->map[3] = 0; + cf->map[4] = 0; + cf->map[5] = 0; + } } /* Return the available size in the file map to write data */ @@ -174,10 +183,12 @@ static int cio_file_format_check(struct cio_chunk *ch, } /* Initialize init bytes */ - write_init_header(cf); + write_init_header(ch, cf); /* Write checksum in context (note: crc32 not finalized) */ - cio_file_calculate_checksum(cf, &cf->crc_cur); + if (ch->ctx->flags & CIO_CHECKSUM) { + cio_file_calculate_checksum(cf, &cf->crc_cur); + } } else { /* Check first two bytes */ @@ -187,22 +198,23 @@ static int cio_file_format_check(struct cio_chunk *ch, return -1; } - /* Initialize CRC variable */ - cf->crc_cur = cio_crc32_init(); + /* Checksum */ + if (ch->ctx->flags & CIO_CHECKSUM) { + /* Initialize CRC variable */ + cf->crc_cur = cio_crc32_init(); - /* Get hash stored in the mmap */ - p = (unsigned char *) cio_file_st_get_hash(cf->map); + /* Get checksum stored in the mmap */ + p = (unsigned char *) cio_file_st_get_hash(cf->map); - /* Calculate data checksum in variable */ - cio_file_calculate_checksum(cf, &crc); + /* Calculate content checksum */ + cio_file_calculate_checksum(cf, &crc); - /* Compare checksum */ - if (ch->ctx->flags & CIO_CHECKSUM) { + /* Compare */ crc_check = cio_crc32_finalize(crc); crc_check = htonl(crc_check); if (memcmp(p, &crc_check, sizeof(crc_check)) != 0) { - cio_log_debug(ch->ctx, "[cio file] invalid crc32 at %s", - ch->name); + cio_log_debug(ch->ctx, "[cio file] invalid crc32 at %s/%s", + ch->name, cf->path); return -1; } cf->crc_cur = crc; @@ -272,15 +284,25 @@ static int mmap_file(struct cio_ctx *ctx, struct cio_chunk *ch, size_t size) return CIO_OK; } - /* Check if some previous content exists */ - ret = fstat(cf->fd, &fst); - if (ret == -1) { - cio_errno(); - return CIO_ERROR; + /* + * 'size' value represents the value of a previous fstat(2) set by a previous + * caller. If the value is greater than zero, just use it, otherwise do a new + * fstat(2) of the file descriptor. + */ + + if (size > 0) { + fs_size = size; } + else { + ret = fstat(cf->fd, &fst); + if (ret == -1) { + cio_errno(); + return CIO_ERROR; + } - /* Get file size from the file system */ - fs_size = fst.st_size; + /* Get file size from the file system */ + fs_size = fst.st_size; + } /* Mmap */ if (cf->flags & CIO_OPEN) { @@ -508,7 +530,7 @@ struct cio_file *cio_file_open(struct cio_ctx *ctx, } /* Map the file */ - ret = mmap_file(ctx, ch, size); + ret = mmap_file(ctx, ch, cf->fs_size); if (ret == CIO_ERROR || ret == CIO_CORRUPTED || ret == CIO_RETRY) { cio_file_close(ch, CIO_FALSE); *err = ret; @@ -678,8 +700,12 @@ int cio_file_write(struct cio_chunk *ch, const void *buf, size_t count) return 0; } + if (!ch) { + return -1; + } + if (cio_chunk_is_up(ch) == CIO_FALSE) { - cio_log_error("[cio file] file is not mmaped: %s:%s", + cio_log_error("[cio file] file is not mmap()ed: %s:%s", ch->st->name, ch->name); return -1; } @@ -689,7 +715,6 @@ int cio_file_write(struct cio_chunk *ch, const void *buf, size_t count) /* validate there is enough space, otherwise resize */ if (av_size < count) { - /* Set the pre-content size (chunk header + metadata) */ pre_content = (CIO_FILE_HEADER_MIN + meta_len); @@ -893,6 +918,7 @@ int cio_file_sync(struct cio_chunk *ch) } } + /* If the mmap size changed, adjust mapping to the proper size */ if (old_size != cf->alloc_size) { #ifndef MREMAP_MAYMOVE /* OSX */ @@ -942,17 +968,18 @@ int cio_file_sync(struct cio_chunk *ch) return 0; } -/* Change the size of file in the file system (not memory map) */ +/* + * Change the size of 'file' in the file system (not memory map). This function + * MUST honor the required new size. + */ int cio_file_fs_size_change(struct cio_file *cf, size_t new_size) { int ret; - if (new_size == cf->alloc_size) { - return 0; - } - - /* macOS does not have fallocate(). - * So, we should use ftruncate always. */ + /* + * fallocate() is not portable an Linux only. Since macOS does not have + * fallocate() we use ftruncate(). + */ #if defined(CIO_HAVE_FALLOCATE) if (new_size > cf->alloc_size) { /* diff --git a/lib/chunkio/tests/fs.c b/lib/chunkio/tests/fs.c index f9e12f4b784..c5a155b6852 100644 --- a/lib/chunkio/tests/fs.c +++ b/lib/chunkio/tests/fs.c @@ -413,10 +413,50 @@ static void test_issue_51() cio_destroy(ctx); } +/* ref: https://github.com/fluent/fluent-bit/2025 */ +static void test_issue_flb_2025() +{ + int i; + int ret; + int err; + int len; + char line[] = "this is a test line\n"; + struct cio_ctx *ctx; + struct cio_chunk *chunk; + struct cio_stream *stream; + + cio_utils_recursive_delete("tmp"); + + /* Create a temporal storage */ + ctx = cio_create("tmp", log_cb, CIO_LOG_DEBUG, CIO_CHECKSUM); + stream = cio_stream_create(ctx, "test", CIO_STORE_FS); + chunk = cio_chunk_open(ctx, stream, "c", CIO_OPEN, 1000, &err); + TEST_CHECK(chunk != NULL); + if (!chunk) { + printf("cannot open chunk\n"); + exit(1); + } + + len = strlen(line); + for (i = 0; i < 1000; i++) { + ret = cio_chunk_write(chunk, line, len); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_down(chunk); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_up(chunk); + TEST_CHECK(ret == CIO_OK); + } + + cio_destroy(ctx); +} + TEST_LIST = { {"fs_write", test_fs_write}, {"fs_checksum", test_fs_checksum}, {"fs_up_down", test_fs_up_down}, {"issue_51", test_issue_51}, + {"issue_flb_2025", test_issue_flb_2025}, { 0 } };