[feature] [tokudb] Issue: #63 Add compression ratio statistics per FT

TokuDB has status to expose the compression ratio:

    mysql> show global status like 'tokudb%COMPRESSION_RATIO';
    +---------------------------------------+----------+
    | Variable_name                         | Value    |
    +---------------------------------------+----------+
    | Tokudb_LEAF_NODE_COMPRESSION_RATIO    | 5.850260 |
    | Tokudb_NONLEAF_NODE_COMPRESSION_RATIO | 0.564453 |
    | Tokudb_OVERALL_NODE_COMPRESSION_RATIO | 4.528809 |
    +---------------------------------------+----------+

but this a global level ratio, and sometimes the users may be interested
in table level compression ratio.

This patch address this request by exposing FT level compresstion ratio
through information_schema.TokuDB_fractal_tree_info table. We add two
columns, leaf_compress_ratio and internal_compress_ratio, which indicate
the compression ratio of leaf nodes and internal nodes accordingly.

Limitations, the comppression ratio infos are maintained in memeory, and
will lost when server is restarted or ft get evicted from memory.
Maybe in future there infos will be persisted in ft file, when this feature
is accepted by PerconaFT upstream.

mysql-test/suite/funcs_1/r/is_columns_is.result

@@ -350,7 +350,9 @@ def information_schema TokuDB_fractal_tree_info bt_num_blocks_in_use 4 0 NO bigi
 def information_schema TokuDB_fractal_tree_info bt_size_allocated 5 0 NO bigint NULL NULL 19 0 NULL NULL NULL bigint(0) select 
 def information_schema TokuDB_fractal_tree_info bt_size_in_use 6 0 NO bigint NULL NULL 19 0 NULL NULL NULL bigint(0) select 
 def information_schema TokuDB_fractal_tree_info dictionary_name 1 NO varchar 256 768 NULL NULL NULL utf8 utf8_general_ci varchar(256) select 
+def information_schema TokuDB_fractal_tree_info internal_compress_ratio 11 0 NO double NULL NULL 12 NULL NULL NULL NULL double select
 def information_schema TokuDB_fractal_tree_info internal_file_name 2 NO varchar 256 768 NULL NULL NULL utf8 utf8_general_ci varchar(256) select 
+def information_schema TokuDB_fractal_tree_info leaf_compress_ratio 10 0 NO double NULL NULL 12 NULL NULL NULL NULL double select
 def information_schema TokuDB_fractal_tree_info table_dictionary_name 9 NO varchar 256 768 NULL NULL NULL utf8 utf8_general_ci varchar(256) select 
 def information_schema TokuDB_fractal_tree_info table_name 8 NO varchar 256 768 NULL NULL NULL utf8 utf8_general_ci varchar(256) select 
 def information_schema TokuDB_fractal_tree_info table_schema 7 NO varchar 256 768 NULL NULL NULL utf8 utf8_general_ci varchar(256) select 
@@ -449,6 +451,7 @@ ORDER BY CHARACTER_SET_NAME, COLLATION_NAME, COL_CML;
 COL_CML DATA_TYPE CHARACTER_SET_NAME COLLATION_NAME
 NULL bigint NULL NULL
 NULL datetime NULL NULL
+NULL double NULL NULL
 NULL int NULL NULL
 NULL tinyint NULL NULL
 --> CHAR(0) is allowed (see manual), and here both CHARACHTER_* values
@@ -819,6 +822,8 @@ NULL information_schema TokuDB_fractal_tree_info bt_size_in_use bigint NULL NULL
 3.0000 information_schema TokuDB_fractal_tree_info table_schema varchar 256 768 utf8 utf8_general_ci varchar(256)
 3.0000 information_schema TokuDB_fractal_tree_info table_name varchar 256 768 utf8 utf8_general_ci varchar(256)
 3.0000 information_schema TokuDB_fractal_tree_info table_dictionary_name varchar 256 768 utf8 utf8_general_ci varchar(256)
+NULL information_schema TokuDB_fractal_tree_info leaf_compress_ratio double NULL NULL NULL NULL double
+NULL information_schema TokuDB_fractal_tree_info internal_compress_ratio double NULL NULL NULL NULL double
 NULL information_schema TokuDB_locks locks_trx_id bigint NULL NULL NULL NULL bigint(0)
 NULL information_schema TokuDB_locks locks_mysql_thread_id bigint NULL NULL NULL NULL bigint(0)
 3.0000 information_schema TokuDB_locks locks_dname varchar 256 768 utf8 utf8_general_ci varchar(256)

mysql-test/suite/rds/r/tokudb_show_compress_ratio_per_ft.result

@@ -0,0 +1,26 @@
+create table t1(c1 int) engine = tokudb;
+create table t2(c1 int) engine = tokudb;
+insert into t1 values (rand());
+select count(*) from t1;
+count(*)
+3145726
+select count(*) from t2;
+count(*)
+6291410
+set global tokudb_checkpoint_on_flush_logs=ON;
+flush logs;
+set global tokudb_checkpoint_on_flush_logs=OFF;
+select leaf_compress_ratio > 3, internal_compress_ratio > 1 from information_schema.TokuDB_fractal_tree_info where table_dictionary_name = 'main';
+leaf_compress_ratio > 3 internal_compress_ratio > 1
+1 0
+1 0
+flush tables;
+set global tokudb_checkpoint_on_flush_logs=ON;
+flush logs;
+set global tokudb_checkpoint_on_flush_logs=OFF;
+select leaf_compress_ratio > 3, internal_compress_ratio > 1 from information_schema.TokuDB_fractal_tree_info where table_dictionary_name = 'main';
+leaf_compress_ratio > 3 internal_compress_ratio > 1
+1 0
+1 0
+drop table t1;
+drop table t2;

mysql-test/suite/rds/t/tokudb_show_compress_ratio_per_ft.test

@@ -0,0 +1,39 @@
+--source include/have_tokudb.inc
+--source include/have_innodb.inc
+
+create table t1(c1 int) engine = tokudb;
+create table t2(c1 int) engine = tokudb;
+
+insert into t1 values (rand());
+
+--let $i=0
+--let $count=20
+--disable_query_log
+while ($i<$count)
+{
+ inc $i;
+ eval insert into t1 values (rand());
+ eval insert into t1 select * from t1;
+ eval insert into t2 select * from t1;
+}
+--enable_query_log
+
+select count(*) from t1;
+select count(*) from t2;
+
+# trigger checkpoint
+set global tokudb_checkpoint_on_flush_logs=ON;
+flush logs;
+set global tokudb_checkpoint_on_flush_logs=OFF;
+
+select leaf_compress_ratio > 3, internal_compress_ratio > 1 from information_schema.TokuDB_fractal_tree_info where table_dictionary_name = 'main';
+flush tables;
+
+set global tokudb_checkpoint_on_flush_logs=ON;
+flush logs;
+set global tokudb_checkpoint_on_flush_logs=OFF;
+
+select leaf_compress_ratio > 3, internal_compress_ratio > 1 from information_schema.TokuDB_fractal_tree_info where table_dictionary_name = 'main';
+
+drop table t1;
+drop table t2;

storage/tokudb/ft-index/buildheader/make_tdb.cc

@@ -553,7 +553,7 @@ static void print_db_struct (void) {
  "int (*verify_with_progress)(DB *, int (*progress_callback)(void *progress_extra, float progress), void *progress_extra, int verbose, int keep_going)",
  "int (*update)(DB *, DB_TXN*, const DBT *key, const DBT *extra, uint32_t flags)",
  "int (*update_broadcast)(DB *, DB_TXN*, const DBT *extra, uint32_t flags)",
- "int (*get_fractal_tree_info64)(DB*,uint64_t*,uint64_t*,uint64_t*,uint64_t*)",
+ "int (*get_fractal_tree_info64)(DB*,uint64_t*,uint64_t*,uint64_t*,uint64_t*,double*,double*)",
  "int (*iterate_fractal_tree_block_map)(DB*,int(*)(uint64_t,int64_t,int64_t,int64_t,int64_t,void*),void*)",
  "const char *(*get_dname)(DB *db)",
  "int (*get_last_key)(DB *db, YDB_CALLBACK_FUNCTION func, void* extra)",

storage/tokudb/ft-index/ft/ft-internal.h

@@ -264,6 +264,12 @@ struct ft {
  // - if the score is high enough, we optimistically attempt to insert directly into the rightmost leaf
  // - if our attempt fails because the key was not in range of the rightmost leaf, we reset the score back to 0
  uint32_t seqinsert_score;
+
+ // counters for uncompressed/compressed size of leaf/internal nodes
+ unsigned long long leaf_uncompressed_bytes;
+ unsigned long long leaf_compressed_bytes;
+ unsigned long long internal_uncompressed_bytes;
+ unsigned long long internal_compressed_bytes;
 };
 
 // Allocate a DB struct off the stack and only set its comparison
@@ -627,6 +633,7 @@ typedef struct {
 
 void toku_ft_status_update_pivot_fetch_reason(ftnode_fetch_extra *bfe);
 void toku_ft_status_update_flush_reason(FTNODE node, uint64_t uncompressed_bytes_flushed, uint64_t bytes_written, tokutime_t write_time, bool for_checkpoint);
+void toku_ft_status_update_flush_for_ft(FTNODE node, FT ft, uint64_t uncompressed_bytes_flushed, uint64_t bytes_written);
 void toku_ft_status_update_serialize_times(FTNODE node, tokutime_t serialize_time, tokutime_t compress_time);
 void toku_ft_status_update_deserialize_times(FTNODE node, tokutime_t deserialize_time, tokutime_t decompress_time);
 void toku_ft_status_note_msn_discard(void);

storage/tokudb/ft-index/ft/ft-ops.cc

@@ -799,6 +799,17 @@ void toku_ft_status_update_flush_reason(FTNODE node,
  }
 }
 
+void toku_ft_status_update_flush_for_ft(FTNODE node, FT ft, uint64_t uncompressed_bytes_flushed, uint64_t bytes_written)
+{
+ if (node->height == 0) {
+ ft->leaf_uncompressed_bytes += uncompressed_bytes_flushed;
+ ft->leaf_compressed_bytes += bytes_written;
+ } else {
+ ft->internal_uncompressed_bytes += uncompressed_bytes_flushed;
+ ft->internal_compressed_bytes += bytes_written;
+ }
+}
+
 void toku_ftnode_checkpoint_complete_callback(void *value_data) {
  FTNODE node = static_cast<FTNODE>(value_data);
  if (node->height > 0) {

storage/tokudb/ft-index/ft/ft-ops.h

@@ -292,6 +292,8 @@ struct ftinfo64 {
  uint64_t num_blocks_in_use; // number of blocks in use by most recent checkpoint
  uint64_t size_allocated; // sum of sizes of blocks in blocktable
  uint64_t size_in_use; // sum of sizes of blocks in use by most recent checkpoint
+ double leaf_ratio; // compress ratio for leaf node
+ double internal_ratio; // compress ratio for internal node
 };
 
 void toku_ft_handle_get_fractal_tree_info64(FT_HANDLE, struct ftinfo64 *);

storage/tokudb/ft-index/ft/ft.cc

@@ -409,6 +409,11 @@ static void ft_init(FT ft, FT_OPTIONS options, CACHEFILE cf) {
  ft_note_unpin_by_checkpoint);
 
  ft->blocktable.verify_no_free_blocknums();
+
+ ft->leaf_uncompressed_bytes = 0;
+ ft->leaf_compressed_bytes = 0;
+ ft->internal_uncompressed_bytes = 0;
+ ft->internal_compressed_bytes = 0;
 }
 
 
@@ -492,6 +497,10 @@ int toku_read_ft_and_store_in_cachefile (FT_HANDLE ft_handle, CACHEFILE cf, LSN
  ft->cmp.create(ft_handle->options.compare_fun, &ft->cmp_descriptor, ft_handle->options.memcmp_magic);
  ft->update_fun = ft_handle->options.update_fun;
  ft->cf = cf;
+ ft->leaf_uncompressed_bytes = 0;
+ ft->leaf_compressed_bytes = 0;
+ ft->internal_uncompressed_bytes = 0;
+ ft->internal_compressed_bytes = 0;
  toku_cachefile_set_userdata(cf,
  reinterpret_cast<void *>(ft),
  ft_log_fassociate_during_checkpoint,
@@ -878,6 +887,10 @@ toku_ft_stat64 (FT ft, struct ftstat64_s *s) {
 
 void toku_ft_get_fractal_tree_info64(FT ft, struct ftinfo64 *info) {
  ft->blocktable.get_info64(info);
+ info->leaf_ratio = ft->leaf_compressed_bytes > 0 ?
+ (double)ft->leaf_uncompressed_bytes / (double)ft->leaf_compressed_bytes : 0;
+ info->internal_ratio = ft->internal_compressed_bytes > 0 ?
+ (double)ft->internal_uncompressed_bytes / (double)ft->internal_compressed_bytes : 0;
 }
 
 int toku_ft_iterate_fractal_tree_block_map(FT ft, int (*iter)(uint64_t,int64_t,int64_t,int64_t,int64_t,void*), void *iter_extra) {

storage/tokudb/ft-index/ft/serialize/ft_node-serialize.cc

@@ -878,6 +878,7 @@ toku_serialize_ftnode_to (int fd, BLOCKNUM blocknum, FTNODE node, FTNODE_DISK_DA
 
  tokutime_t io_time = t1 - t0;
  toku_ft_status_update_flush_reason(node, n_uncompressed_bytes, n_to_write, io_time, for_checkpoint);
+ toku_ft_status_update_flush_for_ft(node, ft, n_uncompressed_bytes, n_to_write);
 
  toku_free(compressed_buf);
  node->dirty = 0; // See #1957. Must set the node to be clean after serializing it so that it doesn't get written again on the next checkpoint or eviction.

storage/tokudb/ft-index/src/ydb_db.cc

@@ -733,14 +733,16 @@ toku_db_set_memcmp_magic(DB *db, uint8_t magic) {
 }
 
 static int
-toku_db_get_fractal_tree_info64(DB *db, uint64_t *num_blocks_allocated, uint64_t *num_blocks_in_use, uint64_t *size_allocated, uint64_t *size_in_use) {
+toku_db_get_fractal_tree_info64(DB *db, uint64_t *num_blocks_allocated, uint64_t *num_blocks_in_use, uint64_t *size_allocated, uint64_t *size_in_use, double *leaf_compress_ratio, double *internal_compress_ratio) {
  HANDLE_PANICKED_DB(db);
  struct ftinfo64 ftinfo;
  toku_ft_handle_get_fractal_tree_info64(db->i->ft_handle, &ftinfo);
  *num_blocks_allocated = ftinfo.num_blocks_allocated;
  *num_blocks_in_use = ftinfo.num_blocks_in_use;
  *size_allocated = ftinfo.size_allocated;
  *size_in_use = ftinfo.size_in_use;
+ *leaf_compress_ratio = ftinfo.leaf_ratio;
+ *internal_compress_ratio = ftinfo.internal_ratio;
  return 0;
 }
 

storage/tokudb/hatoku_hton.cc

@@ -1710,6 +1710,8 @@ static ST_FIELD_INFO tokudb_fractal_tree_info_field_info[] = {
  {"table_schema", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE },
  {"table_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE },
  {"table_dictionary_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE },
+ {"leaf_compress_ratio", MAX_FLOAT_STR_LENGTH, MYSQL_TYPE_FLOAT, 0, 0, NULL, SKIP_OPEN_TABLE },
+ {"internal_compress_ratio", MAX_FLOAT_STR_LENGTH, MYSQL_TYPE_FLOAT, 0, 0, NULL, SKIP_OPEN_TABLE },
  {NULL, 0, MYSQL_TYPE_NULL, 0, 0, NULL, SKIP_OPEN_TABLE}
 };
 
@@ -1719,6 +1721,8 @@ static int tokudb_report_fractal_tree_info_for_db(const DBT *dname, const DBT *i
  uint64_t bt_num_blocks_in_use;
  uint64_t bt_size_allocated;
  uint64_t bt_size_in_use;
+ double leaf_compress_ratio;
+ double internal_compress_ratio;
 
  DB *db = NULL;
  error = db_create(&db, db_env, 0);
@@ -1731,7 +1735,8 @@ static int tokudb_report_fractal_tree_info_for_db(const DBT *dname, const DBT *i
  }
  error = db->get_fractal_tree_info64(db,
  &bt_num_blocks_allocated, &bt_num_blocks_in_use,
- &bt_size_allocated, &bt_size_in_use);
+ &bt_size_allocated, &bt_size_in_use,
+ &leaf_compress_ratio, &internal_compress_ratio);
  if (error) {
  goto exit;
  }
@@ -1760,6 +1765,10 @@ static int tokudb_report_fractal_tree_info_for_db(const DBT *dname, const DBT *i
  table->field[7]->store(table_name.c_ptr(), table_name.length(), system_charset_info);
  table->field[8]->store(dictionary_name.c_ptr(), dictionary_name.length(), system_charset_info);
  }
+
+ table->field[9]->store(leaf_compress_ratio);
+ table->field[10]->store(internal_compress_ratio);
+
  error = schema_table_store_record(thd, table);
 
 exit: