avoid NaN in entropy computation

ronghanghu · May 29, 2018 · 970c766 · 970c766
1 parent f7a6e69
commit 970c766
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 3 deletions.
diff --git a/models_clevr/nmn3_netgen_att.py b/models_clevr/nmn3_netgen_att.py
@@ -256,7 +256,7 @@ def loop_fn(time, cell_output, cell_state, loop_state):
  token_prob = tf.reduce_sum(all_token_probs * tf.cast(mask, tf.float32), axis=1)
  # tf.assert_positive(token_prob)
  neg_entropy = tf.reduce_sum(
- all_token_probs * tf.log(all_token_probs + (1-validity_mult)),
+ all_token_probs * tf.log(tf.maximum(1e-5, all_token_probs + (1-validity_mult))),
  axis=1)
 
  # update states

diff --git a/models_shapes/nmn3_netgen_att.py b/models_shapes/nmn3_netgen_att.py
@@ -205,7 +205,7 @@ def loop_fn(time, cell_output, cell_state, loop_state):
  token_prob = tf.reduce_sum(all_token_probs *
  tf.cast(mask, tf.float32), axis=1)
  neg_entropy = tf.reduce_sum(all_token_probs *
- tf.log(all_token_probs), axis=1)
+ tf.log(tf.maximum(1e-5, all_token_probs)), axis=1)
 
  # is_eos_predicted is a [N] bool tensor, indicating whether
  # <eos> has already been predicted previously in each sequence

diff --git a/models_vqa/nmn3_netgen_att.py b/models_vqa/nmn3_netgen_att.py
@@ -256,7 +256,7 @@ def loop_fn(time, cell_output, cell_state, loop_state):
  token_prob = tf.reduce_sum(all_token_probs * tf.cast(mask, tf.float32), axis=1)
  # tf.assert_positive(token_prob)
  neg_entropy = tf.reduce_sum(
- all_token_probs * tf.log(all_token_probs + (1-validity_mult)),
+ all_token_probs * tf.log(tf.maximum(1e-5, all_token_probs + (1-validity_mult))),
  axis=1)
 
  # update states