Adds tests for float labels in binary classification.

PiperOrigin-RevId: 158716581

tensorflow/python/estimator/canned/dnn_testing_utils.py

@@ -518,6 +518,29 @@ def _input_fn():
  ops.GraphKeys.GLOBAL_STEP: global_step
  }, dnn_classifier.evaluate(input_fn=_input_fn, steps=1))
 
+ def test_float_labels(self):
+ """Asserts evaluation metrics for float labels in binary classification."""
+ global_step = 100
+ create_checkpoint(
+ (([[.6, .5]], [.1, -.1]), ([[1., .8], [-.8, -1.]], [.2, -.2]),
+ ([[-1.], [1.]], [.3]),), global_step, self._model_dir)
+
+ dnn_classifier = self._dnn_classifier_fn(
+ hidden_units=(2, 2),
+ feature_columns=[feature_column.numeric_column('age')],
+ model_dir=self._model_dir)
+ def _input_fn():
+ # batch_size = 2, one false label, and one true.
+ return {'age': [[10.], [10.]]}, [[0.8], [0.4]]
+ # Uses identical numbers as DNNModelTest.test_one_dim_logits.
+ # See that test for calculation of logits.
+ # logits = [[-2.08], [-2.08]] =>
+ # logistic = 1/(1 + exp(-logits)) = [[0.11105597], [0.11105597]]
+ # loss = -0.8 * log(0.111) -0.2 * log(0.889)
+ # -0.4 * log(0.111) -0.6 * log(0.889) = 2.7314420
+ metrics = dnn_classifier.evaluate(input_fn=_input_fn, steps=1)
+ self.assertAlmostEqual(2.7314420, metrics[metric_keys.MetricKeys.LOSS])
+
 
 class BaseDNNRegressorEvaluateTest(object):
 
@@ -939,6 +962,34 @@ def test_binary_classification(self):
  self, base_global_step + num_steps, input_units=1,
  hidden_units=hidden_units, output_units=1, model_dir=self._model_dir)
 
+ def test_binary_classification_float_labels(self):
+ base_global_step = 100
+ hidden_units = (2, 2)
+ create_checkpoint(
+ (([[.6, .5]], [.1, -.1]), ([[1., .8], [-.8, -1.]], [.2, -.2]),
+ ([[-1.], [1.]], [.3]),), base_global_step, self._model_dir)
+
+ # Uses identical numbers as DNNModelFnTest.test_one_dim_logits.
+ # See that test for calculation of logits.
+ # logits = [-2.08] => probabilities = [0.889, 0.111]
+ # loss = -0.8 * log(0.111) -0.2 * log(0.889) = 1.7817210
+ expected_loss = 1.7817210
+ opt = mock_optimizer(
+ self, hidden_units=hidden_units, expected_loss=expected_loss)
+ dnn_classifier = self._dnn_classifier_fn(
+ hidden_units=hidden_units,
+ feature_columns=(feature_column.numeric_column('age'),),
+ optimizer=opt,
+ model_dir=self._model_dir)
+ self.assertEqual(0, opt.minimize.call_count)
+
+ # Train for a few steps, then validate optimizer, summaries, and
+ # checkpoint.
+ num_steps = 5
+ dnn_classifier.train(
+ input_fn=lambda: ({'age': [[10.]]}, [[0.8]]), steps=num_steps)
+ self.assertEqual(1, opt.minimize.call_count)
+
  def test_multi_class(self):
  n_classes = 3
  base_global_step = 100

tensorflow/python/estimator/canned/head_test.py

@@ -83,6 +83,24 @@ def _sigmoid(logits):
  return 1 / (1 + np.exp(-logits))
 
 
+# TODO(roumposg): Reuse the code from dnn_testing_utils.
+def _assert_close(expected, actual, rtol=1e-04, message='',
+ name='assert_close'):
+ with ops.name_scope(name, 'assert_close', (expected, actual, rtol)) as scope:
+ expected = ops.convert_to_tensor(expected, name='expected')
+ actual = ops.convert_to_tensor(actual, name='actual')
+ rdiff = math_ops.abs((expected - actual) / expected, 'diff')
+ rtol = ops.convert_to_tensor(rtol, name='rtol')
+ return check_ops.assert_less(
+ rdiff,
+ rtol,
+ data=(message, 'Condition expected =~ actual did not hold element-wise:'
+ 'expected = ', expected, 'actual = ', actual, 'rdiff = ', rdiff,
+ 'rtol = ', rtol,),
+ summarize=expected.get_shape().num_elements(),
+ name=scope)
+
+
 class MultiClassHeadWithSoftmaxCrossEntropyLoss(test.TestCase):
 
  def test_n_classes_is_none(self):
@@ -983,6 +1001,64 @@ def _train_op_fn(loss):
  metric_keys.MetricKeys.LOSS_MEAN: 20.5,
  }, summary_str)
 
+ def test_float_labels_train(self):
+ head = head_lib._binary_logistic_head_with_sigmoid_cross_entropy_loss()
+
+ # Create estimator spec.
+ logits = np.array([[0.5], [-0.3]], dtype=np.float32)
+ expected_train_result = b'my_train_op'
+ # loss = sum(cross_entropy(labels, logits))
+ # = sum(-label[i]*sigmoid(logit[i]) -(1-label[i])*sigmoid(-logit[i]))
+ # = -0.8 * log(sigmoid(0.5)) -0.2 * log(sigmoid(-0.5))
+ # -0.4 * log(sigmoid(-0.3)) -0.6 * log(sigmoid(0.3))
+ # = 1.2484322
+ expected_loss = 1.2484322
+ def _train_op_fn(loss):
+ with ops.control_dependencies((_assert_close(
+ math_ops.to_float(expected_loss), math_ops.to_float(loss)),)):
+ return constant_op.constant(expected_train_result)
+ spec = head.create_estimator_spec(
+ features={'x': np.array([[42]], dtype=np.float32)},
+ mode=model_fn.ModeKeys.TRAIN,
+ logits=logits,
+ labels=np.array([[0.8], [0.4]], dtype=np.float32),
+ train_op_fn=_train_op_fn)
+
+ # Assert predictions, loss, train_op, and summaries.
+ with self.test_session() as sess:
+ _initialize_variables(self, spec.scaffold)
+ loss, train_result = sess.run((spec.loss, spec.train_op))
+ self.assertAlmostEqual(expected_loss, loss, delta=1.e-5)
+ self.assertEqual(expected_train_result, train_result)
+
+ def test_float_labels_eval(self):
+ head = head_lib._binary_logistic_head_with_sigmoid_cross_entropy_loss()
+
+ # Create estimator spec.
+ logits = np.array([[0.5], [-0.3]], dtype=np.float32)
+ spec = head.create_estimator_spec(
+ features={'x': np.array([[42]], dtype=np.float32)},
+ mode=model_fn.ModeKeys.EVAL,
+ logits=logits,
+ labels=np.array([[0.8], [0.4]], dtype=np.float32))
+
+ # loss = sum(cross_entropy(labels, logits))
+ # = sum(-label[i]*sigmoid(logit[i]) -(1-label[i])*sigmoid(-logit[i]))
+ # = -0.8 * log(sigmoid(0.5)) -0.2 * log(sigmoid(-0.5))
+ # -0.4 * log(sigmoid(-0.3)) -0.6 * log(sigmoid(0.3))
+ # = 1.2484322
+ expected_loss = 1.2484322
+
+ # Assert loss.
+ with self.test_session() as sess:
+ _initialize_variables(self, spec.scaffold)
+ self.assertIsNone(spec.scaffold.summary_op)
+ update_ops = {k: spec.eval_metric_ops[k][1] for k in spec.eval_metric_ops}
+ loss, metrics = sess.run((spec.loss, update_ops))
+ self.assertAlmostEqual(expected_loss, loss, delta=1.e-5)
+ self.assertAlmostEqual(
+ expected_loss / 2., metrics[metric_keys.MetricKeys.LOSS_MEAN])
+
  def test_weighted_multi_example_predict(self):
  """3 examples, 1 batch."""
  head = head_lib._binary_logistic_head_with_sigmoid_cross_entropy_loss(

tensorflow/python/estimator/canned/linear_testing_utils.py

@@ -1137,6 +1137,43 @@ def testFromCheckpoint(self):
  expected_age_weight=age_weight,
  expected_bias=bias)
 
+ def testFromCheckpointFloatLabels(self):
+ """Tests float labels for binary classification."""
+ # Create initial checkpoint.
+ n_classes = self._n_classes
+ if n_classes > 2:
+ return
+ label = 0.8
+ age = 17
+ age_weight = [[2.0]]
+ bias = [-35.0]
+ initial_global_step = 100
+ with ops.Graph().as_default():
+ variables.Variable(age_weight, name=AGE_WEIGHT_NAME)
+ variables.Variable(bias, name=BIAS_NAME)
+ variables.Variable(
+ initial_global_step, name=ops.GraphKeys.GLOBAL_STEP,
+ dtype=dtypes.int64)
+ save_variables_to_ckpt(self._model_dir)
+
+ # logits = age * age_weight + bias = 17 * 2. - 35. = -1.
+ # loss = sigmoid_cross_entropy(logits, label)
+ # => loss = -0.8 * log(sigmoid(-1)) -0.2 * log(sigmoid(+1)) = 1.1132617
+ mock_optimizer = self._mock_optimizer(expected_loss=1.1132617)
+
+ est = linear.LinearClassifier(
+ feature_columns=(feature_column_lib.numeric_column('age'),),
+ n_classes=n_classes,
+ optimizer=mock_optimizer,
+ model_dir=self._model_dir)
+ self.assertEqual(0, mock_optimizer.minimize.call_count)
+
+ # Train for a few steps, and validate optimizer and final checkpoint.
+ num_steps = 10
+ est.train(
+ input_fn=lambda: ({'age': ((age,),)}, ((label,),)), steps=num_steps)
+ self.assertEqual(1, mock_optimizer.minimize.call_count)
+
  def testFromCheckpointMultiBatch(self):
  # Create initial checkpoint.
  n_classes = self._n_classes