Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Removed run_all_in_graph_and_eager_mode in hamming. #1692

Merged
merged 1 commit into from
Apr 17, 2020
Merged

Removed run_all_in_graph_and_eager_mode in hamming. #1692

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
312 changes: 149 additions & 163 deletions tensorflow_addons/metrics/tests/hamming_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,170 +14,156 @@
# ==============================================================================
"""Tests Hamming metrics."""

import numpy as np
import pytest
import tensorflow as tf
from tensorflow_addons.metrics import HammingLoss, hamming_distance
from tensorflow_addons.utils import test_utils
from tensorflow.keras import layers
import numpy as np

from tensorflow_addons.metrics import HammingLoss, hamming_distance


def test_config():
hl_obj = HammingLoss(mode="multilabel", threshold=0.8)
assert hl_obj.name == "hamming_loss"
assert hl_obj.dtype == tf.float32


def check_results(obj, value):
np.testing.assert_allclose(value, obj.result().numpy(), atol=1e-5)


@pytest.mark.xfail(reason="See https://github.com/tensorflow/addons/issues/1200")
def test_mc_4_classes():
actuals = tf.constant(
[
[1, 0, 0, 0],
[0, 0, 1, 0],
[0, 0, 0, 1],
[0, 1, 0, 0],
[0, 1, 0, 0],
[1, 0, 0, 0],
[0, 0, 1, 0],
],
dtype=tf.float32,
)
predictions = tf.constant(
[
[0.85, 0.12, 0.03, 0],
[0, 0, 1, 0],
[0.10, 0.045, 0.045, 0.81],
[1, 0, 0, 0],
[0.80, 0.10, 0.10, 0],
[1, 0, 0, 0],
[0.05, 0, 0.90, 0.05],
],
dtype=tf.float32,
)
# Initialize
hl_obj = HammingLoss("multiclass", threshold=0.8)
hl_obj.update_state(actuals, predictions)
# Check results
check_results(hl_obj, 0.2857143)


@pytest.mark.xfail(reason="See https://github.com/tensorflow/addons/issues/1200")
def test_mc_5_classes():
actuals = tf.constant(
[
[1, 0, 0, 0, 0],
[0, 0, 0, 1, 0],
[0, 0, 0, 0, 1],
[0, 1, 0, 0, 0],
[0, 0, 1, 0, 0],
[0, 0, 1, 0, 0],
[1, 0, 0, 0, 0],
[0, 1, 0, 0, 0],
],
dtype=tf.float32,
)

predictions = tf.constant(
[
[0.85, 0, 0.15, 0, 0],
[0, 0, 0, 1, 0],
[0, 1, 0, 0, 0],
[0.05, 0.90, 0.04, 0, 0.01],
[0.10, 0, 0.81, 0.09, 0],
[0.10, 0.045, 0, 0.81, 0.045],
[1, 0, 0, 0, 0],
[0, 0.85, 0, 0, 0.15],
],
dtype=tf.float32,
)
# Initialize
hl_obj = HammingLoss("multiclass", threshold=0.8)
hl_obj.update_state(actuals, predictions)
# Check results
check_results(hl_obj, 0.25)


@pytest.mark.xfail(reason="See https://github.com/tensorflow/addons/issues/1200")
def test_ml_4_classes():
actuals = tf.constant([[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 0, 1]], dtype=tf.float32)
predictions = tf.constant(
[[0.97, 0.56, 0.83, 0.77], [0.34, 0.95, 0.7, 0.89], [0.95, 0.45, 0.23, 0.56],],
dtype=tf.float32,
)
# Initialize
hl_obj = HammingLoss("multilabel", threshold=0.8)
hl_obj.update_state(actuals, predictions)
# Check results
check_results(hl_obj, 0.16666667)


@pytest.mark.xfail(reason="See https://github.com/tensorflow/addons/issues/1200")
def test_ml_5_classes():
actuals = tf.constant(
[
[1, 0, 0, 0, 0],
[0, 0, 1, 1, 0],
[0, 1, 0, 1, 0],
[0, 1, 1, 0, 0],
[0, 0, 1, 1, 0],
[0, 0, 1, 1, 0],
[1, 0, 0, 0, 1],
[0, 1, 1, 0, 0],
],
dtype=tf.float32,
)
predictions = tf.constant(
[
[1, 0.75, 0.2, 0.55, 0],
[0.65, 0.22, 0.97, 0.88, 0],
[0, 1, 0, 1, 0],
[0, 0.85, 0.9, 0.34, 0.5],
[0.4, 0.65, 0.87, 0, 0.12],
[0.66, 0.55, 1, 0.98, 0],
[0.95, 0.34, 0.67, 0.65, 0.10],
[0.45, 0.97, 0.89, 0.67, 0.46],
],
dtype=tf.float32,
)
# Initialize
hl_obj = HammingLoss("multilabel", threshold=0.7)
hl_obj.update_state(actuals, predictions)
# Check results
check_results(hl_obj, 0.075)


def hamming_distance_test():
actuals = tf.constant([1, 1, 0, 0, 1, 0, 1, 0, 0, 1], dtype=tf.int32)
predictions = tf.constant([1, 0, 0, 0, 1, 0, 0, 1, 0, 1], dtype=tf.int32)
test_result = hamming_distance(actuals, predictions)
np.testing.assert_allclose(0.3, test_result, atol=1e-5)


@test_utils.run_all_in_graph_and_eager_modes
class HammingMetricsTest(tf.test.TestCase):
def test_config(self):
hl_obj = HammingLoss(mode="multilabel", threshold=0.8)
self.assertEqual(hl_obj.name, "hamming_loss")
self.assertEqual(hl_obj.dtype, tf.float32)

def initialize_vars(self, mode, threshold):
hl_obj = HammingLoss(mode=mode, threshold=threshold)
return hl_obj

def update_obj_states(self, obj, actuals, preds):
update_op = obj.update_state(actuals, preds)
self.evaluate(update_op)

def check_results(self, obj, value):
self.assertAllClose(value, self.evaluate(obj.result()), atol=1e-5)

def test_mc_4_classes(self):
self.skipTest("Failing. See https://github.com/tensorflow/addons/issues/1200")
actuals = tf.constant(
[
[1, 0, 0, 0],
[0, 0, 1, 0],
[0, 0, 0, 1],
[0, 1, 0, 0],
[0, 1, 0, 0],
[1, 0, 0, 0],
[0, 0, 1, 0],
],
dtype=tf.float32,
)
predictions = tf.constant(
[
[0.85, 0.12, 0.03, 0],
[0, 0, 1, 0],
[0.10, 0.045, 0.045, 0.81],
[1, 0, 0, 0],
[0.80, 0.10, 0.10, 0],
[1, 0, 0, 0],
[0.05, 0, 0.90, 0.05],
],
dtype=tf.float32,
)
# Initialize
hl_obj = self.initialize_vars("multiclass", 0.8)
# Update
self.update_obj_states(hl_obj, actuals, predictions)
# Check results
self.check_results(hl_obj, 0.2857143)

def test_mc_5_classes(self):
self.skipTest("Failing. See https://github.com/tensorflow/addons/issues/1200")
actuals = tf.constant(
[
[1, 0, 0, 0, 0],
[0, 0, 0, 1, 0],
[0, 0, 0, 0, 1],
[0, 1, 0, 0, 0],
[0, 0, 1, 0, 0],
[0, 0, 1, 0, 0],
[1, 0, 0, 0, 0],
[0, 1, 0, 0, 0],
],
dtype=tf.float32,
)

predictions = tf.constant(
[
[0.85, 0, 0.15, 0, 0],
[0, 0, 0, 1, 0],
[0, 1, 0, 0, 0],
[0.05, 0.90, 0.04, 0, 0.01],
[0.10, 0, 0.81, 0.09, 0],
[0.10, 0.045, 0, 0.81, 0.045],
[1, 0, 0, 0, 0],
[0, 0.85, 0, 0, 0.15],
],
dtype=tf.float32,
)
# Initialize
hl_obj = self.initialize_vars("multiclass", 0.8)
# Update
self.update_obj_states(hl_obj, actuals, predictions)
# Check results
self.check_results(hl_obj, 0.25)

def test_ml_4_classes(self):
self.skipTest("Failing. See https://github.com/tensorflow/addons/issues/1200")
actuals = tf.constant(
[[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 0, 1]], dtype=tf.float32
)
predictions = tf.constant(
[
[0.97, 0.56, 0.83, 0.77],
[0.34, 0.95, 0.7, 0.89],
[0.95, 0.45, 0.23, 0.56],
],
dtype=tf.float32,
)
# Initialize
hl_obj = self.initialize_vars("multilabel", 0.8)
# Update
self.update_obj_states(hl_obj, actuals, predictions)
# Check results
self.check_results(hl_obj, 0.16666667)

def test_ml_5_classes(self):
self.skipTest("Failing. See https://github.com/tensorflow/addons/issues/1200")
actuals = tf.constant(
[
[1, 0, 0, 0, 0],
[0, 0, 1, 1, 0],
[0, 1, 0, 1, 0],
[0, 1, 1, 0, 0],
[0, 0, 1, 1, 0],
[0, 0, 1, 1, 0],
[1, 0, 0, 0, 1],
[0, 1, 1, 0, 0],
],
dtype=tf.float32,
)
predictions = tf.constant(
[
[1, 0.75, 0.2, 0.55, 0],
[0.65, 0.22, 0.97, 0.88, 0],
[0, 1, 0, 1, 0],
[0, 0.85, 0.9, 0.34, 0.5],
[0.4, 0.65, 0.87, 0, 0.12],
[0.66, 0.55, 1, 0.98, 0],
[0.95, 0.34, 0.67, 0.65, 0.10],
[0.45, 0.97, 0.89, 0.67, 0.46],
],
dtype=tf.float32,
)
# Initialize
hl_obj = self.initialize_vars("multilabel", 0.7)
# Update
self.update_obj_states(hl_obj, actuals, predictions)
# Check results
self.check_results(hl_obj, 0.075)

def hamming_distance_test(self):
actuals = tf.constant([1, 1, 0, 0, 1, 0, 1, 0, 0, 1], dtype=tf.int32)
predictions = tf.constant([1, 0, 0, 0, 1, 0, 0, 1, 0, 1], dtype=tf.int32)
test_result = hamming_distance(actuals, predictions)
self.assertAllClose(0.3, test_result, atol=1e-5)

# Keras model check
def test_keras_model(self):
model = tf.keras.Sequential()
model.add(layers.Dense(64, activation="relu"))
model.add(layers.Dense(3, activation="softmax"))
h1 = HammingLoss(mode="multiclass")
model.compile(
optimizer="rmsprop", loss="categorical_crossentropy", metrics=[h1]
)
data = np.random.random((100, 10))
labels = np.random.random((100, 3))
model.fit(data, labels, epochs=1, batch_size=32, verbose=0)
# Keras model check
def test_keras_model():
model = tf.keras.Sequential()
model.add(layers.Dense(64, activation="relu"))
model.add(layers.Dense(3, activation="softmax"))
h1 = HammingLoss(mode="multiclass")
model.compile(optimizer="rmsprop", loss="categorical_crossentropy", metrics=[h1])
data = np.random.random((100, 10))
labels = np.random.random((100, 3))
model.fit(data, labels, epochs=1, batch_size=32, verbose=0)