add check_opt_vs_noopt_jit option to check that results between bas…

…eline and optimized jitted versions match (#249)

* fixes

* rename option

conftest.py

@@ -11,6 +11,9 @@ def pytest_addoption(parser):
  help="Disable checks/assertions for machine configuration for stable benchmarks")
  parser.addoption("--disable_nograd", action='store_true', 
  help="Disable no_grad for eval() runs")
+ parser.addoption("--check_opt_vs_noopt_jit",
+ action='store_true',
+ help="The best attempt to check results for inference runs. Not all models support this!")
 
 def set_fuser(fuser):
  if fuser == "old":

test_bench.py

@@ -80,5 +80,7 @@ def test_eval(self, hub_model, benchmark, pytestconfig):
  hub_model.set_eval()
  benchmark(hub_model.eval)
  benchmark.extra_info['machine_state'] = get_machine_state()
+ if pytestconfig.getoption("check_opt_vs_noopt_jit"):
+ hub_model.check_opt_vs_noopt_jit()
  except NotImplementedError:
  print('Method eval is not implemented, skipping...')

torchbenchmark/util/model.py

@@ -5,7 +5,9 @@
 from collections.abc import Iterable
 import torch
 from contextlib import contextmanager
-
+import warnings
+import inspect
+import os
 
 @contextmanager
 def no_grad(val):
@@ -46,4 +48,61 @@ def _set_mode(self, train):
  (model, _) = self.get_module()
  model.train(train)
 
+ def check_opt_vs_noopt_jit(self):
+ if not self.jit:
+ return
+
+ model_name = inspect.getfile(self.__class__).split(os.sep)[-2]
+ print(f"model_name={model_name} , {inspect.getfile(self.__class__)}")
+ model_blacklist = [
+ 'demucs', # set up issue
+ 'yolov3', # set up issue
+ 'BERT_pytorch', # set up issue
+ 'moco', # set up issue
+ 'Super_SloMo', # results don't match, might be due to the way TE CUDA handles rand?
+ 'attention_is_all_you_need_pytorch', # results don't match, might be due to the way TE CUDA handles rand?
+ ]
+
+ if model_name in model_blacklist:
+ warnings.warn(UserWarning(f"{model_name}.get_module() doesn't support `check_results` yet!"))
+ return
+
+ # if a model doesn't support `get_module`
+ # we should let it throw and then
+ # override `check_results` for that model
+ try:
+ model, inputs = self.get_module()
+ except NotImplementedError:
+ warnings.warn(UserWarning(f"{model_name}.get_module() doesn't support `check_results` yet!"))
+ return
+
+ def bench_allclose(a, b):
+ if isinstance(a, torch.Tensor):
+ assert(isinstance(b, torch.Tensor))
+ assert(a.allclose(b))
+ elif isinstance(a, tuple) or isinstance (b, list):
+ assert(type(a) == type(b))
+ assert(len(a) == len(b))
+ for i in range(len(a)):
+ bench_allclose(a[i], b[i])
+ else:
+ raise RuntimeError("Encountered an supported type.\n" +
+ "Please add the type or override `bench_allclose`")
+
+
+ try:
+ opt = model(*inputs)
+ except Exception as e:
+ print(e)
+ warnings.warn(UserWarning(f"{model_name}.eval() doesn't support `check_results` yet!"))
+ return
+
+ # disable optimizations and force a recompilation
+ # to a baseline version
+ fwd = model._c._get_method("forward")
+ fwd._debug_flush_compilation_cache()
+ torch._C._set_graph_executor_optimize(False)
+ base = model(*inputs)
+ torch._C._set_graph_executor_optimize(True)
 
+ bench_allclose(base, opt)