Move torch version check to test json and update repo fields

    - change incorrect/confusing 'pytorch_commit_*' vars to git_* vars
      since the vars are used in repos such as benchmark
    - record torch pkg versions explicitly regardless of repo used
      for benchmark runs

conftest.py

@@ -1,3 +1,4 @@
+import os
 import pytest
 import torch
 
@@ -21,3 +22,20 @@ def set_fuser(fuser):
 
 def pytest_configure(config):
  set_fuser(config.getoption("fuser"))
+
+def pytest_benchmark_update_machine_info(config, machine_info):
+ machine_info['pytorch_version'] = torch.__version__
+ try:
+ import torchtext
+ machine_info['torchtext_version'] = torchtext.__version__
+ except ImportError:
+ machine_info['torchtext_version'] = '*not-installed*'
+
+ try:
+ import torchvision
+ machine_info['torchvision_version'] = torchvision.__version__
+ except ImportError:
+ machine_info['torchvision_version'] = '*not-installed*'
+
+ machine_info['circle_build_num'] = os.environ.get("CIRCLE_BUILD_NUM")
+ machine_info['circle_project_name'] = os.environ.get("CIRCLE_PROJECT_REPONAME")

scripts/upload_scribe.py

@@ -75,9 +75,12 @@ def __init__(self):
  'time', 'rounds',
  ],
  'normal': [
- 'benchmark_group', 'benchmark_name', 'benchmark_class', 'benchmark_time',
- 'pytorch_commit_id', 'pytorch_branch', 'pytorch_commit_time', 'pytorch_version',
- 'pytorch_git_dirty',
+ 'benchmark_group', 'benchmark_name',
+ 'benchmark_class', 'benchmark_time',
+ 'git_repo', 'git_commit_id', 'git_branch',
+ 'git_commit_time', 'git_dirty',
+ 'pytorch_version', 'python_version',
+ 'torchtext_version', 'torchvision_version',
  'machine_kernel', 'machine_processor', 'machine_hostname',
  'circle_build_num', 'circle_project_reponame',
  ],
@@ -98,16 +101,20 @@ def post_pytest_benchmarks(self, pytest_json):
  "benchmark_name": b['name'],
  "benchmark_class": b['fullname'],
  "benchmark_time": pytest_json['datetime'],
- "pytorch_commit_id": commit_info['id'],
- "pytorch_branch": commit_info['branch'],
- "pytorch_commit_time": commit_info['time'],
- "pytorch_version": None,
- "pytorch_git_dirty": commit_info['dirty'],
+ "git_repo": commit_info['project'],
+ "git_commit_id": commit_info['id'],
+ "git_branch": commit_info['branch'],
+ "git_commit_time": commit_info['time'],
+ "git_dirty": commit_info['dirty'],
+ "pytorch_version": machine_info.get('pytorch_version', None),
+ "torchtext_version": machine_info.get('torchtext_version', None),
+ "torchvision_version": machine_info.get('torchvision_version', None),
+ "python_version": machine_info['python_implementation_version'],
  "machine_kernel": machine_info['release'],
  "machine_processor": machine_info['processor'],
  "machine_hostname": machine_info['node'],
- "circle_build_num": os.environ.get("CIRCLE_BUILD_NUM"),
- "circle_project_reponame": os.environ.get("CIRCLE_PROJECT_REPONAME"),
+ "circle_build_num": machine_info.get('circle_build_num', None),
+ "circle_project_reponame": machine_info.get('circle_project_name', None),
  "stddev": b['stats']['stddev'],
  "rounds": b['stats']['rounds'],
  "min": b['stats']['min'],

test_bench.py

@@ -29,10 +29,11 @@ def pytest_generate_tests(metafunc, display_len=24):
  short = short[:display_len] + "..."
  short_names.append(short)
 
- metafunc.parametrize('model', all_models,
- ids=short_names, scope="class")
- metafunc.parametrize('device', ['cpu', 'cuda'], scope='class')
- metafunc.parametrize('compiler', ['jit', 'eager'], scope='class')
+ if metafunc.cls and metafunc.cls.__name__ == "TestBenchNetwork":
+ metafunc.parametrize('model', all_models,
+ ids=short_names, scope="class")
+ metafunc.parametrize('device', ['cpu', 'cuda'], scope='class')
+ metafunc.parametrize('compiler', ['jit', 'eager'], scope='class')
 
 
 @pytest.fixture(scope='class')