Add v1 nightly CI workflow (#407)

.github/scripts/run.sh

@@ -50,11 +50,6 @@ for c in $(seq 1 $NUM_ITER); do
  taskset -c "${CORE_LIST}" pytest test_bench.py -k "${BENCHMARK_FILTER}" \
  --benchmark-min-rounds "${NUM_ROUNDS}" \
  --benchmark-json ${DATA_DIR}/${DATA_JSON_PREFIX}_${c}.json
- # Fill in circle_build_num and circle_project_reponame
- jq --arg run_id "${GITHUB_RUN_ID}" --arg config_version "githubactions-benchmark-${CONFIG_VER}-metal-fullname" \
- '.machine_info.circle_project_name=$config_version | .machine_info.circle_build_num=$run_id' \
- ${DATA_DIR}/${DATA_JSON_PREFIX}_${c}.json > ${DATA_DIR}/${DATA_JSON_PREFIX}_${c}.json.tmp
- mv ${DATA_DIR}/${DATA_JSON_PREFIX}_${c}.json.tmp ${DATA_DIR}/${DATA_JSON_PREFIX}_${c}.json
 done
 
 echo "Benchmark finished successfully. Output data dir is ${DATA_DIR}."

.github/workflows/v1-nightly.yml

@@ -0,0 +1,64 @@
+name: TorchBench nightly ci v1.0
+on:
+ workflow_dispatch:
+
+jobs:
+ run-benchmark:
+ env:
+ # Set to "v1-alpha" for testing. Will set it to "v1" in release.
+ TORCHBENCH_VER: "v1-alpha"
+ CONFIG_VER: "v1"
+ CONDA_ENV_NAME: "v1-nightly-ci"
+ OUTPUT_DIR: $(echo ${HOME})/.torchbench/v1-nighlty-ci
+ if: ${{ github.repository_owner == 'pytorch' }}
+ runs-on: [self-hosted, bm-runner]
+ env:
+ CONDA_ENV_NAME: torchbench-v1-nightly-ci
+ SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v2
+ with:
+ ref: v1.0
+ - name: Create conda env
+ run: |
+ conda create -y -q --name "${CONDA_ENV_NAME}" python=3.7
+ - name: Install PyTorch nightly
+ run: |
+ . activate "$CONDA_ENV_NAME"
+ # Check if nightly builds are available
+ NIGHTLIES=$(python torchbenchmark/util/torch_nightly.py --packages torch)
+ # If failed, the script will generate empty result
+ if [ -z $NIGHTLIES ]; then
+ echo "Torch nightly build failed. Cancel the workflow."
+ exit 1
+ fi
+ # Install PyTorch nightly from pip
+ pip install --pre torch \
+ -f https://download.pytorch.org/whl/nightly/${CUDA_VERSION}/torch_nightly.html
+ - name: Run benchmark
+ run: |
+ . activate "$CONDA_ENV_NAME"
+ bash ./.github/scripts/run.sh "${OUTPUT_DIR}"
+ - name: Copy artifact and upload to scribe
+ run: |
+ LATEST_RESULT=$(find ${OUTPUT_DIR}/gh${GITHUB_RUN_ID}/ -name "*.json" | sort -r | head -1)
+ echo "Benchmark result file: $LATEST_RESULT"
+ TODAY=$(date "+%Y%m%d%H%M%S")
+ CONFIG_NORM_FILE=${CONFIG_DIR}/${CONFIG_FILE}
+ SCORE_FILE="./benchmark-result-v1-score-${TODAY}.json"
+ # Generate score
+ python compute_score.py --score_version v1 --benchmark_data_file $LATEST_RESULT > $SCORE_FILE
+ # Upload result to Scribe
+ python scripts/upload_scribe.py --pytest_bench_json $LATEST_RESULT --torchbench_score_file $SCORE_FILE 
+ cp ${LATEST_RESULT} ./benchmark-result-v1-${TODAY}.json
+ - name: Upload artifact
+ uses: actions/upload-artifact@v2
+ with:
+ name: Benchmark result
+ path: benchmark-result-v1-*.json
+ - name: Remove conda env
+ run: |
+ conda env remove --name "${CONDA_ENV_NAME}"

conftest.py

@@ -73,8 +73,8 @@ def pytest_benchmark_update_machine_info(config, machine_info):
  except ImportError:
  machine_info['torchvision_version'] = '*not-installed*'
 
- machine_info['circle_build_num'] = os.environ.get("CIRCLE_BUILD_NUM")
- machine_info['circle_project_name'] = os.environ.get("CIRCLE_PROJECT_REPONAME")
+ machine_info['github_run_id'] = os.environ.get("GITHUB_RUN_ID")
+ machine_info['torchbench_score_version'] = os.environ.get("TORCHBENCH_VER")
 
  try:
  # if running on unexpected machine/os, get_machine_config _may_ not work

scripts/upload_scribe.py

@@ -82,10 +82,16 @@ def __init__(self):
  'pytorch_version', 'python_version',
  'torchtext_version', 'torchvision_version',
  'machine_kernel', 'machine_processor', 'machine_hostname',
- 'circle_build_num', 'circle_project_reponame',
+ 'github_run_id', 'torchbench_score_version',
  ],
  'float': [
- 'stddev', 'min', 'median', 'max', 'mean', 'runtime', 'torchbench_score',
+ 'stddev', 'min', 'median', 'max', 'mean', 'runtime',
+ 'torchbench_score',
+ 'torchbench_score_jit_speedup',
+ 'torchbench_subscore_cpu_train', 
+ 'torchbench_subscore_cpu_infer',
+ 'torchbench_subscore_gpu_train',
+ 'torchbench_subscore_gpu_infer',
  ]
  }
 
@@ -113,8 +119,8 @@ def post_pytest_benchmarks(self, pytest_json, max_data_upload=100):
  "machine_kernel": machine_info['release'],
  "machine_processor": machine_info['processor'],
  "machine_hostname": machine_info['node'],
- "circle_build_num": machine_info.get('circle_build_num', None),
- "circle_project_reponame": machine_info.get('circle_project_name', None),
+ "github_run_id": machine_info.get('github_run_id', None),
+ "torchbench_score_version": machine_info.get('torchbench_score_version', None),
  }
 
  stats_msg = {"stddev": b['stats']['stddev'],
@@ -139,7 +145,7 @@ def post_torchbench_score(self, pytest_json, score):
  machine_info = pytest_json['machine_info']
  commit_info = pytest_json['commit_info']
  upload_time = int(time.time())
- m = self.format_message({
+ scribe_message = {
  "time": upload_time,
  "benchmark_time": pytest_json['datetime'],
  "git_repo": commit_info['project'],
@@ -154,10 +160,16 @@ def post_torchbench_score(self, pytest_json, score):
  "machine_kernel": machine_info['release'],
  "machine_processor": machine_info['processor'],
  "machine_hostname": machine_info['node'],
- "circle_build_num": machine_info.get('circle_build_num', None),
- "circle_project_reponame": machine_info.get('circle_project_name', None),
- "torchbench_score": score,
- })
+ "github_run_id": machine_info.get('github_run_id', None),
+ "torchbench_score_version": machine_info.get('torchbench_score_version', None),
+ "torchbench_score": score["score"]["total"],
+ "torchbench_score_jit_speedup": score["score"]["jit-speedup"],
+ "torchbench_subscore_cpu_train": score["score"]["subscore-cpu-train"],
+ "torchbench_subscore_cpu_infer": score["score"]["subscore-cpu-eval"],
+ "torchbench_subscore_gpu_train": score["score"]["subscore-cuda-train"],
+ "torchbench_subscore_gpu_infer": score["score"]["subscore-cuda-eval"],
+ }
+ m = self.format_message(scribe_message)
  self.upload([m])
 
 
@@ -166,13 +178,20 @@ def post_torchbench_score(self, pytest_json, score):
  parser.add_argument("--pytest_bench_json", required=True,
  type=argparse.FileType('r'),
  help='Upload json data formatted by pytest-benchmark module')
- parser.add_argument("--torchbench_score", type=float,
- help="optional torchbench score to include")
+ parser.add_argument("--torchbench_score_file", required=True,
+ type=argparse.FileType('r'),
+ help="torchbench score file to include")
  args = parser.parse_args()
 
+ # Result sanity check
+ json_name = os.path.basename(args.pytest_bench_json.name)
+ json_score = json.load(args.torchbench_score_file)
+ score_data = None
+ for data in json_score:
+ if os.path.basename(data["file"]) == json_name:
+ score_data = data
+ assert score_data, f"Can't find {json_name} score in {args.torchbench_score_file}. Stop."
  benchmark_uploader = PytorchBenchmarkUploader()
  json_data = json.load(args.pytest_bench_json)
  benchmark_uploader.post_pytest_benchmarks(json_data)
-
- if args.torchbench_score is not None:
- benchmark_uploader.post_torchbench_score(json_data, args.torchbench_score)
+ benchmark_uploader.post_torchbench_score(json_data, score_data)

torchbenchmark/score/compute_score_v1.py

@@ -231,12 +231,12 @@ def compute_score(self, data):
  assert not diff_set, f"The request benchmark json doesn't have v1 test: {diff_set}"
 
  summary = {}
- summary["subscore[jit]"] = self.compute_jit_speedup_score(data)
+ summary["jit-speedup"] = self.compute_jit_speedup_score(data)
  devices = ["cpu", "cuda"]
  tests = ["train", "eval"]
  filters = [(a, b) for a in devices for b in tests]
  for f in filters:
- key = f"subscore[{f[0]}-{f[1]}]"
+ key = f"subscore-{f[0]}-{f[1]}"
  summary[key] = self._get_subscore(data_norm, self.norm, self.norm_weights, f) * self.target
  summary["total"] = self._get_score(data_norm, self.norm, self.norm_weights) * self.target
  return summary