Add initial fambench support to e2e_models

.gitmodules

@@ -16,3 +16,6 @@
 [submodule "third_party/tbb"]
 	path = legacy/third_party/tbb
 	url = https://github.com/01org/tbb
+[submodule "submodules/FAMBench"]
+	path = submodules/FAMBench
+	url = https://github.com/facebookresearch/FAMBench.git

run_e2e.py

@@ -9,13 +9,15 @@
 SUPPORT_DEVICE_LIST = ["cpu", "cuda"]
 
 def run(func) -> Dict[str, float]:
-    torch.cuda.synchronize()
+    if torch.cuda.is_available():
+        torch.cuda.synchronize()
     result = {}
     # Collect time_ns() instead of time() which does not provide better precision than 1
     # second according to https://docs.python.org/3/library/time.html#time.time.
     t0 = time.time_ns()
     func()
-    torch.cuda.synchronize()
+    if torch.cuda.is_available():
+        torch.cuda.synchronize()
     t2 = time.time_ns()
     result["latency_ms"] = (t2 - t0) / 1_000_000.0
     return result

torchbenchmark/e2e_models/fambench_xlmr/__init__.py

@@ -0,0 +1,67 @@
+import os
+import sys
+import torch
+import subprocess
+from pathlib import Path
+from dataclasses import dataclass
+
+from torchbenchmark.util.e2emodel import E2EBenchmarkModel
+
+from typing import Optional, List
+
+CURRENT_DIR = Path(os.path.dirname(os.path.realpath(__file__)))
+FAMBENCH_ROOT = CURRENT_DIR.parent.parent.parent.joinpath("submodules", "FAMBench")
+
+def _create_data_dir(data_dir: str):
+    data_dir = Path(data_dir)
+    data_dir.mkdir(parents=True, exist_ok=True)
+    return data_dir
+
+def _get_fambench_test_root(name: str):
+    xlmr_ootb_root = FAMBENCH_ROOT.joinpath("benchmarks")
+    assert xlmr_ootb_root.exists(), f"Can't find FAMBench source at {xlmr_ootb_root.absolute()}," \
+                                    "please check out the submodules."
+    return xlmr_ootb_root
+
+@dataclass
+class FAMBenchXLMREvalConfig:
+    """
+    Original config reference:
+    https://github.com/facebookresearch/FAMBench/blob/main/benchmarks/run_xlmr_ootb.sh
+    """
+    config_name = "default-config"
+    nbatches = 10
+    batchsize = 16
+    seqlength = 16
+    vocabsize = 250000
+    warmupbatches = 1
+    log_dir = os.path.join(CURRENT_DIR, ".data", "logs")
+    config_flags=["--inference-only", f"--num-batches={nbatches}", f"--batch-size={batchsize}", \
+                 f"--sequence-length={seqlength}", f"--vocab-size={vocabsize}", \
+                 f"--famconfig={config_name}", "--half-model", f"--warmup-batches={warmupbatches}", \
+                 f"--logdir={log_dir}"]
+
+class Model(E2EBenchmarkModel):
+    DEFAULT_EVAL_BSIZE = FAMBenchXLMREvalConfig.batchsize
+    def __init__(self, test: str, batch_size: Optional[int]=None, extra_args: List[str]=[]):
+        super().__init__(test=test, batch_size=batch_size, extra_args=extra_args)
+        if not torch.cuda.is_available():
+            raise NotImplementedError("FAMBench only support running on Nvidia GPU.")
+        self.device = "cuda"
+        self.device_num = torch.cuda.device_count()
+        self.name = "xlmr"
+        self.implementation = "ootb"
+        self.code_root = _get_fambench_test_root(self.name)
+        if test == "eval":
+            self.config = FAMBenchXLMREvalConfig()
+            self.config.batchsize = self.batch_size
+            self.num_examples = self.config.nbatches * self.batch_size
+            _create_data_dir(self.config.log_dir)
+
+    def train(self):
+        raise NotImplementedError("FAMBench XLMR train is not implemented yet.")
+
+    def eval(self):
+        prog_args = [sys.executable, f"{self.name}/{self.implementation}/{self.name}.py"]
+        prog_args.extend(self.config.config_flags)
+        subprocess.check_call(prog_args, cwd=self.code_root)

torchbenchmark/e2e_models/fambench_xlmr/install.py

@@ -0,0 +1,8 @@
+import sys
+import subprocess
+
+def pip_install_requirements():
+    subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt'])
+
+if __name__ == '__main__':
+    pip_install_requirements()

torchbenchmark/e2e_models/fambench_xlmr/requirements.txt

@@ -0,0 +1,4 @@
+bitarray
+sacrebleu>=1.4.12
+omegaconf
+hydra-core