Add benchmark entrypoint and noxfile target (aerleon#31)

.gitignore

@@ -2,6 +2,7 @@
 filters/sample_*
 sample_*
 def/AUTOGEN.net
+benchmark_result/
 tests/characterization_data/filters_actual
 tools/new_lint_errors.txt
 

noxfile.py

@@ -1,5 +1,8 @@
 """ Define nox sessions for Aerleon """
 
+from datetime import datetime
+import os
+
 import nox
 from nox_poetry import session, Session
 
@@ -25,6 +28,63 @@ def coverage(session):
     session.run("coverage", "html")
 
 
+@session(python="3.10")
+def benchmark(session):
+    """Runs pyperf and produces a report"""
+    session.run_always("poetry", "install", external=True)
+    tune_system = '__benchmark_tune' in session.posargs
+    suite_name = 'SampleSuiteV1'
+    benchmark_result_path = './benchmark_result'
+    start_time = datetime.now().strftime("%Y-%m-%d-%H.%M.%S")
+    result_filename_base = f'{suite_name}-result-{"tuned-" if tune_system else ""}{start_time}'
+    result_filename = f'{result_filename_base}.json'
+    result_meta_filename = f'{result_filename_base}-metadata.json'
+    session.run(
+        "python",
+        "-c",
+        f"import os; os.makedirs('{benchmark_result_path}', exist_ok=True)",
+    )
+
+    def inner():
+        setup = f'from tools.benchmarks.demo_benchmark import {suite_name}; suite = {suite_name}({session.posargs})'  # noqa E501
+        statement = 'suite.run()'
+
+        session.run(
+            "pyperf",
+            "timeit",
+            "--quiet",
+            "--stats",
+            "--metadata",
+            "--copy-env",
+            "--no-locale",
+            f"--output={os.path.join(benchmark_result_path, result_filename)}",
+            "--setup",
+            setup,
+            statement,
+        )
+        end_time = datetime.now().strftime("%Y-%m-%d-%H.%M.%S")
+        session.run(
+            "python",
+            "-c",
+            f"with open('{result_meta_filename}') as file: file.write('file={result_filename} suite_name={suite_name} tune_system={tune_system} start_time={start_time} end_time={end_time} posargs={session.posargs}')", # noqa E501
+        )
+
+    if tune_system:
+        session.run("pyperf", "system", "tune")
+        try:
+            inner()
+        finally:
+            session.run("pyperf", "system", "reset")
+    else:
+        inner()
+
+
+@session(python="3.10")
+def benchmark_tuned(session):
+    """Runs pyperf with system tuning on and produces a report"""
+    session.notify('benchmark', ['__benchmark_tune'])
+
+
 @session
 def format(session):
     """Runs black"""

pyproject.toml

@@ -38,6 +38,8 @@ nox = {version = "^2022.8.7"}
 nox-poetry = {version = "^1.0.1"}
 pre-commit = {version = "^2.20.0"}
 pytest = {version = "^7.1.3"}
+pyperf = {version = "^2.4.1"}
+psutil = {version = "^5.9.2"}
 
 [tool.poetry.scripts]
 aclgen = "aerleon.aclgen:EntryPoint"

tools/benchmarks/demo_benchmark.py

@@ -0,0 +1,70 @@
+"""Benchmark SampleSuiteV1: Render to string from policy samples."""
+
+import multiprocessing
+import os
+import shutil
+import tempfile
+from unittest import mock
+
+from aerleon import aclgen
+
+
+class SampleSuiteV1:
+    """This benchmark will load all policy files found in ./policies and generate configs.
+    It will not try to write() any configs.
+
+    If any of the following are changed, please bump the version in the class name of this suite:
+
+    * The contents of this test.
+    * The contents of the ./policies folder.
+    * The contents of the ./def folder.
+
+    DO NOT run this benchmark if you have uncomitted files in the ./policies or ./def folders.
+
+    """
+
+    def __init__(self, argv):
+        skip_samples = []
+        only_samples = argv[1:]
+
+        def ignore_policy_files(src, names):
+            if src == 'policies/pol':
+                if len(only_samples) != 0:
+                    return [name for name in names if name not in only_samples]
+                if len(skip_samples) != 0:
+                    return [name for name in names if name in skip_samples]
+            return []
+
+        self.test_subdirectory = tempfile.mkdtemp()
+        self.def_dir = os.path.join(self.test_subdirectory, 'def')
+        self.pol_dir = os.path.join(self.test_subdirectory, 'policies')
+        shutil.rmtree(self.test_subdirectory, ignore_errors=True)
+        os.mkdir(self.test_subdirectory)
+        shutil.copytree('def', self.def_dir)
+        shutil.copytree('policies', self.pol_dir, ignore=ignore_policy_files)
+        self.context = multiprocessing.get_context()
+        self.max_renderers = 1
+        self.exp_info = 2
+        self.ignore_directories = ['DEPRECATED', 'def']
+
+    def run(self):
+        with mock.patch.object(aclgen, '_WriteFile', autospec=True):
+            aclgen.Run(
+                self.pol_dir,
+                self.def_dir,
+                None,
+                self.test_subdirectory,
+                self.exp_info,
+                self.max_renderers,
+                self.ignore_directories,
+                None,
+                None,
+                self.context,
+            )
+
+
+if __name__ == '__main__':
+    import sys
+
+    suite = SampleSuiteV1(sys.argv)
+    suite.run()