[FBcode->GH] Allow all torchvision test rules to run with RE

test/conftest.py

@@ -14,12 +14,19 @@ def pytest_configure(config):
 
 
 def pytest_collection_modifyitems(items):
- # This hook is called by pytest after it has collected the tests (google its name!)
+ # This hook is called by pytest after it has collected the tests (google its name to check out its doc!)
  # We can ignore some tests as we see fit here, or add marks, such as a skip mark.
+ #
+ # Typically here, we try to optimize CI time. In particular, the GPU CI instances don't need to run the
+ # tests that don't need CUDA, because those tests are extensively tested in the CPU CI instances already.
+ # This is true for both CircleCI and the fbcode internal CI.
+ # In the fbcode CI, we have an additional constraint: we try to avoid skipping tests. So instead of relying on
+ # pytest.mark.skip, in fbcode we literally just remove those tests from the `items` list, and it's as if
+ # these tests never existed.
 
  out_items = []
  for item in items:
- # The needs_cuda mark will exist if the test was explicitely decorated with
+ # The needs_cuda mark will exist if the test was explicitly decorated with
  # the @needs_cuda decorator. It will also exist if it was parametrized with a
  # parameter that has the mark: for example if a test is parametrized with
  # @pytest.mark.parametrize('device', cpu_and_gpu())
@@ -57,3 +64,19 @@ def pytest_collection_modifyitems(items):
  out_items.append(item)
 
  items[:] = out_items
+
+
+def pytest_sessionfinish(session, exitstatus):
+ # This hook is called after all tests have run, and just before returning an exit status.
+ # We here change exit code 5 into 0.
+ #
+ # 5 is issued when no tests were actually run, e.g. if you use `pytest -k some_regex_that_is_never_matched`.
+ #
+ # Having no test being run for a given test rule is a common scenario in fbcode, and typically happens on
+ # the GPU test machines which don't run the CPU-only tests (see pytest_collection_modifyitems above). For
+ # example `test_transforms.py` doesn't contain any CUDA test at the time of
+ # writing, so on a GPU test machine, testpilot would invoke pytest on this file and no test would be run.
+ # This would result in pytest returning 5, causing testpilot to raise an error.
+ # To avoid this, we transform this 5 into a 0 to make testpilot happy.
+ if exitstatus == 5:
+ session.exitstatus = 0