neox args test with import in function

tests/Readme.md

@@ -0,0 +1,29 @@
+# Dependencies
+
+Tests use pytests with coverage and forked plugins. Install with:
+
+```bash
+pip install -r requirements/requirements-dev.txt
+```
+
+# Run
+
+Tests can be run using pytest. 
+
+* The argument --forked needs to be provided
+* A coverage report can be created using the optional arguments --cov-report and --cov (see pytest documentation)
+
+```bash
+# run all tests, output coverage report of megatron module in terminal
+pytest --forked --cov-report term --cov=megatron tests
+
+# run tests in tests/model, output coverage report of megatron module as html
+pytest --forked --cov-report html --cov=megatron tests/model
+```
+
+If a html coverage report has been created a simple http server can be run to serve static files.
+
+```bash
+cd htmlcov
+python -m http.server 8000
+```

tests/neox_args/test_neoxargs_commandline.py

@@ -2,16 +2,15 @@
 import sys
 from unittest.mock import patch
 
-
-from megatron.neox_arguments import NeoXArgs
 from ..common import get_root_directory, get_config_directory, get_configs_with_path
 
-
 def test_neoxargs_consume_deepy_args_with_config_dir():
  """
  verify consume_deepy_args processes command line arguments without config dir
  """
 
+ from megatron.neox_arguments import NeoXArgs
+
  # load neox args with command line
  with patch('sys.argv', [str(get_root_directory() / "deepy.py"), "pretrain_gpt2.py"] + get_configs_with_path(["small.yml", "local_setup.yml"])):
  args_loaded_consume = NeoXArgs.consume_deepy_args()
@@ -31,6 +30,8 @@ def test_neoxargs_consume_deepy_args_without_yml_suffix():
  verify consume_deepy_args processes command line arguments without yaml suffix
  """
 
+ from megatron.neox_arguments import NeoXArgs
+
  # load neox args with command line
  with patch('sys.argv', [str(get_root_directory() / "deepy.py"), "pretrain_gpt2.py"] + get_configs_with_path(["small", "local_setup"])):
  args_loaded_consume = NeoXArgs.consume_deepy_args()
@@ -50,6 +51,8 @@ def test_neoxargs_consume_deepy_args_with_config_dir():
  verify consume_deepy_args processes command line arguments including config dir
  """
 
+ from megatron.neox_arguments import NeoXArgs
+
  # load neox args with command line
  with patch('sys.argv', [str(get_root_directory() / "deepy.py"), "pretrain_gpt2.py", '-d', str(get_config_directory())] + ["small.yml", "local_setup.yml"]):
  args_loaded_consume = NeoXArgs.consume_deepy_args()
@@ -68,7 +71,8 @@ def test_neoxargs_consume_neox_args():
  """
  verify megatron args are correctly consumed after sending via deepspeed
  """
-
+ from megatron.neox_arguments import NeoXArgs
+
  # intitially load config from files as would be the case in deepy.py
  yaml_list = get_configs_with_path(["small.yml", "local_setup.yml"])
  args_baseline = NeoXArgs.from_ymls(yaml_list)

tests/neox_args/test_neoxargs_implementation.py

@@ -1,7 +1,6 @@
-from megatron import NeoXArgs
-
 def test_neoxargs_duplicates():
  """
  tests that there are no duplicates among parent classes of NeoXArgs
  """
+ from megatron import NeoXArgs
  assert NeoXArgs.validate_keys(), "test_neoxargs_duplicates"

tests/neox_args/test_neoxargs_load.py

@@ -1,11 +1,11 @@
-from megatron.neox_arguments import NeoXArgs
-
 from ..common import get_configs_with_path
 
 def test_neoxargs_load_arguments_small_local_setup():
  """
  verify small.yml can be loaded without raising validation errors
  """
+ from megatron.neox_arguments import NeoXArgs
+
  yaml_list = get_configs_with_path(["small.yml", "local_setup.yml"])
  args_loaded = NeoXArgs.from_ymls(yaml_list)
  assert isinstance(args_loaded, NeoXArgs)
@@ -14,6 +14,8 @@ def test_neoxargs_load_arguments_small_local_setup_text_generation():
  """
  verify small.yml can be loaded together with text generation without raising validation errors
  """
+ from megatron.neox_arguments import NeoXArgs
+
  yaml_list = get_configs_with_path(["small.yml", "local_setup.yml", "text_generation.yml"])
  args_loaded = NeoXArgs.from_ymls(yaml_list)
  assert isinstance(args_loaded, NeoXArgs)
@@ -22,6 +24,8 @@ def test_neoxargs_load_arguments_medium_local_setup():
  """
  verify medium.yml can be loaded without raising validation errors
  """
+ from megatron.neox_arguments import NeoXArgs
+
  yaml_list = get_configs_with_path(["medium.yml", "local_setup.yml"])
  args_loaded = NeoXArgs.from_ymls(yaml_list)
  assert isinstance(args_loaded, NeoXArgs)
@@ -30,6 +34,8 @@ def test_neoxargs_load_arguments_large_local_setup():
  """
  verify large.yml can be loaded without raising validation errors
  """
+ from megatron.neox_arguments import NeoXArgs
+
  yaml_list = get_configs_with_path(["large.yml", "local_setup.yml"])
  args_loaded = NeoXArgs.from_ymls(yaml_list)
  assert isinstance(args_loaded, NeoXArgs)
@@ -38,6 +44,8 @@ def test_neoxargs_load_arguments_2_7B_local_setup():
  """
  verify 2-7B.yml can be loaded without raising validation errors
  """
+ from megatron.neox_arguments import NeoXArgs
+
  yaml_list = get_configs_with_path(["2-7B.yml", "local_setup.yml"])
  args_loaded = NeoXArgs.from_ymls(yaml_list)
  assert isinstance(args_loaded, NeoXArgs)
@@ -46,6 +54,8 @@ def test_neoxargs_load_arguments_6_7B_local_setup():
  """
  verify 6-7B.yml can be loaded without raising validation errors
  """
+ from megatron.neox_arguments import NeoXArgs
+
  yaml_list = get_configs_with_path(["6-7B.yml", "local_setup.yml"])
  args_loaded = NeoXArgs.from_ymls(yaml_list)
  assert isinstance(args_loaded, NeoXArgs)
@@ -54,6 +64,8 @@ def test_neoxargs_load_arguments_13B_local_setup():
  """
  verify 13B.yml can be loaded without raising validation errors
  """
+ from megatron.neox_arguments import NeoXArgs
+
  yaml_list = get_configs_with_path(["13B.yml", "local_setup.yml"])
  args_loaded = NeoXArgs.from_ymls(yaml_list)
  assert isinstance(args_loaded, NeoXArgs)
@@ -62,6 +74,8 @@ def test_neoxargs_load_arguments_XL_local_setup():
  """
  verify XL.yml can be loaded without raising validation errors
  """
+ from megatron.neox_arguments import NeoXArgs
+
  yaml_list = get_configs_with_path(["XL.yml", "local_setup.yml"])
  args_loaded = NeoXArgs.from_ymls(yaml_list)
  assert isinstance(args_loaded, NeoXArgs)
@@ -70,6 +84,8 @@ def test_neoxargs_load_arguments_175B_local_setup():
  """
  verify 13B.yml can be loaded without raising validation errors
  """
+ from megatron.neox_arguments import NeoXArgs
+
  yaml_list = get_configs_with_path(["175B.yml", "local_setup.yml"])
  args_loaded = NeoXArgs.from_ymls(yaml_list)
  assert isinstance(args_loaded, NeoXArgs)
@@ -78,6 +94,8 @@ def test_neoxargs_fail_instantiate_without_required_params():
  """
  verify assertion error if required arguments are not provided
  """
+ from megatron.neox_arguments import NeoXArgs
+
  try:
  yaml_list = get_configs_with_path(["local_setup.yml"])
  args_loaded = NeoXArgs.from_ymls(yaml_list)
@@ -89,6 +107,8 @@ def test_neoxargs_fail_instantiate_without_any_params():
  """
  verify assertion error if required arguments are not provided
  """
+ from megatron.neox_arguments import NeoXArgs
+
  try:
  args_loaded = NeoXArgs()
  assert False

tests/neox_args/test_neoxargs_usage.py

@@ -1,14 +1,11 @@
 import re
-
-from megatron.neox_arguments import NeoXArgs
 from ..common import get_root_directory
 
-
 def test_neoxargs_usage():
  """"
  checks for code pieces of the pattern "args.*" and verifies that such used arg is defined in NeoXArgs
  """
-
+ from megatron.neox_arguments import NeoXArgs
 
  declared_all = True
  neox_args_attributes = set(NeoXArgs.__dataclass_fields__.keys())