Merge branch 'staging' into staging_gpt2_evals # Please enter a commi…

…t message to explain why this merge is necessary,
# especially if it merges an updated upstream into a topic branch.
#
# Lines starting with '#' will be ignored, and an empty message aborts
# the commit.

scripts/linter.py

@@ -0,0 +1,36 @@
+import os
+import os.path as osp
+import pathlib
+import subprocess
+
+
+def recursively_lint_files():
+ """Recursively lint all python files in chosen subdirectories of megatron-lm"""
+
+ try:
+ import autopep8
+ except ModuleNotFoundError:
+ print("Please first install autopep8 via `pip install autopep8`")
+ return
+
+ # get all python file paths from top level directory
+ file_dir = str(pathlib.Path(__file__).parent.absolute())
+ working_dir = osp.join(file_dir, os.pardir)
+ all_py_paths = set(os.path.join(working_dir, fname)
+ for fname in os.listdir(working_dir) if ".py" in fname)
+
+ # get all python file paths from chosen subdirectories
+ check_dirs = ['docker', 'megatron', 'openwebtext', 'scripts', 'tasks']
+ for sub_dir in check_dirs:
+ for path, _, fnames in os.walk(osp.join(working_dir, sub_dir)):
+ all_py_paths.update(set(osp.join(path, fname) for fname in fnames if ".py" in fname))
+
+ print("Linting the following: ")
+ for py_path in all_py_paths:
+ print(py_path)
+ command = 'autopep8 --max-line-length 100 --aggressive --in-place {}'.format(py_path)
+ subprocess.check_call(command)
+
+
+if __name__ == "__main__":
+ recursively_lint_files()