Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix the bug of tensors not on the same device when running on CUDA device #59

Merged
merged 10 commits into from
Apr 19, 2023
Merged
Next Next commit
feat: add --build into pypots-cli dev, and add check_arguments();
  • Loading branch information
WenjieDu committed Apr 18, 2023
commit dceebe250275e7c4bcb1a85faa929b0d9334c970
83 changes: 70 additions & 13 deletions pypots/utils/commands/dev.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
# License: GLP-v3

import os
import shutil
from argparse import ArgumentParser, Namespace

from pypots.utils.commands import BaseCommand
Expand All @@ -20,6 +21,8 @@

def dev_command_factory(args: Namespace):
return DevCommand(
args.build,
args.cleanup,
args.run_tests,
args.k,
args.show_coverage,
Expand All @@ -44,6 +47,18 @@ def register_subcommand(parser: ArgumentParser):
sub_parser = parser.add_parser(
"dev", help="CLI tools helping develop PyPOTS code"
)
sub_parser.add_argument(
"--build",
dest="build",
action="store_true",
help="Build PyPOTS into a wheel and package the source code into a .tar.gz file for distribution",
)
sub_parser.add_argument(
"--cleanup",
dest="cleanup",
action="store_true",
help="Delete all caches and building files",
)
sub_parser.add_argument(
"--run_tests",
dest="run_tests",
Expand All @@ -59,7 +74,7 @@ def register_subcommand(parser: ArgumentParser):
sub_parser.add_argument(
"-k",
type=str,
default="",
default=None,
help="The -k option of pytest. Description of -k option in pytest: "
"only run tests which match the given substring expression. An expression is a python evaluatable "
"expression where all names are substring-matched against test names and their parent classes. "
Expand All @@ -80,19 +95,65 @@ def register_subcommand(parser: ArgumentParser):

def __init__(
self,
build: bool,
cleanup: bool,
run_tests: bool,
k: str,
show_coverage: bool,
lint_code: bool,
):
self._build = build
self._cleanup = cleanup
self._run_tests = run_tests
self._k = k
self._show_coverage = show_coverage
self._lint_code = lint_code

def check_arguments(self):
"""Run some checks on the arguments to avoid error usages"""
containing_docs = "docs" in os.listdir(".")
containing_pypots = "pypots" in os.listdir(".")
# if currently under dir 'docs', it should have sub-dir 'figs', and 'pypots' should be in the parent dir
whether_under_dir_docs = containing_docs and containing_pypots
# `pypots-cli dev` should only be run under dir 'docs'
# because we probably will compile the doc and generate HTMLs with command `make`
assert whether_under_dir_docs, (
"Command `pypots-cli dev` can only be run under the root directory of project PyPOTS, "
f"but you're running it under the path {os.getcwd()}. Please make a check."
)

if self._k is not None:
assert self._run_tests, (
"Argument `-k` should combine the use of `--run_tests`. "
"Try `pypots-cli dev --run_tests -k your_pattern`"
)

if self._show_coverage:
assert self._run_tests, (
"Argument `--show_coverage` should combine the use of `--run_tests`. "
"Try `pypots-cli dev --run_tests --show_coverage`"
)

if self._cleanup:
assert not self._run_tests and not self._lint_code, (
"Argument `--cleanup` should be used alone. "
"Try `pypots-cli dev --cleanup`"
)

def run(self):
if self._run_tests:
try:
"""Execute the given command."""

# check arguments first
self.check_arguments()

try:
if self._cleanup:
shutil.rmtree("build", ignore_errors=True)
shutil.rmtree("dist", ignore_errors=True)
shutil.rmtree("pypots.egg-info", ignore_errors=True)
elif self._build:
os.system("python setup.py sdist bdist bdist_wheel")
elif self._run_tests:
pytest_command = f"pytest -k {self._k}" if self._k else "pytest"
command_to_run_test = (
f"coverage run -m {pytest_command}"
Expand All @@ -111,17 +172,13 @@ def run(self):
)
os.system("rm -rf .pytest_cache")

except ImportError:
raise ImportError(IMPORT_ERROR_MESSAGE)
except Exception as e:
raise RuntimeError(e)
elif self._lint_code:
try:
elif self._lint_code:
logger.info("Reformatting with Black...")
os.system("black .")
logger.info("Linting with Flake8...")
os.system("flake8 .")
except ImportError:
raise ImportError(IMPORT_ERROR_MESSAGE)
except Exception as e:
raise RuntimeError(e)

except ImportError:
raise ImportError(IMPORT_ERROR_MESSAGE)
except Exception as e:
raise RuntimeError(e)
76 changes: 47 additions & 29 deletions pypots/utils/commands/doc.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,11 @@ def doc_command_factory(args: Namespace):


def purge_statics():
logger.info("Directories _build, _static, and _templates will be deleted if exist")
logger.info(
f"Directories _build, _static, and _templates, and {CLONED_LATEST_PYPOTS} will be deleted if exist"
)
shutil.rmtree("_build", ignore_errors=True)
shutil.rmtree(CLONED_LATEST_PYPOTS, ignore_errors=True)
shutil.rmtree("_static", ignore_errors=True)
shutil.rmtree("_templates", ignore_errors=True)

Expand Down Expand Up @@ -133,21 +136,51 @@ def __init__(
self._port = port
self._cleanup = cleanup

def run(self):
def check_arguments(self):
"""Run some checks on the arguments to avoid error usages"""
parent_dir = os.path.join(".", "..")
containing_figs = "figs" in os.listdir(".")
pypots_in_the_parent_dir = "pypots" in os.listdir(parent_dir)
# if currently under dir 'docs', it should have sub-dir 'figs', and 'pypots' should be in the parent dir
whether_under_dir_docs = containing_figs and pypots_in_the_parent_dir

# `pypots-cli dev` should only be run under dir 'docs'
# because we probably will compile the doc and generate HTMLs with command `make`
assert (
whether_under_dir_docs
), "Command `pypots-cli dev` can only be run under the directory 'docs' in project PyPOTS"
assert whether_under_dir_docs, (
"Command `pypots-cli doc` can only be run under the directory 'docs' in project PyPOTS, "
f"but you're running it under the path {os.getcwd()}. Please make a check."
)

if self._port is not None:
assert self._view_doc, (
"Argument `--port` should combine the use of `--view_doc`. "
"Try `pypots-cli doc --view_doc --port your_port`"
)

if self._branch is not None:
assert self._gene_rst, (
"Argument `--branch` should combine the use of `--gene_rst`. "
"Try `pypots-cli doc --gene_rst --branch your_branch`"
)

if self._cleanup:
assert not self._gene_rst and not self._gene_html and not self._view_doc, (
"Argument `--cleanup` should be used alone. "
"Try `pypots-cli doc --cleanup`"
)

def run(self):
"""Execute the given command."""

# check arguments first
self.check_arguments()

if self._gene_rst:
try:
try:
if self._cleanup:
logger.info("Purging static files...")
purge_statics()
logger.info("Purging finished successfully.")

if self._gene_rst:
if os.path.exists(CLONED_LATEST_PYPOTS):
logger.info(
f"Directory {CLONED_LATEST_PYPOTS} exists, deleting it..."
Expand Down Expand Up @@ -188,36 +221,21 @@ def run(self):
# Delete the useless files.
shutil.rmtree(f"{CLONED_LATEST_PYPOTS}", ignore_errors=True)

except ImportError:
raise ImportError(IMPORT_ERROR_MESSAGE)
except Exception as e:
raise RuntimeError(e)

if self._gene_html:
try:
if self._gene_html:
logger.info("Generating static HTML files...")
purge_statics()
os.system("make html")
except Exception as e:
raise RuntimeError(e)

if self._view_doc:
try:
if self._view_doc:
assert os.path.exists(
"_build/html"
), "_build/html does not exists, please run `pypots-cli doc --gene_html` first"
logger.info("Deploying HTML...")
os.system(
f"python -m http.server {self._port} -d _build/html -b 127.0.0.1"
)
except Exception as e:
raise RuntimeError(e)

if self._cleanup:
try:
logger.info("Purging static files...")
purge_statics()
logger.info("Purging finished successfully.")

except Exception as e:
raise RuntimeError(e)
except ImportError:
raise ImportError(IMPORT_ERROR_MESSAGE)
except Exception as e:
raise RuntimeError(e)