Organize Python tests to use grpcio-tools directly

Moves all tests into a separate package. This does not change existing
supported means of running tests (e.g. through run_tests.py).

.gitignore

@@ -13,6 +13,8 @@ python_build/
 htmlcov/
 dist/
 *.egg
+py27/
+py34/
 
 # Node installation output
 ^node_modules

setup.py

@@ -72,10 +72,6 @@
 ENABLE_CYTHON_TRACING = os.environ.get(
  'GRPC_PYTHON_ENABLE_CYTHON_TRACING', False)
 
-# Environment variable to determine whether or not to include the test files in
-# the installation.
-INSTALL_TESTS = os.environ.get('GRPC_PYTHON_INSTALL_TESTS', False)
-
 CYTHON_EXTENSION_PACKAGE_NAMES = ()
 
 CYTHON_EXTENSION_MODULE_NAMES = ('grpc._cython.cygrpc',)
@@ -183,13 +179,10 @@ def cython_extensions(module_names, extra_sources, include_dirs,
 
 COMMAND_CLASS = {
  'doc': commands.SphinxDocumentation,
- 'build_proto_modules': commands.BuildProtoModules,
  'build_project_metadata': commands.BuildProjectMetadata,
  'build_py': commands.BuildPy,
  'build_ext': commands.BuildExt,
  'gather': commands.Gather,
- 'run_interop': commands.RunInterop,
- 'test_lite': commands.TestLite
 }
 
 # Ensure that package data is copied over before any commands have been run:
@@ -200,32 +193,6 @@ def cython_extensions(module_names, extra_sources, include_dirs,
  pass
 shutil.copyfile('etc/roots.pem', os.path.join(credentials_dir, 'roots.pem'))
 
-TEST_PACKAGE_DATA = {
- 'tests.interop': [
- 'credentials/ca.pem',
- 'credentials/server1.key',
- 'credentials/server1.pem',
- ],
- 'tests.protoc_plugin': [
- 'protoc_plugin_test.proto',
- ],
- 'tests.unit': [
- 'credentials/ca.pem',
- 'credentials/server1.key',
- 'credentials/server1.pem',
- ],
-}
-
-TESTS_REQUIRE = (
- 'oauth2client>=2.1.0',
- 'protobuf>=3.0.0a3',
- 'coverage>=4.0',
-) + INSTALL_REQUIRES
-
-TEST_SUITE = 'tests'
-TEST_LOADER = 'tests:Loader'
-TEST_RUNNER = 'tests:Runner'
-
 PACKAGE_DATA = {
  # Binaries that may or may not be present in the final installation, but are
  # mentioned here for completeness.
@@ -235,12 +202,7 @@ def cython_extensions(module_names, extra_sources, include_dirs,
  '_windows/grpc_c.64.python',
  ],
 }
-if INSTALL_TESTS:
- PACKAGE_DATA = dict(PACKAGE_DATA, **TEST_PACKAGE_DATA)
- PACKAGES = setuptools.find_packages(PYTHON_STEM)
-else:
- PACKAGES = setuptools.find_packages(
- PYTHON_STEM, exclude=['tests', 'tests.*'])
+PACKAGES = setuptools.find_packages(PYTHON_STEM)
 
 setuptools.setup(
  name='grpcio',
@@ -253,8 +215,4 @@ def cython_extensions(module_names, extra_sources, include_dirs,
  install_requires=INSTALL_REQUIRES,
  setup_requires=SETUP_REQUIRES,
  cmdclass=COMMAND_CLASS,
- tests_require=TESTS_REQUIRE,
- test_suite=TEST_SUITE,
- test_loader=TEST_LOADER,
- test_runner=TEST_RUNNER,
 )

src/python/grpcio/commands.py

@@ -134,75 +134,6 @@ def run(self):
  sphinx.main(['', os.path.join('doc', 'src'), os.path.join('doc', 'build')])
 
 
-class BuildProtoModules(setuptools.Command):
- """Command to generate project *_pb2.py modules from proto files."""
-
- description = 'build protobuf modules'
- user_options = [
- ('include=', None, 'path patterns to include in protobuf generation'),
- ('exclude=', None, 'path patterns to exclude from protobuf generation')
- ]
-
- def initialize_options(self):
- self.exclude = None
- self.include = r'.*\.proto$'
- self.protoc_command = None
- self.grpc_python_plugin_command = None
-
- def finalize_options(self):
- self.protoc_command = distutils.spawn.find_executable('protoc')
- self.grpc_python_plugin_command = distutils.spawn.find_executable(
- 'grpc_python_plugin')
-
- def run(self):
- if not self.protoc_command:
- raise CommandError('could not find protoc')
- if not self.grpc_python_plugin_command:
- raise CommandError('could not find grpc_python_plugin '
- '(protoc plugin for GRPC Python)')
-
- if not os.path.exists(PROTO_GEN_STEM):
- os.makedirs(PROTO_GEN_STEM)
-
- include_regex = re.compile(self.include)
- exclude_regex = re.compile(self.exclude) if self.exclude else None
- paths = []
- for walk_root, directories, filenames in os.walk(PROTO_STEM):
- for filename in filenames:
- path = os.path.join(walk_root, filename)
- if include_regex.match(path) and not (
- exclude_regex and exclude_regex.match(path)):
- paths.append(path)
-
- # TODO(kpayson): It would be nice to do this in a batch command,
- # but we currently have name conflicts in src/proto
- for path in paths:
- command = [
- self.protoc_command,
- '--plugin=protoc-gen-python-grpc={}'.format(
- self.grpc_python_plugin_command),
- '-I {}'.format(GRPC_STEM),
- '-I .',
- '-I {}/third_party/protobuf/src'.format(GRPC_STEM),
- '--python_out={}'.format(PROTO_GEN_STEM),
- '--python-grpc_out={}'.format(PROTO_GEN_STEM),
- ] + [path]
- try:
- subprocess.check_output(' '.join(command), cwd=PYTHON_STEM, shell=True,
- stderr=subprocess.STDOUT)
- except subprocess.CalledProcessError as e:
- sys.stderr.write(
- 'warning: Command:\n{}\nMessage:\n{}\nOutput:\n{}'.format(
- command, str(e), e.output))
-
- # Generated proto directories dont include __init__.py, but
- # these are needed for python package resolution
- for walk_root, _, _ in os.walk(PROTO_GEN_STEM):
- if walk_root != PROTO_GEN_STEM:
- path = os.path.join(walk_root, '__init__.py')
- open(path, 'a').close()
-
-
 class BuildProjectMetadata(setuptools.Command):
  """Command to generate project metadata in a module."""
 
@@ -225,10 +156,6 @@ class BuildPy(build_py.build_py):
  """Custom project build command."""
 
  def run(self):
- try:
- self.run_command('build_proto_modules')
- except CommandError as error:
- sys.stderr.write('warning: %s\n' % error.message)
  self.run_command('build_project_metadata')
  build_py.build_py.run(self)
 
@@ -281,76 +208,3 @@ def run(self):
  self.distribution.fetch_build_eggs(self.distribution.install_requires)
  if self.test and self.distribution.tests_require:
  self.distribution.fetch_build_eggs(self.distribution.tests_require)
-
-
-class TestLite(setuptools.Command):
- """Command to run tests without fetching or building anything."""
-
- description = 'run tests without fetching or building anything.'
- user_options = []
-
- def initialize_options(self):
- pass
-
- def finalize_options(self):
- # distutils requires this override.
- pass
-
- def run(self):
- self._add_eggs_to_path()
-
- import tests
- loader = tests.Loader()
- loader.loadTestsFromNames(['tests'])
- runner = tests.Runner()
- result = runner.run(loader.suite)
- if not result.wasSuccessful():
- sys.exit('Test failure')
-
- def _add_eggs_to_path(self):
- """Fetch install and test requirements"""
- self.distribution.fetch_build_eggs(self.distribution.install_requires)
- self.distribution.fetch_build_eggs(self.distribution.tests_require)
-
-
-class RunInterop(test.test):
-
- description = 'run interop test client/server'
- user_options = [
- ('args=', 'a', 'pass-thru arguments for the client/server'),
- ('client', 'c', 'flag indicating to run the client'),
- ('server', 's', 'flag indicating to run the server')
- ]
-
- def initialize_options(self):
- self.args = ''
- self.client = False
- self.server = False
-
- def finalize_options(self):
- if self.client and self.server:
- raise DistutilsOptionError('you may only specify one of client or server')
-
- def run(self):
- if self.distribution.install_requires:
- self.distribution.fetch_build_eggs(self.distribution.install_requires)
- if self.distribution.tests_require:
- self.distribution.fetch_build_eggs(self.distribution.tests_require)
- if self.client:
- self.run_client()
- elif self.server:
- self.run_server()
-
- def run_server(self):
- # We import here to ensure that our setuptools parent has had a chance to
- # edit the Python system path.
- from tests.interop import server
- sys.argv[1:] = self.args.split()
- server.serve()
-
- def run_client(self):
- # We import here to ensure that our setuptools parent has had a chance to
- # edit the Python system path.
- from tests.interop import client
- sys.argv[1:] = self.args.split()
- client.test_interoperability()

src/python/grpcio_health_checking/health_commands.py

@@ -39,51 +39,17 @@
 
 import setuptools
 from setuptools.command import build_py
-from setuptools.command import sdist
 
 ROOT_DIR = os.path.abspath(os.path.dirname(os.path.abspath(__file__)))
 HEALTH_PROTO = os.path.join(ROOT_DIR, '../../proto/grpc/health/v1/health.proto')
 
 
-class BuildProtoModules(setuptools.Command):
- """Command to generate project *_pb2.py modules from proto files."""
+class CopyProtoModules(setuptools.Command):
+ """Command to copy proto modules from grpc/src/proto."""
 
  description = ''
  user_options = []
 
- def initialize_options(self):
- pass
-
- def finalize_options(self):
- self.protoc_command = distutils.spawn.find_executable('protoc')
- self.grpc_python_plugin_command = distutils.spawn.find_executable(
- 'grpc_python_plugin')
-
- def run(self):
- paths = []
- root_directory = os.getcwd()
- for walk_root, directories, filenames in os.walk(root_directory):
- for filename in filenames:
- if filename.endswith('.proto'):
- paths.append(os.path.join(walk_root, filename))
- command = [
- self.protoc_command,
- '--plugin=protoc-gen-python-grpc={}'.format(
- self.grpc_python_plugin_command),
- '-I {}'.format(root_directory),
- '--python_out={}'.format(root_directory),
- '--python-grpc_out={}'.format(root_directory),
- ] + paths
- try:
- subprocess.check_output(' '.join(command), cwd=root_directory, shell=True,
- stderr=subprocess.STDOUT)
- except subprocess.CalledProcessError as e:
- raise Exception('{}\nOutput:\n{}'.format(e.message, e.output))
-
-
-class CopyProtoModules(setuptools.Command):
- """Command to copy proto modules from grpc/src/proto."""
-
  def initialize_options(self):
  pass
 
@@ -101,14 +67,5 @@ class BuildPy(build_py.build_py):
  """Custom project build command."""
 
  def run(self):
- self.run_command('copy_proto_modules')
  self.run_command('build_proto_modules')
  build_py.build_py.run(self)
-
-
-class SDist(sdist.sdist):
- """Custom project build command."""
-
- def run(self):
- self.run_command('copy_proto_modules')
- sdist.sdist.run(self)

src/python/grpcio_health_checking/setup.py

@@ -36,36 +36,44 @@
 from distutils import core as _core
 import setuptools
 
+import grpc.tools.command
+
 # Ensure we're in the proper directory whether or not we're being used by pip.
 os.chdir(os.path.dirname(os.path.abspath(__file__)))
 
 # Break import-style to ensure we can actually find our commands module.
 import health_commands
 
-_PACKAGES = (
+PACKAGES = (
  setuptools.find_packages('.')
 )
 
-_PACKAGE_DIRECTORIES = {
+PACKAGE_DIRECTORIES = {
  '': '.',
 }
 
-_INSTALL_REQUIRES = (
+SETUP_REQUIRES = (
+ 'grpcio-tools>=0.14.0',
+)
+
+INSTALL_REQUIRES = (
  'grpcio>=0.13.1',
 )
 
-_COMMAND_CLASS = {
- 'copy_proto_modules': health_commands.CopyProtoModules,
- 'build_proto_modules': health_commands.BuildProtoModules,
+COMMAND_CLASS = {
+ # Run preprocess from the repository *before* doing any packaging!
+ 'preprocess': health_commands.CopyProtoModules,
+
+ 'build_proto_modules': grpc.tools.command.BuildProtoModules,
  'build_py': health_commands.BuildPy,
- 'sdist': health_commands.SDist,
 }
 
 setuptools.setup(
- name='grpcio_health_checking',
- version='0.14.0b0',
- packages=list(_PACKAGES),
- package_dir=_PACKAGE_DIRECTORIES,
- install_requires=_INSTALL_REQUIRES,
- cmdclass=_COMMAND_CLASS
+ name='grpcio-health-checking',
+ version='0.14.0',
+ packages=list(PACKAGES),
+ package_dir=PACKAGE_DIRECTORIES,
+ install_requires=INSTALL_REQUIRES,
+ setup_requires=SETUP_REQUIRES,
+ cmdclass=COMMAND_CLASS
 )

src/python/grpcio_tests/.gitignore

@@ -0,0 +1,4 @@
+proto/
+src/
+*_pb2.py
+*.egg-info/