Fix beam_PerformanceTests_Python (apache#6142)

* [BEAM-3230] Fix beam_PerformanceTests_Python

.test-infra/jenkins/CommonJobProperties.groovy

@@ -315,31 +315,41 @@ class CommonJobProperties {
  // Adds the standard performance test job steps.
  static def buildPerformanceTest(def context, def argMap) {
  def pkbArgs = genPerformanceArgs(argMap)
+
+ // Absolute path of project root and virtualenv path of Beam and Perfkit.
+ def beam_root = makePathAbsolute(checkoutDir)
+ def perfkit_root = makePathAbsolute("PerfKitBenchmarker")
+ def beam_env = makePathAbsolute("env/.beam_env")
+ def perfkit_env = makePathAbsolute("env/.perfkit_env")
+
  context.steps {
  // Clean up environment.
- shell('rm -rf PerfKitBenchmarker')
- shell('rm -rf .beam_env')
- shell('rm -rf .perfkit_env')
+ shell("rm -rf ${perfkit_root}")
+ shell("rm -rf ${beam_env}")
+ shell("rm -rf ${perfkit_env}")
 
  // create new VirtualEnv, inherit already existing packages
- shell('virtualenv .beam_env --system-site-packages')
- shell('virtualenv .perfkit_env --system-site-packages')
+ shell("virtualenv ${beam_env} --system-site-packages")
+ shell("virtualenv ${perfkit_env} --system-site-packages")
 
  // update setuptools and pip
- shell('.beam_env/bin/pip install --upgrade setuptools pip')
- shell('.perfkit_env/bin/pip install --upgrade setuptools pip')
+ shell("${beam_env}/bin/pip install --upgrade setuptools pip grpcio-tools==1.3.5")
+ shell("${perfkit_env}/bin/pip install --upgrade setuptools pip")
 
  // Clone appropriate perfkit branch
- shell('git clone https://github.com/GoogleCloudPlatform/PerfKitBenchmarker.git')
+ shell("git clone https://github.com/GoogleCloudPlatform/PerfKitBenchmarker.git ${perfkit_root}")
 
  // Install job requirements for Python SDK.
- shell('.beam_env/bin/pip install -e ' + CommonJobProperties.checkoutDir + '/sdks/python/[gcp,test]')
+ shell("${beam_env}/bin/pip install -e ${beam_root}/sdks/python/[gcp,test]")
+
+ // Build PythonSDK tar ball.
+ shell("(cd ${beam_root}/sdks/python && ${beam_env}/bin/python setup.py sdist --dist-dir=target)")
 
  // Install Perfkit benchmark requirements.
- shell('.perfkit_env/bin/pip install -r PerfKitBenchmarker/requirements.txt')
+ shell("${perfkit_env}/bin/pip install -r ${perfkit_root}/requirements.txt")
 
  // Launch performance test.
- shell(".perfkit_env/bin/python PerfKitBenchmarker/pkb.py $pkbArgs")
+ shell("${perfkit_env}/bin/python ${perfkit_root}/pkb.py ${pkbArgs}")
  }
  }
 

.test-infra/jenkins/job_PerformanceTests_Python.groovy

@@ -47,13 +47,13 @@ job('beam_PerformanceTests_Python'){
  def pipelineArgsJoined = pipelineArgList.join(',')
 
  def argMap = [
- beam_sdk : 'python',
- benchmarks: 'beam_integration_benchmark',
- beam_it_args: pipelineArgsJoined
+ beam_sdk : 'python',
+ benchmarks : 'beam_integration_benchmark',
+ bigquery_table : 'beam_performance.wordcount_py_pkb_results',
+ beam_it_class : 'apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_it',
+ beam_prebuilt : 'true', // skip beam prebuild
+ beam_it_args : pipelineArgsJoined,
  ]
 
  commonJobProperties.buildPerformanceTest(delegate, argMap)
-
- // [BEAM-3809] Python performance tests are failing.
- disabled()
 }

sdks/python/setup.py

@@ -38,7 +38,13 @@
 
 def get_version():
  global_names = {}
- exec(open(os.path.normpath('./apache_beam/version.py')).read(), global_names) # pylint: disable=exec-used
+ exec( # pylint: disable=exec-used
+ open(os.path.join(
+ os.path.dirname(os.path.abspath(__file__)),
+ 'apache_beam/version.py')
+ ).read(),
+ global_names
+ )
  return global_names['__version__']