Seperate Build, Test, and Deploy Stages with parallel

Makefile

@@ -454,6 +454,10 @@ else
  CFLAGS += -DMXNET_USE_LIBJPEG_TURBO=0
 endif
 
+ifeq ($(CI), 1)
+ MAVEN_ARGS := -B
+endif
+
 # For quick compile test, used smaller subset
 ALLX_DEP= $(ALL_DEP)
 
@@ -618,27 +622,27 @@ scalatestcompile:
 
 scalapkg:
  (cd $(ROOTDIR)/scala-package && \
- mvn package -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \
+ mvn package $(MAVEN_ARGS) -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \
  -Dbuild.platform="$(SCALA_PKG_PROFILE)" \
  -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
  -Dcurrent_libdir="$(ROOTDIR)/lib" \
  -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")
 
 scalaunittest:
  (cd $(ROOTDIR)/scala-package && \
- mvn integration-test -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),unittest -Dcxx="$(CXX)" \
+ mvn integration-test $(MAVEN_ARGS) -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),unittest -Dcxx="$(CXX)" \
  -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
  -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a" $(SCALA_TEST_ARGS))
 
 scalaintegrationtest:
  (cd $(ROOTDIR)/scala-package && \
- mvn integration-test -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),integrationtest -Dcxx="$(CXX)" \
+ mvn integration-test $(MAVEN_ARGS) -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),integrationtest -Dcxx="$(CXX)" \
  -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
  -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a" $(SCALA_TEST_ARGS))
 
 scalainstall:
  (cd $(ROOTDIR)/scala-package && \
- mvn install -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -DskipTests=true -Dcxx="$(CXX)" \
+ mvn install $(MAVEN_ARGS) -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -DskipTests=true -Dcxx="$(CXX)" \
  -Dbuild.platform="$(SCALA_PKG_PROFILE)" \
  -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
  -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")
@@ -663,14 +667,14 @@ scalarelease-perform:
 
 scaladeploy:
  (cd $(ROOTDIR)/scala-package && \
- mvn deploy -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \
+ mvn deploy $(MAVEN_ARGS) -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \
  -Dbuild.platform="$(SCALA_PKG_PROFILE)" \
  -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
  -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")
 
 scaladeploylocal:
  (cd $(ROOTDIR)/scala-package && \
- mvn deploy -Papache-release,deployLocal,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \
+ mvn deploy $(MAVEN_ARGS) -Papache-release,deployLocal,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \
  -DaltDeploymentRepository=snapshot-repo::default::file:local-snapshot \
  -Dgpg.skip \
  -Dbuild.platform="$(SCALA_PKG_PROFILE)" \

ci/Jenkinsfile_utils.groovy

@@ -64,7 +64,7 @@ def init_git_win() {
 
 // pack libraries for later use
 def pack_lib(name, libs, include_gcov_data = false) {
- sh """
+ sh returnStatus: true, script: """
 set +e
 echo "Packing ${libs} into ${name}"
 echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
@@ -83,7 +83,7 @@ return 0
 def unpack_and_init(name, libs, include_gcov_data = false) {
  init_git()
  unstash name
- sh """
+ sh returnStatus: true, script: """
 set +e
 echo "Unpacked ${libs} from ${name}"
 echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
@@ -147,8 +147,9 @@ def collect_test_results_windows(original_file_name, new_file_name) {
 }
 
 
-def docker_run(platform, function_name, use_nvidia, shared_mem = '500m') {
- def command = "ci/build.py --docker-registry ${env.DOCKER_CACHE_REGISTRY} %USE_NVIDIA% --platform %PLATFORM% --docker-build-retries 3 --shm-size %SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%"
+def docker_run(platform, function_name, use_nvidia, shared_mem = '500m', env_vars = "") {
+ def command = "ci/build.py %ENV_VARS% --docker-registry ${env.DOCKER_CACHE_REGISTRY} %USE_NVIDIA% --platform %PLATFORM% --docker-build-retries 3 --shm-size %SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%"
+ command = command.replaceAll('%ENV_VARS%', env_vars.length() > 0 ? "-e ${env_vars}" : '')
  command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' : '')
  command = command.replaceAll('%PLATFORM%', platform)
  command = command.replaceAll('%FUNCTION_NAME%', function_name)

ci/build.py

@@ -215,20 +215,21 @@ def container_run(platform: str,
  local_ccache_dir: str,
  command: List[str],
  cleanup: Cleanup,
+ environment: Dict[str, str],
  dry_run: bool = False) -> int:
  """Run command in a container"""
  container_wait_s = 600
  #
  # Environment setup
  #
- environment = {
+ environment.update({
  'CCACHE_MAXSIZE': '500G',
  'CCACHE_TEMPDIR': '/tmp/ccache', # temp dir should be local and not shared
  'CCACHE_DIR': '/work/ccache', # this path is inside the container as /work/ccache is
  # mounted
  'CCACHE_LOGFILE': '/tmp/ccache.log', # a container-scoped log, useful for ccache
  # verification.
- }
+ })
  # These variables are passed to the container to the process tree killer can find runaway
  # process inside the container
  # https://wiki.jenkins.io/display/JENKINS/ProcessTreeKiller
@@ -446,6 +447,10 @@ def main() -> int:
  parser.add_argument("--no-cache", action="store_true",
  help="passes --no-cache to docker build")
 
+ parser.add_argument("-e", "--environment", nargs="*", default=[],
+ help="Environment variables for the docker container. "
+ "Specify with a list containing either names or name=value")
+
  parser.add_argument("command",
  help="command to run in the container",
  nargs='*', action='append', type=str)
@@ -474,6 +479,9 @@ def signal_handler(signum, _):
  signal.signal(signal.SIGTERM, signal_handler)
  signal.signal(signal.SIGINT, signal_handler)
 
+ environment = dict([(e.split('=')[:2] if '=' in e else (e, os.environ[e]))
+ for e in args.environment])
+
  if args.list:
  print(list_platforms())
  elif args.platform:
@@ -493,21 +501,21 @@ def signal_handler(signum, _):
  ret = container_run(
  platform=platform, nvidia_runtime=args.nvidiadocker,
  shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry,
- local_ccache_dir=args.ccache_dir, cleanup=cleanup)
+ local_ccache_dir=args.ccache_dir, cleanup=cleanup, environment=environment)
  elif args.print_docker_run:
  command = []
  ret = container_run(
  platform=platform, nvidia_runtime=args.nvidiadocker,
  shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry,
- local_ccache_dir=args.ccache_dir, dry_run=True, cleanup=cleanup)
+ local_ccache_dir=args.ccache_dir, dry_run=True, cleanup=cleanup, environment=environment)
  else:
  # With no commands, execute a build function for the target platform
  command = ["/work/mxnet/ci/docker/runtime_functions.sh", "build_{}".format(platform)]
  logging.info("No command specified, trying default build: %s", ' '.join(command))
  ret = container_run(
  platform=platform, nvidia_runtime=args.nvidiadocker,
  shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry,
- local_ccache_dir=args.ccache_dir, cleanup=cleanup)
+ local_ccache_dir=args.ccache_dir, cleanup=cleanup, environment=environment)
 
  if ret != 0:
  logging.critical("Execution of %s failed with status: %d", command, ret)
@@ -535,7 +543,7 @@ def signal_handler(signum, _):
  container_run(
  platform=platform, nvidia_runtime=args.nvidiadocker,
  shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry,
- local_ccache_dir=args.ccache_dir, cleanup=cleanup)
+ local_ccache_dir=args.ccache_dir, cleanup=cleanup, environment=environment)
  shutil.move(buildir(), plat_buildir)
  logging.info("Built files left in: %s", plat_buildir)
 

ci/docker/install/ubuntu_scala.sh

@@ -33,4 +33,7 @@ apt-get install -y openjdk-8-jre
 apt-get update || true
 apt-get install -y \
  maven \
+ gnupg \
+ gnupg2 \
+ gnupg-agent \
  scala

ci/docker/runtime_functions.sh

@@ -36,6 +36,11 @@ clean_repo() {
  git submodule update --init --recursive
 }
 
+scala_prepare() {
+ # Clean up maven logs
+ export MAVEN_OPTS="-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
+}
+
 build_ccache_wrappers() {
  set -ex
 
@@ -835,21 +840,24 @@ unittest_ubuntu_python3_quantization_gpu() {
 
 unittest_ubuntu_cpu_scala() {
  set -ex
- make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
- make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
+ scala_prepare
+ make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1
+ make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1
 }
 
 unittest_centos7_cpu_scala() {
  set -ex
  cd /work/mxnet
+ scala_prepare
  make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
  make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
 }
 
 unittest_ubuntu_cpu_clojure() {
  set -ex
- make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
- make scalainstall USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
+ scala_prepare
+ make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1
+ make scalainstall USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1
  ./contrib/clojure-package/ci-test.sh
 }
 
@@ -993,8 +1001,9 @@ integrationtest_ubuntu_cpu_dist_kvstore() {
 
 integrationtest_ubuntu_gpu_scala() {
  set -ex
- make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 USE_DIST_KVSTORE=1 SCALA_ON_GPU=1 ENABLE_TESTCOVERAGE=1
- make scalaintegrationtest USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_TEST_ON_GPU=1 USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
+ scala_prepare
+ make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 USE_DIST_KVSTORE=1 SCALA_ON_GPU=1 ENABLE_TESTCOVERAGE=1 CI=1
+ make scalaintegrationtest USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_TEST_ON_GPU=1 USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1
 }
 
 integrationtest_ubuntu_gpu_dist_kvstore() {
@@ -1246,14 +1255,30 @@ deploy_jl_docs() {
  # ...
 }
 
-deploy_nightly_maven() {
+publish_scala_build() {
  set -ex
  pushd .
- cd /work/mxnet
+ scala_prepare
  ./scala-package/dev/build.sh
  popd
 }
 
+publish_scala_test() {
+ set -ex
+ pushd .
+ scala_prepare
+ ./scala-package/dev/test.sh
+ popd
+}
+
+publish_scala_deploy() {
+ set -ex
+ pushd .
+ scala_prepare
+ ./scala-package/dev/deploy.sh
+ popd
+}
+
 # broken_link_checker
 
 broken_link_checker() {

ci/docker_cache.py

@@ -211,8 +211,7 @@ def _get_dockerhub_credentials(): # pragma: no cover
  logging.exception("The request was invalid due to:")
  elif client_error.response['Error']['Code'] == 'InvalidParameterException':
  logging.exception("The request had invalid params:")
- else:
- raise
+ raise
  else:
  secret = get_secret_value_response['SecretString']
  secret_dict = json.loads(secret)

ci/publish/Jenkinsfile

@@ -20,58 +20,86 @@
 // Jenkins pipeline
 // See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
 
+//mxnet libraries
+mx_scala_pub = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, config.mk, scala-package/pom.xml, scala-package/**/pom.xml, scala-package/*/target/test-classes/**, scala-package/local-snapshot/**'
+
 // timeout in minutes
 max_time = 120
 
-node('restricted-mxnetlinux-cpu') {
+node('restricted-utility') {
  // Loading the utilities requires a node context unfortunately
  checkout scm
  utils = load('ci/Jenkinsfile_utils.groovy')
 }
-utils.assign_node_labels(linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu')
+utils.assign_node_labels(utility: 'restricted-utility', linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu')
 
-utils.main_wrapper(
-core_logic: {
- stage('Deploy Nightly Maven') {
- environment {
- MVN_DEPLOY_USER = credentials('mvn-deploy-user')
- MVN_DEPLOY_PASSWORD = credentials('mvn-deploy-password')
- MVN_DEPLOY_PASSPHRASE = credentials('mvn-deploy-passphrase')
- MVN_DEPLOY_MASTERPASS = credentials('mvn-deploy-masterpass')
- MVN_DEPLOY_GPG_KEY = credentials('mvn-deploy-gpg-key')
- }
- parallel 'linux-cpu': {
- node(NODE_LINUX_CPU) {
- ws('workspace/linux-cpu') {
- environment {
- MVN_DEPLOY_OS_TYPE = linux-x86_64-cpu
- }
- utils.init_git()
- timeout(time: max_time, unit: 'MINUTES') {
- sh "ci/build.py -p ubuntu_cpu --docker-registry ${env.DOCKER_CACHE_REGISTRY} --docker-build-retries 3 /work/runtime_functions.sh deploy_nightly_maven"
- }
- }
- }
- },
- 'linux-gpu': {
- node(NODE_LINUX_CPU) {
- ws('workspace/linux-gpu') {
- environment {
- MVN_DEPLOY_OS_TYPE = linux-x86_64-gpu
- }
- utils.init_git()
- timeout(time: max_time, unit: 'MINUTES') {
- sh "ci/build.py -p ubuntu_cpu --docker-registry ${env.DOCKER_CACHE_REGISTRY} --docker-build-retries 3 /work/runtime_functions.sh deploy_nightly_maven"
- }
+// CPU and GPU. OSX nodes are not currently supported by Jenkins
+def nodeMap = ['cpu': NODE_LINUX_CPU, 'gpu': NODE_LINUX_GPU]
+def scalaOSMap = ['cpu': 'linux-x86_64-cpu', 'gpu': 'linux-x86_64-gpu']
+
+def wrapStep(nodeToRun, workspaceName, step) {
+ return {
+ node(nodeToRun) {
+ ws("workspace/${workspaceName}") {
+ timeout(time: max_time, unit: 'MINUTES') {
+ step()
  }
  }
  }
  }
 }
+
+def toBuild = [:]
+def labels = ['cpu'] // , 'gpu']
+for (x in labels) {
+ def label = x // Required due to language
+ toBuild["Scala Build ${label}"] = wrapStep(nodeMap[label], "build-scala-${label}") {
+ env.MAVEN_PUBLISH_OS_TYPE = scalaOSMap[label]
+ utils.init_git()
+ utils.docker_run("ubuntu_${label}", 'publish_scala_build', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE')
+ utils.pack_lib("scala_${label}", mx_scala_pub, false)
+ }
+}
+
+def toTest = [:]
+def systems = ['ubuntu'] // , 'centos7']
+for (x in labels) {
+ def label = x // Required due to language
+ for (y in systems) {
+ def system = y // Required due to language
+ toTest["Scala Test ${system} ${label}"] = wrapStep(nodeMap[label], "test-scala-${system}-${label}") {
+ utils.unpack_and_init("scala_${label}", mx_scala_pub, false)
+ utils.docker_run("${system}_${label}", 'publish_scala_test', label == 'gpu')
+ }
+ }
+}
+
+def toDeploy = [:]
+for (x in labels) {
+ def label = x // Required due to language
+ toDeploy["Scala Deploy ${label}"] = wrapStep(nodeMap[label], "deploy-scala-${label}") {
+ env.MAVEN_PUBLISH_OS_TYPE = scalaOSMap[label]
+ utils.unpack_and_init("scala_${label}", mx_scala_pub, false)
+ utils.docker_run("ubuntu_${label}", 'publish_scala_deploy', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE MAVEN_PUBLISH_SECRET_ENDPOINT_URL MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS MAVEN_PUBLISH_SECRET_NAME_GPG DOCKERHUB_SECRET_ENDPOINT_REGION')
+ }
+}
+
+utils.main_wrapper(
+core_logic: {
+ stage('Build Packages') {
+ parallel toBuild
+ }
+ stage('Test Packages') {
+ parallel toTest
+ }
+ stage('Deploy Packages') {
+ parallel toDeploy
+ }
+}
 ,
 failure_handler: {
  if (currentBuild.result == "FAILURE") {
- emailext body: 'Generating the nightly maven has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY MAVEN FAILED] Build ${BUILD_NUMBER}', to: '${EMAIL}'
+ // emailext body: 'Generating the nightly maven has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY MAVEN FAILED] Build ${BUILD_NUMBER}', to: '${EMAIL}'
  }
 }
 )