ARROW-816: [C++] Travis CI script cleanup, add C++ toolchain env with…

… Flatbuffers, RapidJSON Author: Wes McKinney <[email protected]> Closes apache#537 from wesm/ARROW-816 and squashes the following commits: 16992b6 [Wes McKinney] Disable Travis CI cache on OS X. brew install ccache 4621d2d [Wes McKinney] Fix variable name dc86821 [Wes McKinney] Fixes for integration tests Travis script 5e2c226 [Wes McKinney] Change file mode ed4be57 [Wes McKinney] Travis CI script cleanup, add C++ toolchain env with flatbuffers, rapidjson
Xtuden-com · Apr 13, 2017 · 874666a · 874666a
1 parent 19da86a
commit 874666a
Show file tree

Hide file tree

Showing 7 changed files with 53 additions and 34 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -48,6 +48,7 @@ matrix:
  - compiler: clang
  osx_image: xcode6.4
  os: osx
+ cache:
  addons:
  before_script:
  - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh

diff --git a/ci/travis_before_script_c_glib.sh b/ci/travis_before_script_c_glib.sh
@@ -15,14 +15,14 @@
 
 set -ex
 
+source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
+
 if [ $TRAVIS_OS_NAME == "osx" ]; then
  brew install gtk-doc autoconf-archive gobject-introspection
 fi
 
 gem install gobject-introspection
 
-ARROW_C_GLIB_DIR=$TRAVIS_BUILD_DIR/c_glib
-
 pushd $ARROW_C_GLIB_DIR
 
 : ${ARROW_C_GLIB_INSTALL=$TRAVIS_BUILD_DIR/c-glib-install}

diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
@@ -15,19 +15,20 @@
 
 set -ex
 
-: ${CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build}
+source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
+source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
+
+# Set up C++ toolchain from conda-forge packages for faster builds
+conda create -y -q -p $CPP_TOOLCHAIN python=2.7 flatbuffers rapidjson
 
 if [ $TRAVIS_OS_NAME == "osx" ]; then
  brew update > /dev/null
  brew install jemalloc
+ brew install ccache
 fi
 
-mkdir $CPP_BUILD_DIR
-pushd $CPP_BUILD_DIR
-
-CPP_DIR=$TRAVIS_BUILD_DIR/cpp
-
-: ${ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install}
+mkdir $ARROW_CPP_BUILD_DIR
+pushd $ARROW_CPP_BUILD_DIR
 
 CMAKE_COMMON_FLAGS="\
 -DARROW_BUILD_BENCHMARKS=ON \
@@ -37,11 +38,11 @@ if [ $TRAVIS_OS_NAME == "linux" ]; then
  cmake -DARROW_TEST_MEMCHECK=on \
  $CMAKE_COMMON_FLAGS \
  -DARROW_CXXFLAGS="-Wconversion -Werror" \
- $CPP_DIR
+ $ARROW_CPP_DIR
 else
  cmake $CMAKE_COMMON_FLAGS \
  -DARROW_CXXFLAGS=-Werror \
- $CPP_DIR
+ $ARROW_CPP_DIR
 fi
 
 make -j4

diff --git a/ci/travis_env_common.sh b/ci/travis_env_common.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+
+export MINICONDA=$HOME/miniconda
+export PATH="$MINICONDA/bin:$PATH"
+export CONDA_PKGS_DIRS=$HOME/.conda_packages
+
+# C++ toolchain
+export CPP_TOOLCHAIN=$TRAVIS_BUILD_DIR/cpp-toolchain
+export FLATBUFFERS_HOME=$CPP_TOOLCHAIN
+export RAPIDJSON_HOME=$CPP_TOOLCHAIN
+
+export ARROW_CPP_DIR=$TRAVIS_BUILD_DIR/cpp
+export ARROW_PYTHON_DIR=$TRAVIS_BUILD_DIR/python
+export ARROW_C_GLIB_DIR=$TRAVIS_BUILD_DIR/c_glib
+export ARROW_JAVA_DIR=${TRAVIS_BUILD_DIR}/java
+export ARROW_INTEGRATION_DIR=$TRAVIS_BUILD_DIR/integration
+
+export ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install
+export ARROW_CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build
diff --git a/ci/travis_install_conda.sh b/ci/travis_install_conda.sh
@@ -22,8 +22,7 @@ fi
 
 wget -O miniconda.sh $MINICONDA_URL
 
-export MINICONDA=$HOME/miniconda
-export CONDA_PKGS_DIRS=$HOME/.conda_packages
+source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 mkdir -p $CONDA_PKGS_DIRS
 
 bash miniconda.sh -b -p $MINICONDA

diff --git a/ci/travis_script_integration.sh b/ci/travis_script_integration.sh
@@ -14,23 +14,16 @@
 
 set -e
 
-: ${CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build}
+source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 
-JAVA_DIR=${TRAVIS_BUILD_DIR}/java
-
-pushd $JAVA_DIR
+pushd $ARROW_JAVA_DIR
 
 mvn package
 
 popd
 
-pushd $TRAVIS_BUILD_DIR/integration
-
-export ARROW_CPP_EXE_PATH=$CPP_BUILD_DIR/debug
-
-source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
-export MINICONDA=$HOME/miniconda
-export PATH="$MINICONDA/bin:$PATH"
+pushd $ARROW_INTEGRATION_DIR
+export ARROW_CPP_EXE_PATH=$ARROW_CPP_BUILD_DIR/debug
 
 CONDA_ENV_NAME=arrow-integration-test
 conda create -y -q -n $CONDA_ENV_NAME python=3.5

diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
@@ -14,17 +14,11 @@
 
 set -e
 
-source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
-
-PYTHON_DIR=$TRAVIS_BUILD_DIR/python
-
-# Re-use conda installation from C++
-export MINICONDA=$HOME/miniconda
-export PATH="$MINICONDA/bin:$PATH"
+source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 
 export ARROW_HOME=$ARROW_CPP_INSTALL
 
-pushd $PYTHON_DIR
+pushd $ARROW_PYTHON_DIR
 export PARQUET_HOME=$TRAVIS_BUILD_DIR/parquet-env
 
 build_parquet_cpp() {
@@ -101,10 +95,10 @@ python_version_tests() {
  which python
 
  # faster builds, please
- conda install -y nomkl
+ conda install -y -q nomkl
 
  # Expensive dependencies install from Continuum package repo
- conda install -y pip numpy pandas cython
+ conda install -y -q pip numpy pandas cython
 
  # Build C++ libraries
  build_arrow_libraries arrow-build-$PYTHON_VERSION $ARROW_HOME