.travis.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http:https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

sudo: required
dist: xenial

services:
  - docker

cache:
  directories:
    - $HOME/.ccache
    - $HOME/.m2  # Maven

env:
  - CCACHE_COMPRESS: 1
  - CCACHE_COMPRESSLEVEL: 5

git:
  # ARROW-3083: We use a higher clone depth to also have the latest tag in the cloned history.
  depth: 250

before_install:
  # Common pre-install steps for all builds
  - eval "${MATRIX_EVAL}"
  - ulimit -c unlimited -S
  - |
      if [ $TRAVIS_OS_NAME == "linux" ]; then
        sudo bash -c "echo -e 'Acquire::Retries 10; Acquire::http::Timeout \"20\";' > /etc/apt/apt.conf.d/99-travis-retry"
        sudo apt-get update -qq
      fi
  - eval `python $TRAVIS_BUILD_DIR/ci/detect-changes.py`
  - if [ $TRAVIS_OS_NAME == "linux" ]; then ccache -s; fi

matrix:
  fast_finish: true
  include:
  - name: "Lint, Release tests"
    os: linux
    language: python
    python: "3.6"
    before_script:
    # Always run RAT checks, in case another build in matrix breaks RAT
    - $TRAVIS_BUILD_DIR/ci/travis_release_audit.sh
    - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
    script:
    - $TRAVIS_BUILD_DIR/ci/travis_lint.sh
    - $TRAVIS_BUILD_DIR/ci/travis_release_test.sh
  - name: "C++ unit tests w/ clang 7.0, system packages"
    os: linux
    env:
    - ARROW_TRAVIS_USE_SYSTEM=1
    - ARROW_TRAVIS_PLASMA=1
    - ARROW_TRAVIS_ORC=1
    - ARROW_TRAVIS_PARQUET=1
    - ARROW_TRAVIS_GANDIVA=1
    - ARROW_TRAVIS_VERBOSE=1
    - ARROW_TRAVIS_USE_SYSTEM_JAVA=1
    - ARROW_BUILD_WARNING_LEVEL=CHECKIN
    - ARROW_USE_ASAN=1
    - ARROW_USE_UBSAN=1
    - CC="clang-7"
    - CXX="clang++-7"
    before_script:
    - if [ $ARROW_CI_CPP_AFFECTED != "1" ]; then exit; fi
    - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
    - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
    # If either C++ or Python changed, we must install the C++ libraries
    - git submodule update --init
    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
    script:
    - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1
  # Separating Valgrind and C++ coverage makes individual jobs shorter
  - name: "C++ unit tests w/ conda-forge toolchain, coverage"
    compiler: gcc
    language: cpp
    os: linux
    jdk: openjdk8
    env:
    - ARROW_BUILD_WARNING_LEVEL=CHECKIN
    - ARROW_TRAVIS_COVERAGE=1
    - ARROW_TRAVIS_FLIGHT=1
    - ARROW_TRAVIS_GANDIVA=1
    - ARROW_TRAVIS_GANDIVA_JAVA=1
    - ARROW_TRAVIS_ORC=1
    - ARROW_TRAVIS_PARQUET=1
    - ARROW_TRAVIS_PLASMA=1
    - ARROW_TRAVIS_USE_SYSTEM_JAVA=1
    - ARROW_TRAVIS_USE_TOOLCHAIN=1
    # TODO: This fails in re2 code
    # - ARROW_TRAVIS_VALGRIND=1
    - ARROW_TRAVIS_VERBOSE=1
    before_script:
    - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi
    - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
    - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
    # If either C++ or Python changed, we must install the C++ libraries
    - git submodule update --init
    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
    script:
    - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1
    - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh || travis_terminate 1
    - $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh || travis_terminate 1
  - name: "Python 3.6 unit tests w/ Valgrind, conda-forge toolchain, coverage"
    compiler: gcc
    language: cpp
    os: linux
    jdk: openjdk8
    env:
    # Valgrind is needed for the Plasma store tests
    - ARROW_BUILD_WARNING_LEVEL=CHECKIN
    - ARROW_TRAVIS_COVERAGE=1
    - ARROW_TRAVIS_FLIGHT=1
    - ARROW_TRAVIS_OPTIONAL_INSTALL=1
    - ARROW_TRAVIS_PYTHON_DOCS=1
    - ARROW_TRAVIS_PYTHON_JVM=1
    - ARROW_TRAVIS_USE_SYSTEM_JAVA=1
    - ARROW_TRAVIS_USE_TOOLCHAIN=1
    - ARROW_TRAVIS_VALGRIND=1
    # TODO(wesm): Run the benchmarks outside of Travis
    # - ARROW_TRAVIS_PYTHON_BENCHMARKS=1
    before_script:
    - if [ $ARROW_CI_PYTHON_AFFECTED != "1" ] && [ $ARROW_CI_DOCS_AFFECTED != "1" ]; then exit; fi
    - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
    - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
    - $TRAVIS_BUILD_DIR/ci/travis_install_toolchain.sh
    script:
    - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh || travis_terminate 1
    - export ARROW_TRAVIS_PYTHON_GANDIVA=1
    # Only run Plasma tests with valgrind in one of the Python builds because
    # they are slow
    - export PLASMA_VALGRIND=1
    - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6 || travis_terminate 1
    - $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh
  - name: "[OS X] C++ w/ XCode 9.3"
    compiler: clang
    osx_image: xcode9.3
    os: osx
    cache:
    addons:
    env:
    - ARROW_TRAVIS_USE_TOOLCHAIN=1
    - ARROW_TRAVIS_PLASMA=1
    - ARROW_TRAVIS_FLIGHT=1
    - ARROW_TRAVIS_ORC=1
    - ARROW_TRAVIS_PARQUET=1
    # TODO(ARROW-4763): llvm and llvmdev packages are in conflict:
    # https://github.com/conda-forge/llvmdev-feedstock/issues/60
    # - ARROW_TRAVIS_GANDIVA=1
    # - ARROW_TRAVIS_GANDIVA_JAVA=1
    - ARROW_TRAVIS_OPTIONAL_INSTALL=1
    - ARROW_TRAVIS_VERBOSE=1
    - ARROW_BUILD_WARNING_LEVEL=CHECKIN
    before_script:
    - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi
    # If either C++ or Python changed, we must install the C++ libraries
    - git submodule update --init
    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
    script:
    - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1
    # Disabled because of ARROW-4763
    # - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh
  - name: "[OS X] Python w/ XCode 9.3"
    compiler: clang
    osx_image: xcode9.3
    os: osx
    cache:
    addons:
    env:
    - ARROW_TRAVIS_PLASMA=1
    - ARROW_TRAVIS_USE_TOOLCHAIN=1
    - ARROW_BUILD_WARNING_LEVEL=CHECKIN
    - ARROW_TRAVIS_OPTIONAL_INSTALL=1
    - ARROW_TRAVIS_VERBOSE=1
    - MACOSX_DEPLOYMENT_TARGET="10.9"
    before_script:
    script:
    - if [ $ARROW_CI_PYTHON_AFFECTED != "1" ]; then exit; fi
    - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6
  - name: "[manylinux1] Python"
    language: cpp
    env:
    - PYTHON_VERSIONS="3.6,16 3.7,16"
    before_script:
    - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then docker-compose pull python-manylinux1; fi
    script:
    - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_manylinux.sh; fi
  - name: "Java w/ OpenJDK 8"
    language: java
    os: linux
    jdk: openjdk8
    before_script:
    - if [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi
    - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
    script:
    - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
    - $TRAVIS_BUILD_DIR/ci/travis_script_javadoc.sh
  - name: "Java w/ OpenJDK 9"
    language: java
    os: linux
    jdk: openjdk9
    before_script:
    - if [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi
    script:
    - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
  - name: "Java w/ OpenJDK 11"
    language: java
    os: linux
    jdk: openjdk11
    before_script:
    - if [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi
    script:
    - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
  - name: "Integration w/ OpenJDK 8, conda-forge toolchain"
    language: java
    os: linux
    env: ARROW_TEST_GROUP=integration
    jdk: openjdk8
    env:
    - ARROW_TRAVIS_USE_TOOLCHAIN=1
    - ARROW_TRAVIS_FLIGHT=1
    - ARROW_TRAVIS_INTEGRATION=1
    # TODO(ARROW-4764): Using system java and plasma built with conda doesn't work
    #- ARROW_TRAVIS_PLASMA=1
    #- ARROW_TRAVIS_PLASMA_JAVA_CLIENT=1
    before_script:
    - if [ $ARROW_CI_INTEGRATION_AFFECTED != "1" ]; then exit; fi
    - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
    - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
    # C++: Only build integration targets
    - export ARROW_CPP_BUILD_TARGETS=arrow-integration
    - export ARROW_TRAVIS_OPTIONAL_INSTALL=1
    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
    # Javascript
    - nvm install 11.12
    - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh
    script:
    - $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh
    # - $TRAVIS_BUILD_DIR/ci/travis_script_plasma_java_client.sh
  - name: "NodeJS"
    language: node_js
    os: linux
    node_js:
    - '11.12'
    before_script:
    - if [ $ARROW_CI_JS_AFFECTED != "1" ]; then exit; fi
    - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
    - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh
    script:
    - $TRAVIS_BUILD_DIR/ci/travis_script_js.sh
  - name: "C++ & GLib & Ruby w/ gcc 5.4"
    compiler: gcc
    os: linux
    env:
    - ARROW_TRAVIS_ORC=1
    - ARROW_TRAVIS_GANDIVA=1
    - ARROW_TRAVIS_PARQUET=1
    - ARROW_TRAVIS_PLASMA=1
    before_script:
    - if [ $ARROW_CI_RUBY_AFFECTED != "1" ]; then exit; fi
    - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
    - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library
    - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
    - $TRAVIS_BUILD_DIR/ci/travis_before_script_ruby.sh
    script:
    - if [ $ARROW_CI_C_GLIB_AFFECTED = "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh; fi
    - $TRAVIS_BUILD_DIR/ci/travis_script_ruby.sh
  - name: "[OS X] C++ & GLib & Ruby w/ XCode 10.2 & Homebrew"
    compiler: clang
    osx_image: xcode10.2
    os: osx
    env:
    - ARROW_TRAVIS_FLIGHT=1
    - ARROW_TRAVIS_GANDIVA=1
    - ARROW_TRAVIS_ORC=1
    - ARROW_TRAVIS_PARQUET=1
    - ARROW_TRAVIS_PLASMA=1
    - ARROW_TRAVIS_PYTHON=1
    - ARROW_TRAVIS_VERBOSE=1
    cache:
    addons:
    before_script:
    - if [ $ARROW_CI_RUBY_AFFECTED != "1" ]; then exit; fi
    - $TRAVIS_BUILD_DIR/ci/travis_install_osx.sh
    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library --homebrew
    - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
    - $TRAVIS_BUILD_DIR/ci/travis_before_script_ruby.sh
    script:
    - if [ $ARROW_CI_C_GLIB_AFFECTED = "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh; fi
    - $TRAVIS_BUILD_DIR/ci/travis_script_ruby.sh
  - name: Rust
    language: rust
    cache: cargo
    addons:
      apt:
        packages:
          - libcurl4-openssl-dev
          - libelf-dev
          - libdw-dev
          - binutils-dev
          - cmake
    before_script:
    - if [ $ARROW_CI_RUST_AFFECTED != "1" ]; then exit; fi
    - $TRAVIS_BUILD_DIR/ci/travis_install_cargo.sh
    script:
    - RUSTUP_TOOLCHAIN=nightly $TRAVIS_BUILD_DIR/ci/travis_script_rust.sh
    after_success:
    - pushd ${TRAVIS_BUILD_DIR}/rust
    # Run coverage for codecov.io
    - mkdir -p target/kcov
    - RUST_BACKTRACE=1 RUSTUP_TOOLCHAIN=nightly RUSTFLAGS="-D warnings" cargo coverage --verbose
    - bash <(curl -s https://codecov.io/bash) || echo "Codecov did not collect coverage reports"
  - name: Go
    language: go
    go_import_path: github.com/apache/arrow
    os: linux
    go:
    - 1.12.x
    before_script:
    - if [ $ARROW_CI_GO_AFFECTED != "1" ]; then exit; fi
    script:
    - $TRAVIS_BUILD_DIR/ci/travis_script_go.sh
    after_success:
    - pushd ${TRAVIS_BUILD_DIR}/go/arrow
    - bash <(curl -s https://codecov.io/bash) || echo "Codecov did not collect coverage reports"
  - name: R (with and without libarrow)
    language: r
    cache: packages
    dist: xenial
    env:
    - ARROW_TRAVIS_PARQUET=1
    - ARROW_TRAVIS_USE_SYSTEM=1
    before_install:
    # Have to copy-paste this here because of how R's build steps work
    - eval `python $TRAVIS_BUILD_DIR/ci/detect-changes.py`
    - if [ $ARROW_CI_R_AFFECTED != "1" ]; then exit; fi
    # First check that it builds without libarrow
    - pushd ${TRAVIS_BUILD_DIR}/r
    - R -e 'install.packages("remotes"); remotes::install_deps(dep = TRUE)'
    - R CMD build .
    - R CMD check arrow_*tar.gz
    - rm arrow_*tar.gz
    - popd
    # Now, proceed to install the c++ lib and the rest of the job
    - |
        if [ $TRAVIS_OS_NAME == "linux" ]; then
          sudo bash -c "echo -e 'Acquire::Retries 10; Acquire::http::Timeout \"20\";' > /etc/apt/apt.conf.d/99-travis-retry"
          sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
          sudo apt-get update -qq
        fi
    - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
    # On Xenial, installing clang-7 leads these to get removed,
    # but that breaks R, so let's reinstall them.
    - sudo apt-get install gfortran gfortran-5 libblas-dev libgfortran-5-dev libgfortran3 liblapack-dev liblapack3
    - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library
    - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$TRAVIS_BUILD_DIR/cpp-install/lib
    - export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$TRAVIS_BUILD_DIR/cpp-install/lib/pkgconfig
    - export CXX11FLAGS=-Wall
    # Add this env var so we can assert in the tests that the library is installed correctly
    - export TEST_R_WITH_ARROW=TRUE
    - pushd ${TRAVIS_BUILD_DIR}/r
    after_success:
    - Rscript ../ci/travis_upload_r_coverage.R

after_failure:
- |
    if [ "$TRAVIS_OS_NAME" = "osx" ]; then
      COREFILE=$(find /cores -maxdepth 1 -type f -name "core.*" | head -n 1)
      if [[ -f "$COREFILE" ]]; then
        lldb -c "$COREFILE" --batch --one-line "thread backtrace all -e true"
      fi
      ls -la ~/Library/Logs/DiagnosticReports/
      cat ~/Library/Logs/DiagnosticReports/*.crash
    else
      COREFILE=$(find . -maxdepth 2 -name "core*" | head -n 1)
      if [[ -f "$COREFILE" ]]; then
        gdb -c "$COREFILE" example -ex "thread apply all bt" -ex "set pagination 0" -batch
      fi
    fi

after_script:
  - if [ $TRAVIS_OS_NAME == "linux" ]; then ccache -s; fi

env:
  global:
  - secure: "GcrPtsKUCgNY7HKYjWlHQo8SiFrShDvdZSU8t1m1FJrE+UfK0Dgh9zXmAausM8GmhqSwkF0q4UbLQf2uCnSITWKeEPAL8Mo9eu4ib+ikJx/b3Sk81frgW5ADoHfW1Eyqd8xJNIMwMegJOtRLSDqiXh1CvMlKnY8PyTOGM2DgN9ona/v6p9OFH9Qs0JhBRVXAn0S4ztjumck8E56+01hqRfxbZ88pTfpKghBxYp9PJaMjtGdomjVWlqPaWaWJj+KptT8inV9NK+TVYKx0dXWD+S1Vgr1PytQnLdILOYV23gsOBYqn33ByF/yADl4m3hUjU/qeT0Fi7aWxmVpj+oTJISOSH5N8nIsuNH8mQk2ZzzXHfV7btFvP+cOPRczadoKkT6D6cHA8nQ7b0dphC6bl6SAeSfc/cbhRT+fYnIjg8jFXC8jlyWBr7LR6GXVpc0bND7i300ITo0FuRJhy2OxqPtGo3dKLE7eAcv78tuO0OYJ/kol1PEqFdFkbYbNVbg/cFpbGqiCXDsOtPDbAGBv69YnXdVowSxxs8cRGjSkDydv6ZSytb/Zd4lH/KAomcFNk8adx12O1Lk4sbmVav1cGig5P6OcQKS0jC5IiRb4THcQzVzAkXXbaafKm5sru/NoYxhzmkyhkOc11nTYHKVng+XKWzLCNn7pTTSLitp5+xa4="