Skip to content

Commit

Permalink
ARROW-3150: [Python] Enable Flight in Python wheels for Linux and Win…
Browse files Browse the repository at this point in the history
…dows

Flight enabled in manylinux1 and Windows wheels.

Author: Antoine Pitrou <[email protected]>

Closes apache#4594 from pitrou/ARROW-3150-py-wheels-flight and squashes the following commits:

1fba5ea <Antoine Pitrou> ARROW-3150:  Enable Flight in Python wheels
  • Loading branch information
pitrou authored and wesm committed Jun 20, 2019
1 parent eb23ea9 commit eced2aa
Show file tree
Hide file tree
Showing 22 changed files with 328 additions and 177 deletions.
2 changes: 1 addition & 1 deletion ci/conda_env_cpp.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ flatbuffers
gflags
glog
gmock>=1.8.1
grpc-cpp
grpc-cpp>=1.21.4
gtest>=1.8.1
libprotobuf
lz4-c
Expand Down
2 changes: 1 addition & 1 deletion ci/travis_script_manylinux.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
# specific language governing permissions and limitations
# under the License.


set -ex

# Testing for https://issues.apache.org/jira/browse/ARROW-2657
Expand All @@ -37,6 +36,7 @@ import pyarrow.parquet
import pyarrow.plasma
if sys.version_info.major > 2:
import pyarrow.flight
import pyarrow.gandiva
EOF

Expand Down
21 changes: 0 additions & 21 deletions cpp/cmake_modules/FindArrow.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -70,15 +70,6 @@ find_library(ARROW_PYTHON_LIB_PATH
NO_DEFAULT_PATH)
get_filename_component(ARROW_PYTHON_LIBS ${ARROW_PYTHON_LIB_PATH} DIRECTORY)

if(PYARROW_BUILD_FLIGHT)
find_library(ARROW_FLIGHT_LIB_PATH
NAMES arrow_flight
PATHS ${ARROW_SEARCH_LIB_PATH}
PATH_SUFFIXES ${LIB_PATH_SUFFIXES}
NO_DEFAULT_PATH)
get_filename_component(ARROW_FLIGHT_LIBS ${ARROW_FLIGHT_LIB_PATH} DIRECTORY)
endif()

if(MSVC)
set(CMAKE_FIND_LIBRARY_SUFFIXES ".lib" ".dll")

Expand Down Expand Up @@ -108,7 +99,6 @@ if(ARROW_INCLUDE_DIR AND ARROW_LIBS)
set(ARROW_FOUND TRUE)
set(ARROW_LIB_NAME arrow)
set(ARROW_PYTHON_LIB_NAME arrow_python)
set(ARROW_FLIGHT_LIB_NAME arrow_flight)
if(MSVC)
set(
ARROW_STATIC_LIB
Expand All @@ -118,31 +108,20 @@ if(ARROW_INCLUDE_DIR AND ARROW_LIBS)
ARROW_PYTHON_STATIC_LIB
${ARROW_PYTHON_LIBS}/${ARROW_PYTHON_LIB_NAME}${ARROW_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}
)
set(
ARROW_FLIGHT_STATIC_LIB
${ARROW_FLIGHT_LIBS}/${ARROW_FLIGHT_LIB_NAME}${ARROW_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}
)
set(ARROW_SHARED_LIB
${ARROW_SHARED_LIBS}/${ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
set(
ARROW_PYTHON_SHARED_LIB
${ARROW_PYTHON_SHARED_LIBS}/${ARROW_PYTHON_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
set(
ARROW_FLIGHT_SHARED_LIB
${ARROW_FLIGHT_SHARED_LIBS}/${ARROW_FLIGHT_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
set(ARROW_SHARED_IMP_LIB ${ARROW_LIBS}/${ARROW_LIB_NAME}.lib)
set(ARROW_PYTHON_SHARED_IMP_LIB ${ARROW_PYTHON_LIBS}/${ARROW_PYTHON_LIB_NAME}.lib)
set(ARROW_FLIGHT_SHARED_IMP_LIB ${ARROW_FLIGHT_LIBS}/${ARROW_FLIGHT_LIB_NAME}.lib)
else()
set(ARROW_STATIC_LIB ${ARROW_LIBS}/lib${ARROW_LIB_NAME}.a)
set(ARROW_PYTHON_STATIC_LIB ${ARROW_LIBS}/lib${ARROW_PYTHON_LIB_NAME}.a)
set(ARROW_FLIGHT_STATIC_LIB ${ARROW_LIBS}/lib${ARROW_FLIGHT_LIB_NAME}.a)

set(ARROW_SHARED_LIB ${ARROW_LIBS}/lib${ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
set(ARROW_PYTHON_SHARED_LIB
${ARROW_LIBS}/lib${ARROW_PYTHON_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
set(ARROW_FLIGHT_SHARED_LIB
${ARROW_LIBS}/lib${ARROW_FLIGHT_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
endif()
endif()

Expand Down
5 changes: 4 additions & 1 deletion cpp/cmake_modules/FindArrowFlight.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -69,10 +69,13 @@ find_library(ARROW_FLIGHT_LIB_PATH
get_filename_component(ARROW_FLIGHT_LIBS ${ARROW_FLIGHT_LIB_PATH} DIRECTORY)

if(MSVC)
# Prioritize "/bin" over LIB_PATH_SUFFIXES - DLL files are installed
# in "/bin" and static objects are in "/lib", so we want to search
# "/bin" first
find_library(ARROW_FLIGHT_SHARED_LIBRARIES
NAMES arrow_flight
PATHS ${ARROW_HOME}
PATH_SUFFIXES ${LIB_PATH_SUFFIXES}
PATH_SUFFIXES "bin" ${LIB_PATH_SUFFIXES}
NO_DEFAULT_PATH)
get_filename_component(ARROW_FLIGHT_SHARED_LIBS ${ARROW_FLIGHT_SHARED_LIBRARIES} DIRECTORY)
endif()
Expand Down
113 changes: 55 additions & 58 deletions cpp/cmake_modules/FindBrotli.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -17,37 +17,47 @@
#
# find_package(Brotli)

# Favour static libraries over dynamic libraries, and handle various spellings
set(BROTLI_COMMON_LIB_NAMES
brotlicommon-static
brotlicommon
${CMAKE_STATIC_LIBRARY_PREFIX}brotlicommon-static${CMAKE_STATIC_LIBRARY_SUFFIX}
${CMAKE_STATIC_LIBRARY_PREFIX}brotlicommon_static${CMAKE_STATIC_LIBRARY_SUFFIX}
${CMAKE_SHARED_LIBRARY_PREFIX}brotlicommon${CMAKE_SHARED_LIBRARY_SUFFIX}
${CMAKE_STATIC_LIBRARY_PREFIX}brotlicommon${CMAKE_STATIC_LIBRARY_SUFFIX})

set(BROTLI_ENC_LIB_NAMES
brotlienc-static
brotlienc
${CMAKE_STATIC_LIBRARY_PREFIX}brotlienc-static${CMAKE_STATIC_LIBRARY_SUFFIX}
${CMAKE_STATIC_LIBRARY_PREFIX}brotlienc_static${CMAKE_STATIC_LIBRARY_SUFFIX}
${CMAKE_SHARED_LIBRARY_PREFIX}brotlienc${CMAKE_SHARED_LIBRARY_SUFFIX}
${CMAKE_STATIC_LIBRARY_PREFIX}brotlienc${CMAKE_STATIC_LIBRARY_SUFFIX})

set(BROTLI_DEC_LIB_NAMES
brotlidec-static
brotlidec
${CMAKE_STATIC_LIBRARY_PREFIX}brotlidec-static${CMAKE_STATIC_LIBRARY_SUFFIX}
${CMAKE_STATIC_LIBRARY_PREFIX}brotlidec_static${CMAKE_STATIC_LIBRARY_SUFFIX}
${CMAKE_SHARED_LIBRARY_PREFIX}brotlidec${CMAKE_SHARED_LIBRARY_SUFFIX}
${CMAKE_STATIC_LIBRARY_PREFIX}brotlidec${CMAKE_STATIC_LIBRARY_SUFFIX})

if(BROTLI_ROOT)
find_library(
BROTLI_COMMON_LIBRARY
NAMES brotlicommon
${CMAKE_SHARED_LIBRARY_PREFIX}brotlicommon${CMAKE_SHARED_LIBRARY_SUFFIX}
${CMAKE_STATIC_LIBRARY_PREFIX}brotlicommon${CMAKE_STATIC_LIBRARY_SUFFIX}
${CMAKE_STATIC_LIBRARY_PREFIX}brotlicommon-static${CMAKE_STATIC_LIBRARY_SUFFIX}
${CMAKE_STATIC_LIBRARY_PREFIX}brotlicommon_static${CMAKE_STATIC_LIBRARY_SUFFIX}
PATHS ${BROTLI_ROOT}
PATH_SUFFIXES ${LIB_PATH_SUFFIXES}
NO_DEFAULT_PATH)
find_library(
BROTLI_ENC_LIBRARY
NAMES brotlienc
${CMAKE_SHARED_LIBRARY_PREFIX}brotlienc${CMAKE_SHARED_LIBRARY_SUFFIX}
${CMAKE_STATIC_LIBRARY_PREFIX}brotlienc${CMAKE_STATIC_LIBRARY_SUFFIX}
${CMAKE_STATIC_LIBRARY_PREFIX}brotlienc-static${CMAKE_STATIC_LIBRARY_SUFFIX}
${CMAKE_STATIC_LIBRARY_PREFIX}brotlienc_static${CMAKE_STATIC_LIBRARY_SUFFIX}
PATHS ${BROTLI_ROOT}
PATH_SUFFIXES ${LIB_PATH_SUFFIXES}
NO_DEFAULT_PATH)
find_library(
BROTLI_DEC_LIBRARY
NAMES brotlidec
${CMAKE_SHARED_LIBRARY_PREFIX}brotlidec${CMAKE_SHARED_LIBRARY_SUFFIX}
${CMAKE_STATIC_LIBRARY_PREFIX}brotlidec${CMAKE_STATIC_LIBRARY_SUFFIX}
${CMAKE_STATIC_LIBRARY_PREFIX}brotlidec-static${CMAKE_STATIC_LIBRARY_SUFFIX}
${CMAKE_STATIC_LIBRARY_PREFIX}brotlidec_static${CMAKE_STATIC_LIBRARY_SUFFIX}
PATHS ${BROTLI_ROOT}
PATH_SUFFIXES ${LIB_PATH_SUFFIXES}
NO_DEFAULT_PATH)
find_library(BROTLI_COMMON_LIBRARY
NAMES ${BROTLI_COMMON_LIB_NAMES}
PATHS ${BROTLI_ROOT}
PATH_SUFFIXES ${LIB_PATH_SUFFIXES}
NO_DEFAULT_PATH)
find_library(BROTLI_ENC_LIBRARY
NAMES ${BROTLI_ENC_LIB_NAMES}
PATHS ${BROTLI_ROOT}
PATH_SUFFIXES ${LIB_PATH_SUFFIXES}
NO_DEFAULT_PATH)
find_library(BROTLI_DEC_LIBRARY
NAMES ${BROTLI_DEC_LIB_NAMES}
PATHS ${BROTLI_ROOT}
PATH_SUFFIXES ${LIB_PATH_SUFFIXES}
NO_DEFAULT_PATH)
find_path(BROTLI_INCLUDE_DIR
NAMES brotli/decode.h
PATHS ${BROTLI_ROOT}
Expand All @@ -63,44 +73,31 @@ else()
list(APPEND BROTLI_PC_LIBRARY_DIRS "${BROTLI_PC_libbrotlienc_LIBDIR}")
list(APPEND BROTLI_PC_LIBRARY_DIRS "${BROTLI_PC_libbrotlidec_LIBDIR}")

find_library(BROTLI_COMMON_LIBRARY brotlicommon
find_library(BROTLI_COMMON_LIBRARY
NAMES ${BROTLI_COMMON_LIB_NAMES}
PATHS ${BROTLI_PC_LIBRARY_DIRS}
PATH_SUFFIXES ${LIB_PATH_SUFFIXES}
NO_DEFAULT_PATH)
find_library(BROTLI_ENC_LIBRARY brotlienc
find_library(BROTLI_ENC_LIBRARY
NAMES ${BROTLI_ENC_LIB_NAMES}
PATHS ${BROTLI_PC_LIBRARY_DIRS}
PATH_SUFFIXES ${LIB_PATH_SUFFIXES}
NO_DEFAULT_PATH)
find_library(BROTLI_DEC_LIBRARY brotlidec
find_library(BROTLI_DEC_LIBRARY
NAMES ${BROTLI_DEC_LIB_NAMES}
PATHS ${BROTLI_PC_LIBRARY_DIRS}
PATH_SUFFIXES ${LIB_PATH_SUFFIXES}
NO_DEFAULT_PATH)
else()
find_library(
BROTLI_COMMON_LIBRARY
NAMES
brotlicommon
${CMAKE_SHARED_LIBRARY_PREFIX}brotlicommon${CMAKE_SHARED_LIBRARY_SUFFIX}
${CMAKE_STATIC_LIBRARY_PREFIX}brotlicommon${CMAKE_STATIC_LIBRARY_SUFFIX}
${CMAKE_STATIC_LIBRARY_PREFIX}brotlicommon-static${CMAKE_STATIC_LIBRARY_SUFFIX}
${CMAKE_STATIC_LIBRARY_PREFIX}brotlicommon_static${CMAKE_STATIC_LIBRARY_SUFFIX}
PATH_SUFFIXES ${LIB_PATH_SUFFIXES})
find_library(
BROTLI_ENC_LIBRARY
NAMES brotlienc
${CMAKE_SHARED_LIBRARY_PREFIX}brotlienc${CMAKE_SHARED_LIBRARY_SUFFIX}
${CMAKE_STATIC_LIBRARY_PREFIX}brotlienc${CMAKE_STATIC_LIBRARY_SUFFIX}
${CMAKE_STATIC_LIBRARY_PREFIX}brotlienc-static${CMAKE_STATIC_LIBRARY_SUFFIX}
${CMAKE_STATIC_LIBRARY_PREFIX}brotlienc_static${CMAKE_STATIC_LIBRARY_SUFFIX}
PATH_SUFFIXES ${LIB_PATH_SUFFIXES})
find_library(
BROTLI_DEC_LIBRARY
NAMES brotlidec
${CMAKE_SHARED_LIBRARY_PREFIX}brotlidec${CMAKE_SHARED_LIBRARY_SUFFIX}
${CMAKE_STATIC_LIBRARY_PREFIX}brotlidec${CMAKE_STATIC_LIBRARY_SUFFIX}
${CMAKE_STATIC_LIBRARY_PREFIX}brotlidec-static${CMAKE_STATIC_LIBRARY_SUFFIX}
${CMAKE_STATIC_LIBRARY_PREFIX}brotlidec_static${CMAKE_STATIC_LIBRARY_SUFFIX}
PATH_SUFFIXES ${LIB_PATH_SUFFIXES})
find_library(BROTLI_COMMON_LIBRARY
NAMES ${BROTLI_COMMON_LIB_NAMES}
PATH_SUFFIXES ${LIB_PATH_SUFFIXES})
find_library(BROTLI_ENC_LIBRARY
NAMES ${BROTLI_ENC_LIB_NAMES}
PATH_SUFFIXES ${LIB_PATH_SUFFIXES})
find_library(BROTLI_DEC_LIBRARY
NAMES ${BROTLI_DEC_LIB_NAMES}
PATH_SUFFIXES ${LIB_PATH_SUFFIXES})
find_path(BROTLI_INCLUDE_DIR
NAMES brotli/decode.h
PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES})
Expand Down
4 changes: 2 additions & 2 deletions cpp/cmake_modules/FindRE2.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ elseif(RE2_ROOT)
find_library(
RE2_LIB
NAMES
re2
re2_static re2
"${CMAKE_STATIC_LIBRARY_PREFIX}re2${RE2_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}"
"${CMAKE_SHARED_LIBRARY_PREFIX}re2${CMAKE_SHARED_LIBRARY_SUFFIX}"
PATHS ${RE2_ROOT}
Expand All @@ -52,7 +52,7 @@ else()
find_library(
RE2_LIB
NAMES
re2
re2_static re2
"${CMAKE_STATIC_LIBRARY_PREFIX}re2${RE2_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}"
"${CMAKE_SHARED_LIBRARY_PREFIX}re2${CMAKE_SHARED_LIBRARY_SUFFIX}"
PATH_SUFFIXES ${LIB_PATH_SUFFIXES})
Expand Down
18 changes: 15 additions & 3 deletions dev/tasks/python-wheels/appveyor.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,14 @@ os: Visual Studio 2015
environment:
ARCH: "64"
GENERATOR: Visual Studio 14 2015 Win64
NUMPY_VERSION: "1.14.5"
NUMPY_VERSION: "1.14"
PYTHON_VERSION: "{{ python_version }}"
MSVC_DEFAULT_OPTIONS: ON
ARROW_SRC: C:\apache-arrow
PYARROW_VERSION: {{ arrow.no_rc_version }}
PYARROW_REF: {{ arrow.head }}

init:
- set MINICONDA=C:\Miniconda35-x64
- set MINICONDA=C:\Miniconda3-x64
- set PATH=%MINICONDA%;%MINICONDA%/Scripts;%MINICONDA%/Library/bin;%PATH%

build_script:
Expand All @@ -38,13 +37,26 @@ build_script:
- git clone --no-checkout {{ arrow.remote }} %ARROW_SRC% || exit /B
- git -C %ARROW_SRC% fetch -t {{ arrow.remote }} {{ arrow.branch }} || exit /B
- git -C %ARROW_SRC% checkout {{ arrow.head }} || exit /B
- git -C %ARROW_SRC% submodule update --init || exit /B

# Avoid picking up AppVeyor-installed OpenSSL (linker errors with gRPC)
# XXX Perhaps there is a smarter way of solving this issue?
- rd /s /q C:\OpenSSL-Win32
- rd /s /q C:\OpenSSL-Win64
- rd /s /q C:\OpenSSL-v11-Win32
- rd /s /q C:\OpenSSL-v11-Win64
- rd /s /q C:\OpenSSL-v111-Win32
- rd /s /q C:\OpenSSL-v111-Win64

- call %ARROW_SRC%\dev\tasks\python-wheels\win-build.bat

after_build:
# the artifacts must be uploaded from a directory relative to the build root
- xcopy %ARROW_SRC%\python\dist\* wheels\

#on_finish:
#- ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))

artifacts:
- path: 'wheels\*.whl'

Expand Down
51 changes: 26 additions & 25 deletions dev/tasks/python-wheels/win-build.bat
Original file line number Diff line number Diff line change
Expand Up @@ -20,17 +20,15 @@
@rem create conda environment for compiling
conda update --yes --quiet conda

conda create -n arrow -q -y python=%PYTHON_VERSION% ^
six pytest setuptools numpy=%NUMPY_VERSION% pandas
call conda create -n wheel-build -q -y -c conda-forge ^
--file=%ARROW_SRC%\ci\conda_env_cpp.yml ^
--file=%ARROW_SRC%\ci\conda_env_gandiva.yml ^
--file=%ARROW_SRC%\ci\conda_env_python.yml ^
python=%PYTHON_VERSION% ^
numpy=%NUMPY_VERSION% ^
|| exit /B

conda install -n arrow -q -y -c conda-forge ^
git flatbuffers rapidjson ^
cmake ^
boost-cpp thrift-cpp ^
gflags snappy zlib zstd lz4-c double-conversion ^
llvmdev=7 libprotobuf

call activate arrow
call activate wheel-build

set ARROW_HOME=%CONDA_PREFIX%\Library
set PARQUET_HOME=%CONDA_PREFIX%\Library
Expand All @@ -45,44 +43,47 @@ cmake -G "%GENERATOR%" ^
-DARROW_BOOST_USE_SHARED=OFF ^
-DARROW_BUILD_TESTS=OFF ^
-DCMAKE_BUILD_TYPE=Release ^
-DBrotli_SOURCE=BUNDLED ^
-DRE2_SOURCE=BUNDLED ^
-DARROW_DEPENDENCY_SOURCE=CONDA ^
-DOPENSSL_ROOT_DIR=%CONDA_PREFIX%/Library ^
-DARROW_CXXFLAGS="/MP" ^
-DARROW_FLIGHT=ON ^
-DARROW_PYTHON=ON ^
-DARROW_PARQUET=ON ^
-DARROW_GANDIVA=ON ^
.. || exit /B
cmake --build . --target INSTALL --config Release || exit /B
.. || exit /B
cmake --build . --target install --config Release || exit /B
popd

pushd %ARROW_SRC%\python
set PYARROW_BUILD_TYPE=Release
@rem Gandiva is not supported on Python 2.7, but We don't build 2.7 wheel for windows
set PYARROW_PARALLEL=8
@rem Flight and Gandiva are not supported on Python 2.7,
@rem but we don't build 2.7 wheels for Windows.
set PYARROW_WITH_FLIGHT=1
set PYARROW_WITH_GANDIVA=1
set PYARROW_WITH_PARQUET=1
set PYARROW_WITH_STATIC_BOOST=1
set PYARROW_BUNDLE_ARROW_CPP=1
set SETUPTOOLS_SCM_PRETEND_VERSION=%PYARROW_VERSION%

@rem Newer Cython versions are not available on conda-forge
pip install -U pip
pip install "Cython>=0.29"

python setup.py build_ext bdist_wheel || exit /B
pushd %ARROW_SRC%\python
python setup.py bdist_wheel || exit /B
popd

call deactivate

set ARROW_TEST_DATA=%ARROW_SRC%\testing\data

@rem test the wheel
conda create -n wheel-test -q -y python=%PYTHON_VERSION% ^
numpy=%NUMPY_VERSION% pandas pytest hypothesis
@rem TODO For maximum reliability, we should test in a plain virtualenv instead.
call conda create -n wheel-test -q -y python=%PYTHON_VERSION% ^
numpy=%NUMPY_VERSION% pandas pytest hypothesis || exit /B
call activate wheel-test

@rem install the built wheel
pip install -vv --no-index --find-links=%ARROW_SRC%\python\dist\ pyarrow
pip install -vv --no-index --find-links=%ARROW_SRC%\python\dist\ pyarrow || exit /B

@rem test the imports
python -c "import pyarrow; import pyarrow.parquet; import pyarrow.gandiva;" || exit /B
python -c "import pyarrow; import pyarrow.parquet; import pyarrow.flight; import pyarrow.gandiva;" || exit /B

@rem run the python tests
pytest --pyargs pyarrow || exit /B
2 changes: 1 addition & 1 deletion docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -421,7 +421,7 @@ services:
# $ docker-compose pull python-manylinux1
# an then run:
# $ docker-compose run -e PYTHON_VERSION=3.7 python-manylinux1
image: quay.io/xhochy/arrow_manylinux1_x86_64_base:latest
image: quay.io/ursa-labs/arrow_manylinux1_x86_64_base:latest
build:
context: python/manylinux1
dockerfile: Dockerfile-x86_64_base
Expand Down
Loading

0 comments on commit eced2aa

Please sign in to comment.