switch to CMAKE with building APSI from source in a patched version t…

…o avoid AVX2 (closes LGro#3) (LGro#7)

* switch to cmake LGro#3

* install apsi from source with forced AVX LGro#3

.dockerignore

@@ -0,0 +1,5 @@
+.venv/
+build/
+.github/
+.pytest_cache/
+.vscode/

.github/workflows/cicd.yaml

@@ -10,5 +10,5 @@ jobs:
       - uses: actions/checkout@v2
       - name: Build
         run: docker-compose build
-      - name: Run
-        run: docker-compose run pyapsi
+      - name: Test
+        run: docker-compose run test

CMakeLists.txt

@@ -0,0 +1,22 @@
+cmake_minimum_required(VERSION 3.18.4)
+
+project(_pyapsi)
+
+include(FetchContent)
+
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+FetchContent_Declare(
+    pybind11
+    GIT_REPOSITORY https://github.com/pybind/pybind11.git
+    GIT_TAG v2.9.2
+)
+FetchContent_MakeAvailable(pybind11)
+
+add_subdirectory(external/apsi/)
+
+pybind11_add_module(_pyapsi src/main.cpp)
+
+target_link_libraries(_pyapsi PRIVATE pybind11::module apsi)
+
+target_compile_definitions(_pyapsi PRIVATE)

Dockerfile

@@ -1,27 +1,25 @@
 FROM python:3.9.12-slim-bullseye
 
-RUN apt-get update -q && apt-get install -q -y build-essential tar curl zip unzip git pkg-config
+RUN apt-get update -q && apt-get install -q -y build-essential tar curl zip unzip git pkg-config cmake
+RUN pip install poetry
 
 RUN git clone https://github.com/microsoft/vcpkg /tmp/vcpkg
-
-RUN ./tmp/vcpkg/bootstrap-vcpkg.sh && \
-    export CXXFLAGS="$CXXFLAGS -fPIC" && \
-    ./tmp/vcpkg/vcpkg install seal[no-throw-tran] kuku log4cplus cppzmq flatbuffers jsoncpp apsi
-
-ENV VCPKG_INSTALLED_DIR=/tmp/vcpkg/installed
+RUN /tmp/vcpkg/bootstrap-vcpkg.sh
+RUN /tmp/vcpkg/vcpkg install seal[no-throw-tran] kuku log4cplus cppzmq flatbuffers jsoncpp
+RUN /tmp/vcpkg/vcpkg integrate install
+ENV VCPKG_ROOT_DIR=/tmp/vcpkg
 
 RUN mkdir /tmp/pyapsi
-COPY ./setup.py /tmp/pyapsi/setup.py
-COPY ./pyproject.toml /tmp/pyapsi/pyproject.toml
-COPY ./poetry.lock /tmp/pyapsi/poetry.lock
-COPY ./src /tmp/pyapsi/src
-COPY ./apsi /tmp/pyapsi/apsi
-COPY ./tests /tmp/pyapsi/tests
-
+COPY ./ /tmp/pyapsi
 WORKDIR /tmp/pyapsi
 
-RUN pip install poetry && \
-    poetry install && \
+RUN mkdir /tmp/pyapsi/external
+RUN git clone https://github.com/microsoft/apsi /tmp/pyapsi/external/apsi
+# This is a hack to disable AVX2 use, which causes issues during dynamic linking
+RUN sed -i "s/-D_AVX2_/-D_AVX_/g" /tmp/pyapsi/external/apsi/CMakeLists.txt
+RUN sed -i "s/_AVX2.S/.S/g" /tmp/pyapsi/external/apsi/common/apsi/fourq/amd64/CMakeLists.txt
+
+RUN poetry install && \
     poetry run pip install --verbose .
 
 CMD ["poetry", "run", "pytest", "tests"]

docker-compose.yml

@@ -1,5 +1,5 @@
 services:
-  pyapsi:
+  test:
     build:
       context: ./
       dockerfile: Dockerfile

setup.py

@@ -1,50 +1,177 @@
-"""Setuptools integration of C++ APSI and Python package."""
+"""Setuptools integration of C++ APSI and Python package.
+
+This file is based on https://github.com/pybind/cmake_example/blob/master/setup.py
+which is licensed under the following BSD style license terms.
+
+Copyright (c) 2016 The Pybind Development Team, All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+   may be used to endorse or promote products derived from this software
+   without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+You are under no obligation whatsoever to provide any bug fixes, patches, or
+upgrades to the features, functionality or performance of the source code
+("Enhancements") to anyone; however, if you choose to make your Enhancements
+available either publicly, or directly to the author of this software, without
+imposing a separate written license agreement for such Enhancements, then you
+hereby grant the following license: a non-exclusive, royalty-free perpetual
+license to install, use, modify, prepare derivative works, incorporate into
+other computer software, distribute, and sublicense such enhancements or
+derivative works thereof, in binary and source code form.
+"""
 
 import os
-from glob import glob
+import re
+import subprocess
+import sys
 
-# Available at setup time due to pyproject.toml
-from pybind11.setup_helpers import Pybind11Extension
-from setuptools import setup, find_packages
+from setuptools import Extension, find_packages, setup
+from setuptools.command.build_ext import build_ext
 
 __version__ = "0.1.0"
 
-vcpkg_installed_dir = os.environ["VCPKG_INSTALLED_DIR"]
-
-ext_modules = [
-    Pybind11Extension(
-        "_pyapsi",
-        sorted(glob("src/*.cpp")),
-        define_macros=[
-            ("VERSION_INFO", __version__),
-            ("__LINUX__", 1),
-            ("_X86_", 1),
-            ("GENERIC_IMPLEMENTATION", 1),
-        ],
-        include_dirs=[
-            f"{vcpkg_installed_dir}/x64-linux/include",
-            f"{vcpkg_installed_dir}/x64-linux/include/gsl",
-            f"{vcpkg_installed_dir}/x64-linux/include/Kuku-2.1",
-            f"{vcpkg_installed_dir}/x64-linux/include/SEAL-3.7",
-            f"{vcpkg_installed_dir}/x64-linux/include/APSI-0.7",
-        ],
-        extra_objects=sorted(glob(f"{vcpkg_installed_dir}/x64-linux/lib/*.a")),
-        cxx_std=17,
-        extra_compile_args=[],
-    ),
-]
+
+# Convert distutils Windows platform specifiers to CMake -A arguments
+PLAT_TO_CMAKE = {
+    "win32": "Win32",
+    "win-amd64": "x64",
+    "win-arm32": "ARM",
+    "win-arm64": "ARM64",
+}
+
+# A CMakeExtension needs a sourcedir instead of a file list.
+# The name must be the _single_ output extension from the CMake build.
+# If you need multiple extensions, see scikit-build.
+class CMakeExtension(Extension):
+    def __init__(self, name, sourcedir=""):
+        Extension.__init__(self, name, sources=[])
+        self.sourcedir = os.path.abspath(sourcedir)
+
+
+class CMakeBuild(build_ext):
+    def build_extension(self, ext):
+        extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name)))
+
+        # required for auto-detection & inclusion of auxiliary "native" libs
+        if not extdir.endswith(os.path.sep):
+            extdir += os.path.sep
+
+        debug = int(os.environ.get("DEBUG", 0)) if self.debug is None else self.debug
+        cfg = "Debug" if debug else "Release"
+
+        # CMake lets you override the generator - we need to check this.
+        # Can be set with Conda-Build, for example.
+        cmake_generator = os.environ.get("CMAKE_GENERATOR", "")
+
+        # Set Python_EXECUTABLE instead if you use PYBIND11_FINDPYTHON
+        vcpkgdir = os.environ.get("VCPKG_ROOT_DIR")
+        cmake_args = [
+            f"-DCMAKE_TOOLCHAIN_FILE={vcpkgdir}/scripts/buildsystems/vcpkg.cmake",
+            f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}",
+            f"-DPYTHON_EXECUTABLE={sys.executable}",
+            f"-DCMAKE_BUILD_TYPE={cfg}",  # not used on MSVC, but no harm
+        ]
+        build_args = []
+        # Adding CMake arguments set as environment variable
+        # (needed e.g. to build for ARM OSx on conda-forge)
+        if "CMAKE_ARGS" in os.environ:
+            cmake_args += [item for item in os.environ["CMAKE_ARGS"].split(" ") if item]
+
+        if self.compiler.compiler_type != "msvc":
+            # Using Ninja-build since it a) is available as a wheel and b)
+            # multithreads automatically. MSVC would require all variables be
+            # exported for Ninja to pick it up, which is a little tricky to do.
+            # Users can override the generator with CMAKE_GENERATOR in CMake
+            # 3.15+.
+            if not cmake_generator or cmake_generator == "Ninja":
+                try:
+                    import ninja  # noqa: F401
+
+                    ninja_executable_path = os.path.join(ninja.BIN_DIR, "ninja")
+                    cmake_args += [
+                        "-GNinja",
+                        f"-DCMAKE_MAKE_PROGRAM:FILEPATH={ninja_executable_path}",
+                    ]
+                except ImportError:
+                    pass
+
+        else:
+
+            # Single config generators are handled "normally"
+            single_config = any(x in cmake_generator for x in {"NMake", "Ninja"})
+
+            # CMake allows an arch-in-generator style for backward compatibility
+            contains_arch = any(x in cmake_generator for x in {"ARM", "Win64"})
+
+            # Specify the arch if using MSVC generator, but only if it doesn't
+            # contain a backward-compatibility arch spec already in the
+            # generator name.
+            if not single_config and not contains_arch:
+                cmake_args += ["-A", PLAT_TO_CMAKE[self.plat_name]]
+
+            # Multi-config generators have a different way to specify configs
+            if not single_config:
+                cmake_args += [
+                    f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{cfg.upper()}={extdir}"
+                ]
+                build_args += ["--config", cfg]
+
+        if sys.platform.startswith("darwin"):
+            # Cross-compile support for macOS - respect ARCHFLAGS if set
+            archs = re.findall(r"-arch (\S+)", os.environ.get("ARCHFLAGS", ""))
+            if archs:
+                cmake_args += ["-DCMAKE_OSX_ARCHITECTURES={}".format(";".join(archs))]
+
+        # Set CMAKE_BUILD_PARALLEL_LEVEL to control the parallel build level
+        # across all generators.
+        if "CMAKE_BUILD_PARALLEL_LEVEL" not in os.environ:
+            # self.parallel is a Python 3 only way to set parallel jobs by hand
+            # using -j in the build_ext call, not supported by pip or PyPA-build.
+            if hasattr(self, "parallel") and self.parallel:
+                # CMake 3.12+ only.
+                build_args += [f"-j{self.parallel}"]
+
+        build_temp = os.path.join(self.build_temp, ext.name)
+        if not os.path.exists(build_temp):
+            os.makedirs(build_temp)
+
+        subprocess.check_call(["cmake", ext.sourcedir] + cmake_args, cwd=build_temp)
+        subprocess.check_call(["cmake", "--build", "."] + build_args, cwd=build_temp)
+
 
 setup(
     name="apsi",
     version=__version__,
     author="Lukas Grossberger",
     author_email="[email protected]",
-    url="https://github.com/LGro/pyapsi",
+    url="https://github.com/LGro/PyAPSI",
     description="Python wrapper for labeled and unlabeled asynchronous private set "
-    + "intersection.",
+    + "intersection (APSI).",
     long_description="",
     packages=find_packages(),
-    ext_modules=ext_modules,
+    ext_modules=[CMakeExtension("_pyapsi")],
+    cmdclass={"build_ext": CMakeBuild},
     extras_require={"test": "pytest"},
     zip_safe=False,
     python_requires=">=3.8",

src/main.cpp

@@ -38,15 +38,15 @@
 #include <pybind11/pybind11.h>
 
 // APSI
-#include "apsi/log.h"
-#include "apsi/zmq/sender_dispatcher.h"
-#include "apsi/receiver.h"
-#include "apsi/sender.h"
-#include "apsi/item.h"
-#include "apsi/network/stream_channel.h"
-#include "apsi/psi_params.h"
-#include "apsi/sender_db.h"
-#include "apsi/thread_pool_mgr.h"
+#include <apsi/item.h>
+#include <apsi/log.h>
+#include <apsi/zmq/sender_dispatcher.h>
+#include <apsi/receiver.h>
+#include <apsi/sender.h>
+#include <apsi/network/stream_channel.h>
+#include <apsi/psi_params.h>
+#include <apsi/sender_db.h>
+#include <apsi/thread_pool_mgr.h>
 
 using namespace std;
 using namespace apsi;
@@ -222,6 +222,7 @@ class APSIClient
         return 0;
     }
 
+    // TODO: use std::vector<str> in conjunction with "#include <pybind11/stl.h>" for auto conversion
     py::bytes oprf_request(const py::list &input_items)
     {
         vector<Item> receiver_items;
@@ -442,6 +443,7 @@ PYBIND11_MODULE(_pyapsi, m)
         .def("_run", &APSIServer::run)
         .def("_handle_oprf_request", &APSIServer::handle_oprf_request)
         .def("_handle_query", &APSIServer::handle_query)
+        // TODO: use def_property_readonly instead
         .def_readwrite("_db_label_byte_count", &APSIServer::db_label_byte_count);
     py::class_<APSIClient>(m, "APSIClient")
         .def(py::init<string &>())