Towards better Windows compatibility: TOOLS, DATA_TOOLS & documentati…

…on (valhalla#2521)

* use lua5.3 bitwise operators: https://www.lua.org/manual/5.3/manual.html#3.4.2

* update filename with OS specific path separator

* try to build on azure with TOOLS and DATA_TOOLS

* add kwarg to GraphTile::Suffix to distinguish between path and ID access

* revert previous Lua5.3 syntax

* add Windows section

* stupid: install needed packages.. also try newest vcpkg master again

* [ci skip] nope, still an issue. hope azure triggers, circleci not

* revert graph.lua huge changeset

* [ci skip] more packages missing..

* introduce new facet with OS specific 1000 separator to propely resolve file paths

* [ci skip] explicitly provide LUA_DIR & LUA_LIB paths

* VS Code doesn't auto-save files, thanks jetbrains for spoiling me..

* [ci skip] sorry, wrong vcpkg path..

* run a few exe's on azure CI; upgrade azure CI to VS 2019

* format

* remove redundant mkdir

* add changelog

* double quotes for cmd.exe; try adding toolset name

* change platform for x86 to Win32

* add do_grouping to url_facet and inherit directly from numpunct

.azure-pipelines.yml

@@ -5,20 +5,20 @@
 # TODO: Once CMake 3.13 is available replace -H with -S option
 #
 jobs:
-  - job: VS2017
-    displayName: 'Windows 2016 | VS2017'
+  - job: VS2019
+    displayName: 'Windows 2019 | VS2019'
 
     pool:
-      vmImage: 'vs2017-win2016'
+      vmImage: 'windows-2019'
 
     strategy:
       matrix:
         x86:
-          PLATFORM: 'x86'
-          GENERATOR: 'Visual Studio 15 2017'
+          PLATFORM: 'Win32'
+          TRIPLET: 'x86'
         x64:
           PLATFORM: 'x64'
-          GENERATOR: 'Visual Studio 15 2017 Win64'
+          TRIPLET: 'x64'
 
     variables:
       BUILD_CFG: 'Release'
@@ -31,14 +31,15 @@ jobs:
 
       - script: |
           git clone https://github.com/microsoft/vcpkg %VCPKG_DIR%
+          # TODO: check if https://github.com/microsoft/vcpkg/issues/12606 resolves regularly
           git -C %VCPKG_DIR% checkout f4bd6423
-          echo.set(VCPKG_BUILD_TYPE release)>> %VCPKG_DIR%\triplets\%PLATFORM%-windows.cmake
+          echo.set(VCPKG_BUILD_TYPE release)>> %VCPKG_DIR%\triplets\%TRIPLET%-windows.cmake
           call %VCPKG_DIR%\bootstrap-vcpkg.bat
           %VCPKG_DIR%\vcpkg.exe version
         displayName: 'Install vcpkg'
 
       - script: |
-          %VCPKG_DIR%\vcpkg.exe install boost-algorithm boost-config boost-core boost-heap boost-lexical-cast boost-property-tree boost-variant protobuf zlib --triplet %PLATFORM%-windows
+          %VCPKG_DIR%\vcpkg.exe install boost-algorithm boost-config boost-core boost-heap boost-lexical-cast boost-property-tree boost-variant boost-program-options boost-geometry protobuf zlib curl sqlite3 proj4 luajit libspatialite geos --triplet %TRIPLET%-windows
           %VCPKG_DIR%\vcpkg.exe list
           if exist %VCPKG_DIR%\downloads rmdir /S /Q %VCPKG_DIR%\downloads
           if exist %VCPKG_DIR%\packages rmdir /S /Q %VCPKG_DIR%\packages
@@ -57,9 +58,15 @@ jobs:
 
       - script: |
           cmake --version
-          cmake -H$(Build.SourcesDirectory) -B%BUILD_DIR% -G "%GENERATOR%" -DCMAKE_TOOLCHAIN_FILE=%VCPKG_DIR%\scripts\buildsystems\vcpkg.cmake -DVCPKG_APPLOCAL_DEPS=ON -DENABLE_CCACHE=OFF -DENABLE_DATA_TOOLS=OFF -DENABLE_HTTP=OFF -DENABLE_PYTHON_BINDINGS=OFF -DENABLE_SERVICES=OFF -DENABLE_TOOLS=OFF -DENABLE_BENCHMARKS=OFF
+          cmake -H$(Build.SourcesDirectory) -B%BUILD_DIR% -G "Visual Studio 16 2019" -A %PLATFORM% -T host=%TRIPLET% -DCMAKE_TOOLCHAIN_FILE=%VCPKG_DIR%\scripts\buildsystems\vcpkg.cmake -DVCPKG_APPLOCAL_DEPS=ON -DENABLE_CCACHE=OFF -DENABLE_DATA_TOOLS=ON -DENABLE_HTTP=OFF -DENABLE_PYTHON_BINDINGS=OFF -DENABLE_SERVICES=OFF -DENABLE_TOOLS=ON -DENABLE_BENCHMARKS=OFF -DLUA_INCLUDE_DIR=%VCPKG_DIR%\installed\%TRIPLET%-windows\include\luajit -DLUA_LIBRARIES=%VCPKG_DIR%\installed\%TRIPLET%-windows\lib\lua51.lib
         displayName: 'Run CMake to configure build'
 
       - script: |
           cmake --build %BUILD_DIR% --config %BUILD_CFG%
         displayName: 'Run CMake to build'
+
+      - script: |
+          %BUILD_DIR%\%BUILD_CFG%\valhalla_build_tiles.exe -c .\test\win\valhalla.json .\test\data\utrecht_netherlands.osm.pbf
+          %BUILD_DIR%\%BUILD_CFG%\valhalla_run_route.exe --config .\test\win\valhalla.json -j "{\"locations\": [{\"lat\": 52.10205, \"lon\": 5.114651}, {\"lat\": 52.093113, \"lon\": 5.100918}], \"costing\": \"auto\"}"
+          %BUILD_DIR%\%BUILD_CFG%\\valhalla_run_isochrone.exe --config .\test\win\valhalla.json -j "{\"locations\": [{\"lat\": 52.10205, \"lon\": 5.114651}], \"costing\": \"auto\", \"contours\":[{\"time\":15,\"color\":\"ff0000\"}]}"
+        displayName: 'Test some executables'

CHANGELOG.md

@@ -3,6 +3,7 @@
    * REMOVED:  Remove Node bindings. [#2502](https://github.com/valhalla/valhalla/pull/2502)
 
 * **Bug Fix**
+   * FIXED: More Windows compatibility: build tiles and some run actions work now (including CI tests) [#2300](https://github.com/valhalla/valhalla/issues/2300)
    * FIXED: Crazy ETAs.  If a way has forward speed with no backward speed and it is not oneway, then we must set the default speed.  The reverse logic applies as well.  If a way has no backward speed but has a forward speed and it is not a oneway, then set the default speed. [#2102](https://github.com/valhalla/valhalla/pull/2102)
    * FIXED: Map matching elapsed times spliced amongst different legs and discontinuities are now correct [#2104](https://github.com/valhalla/valhalla/pull/2104)
    * FIXED: Date time information is now propogated amongst different legs and discontinuities [#2107](https://github.com/valhalla/valhalla/pull/2107)

README.md

@@ -156,6 +156,33 @@ ccmake ..
 
 For more information on binaries, see [Command Line Tools](#command-line-tools) section below and the [docs](docs).
 
+Windows builds
+-------------
+
+Support for Windows is not yet fully exploited. Building the Valhalla library works flawlessly, as well as the following application modules:
+
+- `TOOLS`: utilities to query and benchmark various components
+- `DATA_TOOLS`: utilities to build input data and handle transit
+
+It's recommended to work with the following toolset:
+- Visual Studio with C++ support
+- Visual Studio Code (easier and lighter to handle)
+- [vcpkg](https://github.com/Microsoft/vcpkg) to install packages
+- [CMake](https://cmake.org/download/)
+
+1. Install the following packages with `vcpkg` (assuming you're on a 64-bit machine):
+```
+# Basic packages
+C:\path\to\vcpkg.exe install boost-algorithm boost-config boost-core boost-heap boost-lexical-cast boost-property-tree boost-variant boost-geometry boost-program-options curl protobuf zlib --triplet x64-windows
+# If you want to build with DATA_TOOLS & TOOLS
+C:\path\to\vcpkg.exe sqlite3 proj4 luajit libspatialite geos --triplet x64-windows
+```
+2. Let CMake configure the build with the required modules enabled. **Note**, you have to manually link LuaJIT for some reason, e.g. the final command could look like
+```
+"C:\Program Files\CMake\bin\cmake.EXE" --no-warn-unused-cli -DENABLE_TOOLS=ON -DENABLE_DATA_TOOLS=ON -DENABLE_SERVICES=OFF -DENABLE_PYTHON_BINDINGS=OFF -DLUA_LIBRARIES=path\to\vcpkg\installed\x64-windows\lib\lua51.lib -DLUA_INCLUDE_DIR=path\to\vcpkg\installed\x64-windows\include\luajit -DENABLE_BENCHMARKS=OFF -DENABLE_TESTS=OFF -DVCPKG_TARGET_TRIPLET=x64-windows -DCMAKE_TOOLCHAIN_FILE=path\to\vcpkg\scripts\buildsystems\vcpkg.cmake -DCMAKE_EXPORT_COMPILE_COMMANDS:BOOL=TRUE -Hpath/to/project -Bpath/to/project/build -G "Visual Studio 16 2019" -T host=x64 -A x64
+```
+3. Run the build for all targets.
+
 Running
 -------
 

src/baldr/graphtile.cc

@@ -28,6 +28,16 @@ using namespace valhalla::midgard;
 
 namespace {
 struct dir_facet : public std::numpunct<char> {
+protected:
+  virtual char do_thousands_sep() const {
+    return filesystem::path::preferred_separator;
+  }
+
+  virtual std::string do_grouping() const {
+    return "\03";
+  }
+};
+struct url_facet : public std::numpunct<char> {
 protected:
   virtual char do_thousands_sep() const {
     return '/';
@@ -37,13 +47,15 @@ struct dir_facet : public std::numpunct<char> {
     return "\03";
   }
 };
+const std::locale url_locale(std::locale("C"), new url_facet());
 const std::locale dir_locale(std::locale("C"), new dir_facet());
 const AABB2<PointLL> world_box(PointLL(-180, -90), PointLL(180, 90));
 constexpr float COMPRESSION_HINT = 3.5f;
 
 std::string MakeSingleTileUrl(const std::string& tile_url, const valhalla::baldr::GraphId& graphid) {
   auto id_pos = tile_url.find(valhalla::baldr::GraphTile::kTilePathPattern);
-  return tile_url.substr(0, id_pos) + valhalla::baldr::GraphTile::FileSuffix(graphid.Tile_Base()) +
+  return tile_url.substr(0, id_pos) +
+         valhalla::baldr::GraphTile::FileSuffix(graphid.Tile_Base(), false, false) +
          tile_url.substr(id_pos + std::strlen(valhalla::baldr::GraphTile::kTilePathPattern));
 }
 
@@ -384,7 +396,7 @@ void GraphTile::AssociateOneStopIds(const GraphId& graphid) {
   }
 }
 
-std::string GraphTile::FileSuffix(const GraphId& graphid, bool gzipped) {
+std::string GraphTile::FileSuffix(const GraphId& graphid, bool gzipped, bool is_file_path) {
   /*
   if you have a graphid where level == 8 and tileid == 24134109851 you should get:
   8/024/134/109/851.gph since the number of levels is likely to be very small this limits the total
@@ -416,7 +428,11 @@ std::string GraphTile::FileSuffix(const GraphId& graphid, bool gzipped) {
 
   // make a locale to use as a formatter for numbers
   std::ostringstream stream;
-  stream.imbue(dir_locale);
+  if (is_file_path) {
+    stream.imbue(dir_locale);
+  } else {
+    stream.imbue(url_locale);
+  }
 
   // if it starts with a zero the pow trick doesn't work
   if (graphid.level() == 0) {

src/mjolnir/valhalla_fetch_transit.cc

@@ -263,7 +263,7 @@ std::priority_queue<weighted_tile_t> which_tiles(const ptree& pt, const std::str
                           stations_total +
                               10 /* + routes_total * 1000 + pairs_total*/}); // TODO: factor in stop
                                                                              // pairs as well
-      LOG_INFO(GraphTile::FileSuffix(tile) + " should have " + std::to_string(stations_total) +
+      LOG_INFO(GraphTile::FileSuffix(tile, false, false) + " should have " + std::to_string(stations_total) +
                " stations " /* +
           std::to_string(routes_total) +  " routes and " + std::to_string(pairs_total) +  " stop_pairs"*/);
     }

src/mjolnir/valhalla_query_transit.cc

@@ -51,7 +51,7 @@ Transit read_pbf(const std::string& file_name) {
 Transit read_pbf(const GraphId& id, const std::string& transit_dir, std::string& file_name) {
   std::string fname = GraphTile::FileSuffix(id);
   fname = fname.substr(0, fname.size() - 3) + "pbf";
-  file_name = transit_dir + '/' + fname;
+  file_name = transit_dir + filesystem::path::preferred_separator + fname;
   Transit transit;
   transit = read_pbf(file_name);
   return transit;

test/astar.cc

@@ -225,7 +225,8 @@ void make_tile() {
   ASSERT_EQ(tile.FileSuffix(tile_id, false), std::string("2/000/519/120.gph"))
       << "Tile ID didn't match the expected filename";
 
-  ASSERT_PRED1(filesystem::exists, test_dir + "/" + tile.FileSuffix(tile_id, false))
+  ASSERT_PRED1(filesystem::exists,
+               test_dir + filesystem::path::preferred_separator + tile.FileSuffix(tile_id, false))
       << "Expected tile file didn't show up on disk - are the fixtures in the right location?";
 }
 

test/graphreader.cc

@@ -67,7 +67,7 @@ TEST(SimpleCache, CacheLimitsNoOvercommitAfterClear) {
 
 void touch_tile(const uint32_t tile_id, const std::string& tile_dir) {
   auto suffix = GraphTile::FileSuffix({tile_id, 2, 0});
-  auto fullpath = tile_dir + '/' + suffix;
+  auto fullpath = tile_dir + filesystem::path::preferred_separator + suffix;
   filesystem::create_directories(filesystem::path(fullpath).parent_path());
   int fd = open(fullpath.c_str(), O_CREAT | O_WRONLY, 0644);
   if (fd >= 0)