Merge pull request #15195 from ethereum/some_dependencies_as_git_subm…

…odules

Use submodules for some dependencies.

.circleci/config.yml

@@ -866,7 +866,7 @@ jobs:
  name: Check spelling
  command: |
  ~/.local/bin/codespell \
- --skip "*.enc,.git,Dockerfile*,LICENSE,codespell_whitelist.txt,codespell_ignored_lines.txt" \
+ --skip "*.enc,.git,Dockerfile*,LICENSE,codespell_whitelist.txt,codespell_ignored_lines.txt,deps" \
  --ignore-words scripts/codespell_whitelist.txt \
  --exclude-file scripts/codespell_ignored_lines.txt
  - matrix_notify_failure_unless_pr
@@ -1273,7 +1273,7 @@ jobs:
  - run:
  name: Install runtime dependencies
  command: |
- pacman --noconfirm -Syu --noprogressbar --needed z3
+ pacman --noconfirm -Syu --noprogressbar --needed z3 git
  - soltest
 
  t_ubu_clang_soltest: &t_ubu_clang_soltest

.gitmodules

@@ -0,0 +1,9 @@
+[submodule "deps/nlohmann-json"]
+ path = deps/nlohmann-json
+ url = https://github.com/nlohmann/json.git
+[submodule "deps/range-v3"]
+ path = deps/range-v3
+ url = https://github.com/ericniebler/range-v3.git
+[submodule "deps/fmtlib"]
+ path = deps/fmtlib
+ url = https://github.com/fmtlib/fmt.git

Changelog.md

@@ -36,6 +36,8 @@ Bugfixes:
  * Yul Optimizer: Name simplification could lead to forbidden identifiers with a leading and/or trailing dot, e.g., ``x._`` would get simplified into ``x.``.
  * Yul Parser: Fix segfault when parsing very long location comments.
 
+Build System:
+ * Change build system to use git submodules for some dependencies (nlohmann-json, fmtlib & range-v3).
 
 ### 0.8.26 (2024-05-21)
 

cmake/fmtlib.cmake

@@ -1,20 +1,9 @@
-include(FetchContent)
+include(${CMAKE_SOURCE_DIR}/cmake/submodules.cmake)
+initialize_submodule(fmtlib)
 
-FetchContent_Declare(
- fmtlib
- PREFIX "${PROJECT_BINARY_DIR}/deps"
- DOWNLOAD_DIR "${PROJECT_SOURCE_DIR}/deps/downloads"
- DOWNLOAD_NAME fmt-9.1.0.tar.gz
- URL https://github.com/fmtlib/fmt/archive/9.1.0.tar.gz
- URL_HASH SHA256=5dea48d1fcddc3ec571ce2058e13910a0d4a6bab4cc09a809d8b1dd1c88ae6f2
+set(FMT_SYSTEM_HEADERS ON)
+add_subdirectory(
+ ${CMAKE_SOURCE_DIR}/deps/fmtlib
+ EXCLUDE_FROM_ALL
 )
 
-if (CMAKE_VERSION VERSION_LESS "3.14.0")
- FetchContent_GetProperties(fmtlib)
- if (NOT fmtlib_POPULATED)
- FetchContent_Populate(fmtlib)
- add_subdirectory(${fmtlib_SOURCE_DIR} ${fmtlib_BINARY_DIR})
- endif()
-else()
- FetchContent_MakeAvailable(fmtlib)
-endif()

cmake/nlohmann-json.cmake

@@ -1,21 +1,9 @@
-include(ExternalProject)
+include(${CMAKE_SOURCE_DIR}/cmake/submodules.cmake)
+initialize_submodule(nlohmann-json)
 
-ExternalProject_Add(nlohmann-json-project
- DOWNLOAD_DIR "${CMAKE_SOURCE_DIR}/deps/nlohmann/nlohmann"
- DOWNLOAD_NAME json.hpp
- DOWNLOAD_NO_EXTRACT 1
- URL https://github.com/nlohmann/json/releases/download/v3.11.3/json.hpp
- URL_HASH SHA256=9bea4c8066ef4a1c206b2be5a36302f8926f7fdc6087af5d20b417d0cf103ea6
- CMAKE_COMMAND true
- BUILD_COMMAND true
- INSTALL_COMMAND true
+set(JSON_Install OFF CACHE INTERNAL "")
+add_subdirectory(
+ ${CMAKE_SOURCE_DIR}/deps/nlohmann-json
+ EXCLUDE_FROM_ALL
 )
 
-# Create nlohmann-json imported library
-add_library(nlohmann-json INTERFACE IMPORTED)
-file(MAKE_DIRECTORY ${CMAKE_SOURCE_DIR}/deps/nlohmann) # Must exist.
-set_target_properties(nlohmann-json PROPERTIES
- INTERFACE_COMPILE_OPTIONS "\$<\$<CXX_COMPILER_ID:MSVC>:/permissive->"
- INTERFACE_SYSTEM_INCLUDE_DIRECTORIES ${CMAKE_SOURCE_DIR}/deps/nlohmann
- INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_SOURCE_DIR}/deps/nlohmann)
-add_dependencies(nlohmann-json nlohmann-json-project)

cmake/range-v3.cmake

@@ -1,38 +1,11 @@
-include(ExternalProject)
+include(${CMAKE_SOURCE_DIR}/cmake/submodules.cmake)
+initialize_submodule(range-v3)
 
-if (${CMAKE_SYSTEM_NAME} STREQUAL "Emscripten")
- set(RANGE_V3_CMAKE_COMMAND emcmake cmake)
-else()
- set(RANGE_V3_CMAKE_COMMAND ${CMAKE_COMMAND})
-endif()
-
-set(prefix "${PROJECT_BINARY_DIR}/deps")
-set(RANGE_V3_INCLUDE_DIR "${prefix}/include")
-
-ExternalProject_Add(range-v3-project
- PREFIX "${prefix}"
- DOWNLOAD_DIR "${PROJECT_SOURCE_DIR}/deps/downloads"
- DOWNLOAD_NAME range-v3-0.12.0.tar.gz
- URL https://github.com/ericniebler/range-v3/archive/0.12.0.tar.gz
- URL_HASH SHA256=015adb2300a98edfceaf0725beec3337f542af4915cec4d0b89fa0886f4ba9cb
- CMAKE_COMMAND ${RANGE_V3_CMAKE_COMMAND}
- CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>
- -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
- -DBUILD_TESTING=OFF
- -DRANGES_CXX_STD=${CMAKE_CXX_STANDARD}
- -DRANGE_V3_DOCS=OFF
- -DRANGE_V3_EXAMPLES=OFF
- -DRANGE_V3_TESTS=OFF
- -DRANGES_BUILD_CALENDAR_EXAMPLE=OFF
- -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
- BUILD_BYPRODUCTS "${RANGE_V3_INCLUDE_DIR}/range/v3/all.hpp"
-)
-
-# Create range-v3 imported library
 add_library(range-v3 INTERFACE IMPORTED)
-file(MAKE_DIRECTORY ${RANGE_V3_INCLUDE_DIR}) # Must exist.
 set_target_properties(range-v3 PROPERTIES
- INTERFACE_COMPILE_OPTIONS "\$<\$<CXX_COMPILER_ID:MSVC>:/permissive->"
- INTERFACE_SYSTEM_INCLUDE_DIRECTORIES ${RANGE_V3_INCLUDE_DIR}
- INTERFACE_INCLUDE_DIRECTORIES ${RANGE_V3_INCLUDE_DIR})
+ INTERFACE_COMPILE_OPTIONS "\$<\$<CXX_COMPILER_ID:MSVC>:/permissive->"
+ INTERFACE_SYSTEM_INCLUDE_DIRECTORIES ${CMAKE_SOURCE_DIR}/deps/range-v3/include
+ INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_SOURCE_DIR}/deps/range-v3/include
+)
 add_dependencies(range-v3 range-v3-project)
+

cmake/submodules.cmake

@@ -0,0 +1,23 @@
+macro(initialize_submodule SUBMODULE_PATH)
+ if(NOT USE_SYSTEM_LIBRARIES)
+ file(GLOB submodule_contents "${CMAKE_SOURCE_DIR}/deps/${SUBMODULE_PATH}/*")
+
+ if(submodule_contents)
+ message(STATUS "git submodule '${SUBMODULE_PATH}' seem to be already initialized: nothing to do.")
+ else()
+ message(STATUS "git submodule '${SUBMODULE_PATH}' seem not to be initialized: implicitly executing 'git submodule update --init '${CMAKE_SOURCE_DIR}/deps/${SUBMODULE_PATH}'.")
+ find_package(Git)
+ if(NOT Git_FOUND)
+ message(FATAL_ERROR "Failed to initialize submodules: 'git' command not found.")
+ endif()
+ execute_process(
+ COMMAND git submodule update --init ${CMAKE_SOURCE_DIR}/deps/${SUBMODULE_PATH}
+ WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+ RESULT_VARIABLE result
+ )
+ if(NOT result EQUAL 0)
+ message(FATAL_ERROR "Failed to initialize submodules: 'git submodule update --init' failed.")
+ endif()
+ endif()
+ endif()
+endmacro()

libsolutil/CMakeLists.txt

@@ -44,7 +44,7 @@ set(sources
 )
 
 add_library(solutil ${sources})
-target_link_libraries(solutil PUBLIC Boost::boost Boost::filesystem Boost::system range-v3 fmt::fmt-header-only nlohmann-json)
+target_link_libraries(solutil PUBLIC Boost::boost Boost::filesystem Boost::system range-v3 fmt::fmt-header-only nlohmann_json::nlohmann_json)
 target_include_directories(solutil PUBLIC "${PROJECT_SOURCE_DIR}")
 add_dependencies(solutil solidity_BuildInfo.h)
 

scripts/check_style.sh

@@ -6,6 +6,7 @@ ERROR_LOG="$(mktemp -t check_style_XXXXXX.log)"
 
 EXCLUDE_FILES=(
  # The line below is left unquoted to allow the shell globbing path expansion
+ deps/*
  test/cmdlineTests/*/{err,output}
  "libsolutil/picosha2.h"
  "test/cmdlineTests/strict_asm_only_cr/input.yul"

scripts/chk_shellscripts/chk_shellscripts.sh

@@ -13,5 +13,5 @@ if [ ! -f "${SHELLCHECK}" ]; then
  exit 1
 fi
 
-mapfile -t FILES < <(find . -type f -name "*.sh")
+mapfile -t FILES < <(find . -type f -name "*.sh" -not -path "./deps/*")
 "${SHELLCHECK}" "${FILES[@]}"

scripts/create_source_tarball.sh

@@ -23,6 +23,8 @@ REPO_ROOT="$(dirname "$0")"/..
  mkdir "$SOLDIR"
  # Store the current source
  git checkout-index -a --prefix="$SOLDIR"
+ # shellcheck disable=SC2016
+ SOLDIR="$SOLDIR" git submodule foreach 'git checkout-index -a --prefix="$SOLDIR/$sm_path/"'
  # Store the commit hash
  echo "$commithash" > "$SOLDIR/commit_hash.txt"
  if [ -e prerelease.txt ] && [ ! -s prerelease.txt ]

scripts/install_evmone.ps1

@@ -5,5 +5,4 @@ $progressPreference = "silentlyContinue"
 
 Invoke-WebRequest -URI "https://github.com/ethereum/evmone/releases/download/v0.11.0/evmone-0.11.0-windows-amd64.zip" -OutFile "evmone.zip"
 tar -xf evmone.zip "bin/evmone.dll"
-mkdir deps
-mv bin/evmone.dll deps
+mv bin/evmone.dll deps/

scripts/release_ppa.sh

@@ -123,13 +123,6 @@ ppafilesurl=https://launchpad.net/~ethereum/+archive/ubuntu/${pparepo}/+files
 git clone --depth 2 --recursive https://github.com/ethereum/solidity.git -b "$branch"
 mv solidity solc
 
-# Fetch dependencies
-mkdir -p ./solc/deps/downloads/ 2>/dev/null || true
-mkdir -p ./solc/deps/nlohmann/nlohmann/ 2>/dev/null || true
-wget -O ./solc/deps/nlohmann/nlohmann/json.hpp https://github.com/nlohmann/json/releases/download/v3.11.3/json.hpp
-wget -O ./solc/deps/downloads/range-v3-0.12.0.tar.gz https://github.com/ericniebler/range-v3/archive/0.12.0.tar.gz
-wget -O ./solc/deps/downloads/fmt-9.1.0.tar.gz https://github.com/fmtlib/fmt/archive/9.1.0.tar.gz
-
 # Determine version
 cd solc
 version=$("$(dirname "$0")/get_version.sh")