new initial submodule update strategy (#494)

* new initial submodule update strategy

* Azure: increase timeout

* Makefile: change comment [skip ci]

.appveyor.yml

@@ -26,7 +26,6 @@ init:
  - ps: IF ("$env:APPVEYOR_REPO_BRANCH" -ne "master") { $env:APPVEYOR_CACHE_SKIP_SAVE = "true" }
 
 install:
- - git submodule update --init --recursive
  # use the newest versions documented here: https://www.appveyor.com/docs/windows-images-software/#mingw-msys-cygwin
  - IF "%PLATFORM%" == "x86" SET PATH=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin;%PATH%
  - IF "%PLATFORM%" == "x64" SET PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH%

.gitignore

@@ -6,10 +6,6 @@
 # Nimble packages
 /vendor/.nimble
 
-# Go packages
-/vendor/go
-!/vendor/go/src/github.com/libp2p/go-libp2p-daemon
-
 # ntags/ctags output
 /tags
 

Makefile

@@ -1,16 +1,16 @@
-# Copyright (c) 2018-2019 Status Research & Development GmbH. Licensed under
+# Copyright (c) 2018-2020 Status Research & Development GmbH. Licensed under
 # either of:
 # - Apache License, version 2.0
 # - MIT license
 # at your option. This file may not be copied, modified, or distributed except
 # according to those terms.
 
-SHELL := bash # the shell used internally by "make"
+SHELL := bash # the shell used internally by Make
 
 # used inside the included makefiles
 BUILD_SYSTEM_DIR := vendor/nimbus-build-system
 
-# we don't want an error here, so we can handle things later, in the build-system-checks target
+# we don't want an error here, so we can handle things later, in the ".DEFAULT" target
 -include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk
 
 # debugging tools + testing tools
@@ -30,17 +30,9 @@ TOOLS_DIRS := \
 # comma-separated values for the "clean" target
 TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS))
 
-# "--define:release" implies "--stacktrace:off" and it cannot be added to config.nims
-ifeq ($(USE_LIBBACKTRACE), 0)
-NIM_PARAMS := $(NIM_PARAMS) -d:debug -d:disable_libbacktrace
-else
-NIM_PARAMS := $(NIM_PARAMS) -d:release
-endif
-
 .PHONY: \
  all \
  $(TOOLS) \
- build-system-checks \
  deps \
  update \
  nimbus \
@@ -53,20 +45,33 @@ endif
  wrappers-static \
  libbacktrace
 
+ifeq ($(NIM_PARAMS),)
+# "variables.mk" was not included, so we update the submodules.
+GIT_SUBMODULE_UPDATE := git submodule update --init --recursive
+.DEFAULT:
+ +@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \
+ $(GIT_SUBMODULE_UPDATE); \
+ echo
+# Now that the included *.mk files appeared, and are newer than this file, Make will restart itself:
+# https://www.gnu.org/software/make/manual/make.html#Remaking-Makefiles
+#
+# After restarting, it will execute its original goal, so we don't have to start a child Make here
+# with "$(MAKE) $(MAKECMDGOALS)". Isn't hidden control flow great?
+
+else # "variables.mk" was included. Business as usual until the end of this file.
+
 # default target, because it's the first one that doesn't start with '.'
-all: build-system-checks $(TOOLS) nimbus
+all: | $(TOOLS) nimbus
 
 # must be included after the default target
 -include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk
 
-GIT_SUBMODULE_UPDATE := git submodule update --init --recursive
-build-system-checks:
- @[[ -e "$(BUILD_SYSTEM_DIR)/makefiles" ]] || { \
- echo -e "'$(BUILD_SYSTEM_DIR)/makefiles' not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \
- $(GIT_SUBMODULE_UPDATE); \
- echo -e "\nYou can now run '$(MAKE)' again."; \
- exit 1; \
- }
+# "-d:release" implies "--stacktrace:off" and it cannot be added to config.nims
+ifeq ($(USE_LIBBACKTRACE), 0)
+NIM_PARAMS := $(NIM_PARAMS) -d:debug -d:disable_libbacktrace
+else
+NIM_PARAMS := $(NIM_PARAMS) -d:release
+endif
 
 deps: | deps-common nimbus.nims
 ifneq ($(USE_LIBBACKTRACE), 0)
@@ -95,7 +100,7 @@ nimbus.nims:
 
 # nim-libbacktrace
 libbacktrace:
- + $(MAKE) -C vendor/nim-libbacktrace BUILD_CXX_LIB=0 $(HANDLE_OUTPUT)
+ + $(MAKE) -C vendor/nim-libbacktrace --no-print-directory BUILD_CXX_LIB=0
 
 # builds and runs the test suite
 test: | build deps
@@ -167,3 +172,6 @@ wakunode: | build deps
 wakusim: | build deps wakunode
  echo -e $(BUILD_MSG) "build/$@" && \
  $(ENV_SCRIPT) nim wakusim $(NIM_PARAMS) nimbus.nims
+
+endif # "variables.mk" was not included
+

README.md

@@ -61,14 +61,9 @@ nix-shell default.nix
 #### POSIX-compatible OS
 
 ```bash
-make # The first `make` invocation will update all Git submodules and prompt you to run `make` again.
- # It's only required once per Git clone. You'll run `make update` after each `git pull`, in the future,
- # to keep those submodules up to date.
-
-# Run tests
-make test
-
-make nimbus # build the Nimbus binary
+# The first `make` invocation will update all Git submodules.
+# You'll run `make update` after each `git pull`, in the future, to keep those submodules up to date.
+make nimbus
 
 # See available command line options
 build/nimbus -- help
@@ -79,6 +74,9 @@ build/nimbus
 # Update to latest version
 git pull
 make update
+
+# Run tests
+make test
 ```
 
 To run a command that might use binaries from the Status Nim fork:
@@ -112,7 +110,6 @@ Install [Git for Windows](https://gitforwindows.org/) and use a "Git Bash" shell
 
 If you don't want to compile RocksDB and SQLite separately, you can fetch pre-compiled DLLs with:
 ```bash
-mingw32-make # this first invocation will update the Git submodules
 mingw32-make fetch-dlls # this will place the right DLLs for your architecture in the "build/" directory
 ```
 

azure-pipelines.yml

@@ -1,6 +1,8 @@
 jobs:
  - job: Windows
 
+ timeoutInMinutes: 80
+
  pool:
  vmImage: windows-latest
 
@@ -52,10 +54,8 @@ jobs:
  mv "$MINGW_DIR" /c/custom/
  cd ..
  export PATH="/c/custom/${MINGW_DIR}/bin:$PATH"
- echo "Fetching submodules"
  git config --global core.longpaths true
  git config --global core.autocrlf false
- git submodule --quiet update --init --recursive
  mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} CI_CACHE=NimBinaries update
  mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} fetch-dlls
  mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM}