Add new checksum refreshing infrastructure (JuliaLang#38465)

Co-authored-by: Jameson Nash <[email protected]>
inkydragon · Nov 17, 2020 · 85000b1 · 85000b1
1 parent 3365b4e
commit 85000b1
Show file tree

Hide file tree

Showing 24 changed files with 161 additions and 86 deletions.
diff --git a/contrib/refresh_bb_tarballs.sh b/contrib/refresh_bb_tarballs.sh
diff --git a/contrib/refresh_checksums.mk b/contrib/refresh_checksums.mk
@@ -0,0 +1,83 @@
+# This file is a part of Julia. License is MIT: https://julialang.org/license
+
+# Invoke this with no arguments to refresh all tarballs, or with a project name to refresh only that project.
+#
+# Example:
+# make -f contrib/refresh_checksums.mk gmp
+
+SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST))))
+JULIAHOME := $(abspath $(SRCDIR)/..)
+
+# Default target that will have everything else added to it as a dependency
+all:
+
+# Get this list via:
+# using BinaryBuilder
+# print("TRIPLETS=\"$(join(sort(triplet.(BinaryBuilder.supported_platforms(;experimental=true))), " "))\"")
+TRIPLETS=aarch64-apple-darwin aarch64-linux-gnu aarch64-linux-musl armv6l-linux-gnueabihf armv6l-linux-musleabihf armv7l-linux-gnueabihf armv7l-linux-musleabihf i686-linux-gnu i686-linux-musl i686-w64-mingw32 powerpc64le-linux-gnu x86_64-apple-darwin x86_64-linux-gnu x86_64-linux-musl x86_64-unknown-freebsd x86_64-w64-mingw32
+CLANG_TRIPLETS=$(filter %-darwin %-freebsd,$(TRIPLETS))
+NON_CLANG_TRIPLETS=$(filter-out %-darwin %-freebsd,$(TRIPLETS))
+
+# These are the projects currently using BinaryBuilder; both GCC-expanded and non-GCC-expanded:
+BB_PROJECTS=mbedtls libssh2 nghttp2 mpfr curl libgit2 pcre libuv unwind dsfmt objconv p7zip zlib suitesparse openlibm
+BB_GCC_EXPANDED_PROJECTS=openblas
+BB_CXX_EXPANDED_PROJECTS=gmp llvm
+# These are non-BB source-only deps
+NON_BB_PROJECTS=patchelf mozillacert lapack
+
+# Convert `openblas` to `OPENBLAS`
+define upper
+$(shell echo $(1) | tr 'a-z' 'A-Z')
+endef
+
+# If $(2) == `src`, this will generate a `USE_BINARYBUILDER_FOO=0` make flag
+# It will also generate a `FOO_BB_TRIPLET=$(2)` make flag.
+define make_flags
+USE_BINARYBUILDER_$(call upper,$(1))=$(if $(filter src,$(2)),0,1) $(call upper,$(1))_BB_TRIPLET=$(if $(filter src,$(2)),,$(2)) DEPS_GIT=0
+endef
+
+# checksum_bb_dep takes in (name, triplet), and generates a `checksum-$(1)-$(2)` target.
+# note that `"src"` is a special triplet value.
+define checksum_dep
+checksum-$(1)-$(2):
+ @-$(MAKE) -C "$(JULIAHOME)/deps" $(call make_flags,$(1),$(2)) checksum-$(1)
+
+# Add this guy to his project target (e.g. `make -f contrib/refresh_checksums.mk openblas`)
+$(1): checksum-$(1)-$(2)
+
+# Add this guy to the `all` default target
+all: checksum-$(1)-$(2)
+endef
+
+# Generate targets for source hashes for all our projects
+$(foreach project,$(BB_PROJECTS) $(BB_GCC_EXPANDED_PROJECTS) $(BB_CXX_EXPANDED_PROJECTS) $(NON_BB_PROJECTS),$(eval $(call checksum_dep,$(project),src)))
+
+# Generate targets for triplet-specific hashes for all our BB projects
+$(foreach project,$(BB_PROJECTS),$(foreach triplet,$(TRIPLETS),$(eval $(call checksum_dep,$(project),$(triplet)))))
+$(foreach project,$(BB_GCC_EXPANDED_PROJECTS),$(foreach triplet,$(TRIPLETS),$(foreach libgfortran_version,libgfortran3 libgfortran4 libgfortran5,$(eval $(call checksum_dep,$(project),$(triplet)-$(libgfortran_version))))))
+
+# Because MacOS and FreeBSD use clang, they don't actually use cxxstring_abi expansion:
+$(foreach project,$(BB_CXX_EXPANDED_PROJECTS),$(foreach triplet,$(NON_CLANG_TRIPLETS),$(foreach cxxstring_abi,cxx11 cxx03,$(eval $(call checksum_dep,$(project),$(triplet)-$(cxxstring_abi))))))
+$(foreach project,$(BB_CXX_EXPANDED_PROJECTS),$(foreach triplet,$(CLANG_TRIPLETS),$(eval $(call checksum_dep,$(project),$(triplet)))))
+
+# External stdlibs
+checksum-stdlibs:
+ @-$(MAKE) -C "$(JULIAHOME)/stdlib" checksumall
+all: checksum-stdlibs
+
+# doc unicode data
+checksum-doc-unicodedata:
+ @-$(MAKE) -C "$(JULIAHOME)/doc" checksum-unicodedata
+all: checksum-doc-unicodedata
+
+# Special LLVM source hashes for optional targets
+checksum-llvm-special-src:
+ @-$(MAKE) -C "${JULIAHOME}/deps" USE_BINARYBUILDER_LLVM=0 DEPS_GIT=0 BUILD_LLDB=1 BUILD_LLVM_CLANG=1 BUILD_CUSTOM_LIBCXX=1 checksum-llvm
+all: checksum-llvm-special-src
+
+# This file is completely phony
+.PHONY: checksum-*
+
+# Debugging helper
+print-%:
+ @echo '$*=$(subst ','\'',$($*))'
diff --git a/deps/blas.mk b/deps/blas.mk
@@ -1,4 +1,5 @@
 ## OpenBLAS ##
+ifneq ($(USE_BINARYBUILDER_OPENBLAS), 1)
 # LAPACK is built into OpenBLAS by default
 OPENBLAS_GIT_URL := git:https://github.com/xianyi/OpenBLAS.git
 OPENBLAS_TAR_URL = https://api.github.com/repos/xianyi/OpenBLAS/tarball/$1
@@ -92,8 +93,6 @@ endif
 # Do not overwrite the "-j" flag
 OPENBLAS_BUILD_OPTS += MAKE_NB_JOBS=0
 
-ifneq ($(USE_BINARYBUILDER_OPENBLAS), 1)
-
 $(BUILDDIR)/$(OPENBLAS_SRC_DIR)/openblas-winexit.patch-applied: $(BUILDDIR)/$(OPENBLAS_SRC_DIR)/source-extracted
  cd $(BUILDDIR)/$(OPENBLAS_SRC_DIR) && \
  patch -p1 -f < $(SRCDIR)/patches/openblas-winexit.patch
@@ -164,6 +163,9 @@ $(BUILDDIR)/lapack-$(LAPACK_VER)/source-extracted: $(SRCCACHE)/lapack-$(LAPACK_V
  cp $(dir $@)INSTALL/make.inc.gfortran $(dir $@)make.inc
  echo 1 > $@
 
+checksum-lapack: $(SRCCACHE)/lapack-$(LAPACK_VER).tgz
+ $(JLCHECKSUM) $<
+
 ifeq ($(USE_SYSTEM_BLAS), 0)
 $(BUILDDIR)/lapack-$(LAPACK_VER)/build-compiled0: | $(build_prefix)/manifest/openblas
 else ifeq ($(OS),Darwin)

diff --git a/deps/curl.mk b/deps/curl.mk
@@ -26,6 +26,9 @@ $(SRCCACHE)/curl-$(CURL_VER)/source-extracted: $(SRCCACHE)/curl-$(CURL_VER).tar.
  touch -c $(SRCCACHE)/curl-$(CURL_VER)/configure # old target
  echo 1 > $@
 
+checksum-curl: $(SRCCACHE)/curl-$(CURL_VER).tar.bz2
+ $(JLCHECKSUM) $<
+
 $(BUILDDIR)/curl-$(CURL_VER)/build-configured: $(SRCCACHE)/curl-$(CURL_VER)/source-extracted
  mkdir -p $(dir $@)
  cd $(dir $@) && \

diff --git a/deps/dsfmt.mk b/deps/dsfmt.mk
@@ -26,6 +26,9 @@ $(BUILDDIR)/dsfmt-$(DSFMT_VER)/source-extracted: $(SRCCACHE)/dsfmt-$(DSFMT_VER).
  cd $(dir $@) && patch < $(SRCDIR)/patches/dSFMT.c.patch
  echo 1 > $@
 
+checksum-dsfmt: $(SRCCACHE)/dsfmt-$(DSFMT_VER).tar.gz
+ $(JLCHECKSUM) $<
+
 $(BUILDDIR)/dsfmt-$(DSFMT_VER)/build-compiled: $(BUILDDIR)/dsfmt-$(DSFMT_VER)/source-extracted
  cd $(dir $<) && \
  $(CC) $(CPPFLAGS) $(DSFMT_CFLAGS) $(LDFLAGS) dSFMT.c -o libdSFMT.$(SHLIB_EXT)

diff --git a/deps/gmp.mk b/deps/gmp.mk
@@ -19,6 +19,9 @@ $(SRCCACHE)/gmp-$(GMP_VER)/source-extracted: $(SRCCACHE)/gmp-$(GMP_VER).tar.bz2
  touch -c $(SRCCACHE)/gmp-$(GMP_VER)/configure # old target
  echo 1 > $@
 
+checksum-gmp: $(SRCCACHE)/gmp-$(GMP_VER).tar.bz2
+ $(JLCHECKSUM) $<
+
 $(SRCCACHE)/gmp-$(GMP_VER)/build-patched: $(SRCCACHE)/gmp-$(GMP_VER)/source-extracted
  cd $(dir $@) && patch -p1 < $(SRCDIR)/patches/gmp-exception.patch
  cd $(dir $@) && patch -p1 < $(SRCDIR)/patches/gmp_alloc_overflow_func.patch

diff --git a/deps/libgit2.mk b/deps/libgit2.mk
@@ -1,4 +1,5 @@
 ## libgit2
+ifneq ($(USE_BINARYBUILDER_LIBGIT2),1)
 
 LIBGIT2_GIT_URL := git:https://github.com/libgit2/libgit2.git
 LIBGIT2_TAR_URL = https://api.github.com/repos/libgit2/libgit2/tarball/$1
@@ -12,8 +13,6 @@ ifeq ($(USE_SYSTEM_MBEDTLS), 0)
 $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/build-configured: | $(build_prefix)/manifest/mbedtls
 endif
 
-ifneq ($(USE_BINARYBUILDER_LIBGIT2),1)
-
 LIBGIT2_OPTS := $(CMAKE_COMMON) -DCMAKE_BUILD_TYPE=Release -DTHREADSAFE=ON -DUSE_BUNDLED_ZLIB=ON
 ifeq ($(OS),WINNT)
 LIBGIT2_OPTS += -DWIN32=ON -DMINGW=ON
@@ -114,5 +113,8 @@ $(build_datarootdir)/julia/cert.pem: $(SRCCACHE)/cacert-$(MOZILLA_CACERT_VERSION
  mkdir -p $(build_datarootdir)/julia
  cp $< $@
 
+checksum-mozillacert: $(SRCCACHE)/cacert-$(MOZILLA_CACERT_VERSION).pem
+ $(JLCHECKSUM) $<
+
 # When "get"'ing libgit2, download the .pem
 get-libgit2: $(SRCCACHE)/cacert-$(MOZILLA_CACERT_VERSION).pem
diff --git a/deps/libssh2.mk b/deps/libssh2.mk
@@ -1,5 +1,5 @@
 ## libssh2
-
+ifneq ($(USE_BINARYBUILDER_LIBSSH2), 1)
 LIBSSH2_GIT_URL := git:https://github.com/libssh2/libssh2.git
 LIBSSH2_TAR_URL = https://api.github.com/repos/libssh2/libssh2/tarball/$1
 $(eval $(call git-external,libssh2,LIBSSH2,CMakeLists.txt,,$(SRCCACHE)))
@@ -8,7 +8,6 @@ ifeq ($(USE_SYSTEM_MBEDTLS), 0)
 $(BUILDDIR)/$(LIBSSH2_SRC_DIR)/build-configured: | $(build_prefix)/manifest/mbedtls
 endif
 
-ifneq ($(USE_BINARYBUILDER_LIBSSH2), 1)
 LIBSSH2_OPTS := $(CMAKE_COMMON) -DBUILD_SHARED_LIBS=ON -DBUILD_EXAMPLES=OFF \
  -DCMAKE_BUILD_TYPE=Release
 

diff --git a/deps/libuv.mk b/deps/libuv.mk
@@ -1,10 +1,9 @@
 ## LIBUV ##
+ifneq ($(USE_BINARYBUILDER_LIBUV),1)
 LIBUV_GIT_URL:=git:https://github.com/JuliaLang/libuv.git
 LIBUV_TAR_URL=https:https://api.github.com/repos/JuliaLang/libuv/tarball/$1
 $(eval $(call git-external,libuv,LIBUV,configure,,$(SRCCACHE)))
 
-ifneq ($(USE_BINARYBUILDER_LIBUV),1)
-
 UV_CFLAGS := -O2
 ifeq ($(USEMSVC), 1)
 UV_CFLAGS += -DBUILDING_UV_SHARED

diff --git a/deps/llvm.mk b/deps/llvm.mk
@@ -340,6 +340,7 @@ distclean-libcxx:
 distclean-libcxxabi:
  -rm -rf $(LLVM_LIBCXXABI_TAR) $(LLVM_SRC_DIR)/projects/libcxxabi $(LLVM_BUILD_DIR)/libcxxabi-build
 
+
 # We want to ensure that the libcxx linking flags don't get passed to the libcxx build, since it will
 # error on a fresh build
 LLVM_CMAKE += -DCMAKE_EXE_LINKER_FLAGS="$(LLVM_LDFLAGS) $(LLVM_LIBCXX_LDFLAGS)" \
@@ -355,6 +356,23 @@ LIBCXX_DEPENDENCY := $(build_libdir)/libc++abi.so.1.0 $(build_libdir)/libc++.so.
 get-llvm: get-libcxx get-libcxxabi
 endif
 
+checksum-llvm: $(LLVM_TAR) $(LLVM_CLANG_TAR) $(LLVM_COMPILER_RT_TAR) $(LLVM_LIBCXX_TAR) $(LLVM_LLDB_TAR)
+ifneq ($(LLVM_CLANG_TAR),)
+ $(JLCHECKSUM) $(LLVM_CLANG_TAR)
+endif
+ifneq ($(LLVM_COMPILER_RT_TAR),)
+ $(JLCHECKSUM) $(LLVM_COMPILER_RT_TAR)
+endif
+ifneq ($(LLVM_LIBCXX_TAR),)
+ $(JLCHECKSUM) $(LLVM_LIBCXX_TAR)
+endif
+ifneq ($(LLVM_VER),svn)
+ $(JLCHECKSUM) $(LLVM_TAR)
+endif
+ifneq ($(LLVM_LLDB_TAR),)
+ $(JLCHECKSUM) $(LLVM_LLDB_TAR)
+endif
+
 $(LLVM_SRC_DIR)/source-extracted: | $(LLVM_TAR) $(LLVM_CLANG_TAR) $(LLVM_COMPILER_RT_TAR) $(LLVM_LIBCXX_TAR) $(LLVM_LLDB_TAR)
 ifneq ($(LLVM_CLANG_TAR),)
  $(JLCHECKSUM) $(LLVM_CLANG_TAR)

diff --git a/deps/mbedtls.mk b/deps/mbedtls.mk
@@ -28,6 +28,9 @@ $(SRCCACHE)/$(MBEDTLS_SRC)/source-extracted: $(SRCCACHE)/$(MBEDTLS_SRC).tar.gz
  touch -c $(SRCCACHE)/$(MBEDTLS_SRC)/CMakeLists.txt # old target
  echo 1 > $@
 
+checksum-mbedtls: $(SRCCACHE)/$(MBEDTLS_SRC).tar.gz
+ $(JLCHECKSUM) $<
+
 $(BUILDDIR)/$(MBEDTLS_SRC)/build-configured: $(SRCCACHE)/$(MBEDTLS_SRC)/source-extracted
  mkdir -p $(dir $@)
  cd $(dir $@) && \

diff --git a/deps/mpfr.mk b/deps/mpfr.mk
@@ -35,6 +35,9 @@ $(SRCCACHE)/mpfr-$(MPFR_VER)/source-extracted: $(SRCCACHE)/mpfr-$(MPFR_VER).tar.
  touch -c $(SRCCACHE)/mpfr-$(MPFR_VER)/configure # old target
  echo 1 > $@
 
+checksum-mpfr: $(SRCCACHE)/mpfr-$(MPFR_VER).tar.bz2
+ $(JLCHECKSUM) $<
+
 $(BUILDDIR)/mpfr-$(MPFR_VER)/build-configured: $(SRCCACHE)/mpfr-$(MPFR_VER)/source-extracted
  mkdir -p $(dir $@)
  cd $(dir $@) && \

diff --git a/deps/nghttp2.mk b/deps/nghttp2.mk
@@ -11,6 +11,9 @@ $(SRCCACHE)/nghttp2-$(NGHTTP2_VER)/source-extracted: $(SRCCACHE)/nghttp2-$(NGHTT
  touch -c $(SRCCACHE)/nghttp2-$(NGHTTP2_VER)/configure # old target
  echo 1 > $@
 
+checksum-nghttp2: $(SRCCACHE)/nghttp2-$(NGHTTP2_VER).tar.bz2
+ $(JLCHECKSUM) $<
+
 $(BUILDDIR)/nghttp2-$(NGHTTP2_VER)/build-configured: $(SRCCACHE)/nghttp2-$(NGHTTP2_VER)/source-extracted
  mkdir -p $(dir $@)
  cd $(dir $@) && \

diff --git a/deps/openlibm.mk b/deps/openlibm.mk
@@ -1,13 +1,11 @@
 ## openlibm ##
-
-
+ifneq ($(USE_BINARYBUILDER_OPENLIBM), 1)
 OPENLIBM_GIT_URL := git:https://github.com/JuliaMath/openlibm.git
 OPENLIBM_TAR_URL = https://api.github.com/repos/JuliaMath/openlibm/tarball/$1
 $(eval $(call git-external,openlibm,OPENLIBM,,,$(BUILDDIR)))
 
 OPENLIBM_FLAGS := ARCH="$(ARCH)" REAL_ARCH="$(MARCH)" CC="$(CC)" FC="$(FC)" AR="$(AR)" OS="$(OS)" USECLANG=$(USECLANG) USEGCC=$(USEGCC)
 
-ifneq ($(USE_BINARYBUILDER_OPENLIBM), 1)
 $(BUILDDIR)/$(OPENLIBM_SRC_DIR)/build-compiled: $(BUILDDIR)/$(OPENLIBM_SRC_DIR)/source-extracted
  $(MAKE) -C $(dir $<) $(OPENLIBM_FLAGS) $(MAKE_COMMON)
  echo 1 > $@

diff --git a/deps/p7zip.mk b/deps/p7zip.mk
@@ -11,6 +11,9 @@ $(BUILDDIR)/p7zip-$(P7ZIP_VER)/source-extracted: $(SRCCACHE)/p7zip-$(P7ZIP_VER).
  cd $(dir $@) && $(TAR) --strip-components 1 -jxf $<
  echo $1 > $@
 
+checksum-p7zip: $(SRCCACHE)/p7zip-$(P7ZIP_VER).tar.bz2
+ $(JLCHECKSUM) $<
+
 $(BUILDDIR)/p7zip-$(P7ZIP_VER)/p7zip-12-CVE-2016-9296.patch-applied: $(BUILDDIR)/p7zip-$(P7ZIP_VER)/source-extracted
  cd $(dir $@) && patch -p1 -f < $(SRCDIR)/patches/p7zip-12-CVE-2016-9296.patch
  echo 1 > $@

diff --git a/deps/patchelf.mk b/deps/patchelf.mk
@@ -9,6 +9,9 @@ $(SRCCACHE)/patchelf-$(PATCHELF_VER)/source-extracted: $(SRCCACHE)/patchelf-$(PA
  touch -c $(SRCCACHE)/patchelf-$(PATCHELF_VER)/configure # old target
  echo 1 > $@
 
+checksum-patchelf: $(SRCCACHE)/patchelf-$(PATCHELF_VER).tar.gz
+ $(JLCHECKSUM) $<
+
 $(BUILDDIR)/patchelf-$(PATCHELF_VER)/build-configured: $(SRCCACHE)/patchelf-$(PATCHELF_VER)/source-extracted | $(LIBCXX_DEPENDENCY)
  mkdir -p $(dir $@)
  cd $(dir $@) && \

diff --git a/deps/pcre.mk b/deps/pcre.mk
@@ -15,6 +15,9 @@ $(SRCCACHE)/pcre2-$(PCRE_VER)/source-extracted: $(SRCCACHE)/pcre2-$(PCRE_VER).ta
  touch -c $(SRCCACHE)/pcre2-$(PCRE_VER)/configure # old target
  echo $1 > $@
 
+checksum-pcre2: $(SRCCACHE)/pcre2-$(PCRE_VER).tar.bz2
+ $(JLCHECKSUM) $<
+
 $(BUILDDIR)/pcre2-$(PCRE_VER)/build-configured: $(SRCCACHE)/pcre2-$(PCRE_VER)/source-extracted
  mkdir -p $(dir $@)
  cd $(dir $@) && \

diff --git a/deps/suitesparse.mk b/deps/suitesparse.mk
@@ -41,6 +41,9 @@ $(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER)/source-extracted: $(SRCCACHE)/SuiteSp
  $(TAR) -C $(dir $@) --strip-components 1 -zxf $<
  echo 1 > $@
 
+checksum-suitesparse: $(SRCCACHE)/SuiteSparse-$(SUITESPARSE_VER).tar.gz
+ $(JLCHECKSUM) $<
+
 $(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER)/SuiteSparse-winclang.patch-applied: $(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER)/source-extracted
  cd $(dir $@) && patch -p0 < $(SRCDIR)/patches/SuiteSparse-winclang.patch
  echo 1 > $@

diff --git a/deps/tools/bb-install.mk b/deps/tools/bb-install.mk
@@ -56,6 +56,10 @@ $$(build_prefix)/manifest/$(strip $1): $$(SRCCACHE)/$$($(2)_BB_BASENAME) | $(bui
  $(UNTAR) $$< -C $$(build_prefix)
  echo '$$(UNINSTALL_$(strip $1))' > $$@
 
+# Special "checksum-foo" target to speed up `contrib/refresh_checksums.sh`
+checksum-$(1): $$(SRCCACHE)/$$($(2)_BB_BASENAME)
+ $$(JLCHECKSUM) $$<
+
 clean-bb-download-$(1):
  rm -f $$(SRCCACHE)/$$($(2)_BB_BASENAME)