[build] Add symlink_system_library calls for CSL (#40217)

* [build] Add `symlink_system_library` calls for CSL

We differentiate between a `USE_BINARYBUILDER_CSL=0` configuration
(where we copy the CSL libraries into the Julia build tree) versus a
`USE_SYSTEM_CSL=1` configuration (where we symlink them in a manner
similar to other system libararies)

* Expand `JL_PRIVATE_LIBS-1` for versioned library names as well

Without this, we fail to install things like `libgcc_s.so.1` when
installed into the private libdir by `symlink_system_libraries`.

Make.inc

@@ -596,6 +596,21 @@ else
  SHLIB_EXT := so
 endif
 
+ifeq ($(OS),WINNT)
+define versioned_libname
+$$(if $(2),$(1)-$(2).$(SHLIB_EXT),$(1).$(SHLIB_EXT))
+endef
+else ifeq ($(OS),Darwin)
+define versioned_libname
+$$(if $(2),$(1).$(2).$(SHLIB_EXT),$(1).$(SHLIB_EXT))
+endef
+else
+define versioned_libname
+$$(if $(2),$(1).$(SHLIB_EXT).$(2),$(1).$(SHLIB_EXT))
+endef
+endif
+
+
 ifeq ($(SHLIB_EXT), so)
 define SONAME_FLAGS
  -Wl,-soname=$1
@@ -1148,6 +1163,8 @@ BB_TRIPLET_LIBGFORTRAN := $(subst $(SPACE),-,$(filter-out cxx%,$(subst -,$(SPACE
 BB_TRIPLET_CXXABI := $(subst $(SPACE),-,$(filter-out libgfortran%,$(subst -,$(SPACE),$(BB_TRIPLET_LIBGFORTRAN_CXXABI))))
 BB_TRIPLET := $(subst $(SPACE),-,$(filter-out cxx%,$(filter-out libgfortran%,$(subst -,$(SPACE),$(BB_TRIPLET_LIBGFORTRAN_CXXABI)))))
 
+LIBGFORTRAN_VERSION := $(subst libgfortran,,$(filter libgfortran%,$(subst -,$(SPACE),$(BB_TRIPLET_LIBGFORTRAN))))
+
 # This is the set of projects that BinaryBuilder dependencies are hooked up for.
 BB_PROJECTS := BLASTRAMPOLINE OPENBLAS LLVM SUITESPARSE OPENLIBM GMP MBEDTLS LIBSSH2 NGHTTP2 MPFR CURL LIBGIT2 PCRE LIBUV LIBUNWIND DSFMT OBJCONV ZLIB P7ZIP CSL
 define SET_BB_DEFAULT
@@ -1480,8 +1497,12 @@ ifneq ($(findstring $(OS),Linux FreeBSD),)
 LIBGCC_NAME := libgcc_s.$(SHLIB_EXT).1
 endif
 
-
+# USE_SYSTEM_CSL causes it to get symlinked into build_private_shlibdir
+ifeq ($(USE_SYSTEM_CSL),1)
+LIBGCC_BUILD_DEPLIB := $(call dep_lib_path,$(build_libdir),$(build_private_shlibdir)/$(LIBGCC_NAME))
+else
 LIBGCC_BUILD_DEPLIB := $(call dep_lib_path,$(build_libdir),$(build_shlibdir)/$(LIBGCC_NAME))
+endif
 LIBGCC_INSTALL_DEPLIB := $(call dep_lib_path,$(libdir),$(private_shlibdir)/$(LIBGCC_NAME))
 
 # USE_SYSTEM_LIBM and USE_SYSTEM_OPENLIBM causes it to get symlinked into build_private_shlibdir

Makefile

@@ -290,8 +290,11 @@ endif
  done \
  done
  for suffix in $(JL_PRIVATE_LIBS-1) ; do \
- lib=$(build_private_libdir)/$${suffix}.$(SHLIB_EXT); \
- $(INSTALL_M) $$lib $(DESTDIR)$(private_libdir) ; \
+ for lib in $(build_private_libdir)/$${suffix}.$(SHLIB_EXT)*; do \
+ if [ "$${lib##*.}" != "dSYM" ]; then \
+ $(INSTALL_M) $$lib $(DESTDIR)$(private_libdir) ; \
+ fi \
+ done \
  done
 endif
  # Install `7z` into libexec/

base/Makefile

@@ -169,53 +169,69 @@ endif
 # echo "$$P"
 
 define symlink_system_library
-symlink_$1: $$(build_private_libdir)/$1.$$(SHLIB_EXT)
-$$(build_private_libdir)/$1.$$(SHLIB_EXT):
- REALPATH=`$$(call spawn,$$(build_depsbindir)/libwhich) -p $$(notdir $$@)` && \
- $$(call resolve_path,REALPATH) && \
- [ -e "$$$$REALPATH" ] && \
- ([ ! -e "$$@" ] || rm "$$@") && \
- echo ln -sf "$$$$REALPATH" "$$@" && \
- ln -sf "$$$$REALPATH" "$$@"
-ifneq ($2,)
-ifneq ($$(USE_SYSTEM_$2),0)
-SYMLINK_SYSTEM_LIBRARIES += symlink_$1
-endif
+libname_$2 := $$(notdir $(call versioned_libname,$2,$3))
+libpath_$2 := $$(shell $$(call spawn,$$(build_depsbindir)/libwhich) -p $$(libname_$2) 2>/dev/null)
+symlink_$2: $$(build_private_libdir)/$$(libname_$2)
+$$(build_private_libdir)/$$(libname_$2):
+ @if [ -e "$$(libpath_$2)" ]; then \
+ REALPATH=$$(libpath_$2); \
+ $$(call resolve_path,REALPATH) && \
+ [ -e "$$$$REALPATH" ] && \
+ ([ ! -e "$$@" ] || rm "$$@") && \
+ echo ln -sf "$$$$REALPATH" "$$@" && \
+ ln -sf "$$$$REALPATH" "$$@"; \
+ else \
+ if [ "$4" != "ALLOW_FAILURE" ]; then \
+ echo "System library symlink failure: Unable to locate $$(libname_$2) on your system!" >&2; \
+ false; \
+ fi; \
+ fi
+ifneq ($$(USE_SYSTEM_$1),0)
+SYMLINK_SYSTEM_LIBRARIES += symlink_$2
 endif
 endef
 
 # the following excludes: libuv.a, libutf8proc.a
 
-$(eval $(call symlink_system_library,$(LIBMNAME)))
 ifneq ($(USE_SYSTEM_LIBM),0)
-SYMLINK_SYSTEM_LIBRARIES += symlink_$(LIBMNAME)
+$(eval $(call symlink_system_library,LIBM,$(LIBMNAME)))
 else ifneq ($(USE_SYSTEM_OPENLIBM),0)
-SYMLINK_SYSTEM_LIBRARIES += symlink_$(LIBMNAME)
+$(eval $(call symlink_system_library,OPENLIBM,$(LIBMNAME)))
 endif
 
-$(eval $(call symlink_system_library,libpcre2-8,PCRE))
-$(eval $(call symlink_system_library,libdSFMT,DSFMT))
-$(eval $(call symlink_system_library,$(LIBBLASNAME),BLAS))
+$(eval $(call symlink_system_library,CSL,libgcc_s,1))
+ifneq (,$(LIBGFORTRAN_VERSION))
+$(eval $(call symlink_system_library,CSL,libgfortran,$(LIBGFORTRAN_VERSION)))
+endif
+$(eval $(call symlink_system_library,CSL,libquadmath,0))
+$(eval $(call symlink_system_library,CSL,libstdc++,6))
+# We allow libssp, libatomic and libgomp to fail as they are not available on all systems
+$(eval $(call symlink_system_library,CSL,libssp,0,ALLOW_FAILURE))
+$(eval $(call symlink_system_library,CSL,libatomic,1,ALLOW_FAILURE))
+$(eval $(call symlink_system_library,CSL,libgomp,1,ALLOW_FAILURE))
+$(eval $(call symlink_system_library,PCRE,libpcre2-8))
+$(eval $(call symlink_system_library,DSFMT,libdSFMT))
+$(eval $(call symlink_system_library,BLAS,$(LIBBLASNAME)))
 ifneq ($(LIBLAPACKNAME),$(LIBBLASNAME))
-$(eval $(call symlink_system_library,$(LIBLAPACKNAME),LAPACK))
+$(eval $(call symlink_system_library,LAPACK,$(LIBLAPACKNAME)))
 endif
-$(eval $(call symlink_system_library,libgmp,GMP))
-$(eval $(call symlink_system_library,libmpfr,MPFR))
-$(eval $(call symlink_system_library,libmbedtls,MBEDTLS))
-$(eval $(call symlink_system_library,libmbedcrypto,MBEDTLS))
-$(eval $(call symlink_system_library,libmbedx509,MBEDTLS))
-$(eval $(call symlink_system_library,libssh2,LIBSSH2))
-$(eval $(call symlink_system_library,libnghttp2,NGHTTP2))
-$(eval $(call symlink_system_library,libcurl,CURL))
-$(eval $(call symlink_system_library,libgit2,LIBGIT2))
-$(eval $(call symlink_system_library,libamd,SUITESPARSE))
-$(eval $(call symlink_system_library,libcamd,SUITESPARSE))
-$(eval $(call symlink_system_library,libccolamd,SUITESPARSE))
-$(eval $(call symlink_system_library,libcholmod,SUITESPARSE))
-$(eval $(call symlink_system_library,libcolamd,SUITESPARSE))
-$(eval $(call symlink_system_library,libumfpack,SUITESPARSE))
-$(eval $(call symlink_system_library,libspqr,SUITESPARSE))
-$(eval $(call symlink_system_library,libsuitesparseconfig,SUITESPARSE))
+$(eval $(call symlink_system_library,GMP,libgmp))
+$(eval $(call symlink_system_library,MPFR,libmpfr))
+$(eval $(call symlink_system_library,MBEDTLS,libmbedtls))
+$(eval $(call symlink_system_library,MBEDTLS,libmbedcrypto))
+$(eval $(call symlink_system_library,MBEDTLS,libmbedx509))
+$(eval $(call symlink_system_library,LIBSSH2,libssh2))
+$(eval $(call symlink_system_library,NGHTTP2,libnghttp2))
+$(eval $(call symlink_system_library,CURL,libcurl))
+$(eval $(call symlink_system_library,LIBGIT2,libgit2))
+$(eval $(call symlink_system_library,SUITESPARSE,libamd))
+$(eval $(call symlink_system_library,SUITESPARSE,libcamd))
+$(eval $(call symlink_system_library,SUITESPARSE,libccolamd))
+$(eval $(call symlink_system_library,SUITESPARSE,libcholmod))
+$(eval $(call symlink_system_library,SUITESPARSE,libcolamd))
+$(eval $(call symlink_system_library,SUITESPARSE,libumfpack))
+$(eval $(call symlink_system_library,SUITESPARSE,libspqr))
+$(eval $(call symlink_system_library,SUITESPARSE,libsuitesparseconfig))
 # EXCLUDED LIBRARIES (installed/used, but not vendored for use with dlopen):
 # libunwind
 endif # WINNT

deps/csl.mk

@@ -19,48 +19,34 @@ $$(build_shlibdir)/$(1): | $$(build_shlibdir)
  [ -n "$$$${SRC_LIB}" ] && cp $$$${SRC_LIB} $$(build_shlibdir)
 endef
 
-ifeq ($(OS),WINNT)
-define gen_libname
-$$(if $(2),lib$(1)-$(2).$(SHLIB_EXT),lib$(1).$(SHLIB_EXT))
-endef
-else ifeq ($(OS),Darwin)
-define gen_libname
-$$(if $(2),lib$(1).$(2).$(SHLIB_EXT),lib$(1).$(SHLIB_EXT))
-endef
-else
-define gen_libname
-$$(if $(2),lib$(1).$(SHLIB_EXT).$(2),lib$(1).$(SHLIB_EXT))
-endef
-endif
-
 # libgfortran has multiple names; we're just going to copy any version we can find
 # Since we're only looking in the location given by `$(FC)` this should only succeed for one.
-$(eval $(call copy_csl,$(call gen_libname,gfortran,3)))
-$(eval $(call copy_csl,$(call gen_libname,gfortran,4)))
-$(eval $(call copy_csl,$(call gen_libname,gfortran,5)))
+$(eval $(call copy_csl,$(call versioned_libname,libgfortran,3)))
+$(eval $(call copy_csl,$(call versioned_libname,libgfortran,4)))
+$(eval $(call copy_csl,$(call versioned_libname,libgfortran,5)))
 
 # These are all libraries that we should always have
-$(eval $(call copy_csl,$(call gen_libname,quadmath,0)))
-$(eval $(call copy_csl,$(call gen_libname,stdc++,6)))
-$(eval $(call copy_csl,$(call gen_libname,ssp,0)))
-$(eval $(call copy_csl,$(call gen_libname,atomic,1)))
-$(eval $(call copy_csl,$(call gen_libname,gomp,1)))
+$(eval $(call copy_csl,$(call versioned_libname,libquadmath,0)))
+$(eval $(call copy_csl,$(call versioned_libname,libstdc++,6)))
+$(eval $(call copy_csl,$(call versioned_libname,libssp,0)))
+$(eval $(call copy_csl,$(call versioned_libname,libatomic,1)))
+$(eval $(call copy_csl,$(call versioned_libname,libgomp,1)))
 
 ifeq ($(OS),WINNT)
 # Windwos has special gcc_s names
 ifeq ($(ARCH),i686)
-$(eval $(call copy_csl,$(call gen_libname,gcc_s_sjlj,1)))
+$(eval $(call copy_csl,$(call versioned_libname,libgcc_s_sjlj,1)))
 else
-$(eval $(call copy_csl,$(call gen_libname,gcc_s_seh,1)))
+$(eval $(call copy_csl,$(call versioned_libname,libgcc_s_seh,1)))
 endif
 else
-$(eval $(call copy_csl,$(call gen_libname,gcc_s,1)))
+$(eval $(call copy_csl,$(call versioned_libname,libgcc_s,1)))
 endif
 # winpthread is only Windows, pthread is only others
 ifeq ($(OS),WINNT)
-$(eval $(call copy_csl,$(call gen_libname,winpthread,1)))
+$(eval $(call copy_csl,$(call versioned_libname,libwinpthread,1)))
 else
-$(eval $(call copy_csl,$(call gen_libname,pthread,0)))
+$(eval $(call copy_csl,$(call versioned_libname,libpthread,0)))
 endif
 
 get-csl: