Fix executable and more library RPATHs on FreeBSD (JuliaLang#27627)

First of all, we need to add libgfortran, libgcc_s, and libquadmath to
`JL_PRIVATE_LIBS-0` instead of `JL_LIBS`. This ensures that they get
installed into `lib/julia`, where our dependencies are supposed to live,
instead of `lib`, where only libjulia should live.

Now we need to call `fixup-rpath.sh` on more directories. Currently
we're only calling it on `lib`, but we also need to call it on
`lib/julia` and even on `bin`, because FreeBSD seems to load libraries
based on the executable's RPATH rather than libjulia's RPATH.

AND, speaking of the executable's RPATH, we need to make sure we have
`lib/julia` in there; currently we only have `lib`. So in an isolated
environment outside of the build directory, Julia won't be able to load
any of its dependencies, which is bad. This is accomplished with a
simple adjustment in `Make.inc`.

To make sure that our dependencies can find each other properly, we need
to add `RPATH_ESCAPED_ORIGIN` to `LDFLAGS` in `CONFIGURE_COMMON`. That
way, every dependency that uses `configure` will get its RPATH set
appropriately.

Make.inc

@@ -958,6 +958,9 @@ else ifeq ($(OS), Darwin)
  RPATH_LIB := -Wl,-rpath,'@loader_path/julia/' -Wl,-rpath,'@loader_path/'
 else
  RPATH := -Wl,-rpath,'$$ORIGIN/$(build_libdir_rel)' -Wl,-rpath-link,$(build_shlibdir) -Wl,-z,origin
+ifeq ($(OS), FreeBSD)
+ RPATH += -Wl,-rpath,'$$ORIGIN/$(build_private_libdir_rel)'
+endif
  RPATH_ORIGIN := -Wl,-rpath,'$$ORIGIN' -Wl,-z,origin
  RPATH_ESCAPED_ORIGIN := -Wl,-rpath,'\$$\$$ORIGIN' -Wl,-z,origin
  RPATH_LIB := -Wl,-rpath,'$$ORIGIN/julia' -Wl,-rpath,'$$ORIGIN' -Wl,-z,origin

Makefile

@@ -248,17 +248,22 @@ endif
 endif
 endif
 
+# On FreeBSD, /lib/libgcc_s.so.1 is incompatible with Fortran; to use Fortran on FreeBSD,
+# we need to link to the libgcc_s that ships with the same GCC version used by libgfortran.
+# To work around this, we copy the GCC libraries we need, namely libgfortran, libgcc_s,
+# and libquadmath, into our build library directory, $(build_libdir). We also add them to
+# JL_PRIVATE_LIBS-0 so that they know where they need to live at install time.
 ifeq ($(OS),FreeBSD)
 define std_so
-julia-deps: | $$(build_libdir)/lib$(1).so
-$$(build_libdir)/lib$(1).so: | $$(build_libdir)
- $$(INSTALL_M) $$(GCCPATH)/lib$(1).so* $$(build_libdir)
-JL_LIBS += $(1)
+julia-deps: | $$(build_libdir)/$(1).so
+$$(build_libdir)/$(1).so: | $$(build_libdir)
+ $$(INSTALL_M) $$(GCCPATH)/$(1).so* $$(build_libdir)
+JL_PRIVATE_LIBS-0 += $(1)
 endef
 
-$(eval $(call std_so,gfortran))
-$(eval $(call std_so,gcc_s))
-$(eval $(call std_so,quadmath))
+$(eval $(call std_so,libgfortran))
+$(eval $(call std_so,libgcc_s))
+$(eval $(call std_so,libquadmath))
 endif # FreeBSD
 
 ifeq ($(OS),WINNT)
@@ -381,6 +386,15 @@ endif
  # On FreeBSD, remove the build's libdir from each library's RPATH
 ifeq ($(OS),FreeBSD)
  $(JULIAHOME)/contrib/fixup-rpath.sh $(build_depsbindir)/patchelf $(DESTDIR)$(libdir) $(build_libdir)
+ $(JULIAHOME)/contrib/fixup-rpath.sh $(build_depsbindir)/patchelf $(DESTDIR)$(private_libdir) $(build_libdir)
+ $(JULIAHOME)/contrib/fixup-rpath.sh $(build_depsbindir)/patchelf $(DESTDIR)$(bindir) $(build_libdir)
+ # Set libgfortran's RPATH to ORIGIN instead of GCCPATH. It's only libgfortran that
+ # needs to be fixed here, as libgcc_s and libquadmath don't have RPATHs set. If we
+ # don't set libgfortran's RPATH, it won't be able to find its friends on systems
+ # that don't have the exact GCC port installed used for the build.
+ for lib in $(DESTDIR)$(private_libdir)/libgfortran*$(SHLIB_EXT)*; do \
+ $(build_depsbindir)/patchelf --set-rpath '$$ORIGIN' $$lib; \
+ done
 endif
 
  mkdir -p $(DESTDIR)$(sysconfdir)
@@ -409,7 +423,11 @@ ifneq ($(DESTDIR),)
 endif
  @$(MAKE) -C $(BUILDROOT) -f $(JULIAHOME)/Makefile install
  cp $(JULIAHOME)/LICENSE.md $(BUILDROOT)/julia-$(JULIA_COMMIT)
-ifneq ($(OS), WINNT)
+ # Run fixup-libgfortran on all platforms but Windows and FreeBSD. On FreeBSD we
+ # pull in the GCC libraries earlier and use them for the build to make sure we
+ # don't inadvertently link to /lib/libgcc_s.so.1, which is incompatible with
+ # libgfortran.
+ifeq (,$(findstring $(OS),FreeBSD WINNT))
  -$(CUSTOM_LD_LIBRARY_PATH) PATH=$(PATH):$(build_depsbindir) $(JULIAHOME)/contrib/fixup-libgfortran.sh $(DESTDIR)$(private_libdir)
 endif
 ifeq ($(OS), Linux)

contrib/fixup-rpath.sh

@@ -1,17 +1,17 @@
 #!/bin/sh
-# Usage: fixup-rpath.sh <patchelf path> <install libdir> <build libdir>
+# Usage: fixup-rpath.sh <patchelf path> <dir to process> <build libdir>
 
 if [ $# -ne 3 ]; then
  echo "Incorrect number of arguments: Expected 3, got $#"
- echo "Usage: fixup-rpath.sh <patchelf path> <install libdir> <build libdir>"
+ echo "Usage: fixup-rpath.sh <patchelf path> <directory to process> <build libdir>"
  exit 1
 fi
 
 patchelf="$1"
-install_libdir="$2"
+executable_dir="$2"
 build_libdir="$3"
 
-for lib in ${install_libdir}/*.so*; do
+for lib in $(find ${executable_dir} -type f -perm -111); do
  # First get the current RPATH
  rpath="$(${patchelf} --print-rpath ${lib})"
 
@@ -25,6 +25,8 @@ for lib in ${install_libdir}/*.so*; do
  # Drop the trailing :
  new_rpath="${new_rpath%?}"
 
+ echo " Setting RPATH for ${lib} to '${new_rpath}'"
+
  # Now set the new RPATH
  ${patchelf} --set-rpath "${new_rpath}" ${lib}
 done

deps/tools/common.mk

@@ -13,7 +13,7 @@ ifneq ($(USEMSVC), 1)
 CONFIGURE_COMMON += LDFLAGS="$(LDFLAGS) -Wl,--stack,8388608"
 endif
 else
-CONFIGURE_COMMON += LDFLAGS="$(LDFLAGS)"
+CONFIGURE_COMMON += LDFLAGS="$(LDFLAGS) $(RPATH_ESCAPED_ORIGIN)"
 endif
 CONFIGURE_COMMON += F77="$(FC)" CC="$(CC)" CXX="$(CXX)" LD="$(LD)"