Skip to content

Commit

Permalink
Fake JLLs for all packages (JuliaLang#38347)
Browse files Browse the repository at this point in the history
  • Loading branch information
@staticfloat
staticfloat committed Dec 2, 2020
1 parent a178ac2 commit 2eccd8e
Show file tree
Hide file tree
Showing 1,452 changed files with 2,536 additions and 921 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,3 +31,4 @@
/perf*
.DS_Store
.idea/*
.vscode/*
40 changes: 26 additions & 14 deletions Make.inc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ OPENBLAS_USE_THREAD:=1
# Flags for using libraries available on the system instead of building them.
# Please read the notes around usage of SYSTEM flags in README.md
# Issues resulting from use of SYSTEM versions will generally not be accepted.
USE_SYSTEM_CSL:=0
USE_SYSTEM_LLVM:=0
USE_SYSTEM_LIBUNWIND:=0
DISABLE_LIBUNWIND:=0
Expand Down Expand Up @@ -93,6 +94,15 @@ WITH_GC_DEBUG_ENV := 0
# Prevent picking up $ARCH from the environment variables
ARCH:=
# Literal values that are hard to use in Makefiles otherwise:
define newline # a literal \n
endef
COMMA:=,
SPACE:=$(eval) $(eval)
# We need python for things like BB triplet recognition and relative path computation.
# We don't really care about version, generally, so just find something that works:
PYTHON := "$(shell which python 2>/dev/null || which python3 2>/dev/null || which python2 2>/dev/null || echo "{python|python3|python2} not found")"
Expand Down Expand Up @@ -999,19 +1009,25 @@ ifeq ($(USE_SYSTEM_LLVM), 1)
JCPPFLAGS+=-DSYSTEM_LLVM
endif # SYSTEM_LLVM

# Windows builds need a little help finding the LLVM libraries for llvm-config
LLVM_CONFIG_PATH_FIX :=
ifeq ($(OS),WINNT)
LLVM_CONFIG_PATH_FIX := PATH="$(PATH):$(build_bindir)"
endif

ifeq ($(BUILD_OS),$(OS))
LLVM_CONFIG_HOST := $(LLVM_CONFIG)
else
LLVM_CONFIG_HOST := $(basename $(LLVM_CONFIG))-host$(BUILD_EXE)
ifeq (exists, $(shell [ -f '$(LLVM_CONFIG_HOST)' ] && echo exists ))
ifeq ($(shell $(LLVM_CONFIG_HOST) --version),3.3)
ifeq ($(shell $(LLVM_CONFIG_PATH_FIX) $(LLVM_CONFIG_HOST) --version),3.3)
# llvm-config-host <= 3.3 is broken, use llvm-config instead (in an emulator)
# use delayed expansion (= not :=) because spawn isn't defined until later
LLVM_CONFIG_HOST = $(call spawn,$(LLVM_CONFIG))
LLVM_CONFIG_HOST = $(LLVM_CONFIG_PATH_FIX) $(call spawn,$(LLVM_CONFIG))
endif
else
# llvm-config-host does not exist (cmake build)
LLVM_CONFIG_HOST = $(call spawn,$(LLVM_CONFIG))
LLVM_CONFIG_HOST = $(LLVM_CONFIG_PATH_FIX) $(call spawn,$(LLVM_CONFIG))
endif
endif
Expand Down Expand Up @@ -1129,8 +1145,14 @@ endif
USE_BINARYBUILDER ?= 0
endif

# Auto-detect triplet once, create different versions that we use as defaults below for each BB install target
BB_TRIPLET_LIBGFORTRAN_CXXABI := $(shell $(call invoke_python,$(JULIAHOME)/contrib/normalize_triplet.py) $(or $(XC_HOST),$(XC_HOST),$(BUILD_MACHINE)) "$(shell $(FC) --version | head -1)" "$(or $(shell echo '\#include <string>' | $(CXX) $(CXXFLAGS) -x c++ -dM -E - | grep _GLIBCXX_USE_CXX11_ABI | awk '{ print $$3 }' ),1)")
BB_TRIPLET_LIBGFORTRAN := $(subst $(SPACE),-,$(filter-out cxx%,$(subst -,$(SPACE),$(BB_TRIPLET_LIBGFORTRAN_CXXABI))))
BB_TRIPLET_CXXABI := $(subst $(SPACE),-,$(filter-out libgfortran%,$(subst -,$(SPACE),$(BB_TRIPLET_LIBGFORTRAN_CXXABI))))
BB_TRIPLET := $(subst $(SPACE),-,$(filter-out cxx%,$(filter-out libgfortran%,$(subst -,$(SPACE),$(BB_TRIPLET_LIBGFORTRAN_CXXABI)))))

# This is the set of projects that BinaryBuilder dependencies are hooked up for.
BB_PROJECTS := OPENBLAS LLVM SUITESPARSE OPENLIBM GMP MBEDTLS LIBSSH2 NGHTTP2 MPFR CURL LIBGIT2 PCRE LIBUV LIBUNWIND DSFMT OBJCONV ZLIB P7ZIP
BB_PROJECTS := OPENBLAS LLVM SUITESPARSE OPENLIBM GMP MBEDTLS LIBSSH2 NGHTTP2 MPFR CURL LIBGIT2 PCRE LIBUV LIBUNWIND DSFMT OBJCONV ZLIB P7ZIP CSL
define SET_BB_DEFAULT
# First, check to see if BB is disabled on a global setting
ifeq ($$(USE_BINARYBUILDER),0)
Expand Down Expand Up @@ -1576,18 +1598,8 @@ PRINT_FLISP = echo '$(subst ','\'',$(1))'; $(1)
PRINT_JULIA = echo '$(subst ','\'',$(1))'; $(1)
endif
define newline # a literal \n
endef
# Makefile debugging trick:
# call print-VARIABLE to see the runtime value of any variable
# (hardened against any special characters appearing in the output)
print-%:
@echo '$*=$(subst ','\'',$(subst $(newline),\n,$($*)))'
# Literal values that are hard to use in Makefiles otherwise:
COMMA:=,
SPACE:=$(eval) $(eval)
94 changes: 14 additions & 80 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,8 @@ endif
julia-deps: | $(DIRS) $(build_datarootdir)/julia/base $(build_datarootdir)/julia/test
@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/deps

julia-stdlib: | $(DIRS)
# `julia-stdlib` depends on `julia-deps` so that the fake JLL stdlibs can copy in their Artifacts.toml files.
julia-stdlib: | $(DIRS) julia-deps
@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/stdlib

julia-base: julia-deps $(build_sysconfdir)/julia/startup.jl $(build_man1dir)/julia.1 $(build_datarootdir)/julia/julia-config.jl
Expand Down Expand Up @@ -163,17 +164,18 @@ endif
JL_PRIVATE_LIBS-0 := libccalltest libllvmcalltest
ifeq ($(USE_GPL_LIBS), 1)
JL_PRIVATE_LIBS-0 += libsuitesparse_wrapper
JL_PRIVATE_LIBS-$(USE_SYSTEM_SUITESPARSE) += libamd libcamd libccolamd libcholmod libcolamd libumfpack libspqr libsuitesparseconfig
JL_PRIVATE_LIBS-$(USE_SYSTEM_SUITESPARSE) += libamd libbtf libcamd libccolamd libcholmod libcolamd libklu libldl librbio libspqr libsuitesparseconfig libumfpack
endif
JL_PRIVATE_LIBS-$(USE_SYSTEM_PCRE) += libpcre2-8
JL_PRIVATE_LIBS-$(USE_SYSTEM_DSFMT) += libdSFMT
JL_PRIVATE_LIBS-$(USE_SYSTEM_GMP) += libgmp
JL_PRIVATE_LIBS-$(USE_SYSTEM_GMP) += libgmp libgmpxx
JL_PRIVATE_LIBS-$(USE_SYSTEM_MPFR) += libmpfr
JL_PRIVATE_LIBS-$(USE_SYSTEM_LIBSSH2) += libssh2
JL_PRIVATE_LIBS-$(USE_SYSTEM_NGHTTP2) += libnghttp2
JL_PRIVATE_LIBS-$(USE_SYSTEM_MBEDTLS) += libmbedtls libmbedcrypto libmbedx509
JL_PRIVATE_LIBS-$(USE_SYSTEM_CURL) += libcurl
JL_PRIVATE_LIBS-$(USE_SYSTEM_LIBGIT2) += libgit2
JL_PRIVATE_LIBS-$(USE_SYSTEM_LIBUV) += libuv
ifeq ($(OS),WINNT)
JL_PRIVATE_LIBS-$(USE_SYSTEM_ZLIB) += zlib
else
Expand All @@ -197,91 +199,29 @@ ifneq ($(LIBLAPACKNAME),$(LIBBLASNAME))
JL_PRIVATE_LIBS-$(USE_SYSTEM_LAPACK) += $(LIBLAPACKNAME)
endif

JL_PRIVATE_LIBS-$(USE_SYSTEM_CSL) += libgfortran libquadmath libstdc++ libgcc_s libgomp libssp libatomic
ifeq ($(OS),Darwin)
ifeq ($(USE_SYSTEM_BLAS),1)
ifeq ($(USE_SYSTEM_LAPACK),0)
JL_PRIVATE_LIBS-0 += libgfortblas
endif
JL_PRIVATE_LIBS-$(USE_SYSTEM_CSL) += libc++
endif
endif

# On FreeBSD, /lib/libgcc_s.so.1 is incompatible with Fortran; to use Fortran on FreeBSD,
# we need to link to the libgcc_s that ships with the same GCC version used by libgfortran.
# To work around this, we copy the GCC libraries we need, namely libgfortran, libgcc_s,
# and libquadmath, into our build library directory, $(build_libdir). We also add them to
# JL_PRIVATE_LIBS-0 so that they know where they need to live at install time.
ifeq ($(OS),FreeBSD)
define std_so
julia-deps: | $$(build_libdir)/$(1).so
$$(build_libdir)/$(1).so: | $$(build_libdir)
$$(INSTALL_M) $$(GCCPATH)/$(1).so* $$(build_libdir)
JL_PRIVATE_LIBS-0 += $(1)
endef

$(eval $(call std_so,libgfortran))
$(eval $(call std_so,libgcc_s))
$(eval $(call std_so,libquadmath))
endif # FreeBSD

ifeq ($(OS),WINNT)
# find the standard .dll folders
ifeq ($(XC_HOST),)
STD_LIB_PATH ?= $(PATH)
JL_PRIVATE_LIBS-$(USE_SYSTEM_CSL) += libwinpthread
else
STD_LIB_PATH := $(shell LANG=C $(CC) -print-search-dirs | grep '^programs: =' | sed -e "s/^programs: =https://")
STD_LIB_PATH += :$(shell LANG=C $(CC) -print-search-dirs | grep '^libraries: =' | sed -e "s/^libraries: =https://")
ifneq (,$(findstring CYGWIN,$(BUILD_OS))) # the cygwin-mingw32 compiler lies about it search directory paths
STD_LIB_PATH := $(shell echo '$(STD_LIB_PATH)' | sed -e "s!/lib/!/bin/!g")
JL_PRIVATE_LIBS-$(USE_SYSTEM_CSL) += libpthread
endif
endif

pathsearch = $(firstword $(wildcard $(addsuffix /$(1),$(subst :, ,$(2)))))

define std_dll
julia-deps-libs: | $$(build_bindir)/lib$(1).dll $$(build_depsbindir)/lib$(1).dll
$$(build_bindir)/lib$(1).dll: | $$(build_bindir)
cp $$(or $$(call pathsearch,lib$(1).dll,$$(STD_LIB_PATH)),$$(error can't find lib$1.dll)) $$(build_bindir)
$$(build_depsbindir)/lib$(1).dll: | $$(build_depsbindir)
cp $$(or $$(call pathsearch,lib$(1).dll,$$(STD_LIB_PATH)),$$(error can't find lib$1.dll)) $$(build_depsbindir)
JL_TARGETS += $(1)
endef
julia-deps: julia-deps-libs

# Given a list of space-separated libraries, return the first library name that is
# correctly found through `pathsearch`.
define select_std_dll
$(firstword $(foreach name,$(1),$(if $(call pathsearch,lib$(name).dll,$(STD_LIB_PATH)),$(name),)))
endef

$(eval $(call std_dll,$(call select_std_dll,gfortran-3 gfortran-4 gfortran-5)))
$(eval $(call std_dll,quadmath-0))
$(eval $(call std_dll,stdc++-6))
ifeq ($(ARCH),i686)
$(eval $(call std_dll,gcc_s_sjlj-1))
else
$(eval $(call std_dll,gcc_s_seh-1))
ifeq ($(OS),Darwin)
ifeq ($(USE_SYSTEM_BLAS),1)
ifeq ($(USE_SYSTEM_LAPACK),0)
JL_PRIVATE_LIBS-0 += libgfortblas
endif
endif
$(eval $(call std_dll,ssp-0))
$(eval $(call std_dll,winpthread-1))
$(eval $(call std_dll,atomic-1))
endif


define stringreplace
$(build_depsbindir)/stringreplace $$(strings -t x - $1 | grep $2 | awk '{print $$1;}') $3 255 "$(call cygpath_w,$1)"
endef

# Run fixup-libgfortran on all platforms but Windows and FreeBSD. On FreeBSD we
# pull in the GCC libraries earlier and use them for the build to make sure we
# don't inadvertently link to /lib/libgcc_s.so.1, which is incompatible with
# libgfortran, and on Windows we copy them in earlier as well.
ifeq (,$(findstring $(OS),FreeBSD WINNT))
julia-base: $(build_libdir)/libgfortran*.$(SHLIB_EXT)*
$(build_libdir)/libgfortran*.$(SHLIB_EXT)*: | $(build_libdir) julia-deps
-$(CUSTOM_LD_LIBRARY_PATH) PATH="$(PATH):$(build_depsbindir)" PATCHELF="$(PATCHELF)" FC="$(FC)" $(JULIAHOME)/contrib/fixup-libgfortran.sh --verbose $(build_libdir)
JL_PRIVATE_LIBS-0 += libgfortran libgcc_s libquadmath
endif


install: $(build_depsbindir)/stringreplace $(BUILDROOT)/doc/_build/html/en/index.html
ifeq ($(BUNDLE_DEBUG_LIBS),1)
Expand Down Expand Up @@ -446,12 +386,6 @@ endif
ifeq ($(DARWIN_FRAMEWORK),1)
$(MAKE) -C $(JULIAHOME)/contrib/mac/framework frameworknoinstall
endif
ifeq ($(OS),Linux)
ifeq ($(prefix),$(abspath julia-$(JULIA_COMMIT)))
# Only fixup libstdc++ if `prefix` is not set.
-$(JULIAHOME)/contrib/fixup-libstdc++.sh $(DESTDIR)$(libdir) $(DESTDIR)$(private_libdir)
endif
endif

distclean:
-rm -fr $(BUILDROOT)/julia-*.tar.gz $(BUILDROOT)/julia*.exe $(BUILDROOT)/julia-$(JULIA_COMMIT)
Expand Down
1 change: 1 addition & 0 deletions base/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ ifeq ($(DARWIN_FRAMEWORK), 1)
else
@echo "const DARWIN_FRAMEWORK = false" >> $@
endif
@echo "const BUILD_TRIPLET = \"$(BB_TRIPLET_LIBGFORTRAN_CXXABI)\"" >> $@

@# This to ensure that we always rebuild this file, but only when it is modified do we touch build_h.jl,
@# ensuring we rebuild the system image as infrequently as possible
Expand Down
60 changes: 34 additions & 26 deletions base/binaryplatforms.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -232,15 +232,15 @@ function validate_tags(tags::Dict)
throw_version_number("libgfortran_version")
end

# Validate `libstdcxx_version` is a parsable `VersionNumber`
if "libstdcxx_version" in keys(tags) && tryparse(VersionNumber, tags["libstdcxx_version"]) === nothing
throw_version_number("libstdcxx_version")
end

# Validate `cxxstring_abi` is one of the two valid options:
if "cxxstring_abi" in keys(tags) && tags["cxxstring_abi"] ∉ ("cxx03", "cxx11")
throw_invalid_key("cxxstring_abi")
end

# Validate `libstdcxx_version` is a parsable `VersionNumber`
if "libstdcxx_version" in keys(tags) && tryparse(VersionNumber, tags["libstdcxx_version"]) === nothing
throw_version_number("libstdcxx_version")
end
end

function set_compare_strategy!(p::Platform, key::String, f::Function)
Expand Down Expand Up @@ -508,12 +508,12 @@ function triplet(p::AbstractPlatform)
if libgfortran_version(p) !== nothing
str = string(str, "-libgfortran", libgfortran_version(p).major)
end
if libstdcxx_version(p) !== nothing
str = string(str, "-libstdcxx", libstdcxx_version(p).patch)
end
if cxxstring_abi(p) !== nothing
str = string(str, "-", cxxstring_abi(p))
end
if libstdcxx_version(p) !== nothing
str = string(str, "-libstdcxx", libstdcxx_version(p).patch)
end

# Tack on all extra tags
for (tag, val) in tags(p)
Expand Down Expand Up @@ -637,15 +637,15 @@ const libgfortran_version_mapping = Dict(
"libgfortran4" => "(-libgfortran4)|(-gcc7)",
"libgfortran5" => "(-libgfortran5)|(-gcc8)",
)
const libstdcxx_version_mapping = Dict{String,String}(
"libstdcxx_nothing" => "",
"libstdcxx" => "-libstdcxx\\d+",
)
const cxxstring_abi_mapping = Dict(
"cxxstring_nothing" => "",
"cxx03" => "-cxx03",
"cxx11" => "-cxx11",
)
const libstdcxx_version_mapping = Dict{String,String}(
"libstdcxx_nothing" => "",
"libstdcxx" => "-libstdcxx\\d+",
)

"""
parse(::Type{Platform}, triplet::AbstractString)
Expand All @@ -667,8 +667,8 @@ function Base.parse(::Type{Platform}, triplet::AbstractString; validate_strict::
c(call_abi_mapping),
# Next, optional things, like libgfortran/libstdcxx/cxxstring abi
c(libgfortran_version_mapping),
c(libstdcxx_version_mapping),
c(cxxstring_abi_mapping),
c(libstdcxx_version_mapping),
# Finally, the catch-all for extended tags
"(?<tags>(?:-[^-]+\\+[^-]+)*)?",
"\$",
Expand Down Expand Up @@ -736,8 +736,8 @@ function Base.parse(::Type{Platform}, triplet::AbstractString; validate_strict::
libc,
call_abi,
libgfortran_version,
libstdcxx_version,
cxxstring_abi,
libstdcxx_version,
os_version,
tags...,
)
Expand Down Expand Up @@ -929,25 +929,33 @@ detect compiler ABI values such as `libgfortran_version`, `libstdcxx_version` an
we have much of that built.
"""
function host_triplet()
str = Sys.MACHINE
libgfortran_version = detect_libgfortran_version()
if libgfortran_version !== nothing
str = string(str, "-libgfortran", libgfortran_version.major)
str = Base.BUILD_TRIPLET

if !occursin("-libgfortran", str)
libgfortran_version = detect_libgfortran_version()
if libgfortran_version !== nothing
str = string(str, "-libgfortran", libgfortran_version.major)
end
end

libstdcxx_version = detect_libstdcxx_version()
if libstdcxx_version !== nothing
str = string(str, "-libstdcxx", libstdcxx_version.patch)
if !occursin("-cxx", str)
cxxstring_abi = detect_cxxstring_abi()
if cxxstring_abi !== nothing
str = string(str, "-", cxxstring_abi)
end
end

cxxstring_abi = detect_cxxstring_abi()
if cxxstring_abi !== nothing
str = string(str, "-", cxxstring_abi)
if !occursin("-libstdcxx", str)
libstdcxx_version = detect_libstdcxx_version()
if libstdcxx_version !== nothing
str = string(str, "-libstdcxx", libstdcxx_version.patch)
end
end

# Add on julia_version extended tag
str = string(str, "-julia_version+", VersionNumber(VERSION.major, VERSION.minor, VERSION.patch))

if !occursin("-julia_version+", str)
str = string(str, "-julia_version+", VersionNumber(VERSION.major, VERSION.minor, VERSION.patch))
end
return str
end

Expand Down
Loading

0 comments on commit 2eccd8e

Please sign in to comment.