Skip to content

Commit

Permalink
Make sysimg only recompile when stdlibs that are contained within cha…
Browse files Browse the repository at this point in the history 
…nge (#51368)

Otherwise changing a stdlib would trigger a recompilation of the sysimg.
  • Loading branch information
@vchuravy
vchuravy committed Sep 22, 2023
1 parent 9a08d15 commit 027fb37
Show file tree
Hide file tree
Showing 3 changed files with 95 additions and 77 deletions.
141 changes: 66 additions & 75 deletions pkgimage.mk
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST))))
BUILDDIR := .
JULIAHOME := $(SRCDIR)
include $(JULIAHOME)/Make.inc
include $(JULIAHOME)/stdlib/stdlib.mk

VERSDIR := v$(shell cut -d. -f1-2 < $(JULIAHOME)/VERSION)

# set some influential environment variables
export JULIA_DEPOT_PATH := $(build_prefix)/share/julia
Expand All @@ -22,114 +22,105 @@ $(JULIA_DEPOT_PATH):
print-depot-path:
@$(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no -e '@show Base.DEPOT_PATH')

STDLIBS := ArgTools Artifacts Base64 CRC32c FileWatching Libdl NetworkOptions SHA Serialization \
GMP_jll LLVMLibUnwind_jll LibUV_jll LibUnwind_jll MbedTLS_jll OpenLibm_jll PCRE2_jll \
Zlib_jll dSFMT_jll libLLVM_jll libblastrampoline_jll OpenBLAS_jll Printf Random Tar \
LibSSH2_jll MPFR_jll LinearAlgebra Dates Distributed Future LibGit2 Profile SparseArrays UUIDs \
SharedArrays TOML Test LibCURL Downloads Pkg Dates LazyArtifacts Sockets Unicode Markdown \
InteractiveUtils REPL DelimitedFiles Statistics

all-release: $(addprefix cache-release-, $(STDLIBS))
all-debug: $(addprefix cache-debug-, $(STDLIBS))

define pkgimg_builder
$1_SRCS := $$(shell find $$(build_datarootdir)/julia/stdlib/$$(VERSDIR)/$1/src -name \*.jl) \
$$(wildcard $$(build_prefix)/manifest/$$(VERSDIR)/$1)
define stdlib_builder
ifneq ($(filter $(1),$(INDEPENDENT_STDLIBS)),)
$$(BUILDDIR)/stdlib/$1.release.image: $$($1_SRCS) $$(addsuffix .release.image,$$(addprefix $$(BUILDDIR)/stdlib/,$2)) $(build_private_libdir)/sys.$(SHLIB_EXT)
@$$(call PRINT_JULIA, $$(call spawn,$$(JULIA_EXECUTABLE)) --startup-file=no --check-bounds=yes -e 'Base.compilecache(Base.identify_package("$1"))')
@$$(call PRINT_JULIA, $$(call spawn,$$(JULIA_EXECUTABLE)) --startup-file=no --pkgimage=no -e 'Base.compilecache(Base.identify_package("$1"))')
@$$(call PRINT_JULIA, $$(call spawn,$$(JULIA_EXECUTABLE)) --startup-file=no -e 'Base.compilecache(Base.identify_package("$1"))')
touch $$@
cache-release-$1: $$(BUILDDIR)/stdlib/$1.release.image
$$(BUILDDIR)/stdlib/$1.debug.image: $$($1_SRCS) $$(addsuffix .debug.image,$$(addprefix $$(BUILDDIR)/stdlib/,$2)) $(build_private_libdir)/sys-debug.$(SHLIB_EXT)
@$$(call PRINT_JULIA, $$(call spawn,$$(JULIA_EXECUTABLE)) --startup-file=no --check-bounds=yes -e 'Base.compilecache(Base.identify_package("$1"))')
@$$(call PRINT_JULIA, $$(call spawn,$$(JULIA_EXECUTABLE)) --startup-file=no --pkgimage=no -e 'Base.compilecache(Base.identify_package("$1"))')
@$$(call PRINT_JULIA, $$(call spawn,$$(JULIA_EXECUTABLE)) --startup-file=no -e 'Base.compilecache(Base.identify_package("$1"))')
cache-debug-$1: $$(BUILDDIR)/stdlib/$1.debug.image
.SECONDARY: $$(BUILDDIR)/stdlib/$1.release.image $$(BUILDDIR)/stdlib/$1.debug.image
endef

# Used to just define them in the dependency graph
# reside in the system image
define sysimg_builder
touch $$@
else
ifneq ($(filter $(1),$(STDLIBS_WITHIN_SYSIMG)),)
$$(BUILDDIR)/stdlib/$1.release.image:
touch $$@
cache-release-$1: $$(BUILDDIR)/stdlib/$1.release.image
$$(BUILDDIR)/stdlib/$1.debug.image:
touch $$@
else
$$(error $(1) neither in STDLIBS_WITHIN_SYSIMG nor INDEPENDENT_STDLIBS)
endif
endif
cache-release-$1: $$(BUILDDIR)/stdlib/$1.release.image
cache-debug-$1: $$(BUILDDIR)/stdlib/$1.debug.image
.SECONDARY: $$(BUILDDIR)/stdlib/$1.release.image $$(BUILDDIR)/stdlib/$1.debug.image
endef

# no dependencies
$(eval $(call pkgimg_builder,MozillaCACerts_jll,))
$(eval $(call sysimg_builder,ArgTools,))
$(eval $(call sysimg_builder,Artifacts,))
$(eval $(call sysimg_builder,Base64,))
$(eval $(call sysimg_builder,CRC32c,))
$(eval $(call sysimg_builder,FileWatching,))
$(eval $(call sysimg_builder,Libdl,))
$(eval $(call sysimg_builder,Logging,))
$(eval $(call sysimg_builder,Mmap,))
$(eval $(call sysimg_builder,NetworkOptions,))
$(eval $(call sysimg_builder,SHA,))
$(eval $(call sysimg_builder,Serialization,))
$(eval $(call sysimg_builder,Sockets,))
$(eval $(call sysimg_builder,Unicode,))
$(eval $(call pkgimg_builder,Profile,))
$(eval $(call stdlib_builder,MozillaCACerts_jll,))
$(eval $(call stdlib_builder,ArgTools,))
$(eval $(call stdlib_builder,Artifacts,))
$(eval $(call stdlib_builder,Base64,))
$(eval $(call stdlib_builder,CRC32c,))
$(eval $(call stdlib_builder,FileWatching,))
$(eval $(call stdlib_builder,Libdl,))
$(eval $(call stdlib_builder,Logging,))
$(eval $(call stdlib_builder,Mmap,))
$(eval $(call stdlib_builder,NetworkOptions,))
$(eval $(call stdlib_builder,SHA,))
$(eval $(call stdlib_builder,Serialization,))
$(eval $(call stdlib_builder,Sockets,))
$(eval $(call stdlib_builder,Unicode,))
$(eval $(call stdlib_builder,Profile,))

# 1-depth packages
$(eval $(call pkgimg_builder,GMP_jll,Artifacts Libdl))
$(eval $(call pkgimg_builder,LLVMLibUnwind_jll,Artifacts Libdl))
$(eval $(call pkgimg_builder,LibUV_jll,Artifacts Libdl))
$(eval $(call pkgimg_builder,LibUnwind_jll,Artifacts Libdl))
$(eval $(call sysimg_builder,MbedTLS_jll,Artifacts Libdl))
$(eval $(call pkgimg_builder,nghttp2_jll,Artifacts Libdl))
$(eval $(call pkgimg_builder,OpenLibm_jll,Artifacts Libdl))
$(eval $(call pkgimg_builder,PCRE2_jll,Artifacts Libdl))
$(eval $(call pkgimg_builder,Zlib_jll,Artifacts Libdl))
$(eval $(call pkgimg_builder,dSFMT_jll,Artifacts Libdl))
$(eval $(call pkgimg_builder,libLLVM_jll,Artifacts Libdl))
$(eval $(call sysimg_builder,libblastrampoline_jll,Artifacts Libdl))
$(eval $(call sysimg_builder,OpenBLAS_jll,Artifacts Libdl))
$(eval $(call sysimg_builder,Markdown,Base64))
$(eval $(call sysimg_builder,Printf,Unicode))
$(eval $(call sysimg_builder,Random,SHA))
$(eval $(call sysimg_builder,Tar,ArgTools,SHA))
$(eval $(call pkgimg_builder,DelimitedFiles,Mmap))
$(eval $(call stdlib_builder,GMP_jll,Artifacts Libdl))
$(eval $(call stdlib_builder,LLVMLibUnwind_jll,Artifacts Libdl))
$(eval $(call stdlib_builder,LibUV_jll,Artifacts Libdl))
$(eval $(call stdlib_builder,LibUnwind_jll,Artifacts Libdl))
$(eval $(call stdlib_builder,MbedTLS_jll,Artifacts Libdl))
$(eval $(call stdlib_builder,nghttp2_jll,Artifacts Libdl))
$(eval $(call stdlib_builder,OpenLibm_jll,Artifacts Libdl))
$(eval $(call stdlib_builder,PCRE2_jll,Artifacts Libdl))
$(eval $(call stdlib_builder,Zlib_jll,Artifacts Libdl))
$(eval $(call stdlib_builder,dSFMT_jll,Artifacts Libdl))
$(eval $(call stdlib_builder,libLLVM_jll,Artifacts Libdl))
$(eval $(call stdlib_builder,libblastrampoline_jll,Artifacts Libdl))
$(eval $(call stdlib_builder,OpenBLAS_jll,Artifacts Libdl))
$(eval $(call stdlib_builder,Markdown,Base64))
$(eval $(call stdlib_builder,Printf,Unicode))
$(eval $(call stdlib_builder,Random,SHA))
$(eval $(call stdlib_builder,Tar,ArgTools,SHA))
$(eval $(call stdlib_builder,DelimitedFiles,Mmap))

# 2-depth packages
$(eval $(call pkgimg_builder,LLD_jll,Zlib_jll libLLVM_jll Artifacts Libdl))
$(eval $(call sysimg_builder,LibSSH2_jll,Artifacts Libdl MbedTLS_jll))
$(eval $(call pkgimg_builder,MPFR_jll,Artifacts Libdl GMP_jll))
$(eval $(call sysimg_builder,LinearAlgebra,Libdl libblastrampoline_jll OpenBLAS_jll))
$(eval $(call sysimg_builder,Dates,Printf))
$(eval $(call pkgimg_builder,Distributed,Random Serialization Sockets))
$(eval $(call sysimg_builder,Future,Random))
$(eval $(call sysimg_builder,InteractiveUtils,Markdown))
$(eval $(call sysimg_builder,UUIDs,Random SHA))
$(eval $(call stdlib_builder,LLD_jll,Zlib_jll libLLVM_jll Artifacts Libdl))
$(eval $(call stdlib_builder,LibSSH2_jll,Artifacts Libdl MbedTLS_jll))
$(eval $(call stdlib_builder,MPFR_jll,Artifacts Libdl GMP_jll))
$(eval $(call stdlib_builder,LinearAlgebra,Libdl libblastrampoline_jll OpenBLAS_jll))
$(eval $(call stdlib_builder,Dates,Printf))
$(eval $(call stdlib_builder,Distributed,Random Serialization Sockets))
$(eval $(call stdlib_builder,Future,Random))
$(eval $(call stdlib_builder,InteractiveUtils,Markdown))
$(eval $(call stdlib_builder,UUIDs,Random SHA))

# 3-depth packages
$(eval $(call sysimg_builder,LibGit2_jll,MbedTLS_jll LibSSH2_jll Artifacts Libdl))
$(eval $(call pkgimg_builder,LibCURL_jll,LibSSH2_jll nghttp2_jll MbedTLS_jll Zlib_jll Artifacts Libdl))
$(eval $(call sysimg_builder,REPL,InteractiveUtils Markdown Sockets Unicode))
$(eval $(call pkgimg_builder,SharedArrays,Distributed Mmap Random Serialization))
$(eval $(call sysimg_builder,TOML,Dates))
$(eval $(call pkgimg_builder,Test,Logging Random Serialization InteractiveUtils))
$(eval $(call stdlib_builder,LibGit2_jll,MbedTLS_jll LibSSH2_jll Artifacts Libdl))
$(eval $(call stdlib_builder,LibCURL_jll,LibSSH2_jll nghttp2_jll MbedTLS_jll Zlib_jll Artifacts Libdl))
$(eval $(call stdlib_builder,REPL,InteractiveUtils Markdown Sockets Unicode))
$(eval $(call stdlib_builder,SharedArrays,Distributed Mmap Random Serialization))
$(eval $(call stdlib_builder,TOML,Dates))
$(eval $(call stdlib_builder,Test,Logging Random Serialization InteractiveUtils))

# 4-depth packages
$(eval $(call sysimg_builder,LibGit2,LibGit2_jll NetworkOptions Printf SHA Base64))
$(eval $(call sysimg_builder,LibCURL,LibCURL_jll MozillaCACerts_jll))
$(eval $(call stdlib_builder,LibGit2,LibGit2_jll NetworkOptions Printf SHA Base64))
$(eval $(call stdlib_builder,LibCURL,LibCURL_jll MozillaCACerts_jll))

# 5-depth packages
$(eval $(call sysimg_builder,Downloads,ArgTools FileWatching LibCURL NetworkOptions))
$(eval $(call stdlib_builder,Downloads,ArgTools FileWatching LibCURL NetworkOptions))

# 6-depth packages
$(eval $(call pkgimg_builder,Pkg,Dates LibGit2 Libdl Logging Printf Random SHA UUIDs)) # Markdown REPL
$(eval $(call stdlib_builder,Pkg,Dates LibGit2 Libdl Logging Printf Random SHA UUIDs)) # Markdown REPL

# 7-depth packages
$(eval $(call pkgimg_builder,LazyArtifacts,Artifacts Pkg))
$(eval $(call stdlib_builder,LazyArtifacts,Artifacts Pkg))

$(eval $(call pkgimg_builder,SparseArrays,Libdl LinearAlgebra Random Serialization))
$(eval $(call pkgimg_builder,Statistics,LinearAlgebra SparseArrays))
$(eval $(call stdlib_builder,SparseArrays,Libdl LinearAlgebra Random Serialization))
$(eval $(call stdlib_builder,Statistics,LinearAlgebra SparseArrays))
# SuiteSparse_jll
27 changes: 27 additions & 0 deletions stdlib/stdlib.mk
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
STDLIBS_WITHIN_SYSIMG := \
ArgTools Artifacts Base64 CRC32c FileWatching Libdl NetworkOptions SHA Serialization \
MbedTLS_jll libblastrampoline_jll OpenBLAS_jll Printf Random Tar LibSSH2_jll LibGit2_jll \
LinearAlgebra Dates Future LibGit2 UUIDs TOML LibCURL Downloads Pkg Dates Logging \
Sockets Unicode Markdown InteractiveUtils REPL nghttp2_jll LibCURL_jll MozillaCACerts_jll \
Mmap

INDEPENDENT_STDLIBS := \
GMP_jll LLVMLibUnwind_jll LibUV_jll LibUnwind_jll OpenLibm_jll PCRE2_jll \
Zlib_jll dSFMT_jll libLLVM_jll LLD_jll MPFR_jll \
DelimitedFiles Distributed SharedArrays SparseArrays Statistics Test LazyArtifacts \
Profile


STDLIBS := $(STDLIBS_WITHIN_SYSIMG) $(INDEPENDENT_STDLIBS)
VERSDIR := v$(shell cut -d. -f1-2 < $(JULIAHOME)/VERSION)

SYSIMG_STDLIB_SRCS =
define STDLIB_srcs
$1_SRCS := $$(shell find $$(build_datarootdir)/julia/stdlib/$$(VERSDIR)/$1/src -name \*.jl) \
$$(wildcard $$(build_prefix)/manifest/$$(VERSDIR)/$1)
ifneq ($(filter $(1),$(STDLIBS_WITHIN_SYSIMG)),)
SYSIMG_STDLIB_SRCS += $$($1_SRCS)
endif
endef

$(foreach stdlib,$(STDLIBS),$(eval $(call STDLIB_srcs,$(stdlib))))
4 changes: 2 additions & 2 deletions sysimage.mk
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST))))
BUILDDIR := .
JULIAHOME := $(SRCDIR)
include $(JULIAHOME)/Make.inc
include $(JULIAHOME)/stdlib/stdlib.mk

default: sysimg-$(JULIA_BUILD_MODE) # contains either "debug" or "release"
all: sysimg-release sysimg-debug
Expand Down Expand Up @@ -53,8 +54,7 @@ COMPILER_SRCS += $(shell find $(JULIAHOME)/base/compiler -name \*.jl)
# sort these to remove duplicates
BASE_SRCS := $(sort $(shell find $(JULIAHOME)/base -name \*.jl -and -not -name sysimg.jl) \
$(shell find $(BUILDROOT)/base -name \*.jl -and -not -name sysimg.jl))
STDLIB_SRCS := $(JULIAHOME)/base/sysimg.jl $(shell find $(build_datarootdir)/julia/stdlib/$(VERSDIR)/*/src -name \*.jl) \
$(wildcard $(build_prefix)/manifest/$(VERSDIR)/*)
STDLIB_SRCS := $(JULIAHOME)/base/sysimg.jl $(SYSIMG_STDLIB_SRCS)
RELBUILDROOT := $(call rel_path,$(JULIAHOME)/base,$(BUILDROOT)/base)/ # <-- make sure this always has a trailing slash

$(build_private_libdir)/corecompiler.ji: $(COMPILER_SRCS)
Expand Down

0 comments on commit 027fb37

Please sign in to comment.