Increase build precompilation (#53682)

- Bake precompilepkgs into the sysimage
- Enable compilation in stdlib JLLs @staticfloat I guess this is ok now
we have pkgimages?
- Add some hardcoded precompiles encountered locally
- Disable a problematic llvm test, with a note

With this there's no compilation loading Pkg in interactive mode
```
% ./julia --trace-compile=stderr --start=no -q
julia> ]
(@v1.12) pkg> 
```

Before
```
% time julia +nightly --start=no --trace-compile=stderr -q -e "using Pkg"
precompile(Tuple{typeof(Base.setindex!), Base.Dict{Symbol, Union{Nothing, Bool, Int64, String}}, Int64, Symbol})
precompile(Tuple{typeof(Base.haskey), Base.Dict{String, Any}, String})
precompile(Tuple{typeof(MbedTLS_jll.__init__)})
precompile(Tuple{typeof(LibSSH2_jll.__init__)})
precompile(Tuple{typeof(LibGit2_jll.__init__)})
precompile(Tuple{typeof(nghttp2_jll.__init__)})
precompile(Tuple{typeof(LibCURL_jll.__init__)})
precompile(Tuple{typeof(MozillaCACerts_jll.__init__)})
precompile(Tuple{typeof(Base.Filesystem.normpath), String, String, Vararg{String}})
precompile(Tuple{typeof(p7zip_jll.__init__)})
precompile(Tuple{typeof(p7zip_jll.init_p7zip_path)})
precompile(Tuple{typeof(Base.append!), Array{String, 1}, Array{String, 1}})
precompile(Tuple{typeof(Base.join), Array{String, 1}, Char})
julia +nightly --start=no --trace-compile=stderr -q -e "using Pkg"  1.48s user 0.40s system 206% cpu 0.910 total
```
PR
```
% time ./julia --start=no --trace-compile=stderr -q -e "using Pkg"
./julia --start=no --trace-compile=stderr -q -e "using Pkg"  1.62s user 0.13s system 402% cpu 0.433 total
```

base/precompilation.jl

@@ -354,12 +354,10 @@ function precompilepkgs(pkgs::Vector{String}=String[];
  _from_loading::Bool=false,
  configs::Union{Config,Vector{Config}}=(``=>Base.CacheFlags()),
  io::IO=stderr,
- flags_cacheflags=nothing)
+ # asking for timing disables fancy mode, as timing is shown in non-fancy mode
+ fancyprint::Bool = can_fancyprint(io) && !timing
+ )
 
- if flags_cacheflags !== nothing
- # TODO: Remove `flags_cacheflags` once Pkg moves to the renamed kwarg `configs`
- configs = flags_cacheflags
- end
  configs = configs isa Config ? [configs] : configs
 
  time_start = time_ns()
@@ -374,9 +372,6 @@ function precompilepkgs(pkgs::Vector{String}=String[];
  num_tasks = parse(Int, get(ENV, "JULIA_NUM_PRECOMPILE_TASKS", string(default_num_tasks)))
  parallel_limiter = Base.Semaphore(num_tasks)
 
- # asking for timing disables fancy mode, as timing is shown in non-fancy mode
- fancyprint = can_fancyprint(io) && !timing
-
  if _from_loading && !Sys.isinteractive() && Base.get_bool_env("JULIA_TESTS", false)
  # suppress passive loading printing in julia test suite. `JULIA_TESTS` is set in Base.runtests
  io = devnull
@@ -434,7 +429,7 @@ function precompilepkgs(pkgs::Vector{String}=String[];
  end
  end
 
- @debug "precompile: deps collected" _group=:precompile
+ @debug "precompile: deps collected"
  # this loop must be run after the full depsmap has been populated
  for (pkg, pkg_exts) in pkg_exts_map
  # find any packages that depend on the extension(s)'s deps and replace those deps in their deps list with the extension(s),
@@ -446,7 +441,7 @@ function precompilepkgs(pkgs::Vector{String}=String[];
  end
  end
  end
- @debug "precompile: extensions collected" _group=:precompile
+ @debug "precompile: extensions collected"
 
  # return early if no deps
  if isempty(depsmap)
@@ -474,7 +469,7 @@ function precompilepkgs(pkgs::Vector{String}=String[];
  was_recompiled[dep_config] = false
  end
  end
- @debug "precompile: signalling initialized" _group=:precompile
+ @debug "precompile: signalling initialized"
 
 
  # find and guard against circular deps
@@ -515,7 +510,7 @@ function precompilepkgs(pkgs::Vector{String}=String[];
  if !isempty(circular_deps)
  @warn """Circular dependency detected. Precompilation will be skipped for:\n $(join(string.(circular_deps), "\n "))"""
  end
- @debug "precompile: circular dep check done" _group=:precompile
+ @debug "precompile: circular dep check done"
 
  # if a list of packages is given, restrict to dependencies of given packages
  if !isempty(pkgs)
@@ -562,7 +557,7 @@ function precompilepkgs(pkgs::Vector{String}=String[];
  else
  target *= "..."
  end
- @debug "precompile: packages filtered" _group=:precompile
+ @debug "precompile: packages filtered"
 
  pkg_queue = PkgConfig[]
  failed_deps = Dict{PkgConfig, String}()
@@ -737,7 +732,7 @@ function precompilepkgs(pkgs::Vector{String}=String[];
  if !_from_loading
  Base.LOADING_CACHE[] = Base.LoadingCache()
  end
- @debug "precompile: starting precompilation loop"  _group=:precompile
+ @debug "precompile: starting precompilation loop" depsmap direct_deps
  ## precompilation loop
  for (pkg, deps) in depsmap
  cachepaths = Base.find_all_in_cache_path(pkg)

contrib/generate_precompile.jl

@@ -39,6 +39,17 @@ precompile(Base.unsafe_string, (Ptr{Int8},))
 # loading.jl
 precompile(Base.__require_prelocked, (Base.PkgId, Nothing))
 precompile(Base._require, (Base.PkgId, Nothing))
+precompile(Base.indexed_iterate, (Pair{Symbol, Union{Nothing, String}}, Int))
+precompile(Base.indexed_iterate, (Pair{Symbol, Union{Nothing, String}}, Int, Int))
+
+# Pkg loading
+precompile(Tuple{typeof(Base.Filesystem.normpath), String, String, Vararg{String}})
+precompile(Tuple{typeof(Base.append!), Array{String, 1}, Array{String, 1}})
+precompile(Tuple{typeof(Base.join), Array{String, 1}, Char})
+precompile(Tuple{typeof(Base.getindex), Base.Dict{Any, Any}, Char})
+precompile(Tuple{typeof(Base.delete!), Base.Set{Any}, Char})
+precompile(Tuple{typeof(Base.convert), Type{Base.Dict{String, Base.Dict{String, String}}}, Base.Dict{String, Any}})
+precompile(Tuple{typeof(Base.convert), Type{Base.Dict{String, Array{String, 1}}}, Base.Dict{String, Any}})
 
 # REPL
 precompile(isequal, (String, String))
@@ -67,6 +78,10 @@ precompile(Tuple{typeof(haskey), Dict{Base.PkgId,Vector{Function}}, Base.PkgId})
 precompile(Tuple{typeof(delete!), Dict{Base.PkgId,Vector{Function}}, Base.PkgId})
 precompile(Tuple{typeof(push!), Vector{Function}, Function})
 
+# preferences
+precompile(Base.get_preferences, (Base.UUID,))
+precompile(Base.record_compiletime_preference, (Base.UUID, String))
+
 # miscellaneous
 precompile(Tuple{typeof(Base.exit)})
 precompile(Tuple{typeof(Base.require), Base.PkgId})
@@ -84,6 +99,23 @@ precompile(Base.CoreLogging.current_logger_for_env, (Base.CoreLogging.LogLevel,
 precompile(Base.CoreLogging.env_override_minlevel, (Symbol, Module))
 precompile(Base.StackTraces.lookup, (Ptr{Nothing},))
 precompile(Tuple{typeof(Base.run_module_init), Module, Int})
+
+# precompilepkgs
+precompile(Tuple{typeof(Base.get), Type{Array{String, 1}}, Base.Dict{String, Any}, String})
+precompile(Tuple{typeof(Base.get), Type{Base.Dict{String, Any}}, Base.Dict{String, Any}, String})
+precompile(Tuple{typeof(Base.haskey), Base.Dict{String, Any}, String})
+precompile(Tuple{typeof(Base.indexed_iterate), Tuple{Base.TTY, Bool}, Int, Int})
+precompile(Tuple{typeof(Base.indexed_iterate), Tuple{Base.TTY, Bool}, Int})
+precompile(Tuple{typeof(Base.open), Base.CmdRedirect, String, Base.TTY})
+precompile(Tuple{typeof(Base.Precompilation.precompilepkgs)})
+precompile(Tuple{typeof(Base.Precompilation.printpkgstyle), Base.TTY, Symbol, String})
+precompile(Tuple{typeof(Base.rawhandle), Base.TTY})
+precompile(Tuple{typeof(Base.setindex!), Base.Dict{String, Array{String, 1}}, Array{String, 1}, String})
+precompile(Tuple{typeof(Base.setindex!), GenericMemory{:not_atomic, Union{Base.Libc.RawFD, Base.SyncCloseFD, IO}, Core.AddrSpace{Core}(0x00)}, Base.TTY, Int})
+precompile(Tuple{typeof(Base.setup_stdio), Base.TTY, Bool})
+precompile(Tuple{typeof(Base.spawn_opts_inherit), Base.DevNull, Base.TTY, Base.TTY})
+precompile(Tuple{typeof(Core.kwcall), NamedTuple{(:context,), Tuple{Base.TTY}}, typeof(Base.sprint), Function})
+precompile(Tuple{Type{Base.UUID}, Base.UUID})
 """
 
 for T in (Float16, Float32, Float64), IO in (IOBuffer, IOContext{IOBuffer}, Base.TTY, IOContext{Base.TTY})
@@ -131,17 +163,17 @@ for match = Base._methods(+, (Int, Int), -1, Base.get_world_counter())
  push!(Expr[], Expr(:return, false))
  vcat(String[], String[])
  k, v = (:hello => nothing)
- precompile(Base.indexed_iterate, (Pair{Symbol, Union{Nothing, String}}, Int))
- precompile(Base.indexed_iterate, (Pair{Symbol, Union{Nothing, String}}, Int, Int))
+
  # Preferences uses these
- precompile(Base.get_preferences, (Base.UUID,))
- precompile(Base.record_compiletime_preference, (Base.UUID, String))
  get(Dict{String,Any}(), "missing", nothing)
  delete!(Dict{String,Any}(), "missing")
  for (k, v) in Dict{String,Any}()
  println(k)
  end
 
+ # interactive statup uses this
+ write(IOBuffer(), "")
+
  break # only actually need to do this once
 end
 """
@@ -223,10 +255,10 @@ ansi_disablecursor = "\e[?25l"
 blackhole = Sys.isunix() ? "/dev/null" : "nul"
 procenv = Dict{String,Any}(
  "JULIA_HISTORY" => blackhole,
- "JULIA_PROJECT" => nothing, # remove from environment
- "JULIA_LOAD_PATH" => "@stdlib",
+ "JULIA_LOAD_PATH" => "@$(Sys.iswindows() ? ";" : ":")@stdlib",
  "JULIA_DEPOT_PATH" => Sys.iswindows() ? ";" : ":",
  "TERM" => "",
+ # "JULIA_DEBUG" => "precompilation",
  "JULIA_FALLBACK_REPL" => "true")
 
 generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printed
@@ -263,24 +295,32 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe
  print_state("step1" => "R")
  # Also precompile a package here
  pkgname = "__PackagePrecompilationStatementModule"
- mkpath(joinpath(prec_path, pkgname, "src"))
- path = joinpath(prec_path, pkgname, "src", "$pkgname.jl")
- write(path,
- """
- module $pkgname
- end
- """)
+ pkguuid = "824efdaf-a0e9-431c-8ee7-3d356b2531c2"
+ pkgpath = joinpath(prec_path, pkgname)
+ mkpath(joinpath(pkgpath, "src"))
+ write(joinpath(pkgpath, "src", "$pkgname.jl"),
+ """
+ module $pkgname
+ println("Precompiling $pkgname")
+ end
+ """)
+ write(joinpath(pkgpath, "Project.toml"),
+ """
+ name = "$pkgname"
+ uuid = "$pkguuid"
+ """)
+ touch(joinpath(pkgpath, "Manifest.toml"))
  tmp_prec = tempname(prec_path)
  tmp_proc = tempname(prec_path)
  s = """
- pushfirst!(DEPOT_PATH, $(repr(prec_path)));
+ pushfirst!(DEPOT_PATH, $(repr(joinpath(prec_path,"depot"))));
  Base.PRECOMPILE_TRACE_COMPILE[] = $(repr(tmp_prec));
- Base.compilecache(Base.PkgId($(repr(pkgname))), $(repr(path)))
+ Base.Precompilation.precompilepkgs(;fancyprint=true);
  $precompile_script
  """
  p = run(pipeline(addenv(`$(julia_exepath()) -O0 --trace-compile=$tmp_proc --sysimage $sysimg
- --cpu-target=native --startup-file=no --color=yes`, procenv),
- stdin=IOBuffer(s), stdout=debug_output))
+ --cpu-target=native --startup-file=no --color=yes --project=$(pkgpath)`, procenv),
+ stdin=IOBuffer(s), stderr=debug_output, stdout=debug_output))
  n_step1 = 0
  for f in (tmp_prec, tmp_proc)
  isfile(f) || continue
@@ -334,9 +374,8 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe
  yield() # Make clock spinning
  print_state("step3" => string("R$n_succeeded", failed > 0 ? " ($failed failed)" : ""))
  catch ex
- @show backtrace()
  # See #28808
- @warn "Failed to precompile expression" form=statement exception=ex _module=nothing _file=nothing _line=0
+ @warn "Failed to precompile expression" form=statement exception=(ex,catch_backtrace()) _module=nothing _file=nothing _line=0
  end
  end
  wait(clock) # Stop asynchronous printing

stdlib/CompilerSupportLibraries_jll/src/CompilerSupportLibraries_jll.jl

@@ -4,7 +4,6 @@
 
 baremodule CompilerSupportLibraries_jll
 using Base, Libdl, Base.BinaryPlatforms
-Base.Experimental.@compiler_options compile=min optimize=0 infer=false
 
 const PATH_list = String[]
 const LIBPATH_list = String[]

stdlib/GMP_jll/src/GMP_jll.jl

@@ -3,7 +3,6 @@
 ## dummy stub for https://github.com/JuliaBinaryWrappers/GMP_jll.jl
 baremodule GMP_jll
 using Base, Libdl
-Base.Experimental.@compiler_options compile=min optimize=0 infer=false
 
 const PATH_list = String[]
 const LIBPATH_list = String[]

stdlib/LLD_jll/src/LLD_jll.jl

@@ -4,7 +4,6 @@
 
 baremodule LLD_jll
 using Base, Libdl
-Base.Experimental.@compiler_options compile=min optimize=0 infer=false
 
 const PATH_list = String[]
 const LIBPATH_list = String[]

stdlib/LLVMLibUnwind_jll/src/LLVMLibUnwind_jll.jl

@@ -4,7 +4,6 @@
 
 baremodule LLVMLibUnwind_jll
 using Base, Libdl
-Base.Experimental.@compiler_options compile=min optimize=0 infer=false
 
 const PATH_list = String[]
 const LIBPATH_list = String[]

stdlib/LibCURL_jll/src/LibCURL_jll.jl

@@ -4,7 +4,6 @@
 
 baremodule LibCURL_jll
 using Base, Libdl, nghttp2_jll
-Base.Experimental.@compiler_options compile=min optimize=0 infer=false
 
 const PATH_list = String[]
 const LIBPATH_list = String[]

stdlib/LibGit2_jll/src/LibGit2_jll.jl

@@ -4,7 +4,6 @@
 
 baremodule LibGit2_jll
 using Base, Libdl, MbedTLS_jll, LibSSH2_jll
-Base.Experimental.@compiler_options compile=min optimize=0 infer=false
 
 const PATH_list = String[]
 const LIBPATH_list = String[]

stdlib/LibSSH2_jll/src/LibSSH2_jll.jl

@@ -4,7 +4,6 @@
 
 baremodule LibSSH2_jll
 using Base, Libdl, MbedTLS_jll
-Base.Experimental.@compiler_options compile=min optimize=0 infer=false
 
 const PATH_list = String[]
 const LIBPATH_list = String[]

stdlib/LibUV_jll/src/LibUV_jll.jl

@@ -4,7 +4,6 @@
 
 baremodule LibUV_jll
 using Base, Libdl
-Base.Experimental.@compiler_options compile=min optimize=0 infer=false
 
 # NOTE: This file is currently empty, as we link libuv statically for now.
 

stdlib/LibUnwind_jll/src/LibUnwind_jll.jl

@@ -4,7 +4,6 @@
 
 baremodule LibUnwind_jll
 using Base, Libdl
-Base.Experimental.@compiler_options compile=min optimize=0 infer=false
 
 const PATH_list = String[]
 const LIBPATH_list = String[]

stdlib/MPFR_jll/src/MPFR_jll.jl

@@ -3,7 +3,6 @@
 ## dummy stub for https://github.com/JuliaBinaryWrappers/MPFR_jll.jl
 baremodule MPFR_jll
 using Base, Libdl, GMP_jll
-Base.Experimental.@compiler_options compile=min optimize=0 infer=false
 
 const PATH_list = String[]
 const LIBPATH_list = String[]

stdlib/MbedTLS_jll/src/MbedTLS_jll.jl

@@ -4,7 +4,6 @@
 
 baremodule MbedTLS_jll
 using Base, Libdl
-Base.Experimental.@compiler_options compile=min optimize=0 infer=false
 
 const PATH_list = String[]
 const LIBPATH_list = String[]

stdlib/MozillaCACerts_jll/src/MozillaCACerts_jll.jl

@@ -4,7 +4,6 @@
 
 baremodule MozillaCACerts_jll
 using Base
-Base.Experimental.@compiler_options compile=min optimize=0 infer=false
 
 const PATH_list = String[]
 const LIBPATH_list = String[]

stdlib/OpenBLAS_jll/src/OpenBLAS_jll.jl

@@ -13,7 +13,6 @@ using Base, Libdl, Base.BinaryPlatforms
 # using CompilerSupportLibraries_jll
 # Because of this however, we have to manually load the libraries we
 # _do_ care about, namely libgfortran
-Base.Experimental.@compiler_options compile=min optimize=0 infer=false
 
 const PATH_list = String[]
 const LIBPATH_list = String[]

stdlib/OpenLibm_jll/src/OpenLibm_jll.jl

@@ -3,7 +3,6 @@
 ## dummy stub for https://github.com/JuliaBinaryWrappers/OpenLibm_jll.jl
 baremodule OpenLibm_jll
 using Base, Libdl
-Base.Experimental.@compiler_options compile=min optimize=0 infer=false
 
 const PATH_list = String[]
 const LIBPATH_list = String[]

stdlib/PCRE2_jll/src/PCRE2_jll.jl

@@ -3,7 +3,6 @@
 ## dummy stub for https://github.com/JuliaBinaryWrappers/PCRE2_jll.jl
 baremodule PCRE2_jll
 using Base, Libdl
-Base.Experimental.@compiler_options compile=min optimize=0 infer=false
 
 const PATH_list = String[]
 const LIBPATH_list = String[]

stdlib/REPL/test/precompilation.jl

@@ -27,7 +27,10 @@ if !Sys.iswindows()
  tracecompile_out = read(f, String)
  close(ptm) # close after reading so we don't get precompiles from error shutdown
 
- expected_precompiles = 2
+ # given this test checks that startup is snappy, it's best to add workloads to
+ # contrib/generate_precompile.jl rather than increase this number. But if that's not
+ # possible, it'd be helpful to add a comment with the statement and a reason below
+ expected_precompiles = 0
 
  n_precompiles = count(r"precompile\(", tracecompile_out)
 

stdlib/SuiteSparse_jll/src/SuiteSparse_jll.jl

@@ -3,7 +3,6 @@
 ## dummy stub for https://github.com/JuliaBinaryWrappers/SuiteSparse_jll.jl
 baremodule SuiteSparse_jll
 using Base, Libdl, libblastrampoline_jll
-Base.Experimental.@compiler_options compile=min optimize=0 infer=false
 
 const PATH_list = String[]
 const LIBPATH_list = String[]

stdlib/Zlib_jll/src/Zlib_jll.jl

@@ -3,7 +3,6 @@
 ## dummy stub for https://github.com/JuliaBinaryWrappers/Zlib_jll.jl
 baremodule Zlib_jll
 using Base, Libdl
-Base.Experimental.@compiler_options compile=min optimize=0 infer=false
 
 const PATH_list = String[]
 const LIBPATH_list = String[]

stdlib/dSFMT_jll/src/dSFMT_jll.jl

@@ -4,7 +4,6 @@
 
 baremodule dSFMT_jll
 using Base, Libdl
-Base.Experimental.@compiler_options compile=min optimize=0 infer=false
 
 const PATH_list = String[]
 const LIBPATH_list = String[]

stdlib/libLLVM_jll/src/libLLVM_jll.jl

@@ -4,7 +4,6 @@
 
 baremodule libLLVM_jll
 using Base, Libdl
-Base.Experimental.@compiler_options compile=min optimize=0 infer=false
 
 const PATH_list = String[]
 const LIBPATH_list = String[]

stdlib/libblastrampoline_jll/src/libblastrampoline_jll.jl

@@ -4,7 +4,6 @@
 
 baremodule libblastrampoline_jll
 using Base, Libdl
-Base.Experimental.@compiler_options compile=min optimize=0 infer=false
 
 const PATH_list = String[]
 const LIBPATH_list = String[]