The return of the "Generate precompilation as part of build process" (#…

…28319)

*"generate precompile as part of build process (#28118)" (#28318)"

Makefile

@@ -206,7 +206,7 @@ define sysimg_builder
 $$(build_private_libdir)/sys$1-o.a $$(build_private_libdir)/sys$1-bc.a : $$(build_private_libdir)/sys$1-%.a : $$(build_private_libdir)/sys.ji
  @$$(call PRINT_JULIA, cd $$(JULIAHOME)/base && \
  if ! $$(call spawn,$3) $2 -C "$$(JULIA_CPU_TARGET)" --output-$$* $$(call cygpath_w,$$@).tmp $$(JULIA_SYSIMG_BUILD_FLAGS) \
- --startup-file=no --warn-overwrite=yes --sysimage $$(call cygpath_w,$$<) -e nothing; then \
+ --startup-file=no --warn-overwrite=yes --sysimage $$(call cygpath_w,$$<) $$(call cygpath_w,$$(JULIAHOME)/contrib/generate_precompile.jl) $$(call cygpath_w,$$<); then \
  echo '*** This error is usually fixed by running `make clean`. If the error persists$$(COMMA) try `make cleanall`. ***'; \
  false; \
  fi )

base/options.jl

@@ -26,6 +26,7 @@ struct JLOptions
  warn_overwrite::Int8
  can_inline::Int8
  polly::Int8
+ trace_compile::Int8
  fast_math::Int8
  worker::Int8
  cookie::Ptr{UInt8}

base/sysimg.jl

@@ -522,9 +522,9 @@ let
  :SuiteSparse,
  :Distributed,
  :SharedArrays,
- :Pkg,
  :Test,
  :REPL,
+ :Pkg,
  :Statistics,
  ]
 
@@ -922,11 +922,9 @@ Base.init_load_path() # want to be able to find external packages in userimg.jl
 
 let
 tot_time_userimg = @elapsed (Base.isfile("userimg.jl") && Base.include(Main, "userimg.jl"))
-tot_time_precompile = Base.is_primary_base_module ? (@elapsed Base.include(Base, "precompile.jl")) : 0.0
-
 
 tot_time_base = (Base.end_base_include - Base.start_base_include) * 10.0^(-9)
-tot_time = tot_time_base + Base.tot_time_stdlib[] + tot_time_userimg + tot_time_precompile
+tot_time = tot_time_base + Base.tot_time_stdlib[] + tot_time_userimg
 
 println("Sysimage built. Summary:")
 print("Total ─────── "); Base.time_print(tot_time * 10^9); print(" \n");
@@ -935,7 +933,6 @@ print("Stdlibs: ──── "); Base.time_print(Base.tot_time_stdlib[] * 10^9);
 if isfile("userimg.jl")
 print("Userimg: ──── "); Base.time_print(tot_time_userimg * 10^9); print(" "); show(IOContext(stdout, :compact=>true), (tot_time_userimg / tot_time) * 100); println("%")
 end
-print("Precompile: ─ "); Base.time_print(tot_time_precompile * 10^9); print(" "); show(IOContext(stdout, :compact=>true), (tot_time_precompile / tot_time) * 100); println("%")
 end
 
 empty!(LOAD_PATH)

contrib/add_license_to_files.jl

@@ -30,7 +30,6 @@ const excludedirs = [
 
 const skipfiles = [
  "../contrib/add_license_to_files.jl",
- "../contrib/fixup_precompile.jl",
  # files to check - already copyright
  # see: https://github.com/JuliaLang/julia/pull/11073#issuecomment-98099389
  "../base/special/trig.jl",

contrib/generate_precompile.jl

@@ -0,0 +1,139 @@
+# Prevent this from being put into the Main namespace
+Base.reinit_stdio()
+let
+M = Module()
+@eval M begin
+julia_cmd() = (julia = joinpath(Sys.BINDIR, Base.julia_exename()); `$julia`)
+
+function generate_precompilable_package(path, pkgname)
+ mkpath(joinpath(path, pkgname, "src"))
+ write(joinpath(path, pkgname, "src", "$pkgname.jl"),
+ """
+ __precompile__()
+ module $pkgname
+ end
+ """)
+end
+
+function generate_precompile_statements()
+ t = time()
+ println("Generating precompile statements...")
+ println("──────────────────────────────────────")
+
+ # Reset code loading vars
+ push!(LOAD_PATH, "@stdlib")
+
+ tmpd = mktempdir()
+ push!(DEPOT_PATH, tmpd)
+ push!(LOAD_PATH, tmpd)
+ pkgname = "__PackagePrecompilationStatementModule"
+ generate_precompilable_package(tmpd, pkgname)
+ @eval using __PackagePrecompilationStatementModule
+ pop!(LOAD_PATH)
+ pop!(DEPOT_PATH)
+ rm(tmpd; recursive=true)
+
+ if isempty(ARGS)
+ sysimg = joinpath(dirname(Sys.BINDIR), "lib", "julia", "sys.ji")
+ else
+ sysimg = ARGS[1]
+ end
+
+ tmp = tempname()
+ touch(tmp)
+ have_repl = haskey(Base.loaded_modules,
+ Base.PkgId(Base.UUID("3fa0cd96-eef1-5676-8a61-b3b8758bbffb"), "REPL"))
+ if have_repl
+ # Have a REPL, run the repl replayer and an interactive session that we immidiately kill
+ setup = """
+ include($(repr(joinpath(@__DIR__, "precompile_replay.jl"))))
+ @async while true
+ sleep(0.05)
+ if isdefined(Base, :active_repl)
+ exit(0)
+ end
+ end
+ """
+ # Do not redirect stdin unless it is to a tty, because that changes code paths
+ ok = try
+ run(pipeline(`$(julia_cmd()) --sysimage $sysimg --trace-compile=yes -O0
+ --startup-file=no --q -e $setup -i`; stderr=tmp))
+ true
+ catch
+ false
+ end
+ else
+ # No REPL, just record the startup
+ ok = try
+ run(pipeline(`$(julia_cmd()) --sysimage $sysimg --trace-compile=yes -O0
+ --startup-file=no --q -e0`; stderr=tmp))
+ true
+ catch
+ false
+ end
+ end
+
+ s = read(tmp, String)
+ if !ok
+ error("precompilation process failed, stderr is:\n$s")
+ end
+ new_precompiles = Set{String}()
+ for statement in split(s, '\n')
+ startswith(statement, "precompile(Tuple{") || continue
+ # Replace the FakeTerminal with a TTYTerminal and filter out everything we compiled in Main
+ statement = replace(statement, "FakeTerminals.FakeTerminal" => "REPL.Terminals.TTYTerminal")
+ (occursin(r"Main.", statement) || occursin(r"FakeTerminals.", statement)) && continue
+ # AppVeyor CI emits a single faulty precompile statement:
+ # precompile(Tuple{getfield(precompile(Tuple{typeof(Base.uvfinalize), Base.PipeEndpoint})
+ # which lacks the correct closing brackets.
+ # Filter out such lines here.
+ for (l, r) in ('(' => ')', '{' => '}')
+ if count(isequal(l), statement) != count(isequal(r), statement)
+ continue
+ end
+ push!(new_precompiles, statement)
+ end
+ end
+
+ PrecompileStagingArea = Module()
+ for (_pkgid, _mod) in Base.loaded_modules
+ if !(_pkgid.name in ("Main", "Core", "Base"))
+ eval(PrecompileStagingArea, :($(Symbol(_mod)) = $_mod))
+ end
+ end
+ # Load the precompile statements
+ Base.include_string(PrecompileStagingArea, join(collect(new_precompiles), '\n'))
+
+ # Add a few manual things, run `julia` with `--trace-compile` to find these.
+ if have_repl
+ @eval PrecompileStagingArea begin
+ # Could startup with REPL banner instead but it is a bit visually noisy,
+ # so just precompile it manualally here.
+ precompile(Tuple{typeof(Base.banner), REPL.Terminals.TTYTerminal})
+
+ # This is probablably important for precompilation (0.2s precompile time)
+ # but doesn't seem to get caught in the script above
+ precompile(Tuple{typeof(Base.create_expr_cache), String, String, Array{Base.Pair{Base.PkgId, UInt64}, 1}, Base.UUID})
+
+ # Manual things from starting and exiting julia
+ precompile(Tuple{Type{Logging.ConsoleLogger}, Base.TTY})
+ precompile(Tuple{Type{REPL.Terminals.TTYTerminal}, String, Base.TTY, Base.TTY, Base.TTY})
+ precompile(Tuple{typeof(Base.displaysize), Base.TTY})
+ precompile(Tuple{typeof(REPL.LineEdit.prompt_string), typeof(Base.input_color)})
+ precompile(Tuple{typeof(Base.input_color)})
+ precompile(Tuple{typeof(Base.eof), Base.TTY})
+ precompile(Tuple{typeof(Base.alloc_buf_hook), Base.TTY, UInt64})
+ precompile(Tuple{typeof(Base.read), Base.TTY, Type{UInt8}})
+ end
+ end
+
+ println("──────────────────────────────────────")
+ print("$(length(new_precompiles)) precompile statements generated in "), Base.time_print((time() - t) * 10^9)
+ println()
+ return
+end
+
+generate_precompile_statements()
+
+end # @eval
+end # let

contrib/precompile_replay.jl

@@ -0,0 +1,73 @@
+import REPL
+include(joinpath(Sys.STDLIB, "REPL", "test", "FakeTerminals.jl"))
+import .FakeTerminals.FakeTerminal
+
+const CTRL_C = '\x03'
+const UP_ARROW = "\e[A"
+const DOWN_ARROW = "\e[B"
+
+# TODO: Have a utility to generate this from a real REPL session?
+precompile_script = """
+2+2
+print("")
+@time 1+1
+?reinterpret
+;pwd
+using Ra\t$CTRL_C
+\\alpha\t$CTRL_C
+\e[200~paste here ;)\e[201~"$CTRL_C
+$UP_ARROW$DOWN_ARROW
+123\b\b\b$CTRL_C
+\b\b
+f(x) = x03
+f(1,2)
+[][1]
+cd("complet_path\t\t$CTRL_C
+"""
+
+function fake_repl(@nospecialize(f))
+ input = Pipe()
+ output = Pipe()
+ err = Pipe()
+ Base.link_pipe!(input, reader_supports_async=true, writer_supports_async=true)
+ Base.link_pipe!(output, reader_supports_async=true, writer_supports_async=true)
+ Base.link_pipe!(err, reader_supports_async=true, writer_supports_async=true)
+
+ repl = REPL.LineEditREPL(FakeTerminal(input.out, output.in, err.in), true)
+
+ f(input.in, output.out, repl)
+ t = @async begin
+ close(input.in)
+ close(output.in)
+ close(err.in)
+ end
+ #print(read(output.out, String))
+ Base._wait(t)
+ nothing
+end
+
+function run_repl()
+ # Writing ^C to the repl will cause sigint, so let's not die on that
+ ccall(:jl_exit_on_sigint, Cvoid, (Cint,), 0)
+
+ fake_repl() do stdin_write, stdout_read, repl
+ repl.specialdisplay = REPL.REPLDisplay(repl)
+ repl.history_file = false
+
+ repltask = @async begin
+ REPL.run_repl(repl)
+ end
+
+ write(stdin_write, precompile_script)
+ # Close REPL ^D
+ write(stdin_write, '\x04')
+
+ readavailable(stdout_read)
+
+ Base._wait(repltask)
+ end
+
+ ccall(:jl_exit_on_sigint, Cvoid, (Cint,), 1)
+end
+
+run_repl()

src/gf.c

@@ -1088,13 +1088,13 @@ static jl_method_instance_t *jl_mt_assoc_by_type(jl_methtable_t *mt, jl_datatype
  if (entry != NULL) {
  jl_method_t *m = entry->func.method;
  if (!jl_has_call_ambiguities((jl_value_t*)tt, m)) {
-#ifdef TRACE_COMPILE
- if (!jl_has_free_typevars((jl_value_t*)tt)) {
- jl_printf(JL_STDERR, "precompile(");
- jl_static_show(JL_STDERR, (jl_value_t*)tt);
- jl_printf(JL_STDERR, ")\n");
+ if (jl_options.trace_compile) {
+ if (!jl_has_free_typevars((jl_value_t*)tt)) {
+ jl_printf(JL_STDERR, "precompile(");
+ jl_static_show(JL_STDERR, (jl_value_t*)tt);
+ jl_printf(JL_STDERR, ")\n");
+ }
  }
-#endif
  if (!mt_cache) {
  intptr_t nspec = (mt == jl_type_type_mt ? m->nargs + 1 : mt->max_args + 2);
  jl_compilation_sig(tt, env, m, nspec, &newparams);