From 924484f2f0042182b4ce28772b6a705ba2b472e8 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 7 Oct 2020 20:36:17 -0700 Subject: [PATCH] build: improve precompile generation script (#37918) --- contrib/generate_precompile.jl | 113 +++++++++++++++++---------------- 1 file changed, 59 insertions(+), 54 deletions(-) diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl index acc05b8881c9e..6d556799d165f 100644 --- a/contrib/generate_precompile.jl +++ b/contrib/generate_precompile.jl @@ -29,7 +29,7 @@ hardcoded_precompile_statements = """ @assert precompile(Tuple{typeof(Base.Experimental.register_error_hint), Any, Type}) """ -precompile_script = """ +repl_script = """ 2+2 print("") @time 1+1 @@ -45,6 +45,9 @@ f(x) = x03 f(1,2) [][1] cd("complet_path\t\t$CTRL_C +""" + +precompile_script = """ # Used by JuliaInterpreter push!(Set{Module}(), Main) push!(Set{Method}(), first(methods(collect))) @@ -88,7 +91,7 @@ if Artifacts !== nothing precompile_script *= """ using Artifacts, Base.BinaryPlatforms, Libdl artifacts_toml = abspath(joinpath(Sys.STDLIB, "Artifacts", "test", "Artifacts.toml")) - cd(() -> @artifact_str("c_simple"), dirname(artifacts_toml)) + # cd(() -> (name = "c_simple"; @artifact_str(name)), dirname(artifacts_toml)) artifacts = Artifacts.load_artifacts_toml(artifacts_toml) platforms = [Artifacts.unpack_platform(e, "c_simple", artifacts_toml) for e in artifacts["c_simple"]] best_platform = select_platform(Dict(p => triplet(p) for p in platforms)) @@ -102,7 +105,8 @@ Pkg = get(Base.loaded_modules, nothing) if Pkg !== nothing - precompile_script *= Pkg.precompile_script + # TODO: Split Pkg precompile script into REPL and script part + repl_script *= Pkg.precompile_script end FileWatching = get(Base.loaded_modules, @@ -141,10 +145,17 @@ function generate_precompile_statements() debug_output = devnull # or stdout sysimg = Base.unsafe_string(Base.JLOptions().image_file) - # Precompile a package - global hardcoded_precompile_statements + # Extract the precompile statements from the precompile file + statements = Set{String}() + + # From hardcoded statements + for statement in split(hardcoded_precompile_statements::String, '\n') + push!(statements, statement) + end + # Collect statements from running the script mktempdir() do prec_path + # Also precompile a package here pkgname = "__PackagePrecompilationStatementModule" mkpath(joinpath(prec_path, pkgname, "src")) path = joinpath(prec_path, pkgname, "src", "$pkgname.jl") @@ -154,20 +165,20 @@ function generate_precompile_statements() end """) tmp = tempname() - # Running compilecache on buildbots fails with - # `More than one command line CPU targets specified without a `--output-` flag specified` - # so start a new process without a CPU target specified s = """ push!(DEPOT_PATH, $(repr(prec_path))); Base.PRECOMPILE_TRACE_COMPILE[] = $(repr(tmp)); Base.compilecache(Base.PkgId($(repr(pkgname))), $(repr(path))) + $precompile_script """ run(`$(julia_exepath()) -O0 --sysimage $sysimg --startup-file=no -Cnative -e $s`) - hardcoded_precompile_statements *= "\n" * read(tmp, String) + for statement in split(read(tmp, String), '\n') + push!(statements, statement) + end end mktemp() do precompile_file, precompile_file_h - # Run a repl process and replay our script + # Collect statements from running a REPL process and replaying our REPL script pts, ptm = open_fake_pty() blackhole = Sys.isunix() ? "/dev/null" : "nul" if have_repl @@ -212,12 +223,12 @@ function generate_precompile_statements() readavailable(output_copy) # Input our script if have_repl - precompile_lines = split(precompile_script::String, '\n'; keepempty=false) + precompile_lines = split(repl_script::String, '\n'; keepempty=false) curr = 0 for l in precompile_lines sleep(0.1) curr += 1 - print("\rGenerating precompile statements... $curr/$(length(precompile_lines))") + print("\rGenerating REPL precompile statements... $curr/$(length(precompile_lines))") # consume any other output bytesavailable(output_copy) > 0 && readavailable(output_copy) # push our input @@ -237,57 +248,51 @@ function generate_precompile_statements() close(ptm) write(debug_output, "\n#### FINISHED ####\n") - # Extract the precompile statements from the precompile file - statements = Set{String}() - for statement in eachline(precompile_file_h) + for statement in split(read(precompile_file, String), '\n') # Main should be completely clean occursin("Main.", statement) && continue push!(statements, statement) end + end - for statement in split(hardcoded_precompile_statements::String, '\n') - push!(statements, statement) - end - - # Create a staging area where all the loaded packages are available - PrecompileStagingArea = Module() - for (_pkgid, _mod) in Base.loaded_modules - if !(_pkgid.name in ("Main", "Core", "Base")) - eval(PrecompileStagingArea, :(const $(Symbol(_mod)) = $_mod)) - end + # Create a staging area where all the loaded packages are available + PrecompileStagingArea = Module() + for (_pkgid, _mod) in Base.loaded_modules + if !(_pkgid.name in ("Main", "Core", "Base")) + eval(PrecompileStagingArea, :(const $(Symbol(_mod)) = $_mod)) end + end - # Execute the collected precompile statements - n_succeeded = 0 - include_time = @elapsed for statement in sort(collect(statements)) - # println(statement) - # The compiler has problem caching signatures with `Vararg{?, N}`. Replacing - # N with a large number seems to work around it. - statement = replace(statement, r"Vararg{(.*?), N} where N" => s"Vararg{\1, 100}") - try - Base.include_string(PrecompileStagingArea, statement) - n_succeeded += 1 - print("\rExecuting precompile statements... $n_succeeded/$(length(statements))") - catch - # See #28808 - # @error "Failed to precompile $statement" - end + # Execute the collected precompile statements + n_succeeded = 0 + include_time = @elapsed for statement in sort(collect(statements)) + # println(statement) + # The compiler has problem caching signatures with `Vararg{?, N}`. Replacing + # N with a large number seems to work around it. + statement = replace(statement, r"Vararg{(.*?), N} where N" => s"Vararg{\1, 100}") + try + Base.include_string(PrecompileStagingArea, statement) + n_succeeded += 1 + print("\rExecuting precompile statements... $n_succeeded/$(length(statements))") + catch + # See #28808 + # @error "Failed to precompile $statement" end - println() - if have_repl - # Seems like a reasonable number right now, adjust as needed - # comment out if debugging script - @assert n_succeeded > 1200 - end - - tot_time = time_ns() - start_time - include_time *= 1e9 - gen_time = tot_time - include_time - println("Precompilation complete. Summary:") - print("Total ─────── "); Base.time_print(tot_time); println() - print("Generation ── "); Base.time_print(gen_time); print(" "); show(IOContext(stdout, :compact=>true), gen_time / tot_time * 100); println("%") - print("Execution ─── "); Base.time_print(include_time); print(" "); show(IOContext(stdout, :compact=>true), include_time / tot_time * 100); println("%") end + println() + if have_repl + # Seems like a reasonable number right now, adjust as needed + # comment out if debugging script + @assert n_succeeded > 1200 + end + + tot_time = time_ns() - start_time + include_time *= 1e9 + gen_time = tot_time - include_time + println("Precompilation complete. Summary:") + print("Total ─────── "); Base.time_print(tot_time); println() + print("Generation ── "); Base.time_print(gen_time); print(" "); show(IOContext(stdout, :compact=>true), gen_time / tot_time * 100); println("%") + print("Execution ─── "); Base.time_print(include_time); print(" "); show(IOContext(stdout, :compact=>true), include_time / tot_time * 100); println("%") return end