build: improve precompile generation script (JuliaLang#37918)

contrib/generate_precompile.jl

@@ -29,7 +29,7 @@ hardcoded_precompile_statements = """
 @assert precompile(Tuple{typeof(Base.Experimental.register_error_hint), Any, Type})
 """
 
-precompile_script = """
+repl_script = """
 2+2
 print("")
 @time 1+1
@@ -45,6 +45,9 @@ f(x) = x03
 f(1,2)
 [][1]
 cd("complet_path\t\t$CTRL_C
+"""
+
+precompile_script = """
 # Used by JuliaInterpreter
 push!(Set{Module}(), Main)
 push!(Set{Method}(), first(methods(collect)))
@@ -88,7 +91,7 @@ if Artifacts !== nothing
  precompile_script *= """
  using Artifacts, Base.BinaryPlatforms, Libdl
  artifacts_toml = abspath(joinpath(Sys.STDLIB, "Artifacts", "test", "Artifacts.toml"))
- cd(() -> @artifact_str("c_simple"), dirname(artifacts_toml))
+ # cd(() -> (name = "c_simple"; @artifact_str(name)), dirname(artifacts_toml))
  artifacts = Artifacts.load_artifacts_toml(artifacts_toml)
  platforms = [Artifacts.unpack_platform(e, "c_simple", artifacts_toml) for e in artifacts["c_simple"]]
  best_platform = select_platform(Dict(p => triplet(p) for p in platforms))
@@ -102,7 +105,8 @@ Pkg = get(Base.loaded_modules,
  nothing)
 
 if Pkg !== nothing
- precompile_script *= Pkg.precompile_script
+ # TODO: Split Pkg precompile script into REPL and script part
+ repl_script *= Pkg.precompile_script
 end
 
 FileWatching = get(Base.loaded_modules,
@@ -141,10 +145,17 @@ function generate_precompile_statements()
  debug_output = devnull # or stdout
  sysimg = Base.unsafe_string(Base.JLOptions().image_file)
 
- # Precompile a package
- global hardcoded_precompile_statements
+ # Extract the precompile statements from the precompile file
+ statements = Set{String}()
+
+ # From hardcoded statements
+ for statement in split(hardcoded_precompile_statements::String, '\n')
+ push!(statements, statement)
+ end
 
+ # Collect statements from running the script
  mktempdir() do prec_path
+ # Also precompile a package here
  pkgname = "__PackagePrecompilationStatementModule"
  mkpath(joinpath(prec_path, pkgname, "src"))
  path = joinpath(prec_path, pkgname, "src", "$pkgname.jl")
@@ -154,20 +165,20 @@ function generate_precompile_statements()
  end
  """)
  tmp = tempname()
- # Running compilecache on buildbots fails with
- # `More than one command line CPU targets specified without a `--output-` flag specified`
- # so start a new process without a CPU target specified
  s = """
  push!(DEPOT_PATH, $(repr(prec_path)));
  Base.PRECOMPILE_TRACE_COMPILE[] = $(repr(tmp));
  Base.compilecache(Base.PkgId($(repr(pkgname))), $(repr(path)))
+ $precompile_script
  """
  run(`$(julia_exepath()) -O0 --sysimage $sysimg --startup-file=no -Cnative -e $s`)
- hardcoded_precompile_statements *= "\n" * read(tmp, String)
+ for statement in split(read(tmp, String), '\n')
+ push!(statements, statement)
+ end
  end
 
  mktemp() do precompile_file, precompile_file_h
- # Run a repl process and replay our script
+ # Collect statements from running a REPL process and replaying our REPL script
  pts, ptm = open_fake_pty()
  blackhole = Sys.isunix() ? "/dev/null" : "nul"
  if have_repl
@@ -212,12 +223,12 @@ function generate_precompile_statements()
  readavailable(output_copy)
  # Input our script
  if have_repl
- precompile_lines = split(precompile_script::String, '\n'; keepempty=false)
+ precompile_lines = split(repl_script::String, '\n'; keepempty=false)
  curr = 0
  for l in precompile_lines
  sleep(0.1)
  curr += 1
- print("\rGenerating precompile statements... $curr/$(length(precompile_lines))")
+ print("\rGenerating REPL precompile statements... $curr/$(length(precompile_lines))")
  # consume any other output
  bytesavailable(output_copy) > 0 && readavailable(output_copy)
  # push our input
@@ -237,57 +248,51 @@ function generate_precompile_statements()
  close(ptm)
  write(debug_output, "\n#### FINISHED ####\n")
 
- # Extract the precompile statements from the precompile file
- statements = Set{String}()
- for statement in eachline(precompile_file_h)
+ for statement in split(read(precompile_file, String), '\n')
  # Main should be completely clean
  occursin("Main.", statement) && continue
  push!(statements, statement)
  end
+ end
 
- for statement in split(hardcoded_precompile_statements::String, '\n')
- push!(statements, statement)
- end
-
- # Create a staging area where all the loaded packages are available
- PrecompileStagingArea = Module()
- for (_pkgid, _mod) in Base.loaded_modules
- if !(_pkgid.name in ("Main", "Core", "Base"))
- eval(PrecompileStagingArea, :(const $(Symbol(_mod)) = $_mod))
- end
+ # Create a staging area where all the loaded packages are available
+ PrecompileStagingArea = Module()
+ for (_pkgid, _mod) in Base.loaded_modules
+ if !(_pkgid.name in ("Main", "Core", "Base"))
+ eval(PrecompileStagingArea, :(const $(Symbol(_mod)) = $_mod))
  end
+ end
 
- # Execute the collected precompile statements
- n_succeeded = 0
- include_time = @elapsed for statement in sort(collect(statements))
- # println(statement)
- # The compiler has problem caching signatures with `Vararg{?, N}`. Replacing
- # N with a large number seems to work around it.
- statement = replace(statement, r"Vararg{(.*?), N} where N" => s"Vararg{\1, 100}")
- try
- Base.include_string(PrecompileStagingArea, statement)
- n_succeeded += 1
- print("\rExecuting precompile statements... $n_succeeded/$(length(statements))")
- catch
- # See #28808
- # @error "Failed to precompile $statement"
- end
+ # Execute the collected precompile statements
+ n_succeeded = 0
+ include_time = @elapsed for statement in sort(collect(statements))
+ # println(statement)
+ # The compiler has problem caching signatures with `Vararg{?, N}`. Replacing
+ # N with a large number seems to work around it.
+ statement = replace(statement, r"Vararg{(.*?), N} where N" => s"Vararg{\1, 100}")
+ try
+ Base.include_string(PrecompileStagingArea, statement)
+ n_succeeded += 1
+ print("\rExecuting precompile statements... $n_succeeded/$(length(statements))")
+ catch
+ # See #28808
+ # @error "Failed to precompile $statement"
  end
- println()
- if have_repl
- # Seems like a reasonable number right now, adjust as needed
- # comment out if debugging script
- @assert n_succeeded > 1200
- end
-
- tot_time = time_ns() - start_time
- include_time *= 1e9
- gen_time = tot_time - include_time
- println("Precompilation complete. Summary:")
- print("Total ─────── "); Base.time_print(tot_time); println()
- print("Generation ── "); Base.time_print(gen_time); print(" "); show(IOContext(stdout, :compact=>true), gen_time / tot_time * 100); println("%")
- print("Execution ─── "); Base.time_print(include_time); print(" "); show(IOContext(stdout, :compact=>true), include_time / tot_time * 100); println("%")
  end
+ println()
+ if have_repl
+ # Seems like a reasonable number right now, adjust as needed
+ # comment out if debugging script
+ @assert n_succeeded > 1200
+ end
+
+ tot_time = time_ns() - start_time
+ include_time *= 1e9
+ gen_time = tot_time - include_time
+ println("Precompilation complete. Summary:")
+ print("Total ─────── "); Base.time_print(tot_time); println()
+ print("Generation ── "); Base.time_print(gen_time); print(" "); show(IOContext(stdout, :compact=>true), gen_time / tot_time * 100); println("%")
+ print("Execution ─── "); Base.time_print(include_time); print(" "); show(IOContext(stdout, :compact=>true), include_time / tot_time * 100); println("%")
 
  return
 end