emit precompile statements to separate file (JuliaLang#28419)

if the filename to trace--compile is stdout, print to the stdout stream

base/options.jl

@@ -26,7 +26,7 @@ struct JLOptions
  warn_overwrite::Int8
  can_inline::Int8
  polly::Int8
- trace_compile::Int8
+ trace_compile::Ptr{UInt8}
  fast_math::Int8
  worker::Int8
  cookie::Ptr{UInt8}

contrib/generate_precompile.jl

@@ -82,65 +82,63 @@ function generate_precompile_statements()
  print("Generating precompile statements...")
  sysimg = isempty(ARGS) ? joinpath(dirname(Sys.BINDIR), "lib", "julia", "sys.ji") : ARGS[1]
 
- # Run a repl process and replay our script
- stdout_accumulator, stderr_accumulator = IOBuffer(), IOBuffer()
- with_fake_pty() do slave, master
- with_fake_pty() do slave_err, master_err
- done = false
- withenv("JULIA_HISTORY" => tempname(), "JULIA_PROJECT" => nothing,
- "TERM" => "") do
- p = run(`$(julia_cmd()) -O0 --trace-compile=yes --sysimage $sysimg
- --startup-file=no --color=yes`,
- slave, slave, slave_err; wait=false)
- readuntil(master, "julia>", keep=true)
- for (tty, accumulator) in (master => stdout_accumulator,
- master_err => stderr_accumulator)
- @async begin
- while true
- done && break
- write(accumulator, readavailable(tty))
+ mktemp() do precompile_file, _
+ # Run a repl process and replay our script
+ stdout_accumulator, stderr_accumulator = IOBuffer(), IOBuffer()
+ with_fake_pty() do slave, master
+ with_fake_pty() do slave_err, master_err
+ done = false
+ withenv("JULIA_HISTORY" => tempname(), "JULIA_PROJECT" => nothing,
+ "TERM" => "") do
+ p = run(`$(julia_cmd()) -O0 --trace-compile=$precompile_file --sysimage $sysimg
+ --startup-file=no --color=yes`,
+ slave, slave, slave_err; wait=false)
+ readuntil(master, "julia>", keep=true)
+ for (tty, accumulator) in (master => stdout_accumulator,
+ master_err => stderr_accumulator)
+ @async begin
+ while true
+ done && break
+ write(accumulator, readavailable(tty))
+ end
  end
  end
- end
- if have_repl
- for l in split(precompile_script, '\n'; keepempty=false)
- write(master, l, '\n')
+  if have_repl
+  for l in split(precompile_script, '\n'; keepempty=false)
+  write(master, l, '\n')
+ end
  end
+ write(master, "exit()\n")
+ wait(p)
+ done = true
  end
- write(master, "exit()\n")
- wait(p)
- done = true
  end
  end
- end
 
- stderr_output = String(take!(stderr_accumulator))
- # println(stderr_output)
- # stdout_output = String(take!(stdout_accumulator))
- # println(stdout_output)
-
- # Extract the precompile statements from stderr
- statements = Set{String}()
- for statement in split(stderr_output, '\n')
- m = match(r"(precompile\(Tuple{.*)", statement)
- m === nothing && continue
- statement = m.captures[1]
- occursin(r"Main.", statement) && continue
- push!(statements, statement)
- end
+ # Check what the REPL displayed
+ # stdout_output = String(take!(stdout_accumulator))
+ # println(stdout_output)
 
- # Load the precompile statements
- statements_ordered = join(sort(collect(statements)), '\n')
- # println(statements_ordered)
- if have_repl
- # Seems like a reasonable number right now, adjust as needed
- @assert length(statements) > 700
- end
+ # Extract the precompile statements from stderr
+ statements = Set{String}()
+ for statement in split(read(precompile_file, String), '\n')
+ occursin("Main.", statement) && continue
+ push!(statements, statement)
+ end
+
+ # Load the precompile statements
+ statements_ordered = join(sort(collect(statements)), '\n')
+ # println(statements_ordered)
+ if have_repl
+ # Seems like a reasonable number right now, adjust as needed
+ @assert length(statements) > 700
+ end
 
- Base.include_string(PrecompileStagingArea, statements_ordered)
- print(" $(length(statements)) generated in ")
- Base.time_print((time() - start_time) * 10^9)
- println()
+ Base.include_string(PrecompileStagingArea, statements_ordered)
+ print(" $(length(statements)) generated in ")
+ Base.time_print((time() - start_time) * 10^9)
+ println()
+ end
  end
 
  # Fall back to explicit list on Windows, might as well include them

contrib/precompile_explicit.jl

@@ -3,7 +3,7 @@
 # Steps to regenerate this file:
 # 1. Remove all `precompile` calls
 # 2. Rebuild system image
-# 3. Start julia with `--trace-compile=yes and do some stuff
+# 3. Start julia with `--trace-compile=precompiles.txt and do some stuff
 # 5. Run `grep -v '#[0-9]' precompiles.txt >> contrib/precompile_explicit.jl`
 # (filters out closures, which might have different generated names in different environments)
 # This list is only used on Windows, otherwise precompile statements are generated dynamically.

src/gf.c

@@ -1071,6 +1071,10 @@ static jl_method_instance_t *cache_method(
 
 static jl_method_instance_t *jl_mt_assoc_by_type(jl_methtable_t *mt, jl_datatype_t *tt, int mt_cache, int allow_exec, size_t world)
 {
+ // TODO: Merge with jl_dump_compiles?
+ static ios_t f_precompile;
+ static JL_STREAM* s_precompile = NULL;
+
  // caller must hold the mt->writelock
  jl_typemap_entry_t *entry = NULL;
  entry = jl_typemap_assoc_by_type(mt->cache, (jl_value_t*)tt, NULL, /*subtype*/1, jl_cachearg_offset(mt), world, /*max_world_mask*/0);
@@ -1088,11 +1092,24 @@ static jl_method_instance_t *jl_mt_assoc_by_type(jl_methtable_t *mt, jl_datatype
  if (entry != NULL) {
  jl_method_t *m = entry->func.method;
  if (!jl_has_call_ambiguities((jl_value_t*)tt, m)) {
- if (jl_options.trace_compile) {
+ if (jl_options.trace_compile != NULL) {
+ if (s_precompile == NULL) {
+ const char* t = jl_options.trace_compile;
+ if (!strncmp(t, "stderr", 6))
+ s_precompile = JL_STDERR;
+ else {
+ if (ios_file(&f_precompile, t, 1, 1, 1, 1) == NULL)
+ jl_errorf("cannot open precompile statement file \"%s\" for writing", t);
+ s_precompile = (JL_STREAM*) &f_precompile;
+ }
+ }
  if (!jl_has_free_typevars((jl_value_t*)tt)) {
- jl_printf(JL_STDERR, "precompile(");
- jl_static_show(JL_STDERR, (jl_value_t*)tt);
- jl_printf(JL_STDERR, ")\n");
+ jl_printf(s_precompile, "precompile(");
+ jl_static_show(s_precompile, (jl_value_t*)tt);
+ jl_printf(s_precompile, ")\n");
+
+ if (s_precompile != JL_STDERR)
+ ios_flush(&f_precompile);
  }
  }
  if (!mt_cache) {

src/jloptions.c

@@ -55,11 +55,7 @@ jl_options_t jl_options = { 0, // quiet
  0, // method overwrite warning
  1, // can_inline
  JL_OPTIONS_POLLY_ON, // polly
-#ifdef TRACE_COMPILE
- 1, // trace_compile
-#else
- 0, // trace_compile
-#endif
+ NULL, // trace_compile
  JL_OPTIONS_FAST_MATH_DEFAULT,
  0, // worker
  NULL, // cookie
@@ -575,13 +571,9 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp)
  }
  break;
  case opt_trace_compile:
- if (!strcmp(optarg,"yes"))
- jl_options.trace_compile = 1;
- else if (!strcmp(optarg,"no"))
- jl_options.trace_compile = 0;
- else {
- jl_errorf("julia: invalid argument to --trace-compile (%s)", optarg);
- }
+ jl_options.trace_compile = strdup(optarg);
+ if (!jl_options.trace_compile)
+ jl_errorf("fatal error: failed to allocate memory: %s", strerror(errno));
  break;
  case opt_math_mode:
  if (!strcmp(optarg,"ieee"))

src/julia.h

@@ -1799,7 +1799,7 @@ typedef struct {
  int8_t warn_overwrite;
  int8_t can_inline;
  int8_t polly;
- int8_t trace_compile;
+ const char *trace_compile;
  int8_t fast_math;
  int8_t worker;
  const char *cookie;

src/options.h

@@ -89,7 +89,6 @@
 
 // print all signatures type inference is invoked on
 //#define TRACE_INFERENCE
-//#define TRACE_COMPILE
 
 // print all generic method dispatches (excludes inlined and specialized call
 // sites). this generally prints too much output to be useful.