forked from JuliaLang/julia
-
Notifications
You must be signed in to change notification settings - Fork 0
/
generate_precompile.jl
258 lines (234 loc) · 9.21 KB
/
generate_precompile.jl
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
# This file is a part of Julia. License is MIT: https://julialang.org/license
if isempty(ARGS) || ARGS[1] !== "0"
Sys.__init_build()
# Prevent this from being put into the Main namespace
@eval Module() begin
if !isdefined(Base, :uv_eventloop)
Base.reinit_stdio()
end
Base.include(@__MODULE__, joinpath(Sys.BINDIR, "..", "share", "julia", "test", "testhelpers", "FakePTYs.jl"))
import .FakePTYs: open_fake_pty
CTRL_C = '\x03'
UP_ARROW = "\e[A"
DOWN_ARROW = "\e[B"
hardcoded_precompile_statements = """
@assert precompile(Tuple{typeof(Base.stale_cachefile), String, String})
@assert precompile(Tuple{typeof(push!), Set{Module}, Module})
@assert precompile(Tuple{typeof(push!), Set{Method}, Method})
@assert precompile(Tuple{typeof(push!), Set{Base.PkgId}, Base.PkgId})
@assert precompile(Tuple{typeof(setindex!), Dict{String,Base.PkgId}, Base.PkgId, String})
@assert precompile(Tuple{typeof(get!), Type{Vector{Function}}, Dict{Base.PkgId,Vector{Function}}, Base.PkgId})
@assert precompile(Tuple{typeof(isassigned), Core.SimpleVector, Int})
@assert precompile(Tuple{typeof(pushfirst!), Vector{Any}, Function})
"""
precompile_script = """
2+2
print("")
@time 1+1
; pwd
? reinterpret
using Ra\t$CTRL_C
\\alpha\t$CTRL_C
\e[200~paste here ;)\e[201~"$CTRL_C
$UP_ARROW$DOWN_ARROW$CTRL_C
123\b\b\b$CTRL_C
\b\b$CTRL_C
f(x) = x03
f(1,2)
[][1]
cd("complet_path\t\t$CTRL_C
"""
julia_exepath() = joinpath(Sys.BINDIR, Base.julia_exename())
have_repl = haskey(Base.loaded_modules,
Base.PkgId(Base.UUID("3fa0cd96-eef1-5676-8a61-b3b8758bbffb"), "REPL"))
if have_repl
hardcoded_precompile_statements *= """
@assert precompile(Tuple{typeof(getproperty), REPL.REPLBackend, Symbol})
"""
end
Distributed = get(Base.loaded_modules,
Base.PkgId(Base.UUID("8ba89e20-285c-5b6f-9357-94700520ee1b"), "Distributed"),
nothing)
if Distributed !== nothing
precompile_script *= """
using Distributed
addprocs(2)
pmap(x->iseven(x) ? 1 : 0, 1:4)
@distributed (+) for i = 1:100 Int(rand(Bool)) end
"""
end
Artifacts = get(Base.loaded_modules,
Base.PkgId(Base.UUID("56f22d72-fd6d-98f1-02f0-08ddc0907c33"), "Artifacts"),
nothing)
if Artifacts !== nothing
precompile_script *= """
using Artifacts, Base.BinaryPlatforms
artifacts_toml = abspath($(repr(joinpath(Sys.STDLIB, "Artifacts", "test", "Artifacts.toml"))))
cd(() -> @artifact_str("c_simple"), dirname(artifacts_toml))
artifacts = Artifacts.load_artifacts_toml(artifacts_toml)
platforms = [Artifacts.unpack_platform(e, "c_simple", artifacts_toml) for e in artifacts["c_simple"]]
best_platform = select_platform(Dict(p => triplet(p) for p in platforms))
"""
end
Pkg = get(Base.loaded_modules,
Base.PkgId(Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg"),
nothing)
if Pkg !== nothing
precompile_script *= Pkg.precompile_script
end
FileWatching = get(Base.loaded_modules,
Base.PkgId(Base.UUID("7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee"), "FileWatching"),
nothing)
if FileWatching !== nothing
hardcoded_precompile_statements *= """
@assert precompile(Tuple{typeof(FileWatching.watch_file), String, Float64})
@assert precompile(Tuple{typeof(FileWatching.watch_file), String, Int})
"""
end
function generate_precompile_statements()
start_time = time_ns()
debug_output = devnull # or stdout
# Precompile a package
mktempdir() do prec_path
push!(DEPOT_PATH, prec_path)
push!(LOAD_PATH, prec_path)
pkgname = "__PackagePrecompilationStatementModule"
mkpath(joinpath(prec_path, pkgname, "src"))
write(joinpath(prec_path, pkgname, "src", "$pkgname.jl"),
"""
module $pkgname
end
""")
@eval using __PackagePrecompilationStatementModule
empty!(LOAD_PATH)
empty!(DEPOT_PATH)
end
mktemp() do precompile_file, precompile_file_h
# Run a repl process and replay our script
pts, ptm = open_fake_pty()
blackhole = Sys.isunix() ? "/dev/null" : "nul"
if have_repl
cmdargs = ```--color=yes
-e 'import REPL; REPL.Terminals.is_precompiling[] = true'
```
else
cmdargs = `-e nothing`
end
p = withenv("JULIA_HISTORY" => blackhole,
"JULIA_PROJECT" => nothing, # remove from environment
"JULIA_LOAD_PATH" => Sys.iswindows() ? "@;@stdlib" : "@:@stdlib",
"TERM" => "") do
sysimg = Base.unsafe_string(Base.JLOptions().image_file)
run(```$(julia_exepath()) -O0 --trace-compile=$precompile_file --sysimage $sysimg
--cpu-target=native --startup-file=no --color=yes
-e 'import REPL; REPL.Terminals.is_precompiling[] = true'
-i $cmdargs```,
pts, pts, pts; wait=false)
end
Base.close_stdio(pts)
# Prepare a background process to copy output from process until `pts` is closed
output_copy = Base.BufferStream()
tee = @async try
while !eof(ptm)
l = readavailable(ptm)
write(debug_output, l)
Sys.iswindows() && (sleep(0.1); yield(); yield()) # workaround hang - probably a libuv issue?
write(output_copy, l)
end
close(output_copy)
close(ptm)
catch ex
close(output_copy)
close(ptm)
if !(ex isa Base.IOError && ex.code == Base.UV_EIO)
rethrow() # ignore EIO on ptm after pts dies
end
end
# wait for the definitive prompt before start writing to the TTY
readuntil(output_copy, "julia>")
sleep(0.1)
readavailable(output_copy)
# Input our script
if have_repl
precompile_lines = split(precompile_script, '\n'; keepempty=false)
curr = 0
for l in precompile_lines
sleep(0.1)
curr += 1
print("\rGenerating precompile statements... $curr/$(length(precompile_lines))")
# consume any other output
bytesavailable(output_copy) > 0 && readavailable(output_copy)
# push our input
write(debug_output, "\n#### inputting statement: ####\n$(repr(l))\n####\n")
write(ptm, l, "\n")
readuntil(output_copy, "\n")
# wait for the next prompt-like to appear
# NOTE: this is rather innaccurate because the Pkg REPL mode is a special flower
readuntil(output_copy, "\n")
readuntil(output_copy, "> ")
end
println()
end
write(ptm, "exit()\n")
wait(tee)
success(p) || Base.pipeline_error(p)
close(ptm)
write(debug_output, "\n#### FINISHED ####\n")
# Extract the precompile statements from the precompile file
statements = Set{String}()
for statement in eachline(precompile_file_h)
# Main should be completely clean
occursin("Main.", statement) && continue
push!(statements, statement)
end
for statement in split(hardcoded_precompile_statements, '\n')
push!(statements, statement)
end
# Create a staging area where all the loaded packages are available
PrecompileStagingArea = Module()
for (_pkgid, _mod) in Base.loaded_modules
if !(_pkgid.name in ("Main", "Core", "Base"))
eval(PrecompileStagingArea, :(const $(Symbol(_mod)) = $_mod))
end
end
# Execute the collected precompile statements
n_succeeded = 0
include_time = @elapsed for statement in sort(collect(statements))
# println(statement)
try
Base.include_string(PrecompileStagingArea, statement)
n_succeeded += 1
print("\rExecuting precompile statements... $n_succeeded/$(length(statements))")
catch
# See #28808
# @error "Failed to precompile $statement"
end
end
println()
if have_repl
# Seems like a reasonable number right now, adjust as needed
# comment out if debugging script
@assert n_succeeded > 1200
end
tot_time = time_ns() - start_time
include_time *= 1e9
gen_time = tot_time - include_time
println("Precompilation complete. Summary:")
print("Total ─────── "); Base.time_print(tot_time); println()
print("Generation ── "); Base.time_print(gen_time); print(" "); show(IOContext(stdout, :compact=>true), gen_time / tot_time * 100); println("%")
print("Execution ─── "); Base.time_print(include_time); print(" "); show(IOContext(stdout, :compact=>true), include_time / tot_time * 100); println("%")
end
return
end
generate_precompile_statements()
# As a last step in system image generation,
# remove some references to build time environment for a more reproducible build.
@eval Base PROGRAM_FILE = ""
@eval Sys begin
BINDIR = ""
STDLIB = ""
end
empty!(Base.ARGS)
empty!(Core.ARGS)
end # @eval
end