Merge pull request JuliaLang#8745 from JuliaLang/jn/static_compile_4

incremental precompilation (part 4: user-interface)!

NEWS.md

@@ -31,6 +31,12 @@ New language features
  * The syntax `function foo end` can be used to introduce a generic function without
  yet adding any methods ([#8283]).
 
+ * Incremental compilation of modules: ``Base.compile(module::Symbol)`` (stored in `~/.julia/lib/v0.4`)
+
+ * See manual section on `Module initialization and precompilation` (under `Modules`) for details and errata.
+
+ * New option `--compile-incremental={yes|no}` added to invoke the equivalent of ``Base.compile`` from the command line.
+
 Language changes
 ----------------
 
@@ -450,6 +456,8 @@ Deprecated or removed
 
  * `sync_gc_total_bytes` -> `jl_gc_sync_total_bytes`
 
+ * `require(::AbstractString)` and `reload` (see news about addition of `compile`)
+
 Julia v0.3.0 Release Notes
 ==========================
 

base/boot.jl

@@ -71,8 +71,8 @@
 # module::Module
 #end
 
-#type Box{T}
-# contents::T
+#type Box
+# contents::Any
 #end
 
 #abstract Ref{T}

base/client.jl

@@ -274,7 +274,9 @@ let reqarg = Set(UTF8String["--home", "-H",
  end
  # load file immediately on all processors
  if opts.load != C_NULL
- require(bytestring(opts.load))
+ @sync for p in procs()
+ @async remotecall_fetch(p, include, bytestring(opts.load))
+ end
  end
  # eval expression
  if opts.eval != C_NULL
@@ -322,13 +324,16 @@ is_interactive = false
 isinteractive() = (is_interactive::Bool)
 
 const LOAD_PATH = ByteString[]
+const LOAD_CACHE_PATH = ByteString[]
 function init_load_path()
  vers = "v$(VERSION.major).$(VERSION.minor)"
  if haskey(ENV,"JULIA_LOAD_PATH")
  prepend!(LOAD_PATH, split(ENV["JULIA_LOAD_PATH"], @windows? ';' : ':'))
  end
  push!(LOAD_PATH,abspath(JULIA_HOME,"..","local","share","julia","site",vers))
  push!(LOAD_PATH,abspath(JULIA_HOME,"..","share","julia","site",vers))
+ push!(LOAD_CACHE_PATH,abspath(homedir(),".julia","lib",vers))
+ push!(LOAD_CACHE_PATH,abspath(JULIA_HOME,"..","usr","lib","julia")) #TODO: fixme
 end
 
 function load_juliarc()

base/deprecated.jl

@@ -635,7 +635,6 @@ end
 @deprecate mmap_bitarray{N}(::Type{Bool}, dims::NTuple{N,Integer}, s::IOStream, offset::FileOffset=position(s)) mmap(s, BitArray, dims, offset)
 @deprecate mmap_bitarray{N}(dims::NTuple{N,Integer}, s::IOStream, offset=position(s)) mmap(s, BitArray, dims, offset)
 
-
 # T[a:b] and T[a:s:b]
 @noinline function getindex{T<:Union{Char,Number}}(::Type{T}, r::Range)
  depwarn("T[a:b] concatenation is deprecated; use T[a:b;] instead", :getindex)
@@ -654,3 +653,34 @@ function getindex{T<:Union{Char,Number}}(::Type{T}, r1::Range, rs::Range...)
  end
  return a
 end
+
+function require(mod::AbstractString)
+ depwarn("`require` is deprecated, use `using` or `import` instead", :require)
+ require(symbol(require_filename(mod)))
+end
+function require(f::AbstractString, fs::AbstractString...)
+ require(f)
+ for fn in fs
+ require(fn)
+ end
+end
+export require
+function require_filename(name::AbstractString)
+ # This function can be deleted when the deprecation for `require`
+ # is deleted.
+ # While we could also strip off the absolute path, the user may be
+ # deliberately directing to a different file than what got
+ # cached. So this takes a conservative approach.
+ if endswith(name, ".jl")
+ tmp = name[1:end-3]
+ for prefix in LOAD_CACHE_PATH
+ path = joinpath(prefix, tmp*".ji")
+ if isfile(path)
+ return tmp
+ end
+ end
+ end
+ name
+end
+const reload = require
+export reload

base/exports.jl

@@ -1096,8 +1096,6 @@ export
  evalfile,
  include,
  include_string,
- reload,
- require,
 
 # RTS internals
  finalizer,

base/interactiveutil.jl

@@ -385,7 +385,6 @@ function workspace()
  Expr(:toplevel,
  :(const Base = $(Expr(:quote, b))),
  :(const LastMain = $(Expr(:quote, last)))))
- empty!(package_list)
  empty!(package_locks)
  nothing
 end

base/loading.jl

@@ -1,6 +1,6 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-# require
+# Base.require is the implementation for the `import` statement
 
 function find_in_path(name::AbstractString)
  isabspath(name) && return name
@@ -23,8 +23,8 @@ function find_in_path(name::AbstractString)
  return nothing
 end
 
-find_in_node1_path(name) = myid()==1 ?
- find_in_path(name) : remotecall_fetch(1, find_in_path, name)
+find_in_node_path(name, node::Int=1) = myid() == node ?
+ find_in_path(name) : remotecall_fetch(node, find_in_path, name)
 
 function find_source_file(file)
  (isabspath(file) || isfile(file)) && return file
@@ -34,62 +34,114 @@ function find_source_file(file)
  isfile(file2) ? file2 : nothing
 end
 
-# Store list of files and their load time
-package_list = Dict{ByteString,Float64}()
-# to synchronize multiple tasks trying to require something
-package_locks = Dict{ByteString,Any}()
-require(f::AbstractString, fs::AbstractString...) = (require(f); for x in fs require(x); end)
-
-# only broadcast top-level (not nested) requires and reloads
-toplevel_load = true
+function find_in_cache_path(mod::Symbol)
+ name = string(mod)
+ for prefix in LOAD_CACHE_PATH
+ path = joinpath(prefix, name*".ji")
+ if isfile(path)
+ produce(path)
+ end
+ end
+ nothing
+end
 
-function require(name::AbstractString)
- path = find_in_node1_path(name)
- path == nothing && throw(ArgumentError("$name not found in path"))
+function _include_from_serialized(content::Vector{UInt8})
+ m = ccall(:jl_restore_incremental_from_buf, UInt, (Ptr{Uint8},Int), content, sizeof(content))
+ return m != 0
+end
 
- if myid() == 1 && toplevel_load
- refs = Any[ @spawnat p _require(path) for p in filter(x->x!=1, procs()) ]
- _require(path)
- for r in refs; wait(r); end
+function _require_from_serialized(node::Int, path_to_try::ByteString, toplevel_load::Bool)
+ if toplevel_load && myid() == 1 && nprocs() > 1
+ # broadcast top-level import/using from node 1 (only)
+ if node == myid()
+ content = open(readbytes, path_to_try)
+ else
+ content = remotecall_fetch(node, open, readbytes, path_to_try)
+ end
+ if _include_from_serialized(content)
+ others = filter(x -> x != myid(), procs())
+ refs = Any[ @spawnat p _include_from_serialized(content) for p in others]
+ for (id, ref) in zip(others, refs)
+ if !fetch(ref)
+ warn("node state is inconsistent: node $id failed to load cache from $path_to_try")
+ end
+ end
+ return true
+ end
+ elseif node == myid()
+ if ccall(:jl_restore_incremental, UInt, (Ptr{Uint8},), path_to_try) != 0
+ return true
+ end
  else
- _require(path)
+ content = remotecall_fetch(node, open, readbytes, path_to_try)
+ if _include_from_serialized(content)
+ return true
+ end
  end
- nothing
+ # otherwise, continue search
+ return false
 end
 
-function _require(path)
- global toplevel_load
- if haskey(package_list,path)
- loaded, c = package_locks[path]
- !loaded && wait(c)
- else
- last = toplevel_load
- toplevel_load = false
- try
- reload_path(path)
- finally
- toplevel_load = last
+function _require_from_serialized(node::Int, mod::Symbol, toplevel_load::Bool)
+ name = string(mod)
+ finder = @spawnat node @task find_in_cache_path(mod) # TODO: switch this to an explicit Channel
+ while true
+ path_to_try = remotecall_fetch(node, finder->consume(fetch(finder)), finder)
+ path_to_try === nothing && return false
+ if _require_from_serialized(node, path_to_try, toplevel_load)
+ return true
+ else
+ warn("deserialization checks failed while attempting to load cache from $path_to_try")
  end
  end
 end
 
-function reload(name::AbstractString)
+# to synchronize multiple tasks trying to import/using something
+package_locks = Dict{Symbol,Condition}()
+package_loaded = Set{Symbol}()
+
+# require always works in Main scope and loads files from node 1
+toplevel_load = true
+function require(mod::Symbol)
  global toplevel_load
- path = find_in_node1_path(name)
- path == nothing && throw(ArgumentError("$name not found in path"))
- refs = nothing
- if myid() == 1 && toplevel_load
- refs = Any[ @spawnat p reload_path(path) for p in filter(x->x!=1, procs()) ]
+ loading = get(package_locks, mod, false)
+ if loading !== false
+  # load already in progress for this module
+  wait(loading)
+ return
  end
+ package_locks[mod] = Condition()
+
  last = toplevel_load
- toplevel_load = false
  try
- reload_path(path)
+ toplevel_load = false
+ if _require_from_serialized(1, mod, last)
+ return true
+ end
+ if JLOptions().incremental != 0
+ # spawn off a new incremental compile task from node 1 for recursive `require` calls
+ cachefile = compile(mod)
+ if !_require_from_serialized(1, cachefile, last)
+ warn("require failed to create a precompiled cache file")
+ end
+ return
+ end
+
+ name = string(mod)
+ path = find_in_node_path(name, 1)
+ path === nothing && throw(ArgumentError("$name not found in path"))
+ if last && myid() == 1 && nprocs() > 1
+ # broadcast top-level import/using from node 1 (only)
+ content = open(readall, path)
+ refs = Any[ @spawnat p eval(Main, :(Base.include_from_node1($path))) for p in procs() ]
+ for r in refs; wait(r); end
+ else
+ eval(Main, :(Base.include_from_node1($path)))
+ end
  finally
  toplevel_load = last
- end
- if refs !== nothing
- for r in refs; wait(r); end
+ loading = pop!(package_locks, mod)
+ notify(loading, all=true)
  end
  nothing
 end
@@ -145,38 +197,62 @@ function include_from_node1(path::AbstractString)
  result
 end
 
-function reload_path(path::AbstractString)
- had = haskey(package_list, path)
- if !had
- package_locks[path] = (false, Condition())
- end
- package_list[path] = time()
- tls = task_local_storage()
- prev = pop!(tls, :SOURCE_PATH, nothing)
- try
- eval(Main, :(Base.include_from_node1($path)))
- catch e
- had || delete!(package_list, path)
- rethrow(e)
- finally
- if prev != nothing
- tls[:SOURCE_PATH] = prev
- end
- end
- reloaded, c = package_locks[path]
- if !reloaded
- package_locks[path] = (true, c)
- notify(c, all=true)
- end
- nothing
-end
-
 function evalfile(path::AbstractString, args::Vector{UTF8String}=UTF8String[])
  return eval(Module(:__anon__),
  Expr(:toplevel,
  :(const ARGS = $args),
- :(eval(x) = Core.eval(__anon__,x)),
- :(eval(m,x) = Core.eval(m,x)),
- :(include($path))))
+ :(eval(x) = Main.Core.eval(__anon__,x)),
+ :(eval(m,x) = Main.Core.eval(m,x)),
+ :(Main.Base.include($path))))
 end
 evalfile(path::AbstractString, args::Vector) = evalfile(path, UTF8String[args...])
+
+function create_expr_cache(input::AbstractString, output::AbstractString)
+ code_object = """
+ while !eof(STDIN)
+ eval(Main, deserialize(STDIN))
+ end
+ """
+ io, pobj = open(detach(setenv(`$(julia_cmd())
+ --output-ji $output --output-incremental=yes
+ --startup-file=no --history-file=no
+ --eval $code_object`,
+ ["JULIA_HOME=$JULIA_HOME", "HOME=$(homedir())"])), "w", STDOUT)
+ serialize(io, quote
+ empty!(Base.LOAD_PATH)
+ append!(Base.LOAD_PATH, $LOAD_PATH)
+ empty!(Base.LOAD_CACHE_PATH)
+ append!(Base.LOAD_CACHE_PATH, $LOAD_CACHE_PATH)
+ empty!(Base.DL_LOAD_PATH)
+ append!(Base.DL_LOAD_PATH, $DL_LOAD_PATH)
+ end)
+ source = source_path(nothing)
+ if source !== nothing
+ serialize(io, quote
+ task_local_storage()[:SOURCE_PATH] = $(source)
+ end)
+ end
+ serialize(io, :(Base.include($(abspath(input)))))
+ if source !== nothing
+ serialize(io, quote
+ delete!(task_local_storage(), :SOURCE_PATH)
+ end)
+ end
+ close(io)
+ wait(pobj)
+ return pobj
+end
+
+function compile(mod::Symbol)
+ myid() == 1 || error("can only compile from node 1")
+ name = string(mod)
+ path = find_in_path(name)
+ path === nothing && throw(ArgumentError("$name not found in path"))
+ cachepath = LOAD_CACHE_PATH[1]
+ if !isdir(cachepath)
+ mkpath(cachepath)
+ end
+ cachefile = abspath(cachepath, name*".ji")
+ create_expr_cache(path, cachefile)
+ return cachefile
+end

base/options.jl

@@ -32,6 +32,7 @@ immutable JLOptions
  outputbc::Ptr{UInt8}
  outputo::Ptr{UInt8}
  outputji::Ptr{UInt8}
+ incremental::Int8
 end
 
 JLOptions() = unsafe_load(cglobal(:jl_options, JLOptions))