diff --git a/base/loading.jl b/base/loading.jl index 920d4cb8cfa43..b5f44d37ca766 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -824,9 +824,19 @@ function require(into::Module, mod::Symbol) return require(uuidkey, cache) end +struct PkgOrigin + # version::VersionNumber + # path::String + cachepath::Union{String,Nothing} +end +const pkgorigins = Dict{PkgId,PkgOrigin}() + function require(uuidkey::PkgId, cache::TOMLCache=TOMLCache()) if !root_module_exists(uuidkey) - _require(uuidkey, cache) + cachefile = _require(uuidkey, cache) + if cachefile !== nothing + pkgorigins[uuidkey] = PkgOrigin(cachefile) + end # After successfully loading, notify downstream consumers for callback in package_callbacks invokelatest(callback, uuidkey) @@ -881,6 +891,7 @@ function unreference_module(key::PkgId) end end +# Returns `nothing` or the name of the newly-created cachefile function _require(pkg::PkgId, cache::TOMLCache) # handle recursive calls to require loading = get(package_locks, pkg, false) @@ -942,7 +953,7 @@ function _require(pkg::PkgId, cache::TOMLCache) if isa(m, Exception) @warn "The call to compilecache failed to create a usable precompiled cache file for $pkg" exception=m else - return + return cachefile end end end @@ -1257,6 +1268,13 @@ module_build_id(m::Module) = ccall(:jl_module_build_id, UInt64, (Any,), m) isvalid_cache_header(f::IOStream) = (0 != ccall(:jl_read_verify_header, Cint, (Ptr{Cvoid},), f.ios)) isvalid_file_crc(f::IOStream) = (_crc32c(seekstart(f), filesize(f) - 4) == read(f, UInt32)) +struct CacheHeaderIncludes + id::PkgId + filename::String + mtime::Float64 + modpath::Vector{String} # seemingly not needed in Base, but used by Revise +end + function parse_cache_header(f::IO) modules = Vector{Pair{PkgId, UInt64}}() while true @@ -1270,7 +1288,7 @@ function parse_cache_header(f::IO) totbytes = read(f, Int64) # total bytes for file dependencies # read the list of requirements # and split the list into include and requires statements - includes = Tuple{PkgId, String, Float64}[] + includes = CacheHeaderIncludes[] requires = Pair{PkgId, PkgId}[] while true n2 = read(f, Int32) @@ -1280,20 +1298,21 @@ function parse_cache_header(f::IO) n1 = read(f, Int32) # map ids to keys modkey = (n1 == 0) ? PkgId("") : modules[n1].first + modpath = String[] if n1 != 0 - # consume (and ignore) the module path too + # determine the complete module path while true n1 = read(f, Int32) totbytes -= 4 n1 == 0 && break - skip(f, n1) # String(read(f, n1)) + push!(modpath, String(read(f, n1))) totbytes -= n1 end end if depname[1] == '\0' push!(requires, modkey => binunpack(depname)) else - push!(includes, (modkey, depname, mtime)) + push!(includes, CacheHeaderIncludes(modkey, depname, mtime, modpath)) end totbytes -= 4 + 4 + n2 + 8 end @@ -1312,11 +1331,20 @@ function parse_cache_header(f::IO) return modules, (includes, requires), required_modules, srctextpos end -function parse_cache_header(cachefile::String) +function parse_cache_header(cachefile::String; srcfiles_only::Bool=false) io = open(cachefile, "r") try !isvalid_cache_header(io) && throw(ArgumentError("Invalid header in cache file $cachefile.")) - return parse_cache_header(io) + ret = parse_cache_header(io) + srcfiles_only || return ret + modules, (includes, requires), required_modules, srctextpos = ret + srcfiles = srctext_files(io, srctextpos) + delidx = Int[] + for (i, chi) in enumerate(includes) + chi.filename ∈ srcfiles || push!(delidx, i) + end + deleteat!(includes, delidx) + return modules, (includes, requires), required_modules, srctextpos finally close(io) end @@ -1324,7 +1352,7 @@ end function cache_dependencies(f::IO) defs, (includes, requires), modules = parse_cache_header(f) - return modules, map(mod_fl_mt -> (mod_fl_mt[2], mod_fl_mt[3]), includes) # discard the module + return modules, map(chi -> (chi.filename, chi.mtime), includes) # return just filename and mtime end function cache_dependencies(cachefile::String) @@ -1368,6 +1396,21 @@ function read_dependency_src(cachefile::String, filename::AbstractString) end end +function srctext_files(f::IO, srctextpos::Int64) + files = Set{String}() + srctextpos == 0 && return files + seek(f, srctextpos) + while !eof(f) + filenamelen = read(f, Int32) + filenamelen == 0 && break + fn = String(read(f, filenamelen)) + len = read(f, UInt64) + push!(files, fn) + seek(f, position(f) + len) + end + return files +end + # returns true if it "cachefile.ji" is stale relative to "modpath.jl" # otherwise returns the list of dependencies to also check stale_cachefile(modpath::String, cachefile::String) = stale_cachefile(modpath, cachefile, TOMLCache()) @@ -1379,6 +1422,7 @@ function stale_cachefile(modpath::String, cachefile::String, cache::TOMLCache) return true # invalid cache file end (modules, (includes, requires), required_modules) = parse_cache_header(io) + id = isempty(modules) ? nothing : first(modules).first modules = Dict{PkgId, UInt64}(modules) # Check if transitive dependencies can be fulfilled @@ -1423,8 +1467,8 @@ function stale_cachefile(modpath::String, cachefile::String, cache::TOMLCache) # now check if this file is fresh relative to its source files if !skip_timecheck - if !samefile(includes[1][2], modpath) - @debug "Rejecting cache file $cachefile because it is for file $(includes[1][2])) not file $modpath" + if !samefile(includes[1].filename, modpath) + @debug "Rejecting cache file $cachefile because it is for file $(includes[1].filename)) not file $modpath" return true # cache file was compiled from a different path end for (modkey, req_modkey) in requires @@ -1434,7 +1478,8 @@ function stale_cachefile(modpath::String, cachefile::String, cache::TOMLCache) return true end end - for (_, f, ftime_req) in includes + for chi in includes + f, ftime_req = chi.filename, chi.mtime # Issue #13606: compensate for Docker images rounding mtimes # Issue #20837: compensate for GlusterFS truncating mtimes to microseconds ftime = mtime(f) @@ -1450,6 +1495,10 @@ function stale_cachefile(modpath::String, cachefile::String, cache::TOMLCache) return true end + if isa(id, PkgId) + pkgorigins[id] = PkgOrigin(cachefile) + end + return depmods # fresh cachefile finally close(io) diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl index 6cdb09953d1e6..6fc91f487bbd8 100644 --- a/contrib/generate_precompile.jl +++ b/contrib/generate_precompile.jl @@ -19,10 +19,12 @@ hardcoded_precompile_statements = """ @assert precompile(Tuple{typeof(push!), Set{Module}, Module}) @assert precompile(Tuple{typeof(push!), Set{Method}, Method}) @assert precompile(Tuple{typeof(push!), Set{Base.PkgId}, Base.PkgId}) +@assert precompile(Tuple{typeof(getindex), Dict{Base.PkgId,String}, Base.PkgId}) @assert precompile(Tuple{typeof(setindex!), Dict{String,Base.PkgId}, Base.PkgId, String}) @assert precompile(Tuple{typeof(get!), Type{Vector{Function}}, Dict{Base.PkgId,Vector{Function}}, Base.PkgId}) @assert precompile(Tuple{typeof(isassigned), Core.SimpleVector, Int}) @assert precompile(Tuple{typeof(pushfirst!), Vector{Any}, Function}) +@assert precompile(Tuple{typeof(Base.parse_cache_header), String}) """ precompile_script = """ diff --git a/test/precompile.jl b/test/precompile.jl index bbd642a625060..9739704ee2c16 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -264,9 +264,9 @@ try @test string(Base.Docs.doc(Foo.Bar.bar)) == "bar function\n" modules, (deps, requires), required_modules = Base.parse_cache_header(cachefile) - discard_module = mod_fl_mt -> (mod_fl_mt[2], mod_fl_mt[3]) + discard_module = mod_fl_mt -> (mod_fl_mt.filename, mod_fl_mt.mtime) @test modules == [ Base.PkgId(Foo) => Base.module_build_id(Foo) ] - @test map(x -> x[2], deps) == [ Foo_file, joinpath(dir, "foo.jl"), joinpath(dir, "bar.jl") ] + @test map(x -> x.filename, deps) == [ Foo_file, joinpath(dir, "foo.jl"), joinpath(dir, "bar.jl") ] @test requires == [ Base.PkgId(Foo) => Base.PkgId(string(FooBase_module)), Base.PkgId(Foo) => Base.PkgId(Foo2), Base.PkgId(Foo) => Base.PkgId(Test), @@ -299,6 +299,8 @@ try end ) @test discard_module.(deps) == deps1 + modules, (deps, requires), required_modules = Base.parse_cache_header(cachefile; srcfiles_only=true) + @test map(x -> x.filename, deps) == [Foo_file] @test current_task()(0x01, 0x4000, 0x30031234) == 2 @test sin(0x01, 0x4000, 0x30031234) == 52 @@ -339,6 +341,55 @@ try @test pointer_from_objref(PV) === pointer_from_objref(ft(ft(PV)[1].parameters[1])[1]) end + Nest_module = :Nest4b3a94a1a081a8cb + Nest_file = joinpath(dir, "$Nest_module.jl") + NestInner_file = joinpath(dir, "$(Nest_module)Inner.jl") + NestInner2_file = joinpath(dir, "$(Nest_module)Inner2.jl") + write(Nest_file, + """ + module $Nest_module + include("$(escape_string(NestInner_file))") + end + """) + write(NestInner_file, + """ + module NestInner + include("$(escape_string(NestInner2_file))") + end + """) + write(NestInner2_file, + """ + f() = 22 + """) + Nest = Base.require(Main, Nest_module) + cachefile = joinpath(cachedir, "$Nest_module.ji") + modules, (deps, requires), required_modules = Base.parse_cache_header(cachefile) + @test last(deps).modpath == ["NestInner"] + + UsesB_module = :UsesB4b3a94a1a081a8cb + B_module = :UsesB4b3a94a1a081a8cb_B + UsesB_file = joinpath(dir, "$UsesB_module.jl") + B_file = joinpath(dir, "$(B_module).jl") + write(UsesB_file, + """ + module $UsesB_module + using $B_module + end + """) + write(B_file, + """ + module $B_module + export bfunc + bfunc() = 33 + end + """) + UsesB = Base.require(Main, UsesB_module) + cachefile = joinpath(cachedir, "$UsesB_module.ji") + modules, (deps, requires), required_modules = Base.parse_cache_header(cachefile) + id1, id2 = only(requires) + @test Base.pkgorigins[id1].cachepath == cachefile + @test Base.pkgorigins[id2].cachepath == joinpath(cachedir, "$B_module.ji") + Baz_file = joinpath(dir, "Baz.jl") write(Baz_file, """