From 240a190a48ad33bc08f54f4aa6d8ee1397f959dc Mon Sep 17 00:00:00 2001 From: KristofferC Date: Sun, 24 May 2020 22:44:41 +0200 Subject: [PATCH] move the TOML parser to Base and implement code loading on top of the Base TOML parser --- base/Base.jl | 2 + base/initdefs.jl | 2 +- base/loading.jl | 443 +++++++----------- base/pkgid.jl | 43 ++ .../TOML/src/parser.jl => base/toml_parser.jl | 6 +- base/uuid.jl | 1 + stdlib/Profile/src/Profile.jl | 2 +- stdlib/REPL/src/REPLCompletions.jl | 26 +- stdlib/TOML/src/TOML.jl | 7 +- test/loading.jl | 36 +- test/precompile.jl | 2 +- 11 files changed, 261 insertions(+), 309 deletions(-) create mode 100644 base/pkgid.jl rename stdlib/TOML/src/parser.jl => base/toml_parser.jl (99%) diff --git a/base/Base.jl b/base/Base.jl index 5cb0d558c228d..b95ac8732ed09 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -340,6 +340,8 @@ include("threadcall.jl") # code loading include("uuid.jl") +include("pkgid.jl") +include("toml_parser.jl") include("loading.jl") # misc useful functions & macros diff --git a/base/initdefs.jl b/base/initdefs.jl index b557e7e4892db..53a056595e7ac 100644 --- a/base/initdefs.jl +++ b/base/initdefs.jl @@ -234,7 +234,7 @@ function load_path_expand(env::AbstractString)::Union{String, Nothing} # if you put a `@` in LOAD_PATH manually, it's expanded late env == "@" && return active_project(false) env == "@." && return current_project() - env == "@stdlib" && return Sys.STDLIB + env == "@stdlib" && return Sys.STDLIB::String env = replace(env, '#' => VERSION.major, count=1) env = replace(env, '#' => VERSION.minor, count=1) env = replace(env, '#' => VERSION.patch, count=1) diff --git a/base/loading.jl b/base/loading.jl index 14fcbee95e867..ec554be52bed7 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -151,68 +151,38 @@ function version_slug(uuid::UUID, sha1::SHA1, p::Int=5) return slug(crc, p) end -## package identification: determine unique identity of package to be loaded ## - -function find_package(args...) - pkg = identify_package(args...) - pkg === nothing && return nothing - return locate_package(pkg) -end - -struct PkgId - uuid::Union{UUID,Nothing} - name::String - - PkgId(u::UUID, name::AbstractString) = new(UInt128(u) == 0 ? nothing : u, name) - PkgId(::Nothing, name::AbstractString) = new(nothing, name) -end -PkgId(name::AbstractString) = PkgId(nothing, name) - -function PkgId(m::Module, name::String = String(nameof(moduleroot(m)))) - uuid = UUID(ccall(:jl_module_uuid, NTuple{2, UInt64}, (Any,), m)) - UInt128(uuid) == 0 ? PkgId(name) : PkgId(uuid, name) +struct TOMLCache + p::TOML.Parser + d::Dict{String, Dict{String, Any}} end +TOMLCache() = TOMLCache(TOML.Parser(), Dict{String, Dict{String, Any}}()) -==(a::PkgId, b::PkgId) = a.uuid == b.uuid && a.name == b.name - -function hash(pkg::PkgId, h::UInt) - h += 0xc9f248583a0ca36c % UInt - h = hash(pkg.uuid, h) - h = hash(pkg.name, h) - return h +function parsed_toml(cache::TOMLCache, project_file::String) + get!(cache.d, project_file) do + TOML.reinit!(cache.p, read(project_file, String); filepath=project_file) + TOML.parse(cache.p) + end end -show(io::IO, pkg::PkgId) = - print(io, pkg.name, " [", pkg.uuid === nothing ? "top-level" : pkg.uuid, "]") - -function binpack(pkg::PkgId) - io = IOBuffer() - write(io, UInt8(0)) - uuid = pkg.uuid - write(io, uuid === nothing ? UInt128(0) : UInt128(uuid)) - write(io, pkg.name) - return String(take!(io)) -end +## package identification: determine unique identity of package to be loaded ## -function binunpack(s::String) - io = IOBuffer(s) - @assert read(io, UInt8) === 0x00 - uuid = read(io, UInt128) - name = read(io, String) - return PkgId(UUID(uuid), name) +# Used by Pkg in but not used in loading itself +function find_package(arg) + pkg = identify_package(arg, TOMLCache()) + pkg === nothing && return nothing + return locate_package(pkg, cache) end ## package identity: given a package name and a context, try to return its identity ## - -identify_package(where::Module, name::String) = identify_package(PkgId(where), name) +identify_package(where::Module, name::String, cache::TOMLCache = TOMLCache()) = identify_package(PkgId(where), name, cache) # identify_package computes the PkgId for `name` from the context of `where` # or return `nothing` if no mapping exists for it -function identify_package(where::PkgId, name::String)::Union{Nothing,PkgId} +function identify_package(where::PkgId, name::String, cache::TOMLCache=TOMLCache())::Union{Nothing,PkgId} where.name === name && return where - where.uuid === nothing && return identify_package(name) # ignore `where` + where.uuid === nothing && return identify_package(name, cache) # ignore `where` for env in load_path() - uuid = manifest_deps_get(env, where, name) + uuid = manifest_deps_get(env, where, name, cache) uuid === nothing && continue # not found--keep looking uuid.uuid === nothing || return uuid # found in explicit environment--use it return nothing # found in implicit environment--return "not found" @@ -222,47 +192,33 @@ end # identify_package computes the PkgId for `name` from toplevel context # by looking through the Project.toml files and directories -function identify_package(name::String)::Union{Nothing,PkgId} +function identify_package(name::String, cache::TOMLCache=TOMLCache())::Union{Nothing,PkgId} for env in load_path() - uuid = project_deps_get(env, name) + uuid = project_deps_get(env, name, cache) uuid === nothing || return uuid # found--return it end return nothing end -function identify_package(name::String, names::String...) - pkg = identify_package(name) - pkg === nothing && return nothing - return identify_package(pkg, names...) -end - -# locate `tail(names)` package by following the search path graph through `names` starting from `where` -function identify_package(where::PkgId, name::String, names::String...) - pkg = identify_package(where, name) - pkg === nothing && return nothing - return identify_package(pkg, names...) -end - ## package location: given a package identity, find file to load ## - -function locate_package(pkg::PkgId)::Union{Nothing,String} +function locate_package(pkg::PkgId, cache::TOMLCache=TOMLCache())::Union{Nothing,String} if pkg.uuid === nothing for env in load_path() # look for the toplevel pkg `pkg.name` in this entry - found = project_deps_get(env, pkg.name) + found = project_deps_get(env, pkg.name, cache) found === nothing && continue if pkg == found # pkg.name is present in this directory or project file, # return the path the entry point for the code, if it could be found # otherwise, signal failure - return implicit_manifest_uuid_path(env, pkg) + return implicit_manifest_uuid_path(env, pkg, cache) end @assert found.uuid !== nothing - return locate_package(found) # restart search now that we know the uuid for pkg + return locate_package(found, cache) # restart search now that we know the uuid for pkg end else for env in load_path() - path = manifest_uuid_path(env, pkg) + path = manifest_uuid_path(env, pkg, cache) path === nothing || return entry_path(path, pkg.name) end end @@ -319,136 +275,83 @@ function env_project_file(env::String)::Union{Bool,String} return false end -function project_deps_get(env::String, name::String)::Union{Nothing,PkgId} +function project_deps_get(env::String, name::String, cache::TOMLCache)::Union{Nothing,PkgId} project_file = env_project_file(env) if project_file isa String - pkg_uuid = explicit_project_deps_get(project_file, name) + pkg_uuid = explicit_project_deps_get(project_file, name, cache) pkg_uuid === nothing || return PkgId(pkg_uuid, name) elseif project_file - return implicit_project_deps_get(env, name) + return implicit_project_deps_get(env, name, cache) end return nothing end -function manifest_deps_get(env::String, where::PkgId, name::String)::Union{Nothing,PkgId} +function manifest_deps_get(env::String, where::PkgId, name::String, cache::TOMLCache)::Union{Nothing,PkgId} @assert where.uuid !== nothing project_file = env_project_file(env) if project_file isa String # first check if `where` names the Project itself - proj = project_file_name_uuid(project_file, where.name) + proj = project_file_name_uuid(project_file, where.name, cache) if proj == where # if `where` matches the project, use [deps] section as manifest, and stop searching - pkg_uuid = explicit_project_deps_get(project_file, name) + pkg_uuid = explicit_project_deps_get(project_file, name, cache) return PkgId(pkg_uuid, name) end # look for manifest file and `where` stanza - return explicit_manifest_deps_get(project_file, where.uuid, name) + return explicit_manifest_deps_get(project_file, where.uuid, name, cache) elseif project_file # if env names a directory, search it - return implicit_manifest_deps_get(env, where, name) + return implicit_manifest_deps_get(env, where, name, cache) end return nothing end -function manifest_uuid_path(env::String, pkg::PkgId)::Union{Nothing,String} +function manifest_uuid_path(env::String, pkg::PkgId, cache::TOMLCache)::Union{Nothing,String} project_file = env_project_file(env) if project_file isa String - proj = project_file_name_uuid(project_file, pkg.name) + proj = project_file_name_uuid(project_file, pkg.name, cache) if proj == pkg # if `pkg` matches the project, return the project itself - return project_file_path(project_file, pkg.name) + return project_file_path(project_file, pkg.name, cache) end # look for manifest file and `where` stanza - return explicit_manifest_uuid_path(project_file, pkg) + return explicit_manifest_uuid_path(project_file, pkg, cache) elseif project_file # if env names a directory, search it - return implicit_manifest_uuid_path(env, pkg) + return implicit_manifest_uuid_path(env, pkg, cache) end return nothing end -# regular expressions for scanning project & manifest files - -const re_section = r"^\s*\[" -const re_array_of_tables = r"^\s*\[\s*\[" -const re_section_deps = r"^\s*\[\s*\"?deps\"?\s*\]\s*(?:#|$)" -const re_section_capture = r"^\s*\[\s*\[\s*\"?(\w+)\"?\s*\]\s*\]\s*(?:#|$)" -const re_subsection_deps = r"^\s*\[\s*\"?(\w+)\"?\s*\.\s*\"?deps\"?\s*\]\s*(?:#|$)" -const re_key_to_string = r"^\s*(\w+)\s*=\s*\"(.*)\"\s*(?:#|$)" -const re_uuid_to_string = r"^\s*uuid\s*=\s*\"(.*)\"\s*(?:#|$)" -const re_name_to_string = r"^\s*name\s*=\s*\"(.*)\"\s*(?:#|$)" -const re_path_to_string = r"^\s*path\s*=\s*\"(.*)\"\s*(?:#|$)" -const re_hash_to_string = r"^\s*git-tree-sha1\s*=\s*\"(.*)\"\s*(?:#|$)" -const re_manifest_to_string = r"^\s*manifest\s*=\s*\"(.*)\"\s*(?:#|$)" -const re_deps_to_any = r"^\s*deps\s*=\s*(.*?)\s*(?:#|$)" - # find project file's top-level UUID entry (or nothing) -function project_file_name_uuid(project_file::String, name::String)::PkgId - pkg = open(project_file) do io - uuid = dummy_uuid(project_file) - for line in eachline(io) - occursin(re_section, line) && break - if (m = match(re_name_to_string, line)) !== nothing - name = String(m.captures[1]) - elseif (m = match(re_uuid_to_string, line)) !== nothing - uuid = UUID(m.captures[1]) - end - end - return PkgId(uuid, name) - end - return pkg +function project_file_name_uuid(project_file::String, name::String, cache::TOMLCache)::PkgId + uuid = dummy_uuid(project_file) + d = parsed_toml(cache, project_file) + uuid′ = get(d, "uuid", nothing)::Union{String, Nothing} + uuid′ === nothing || (uuid = UUID(uuid′)) + name = get(d, "name", name)::String + return PkgId(uuid, name) end -function project_file_path(project_file::String, name::String)::String - path = open(project_file) do io - for line in eachline(io) - occursin(re_section, line) && break - if (m = match(re_path_to_string, line)) !== nothing - return String(m.captures[1]) - end - end - return "" - end - return joinpath(dirname(project_file), path) +function project_file_path(project_file::String, name::String, cache) + d = parsed_toml(cache, project_file) + joinpath(dirname(project_file), get(d, "path", "")::String) end - # find project file's corresponding manifest file -function project_file_manifest_path(project_file::String)::Union{Nothing,String} - open(project_file) do io - dir = abspath(dirname(project_file)) - for line in eachline(io) - occursin(re_section, line) && break - if (m = match(re_manifest_to_string, line)) !== nothing - manifest_file = normpath(joinpath(dir, m.captures[1])) - isfile_casesensitive(manifest_file) && return manifest_file - return nothing # silently stop if the explicitly listed manifest file is not present - end - end - for mfst in manifest_names - manifest_file = joinpath(dir, mfst) - isfile_casesensitive(manifest_file) && return manifest_file - end - return nothing +function project_file_manifest_path(project_file::String, cache::TOMLCache)::Union{Nothing,String} + dir = abspath(dirname(project_file)) + d = parsed_toml(cache, project_file) + explicit_manifest = get(d, "manifest", nothing)::Union{String, Nothing} + if explicit_manifest !== nothing + manifest_file = normpath(joinpath(dir, explicit_manifest)) + isfile_casesensitive(manifest_file) && return manifest_file end -end - -# find `name` in a manifest file and return its UUID -# return `nothing` on failure -function manifest_file_name_uuid(manifest_file::IO, name::String)::Union{Nothing,UUID} - name_section = false - uuid = nothing - for line in eachline(manifest_file) - if (m = match(re_section_capture, line)) !== nothing - name_section && break - name_section = (m.captures[1] == name) - elseif name_section - if (m = match(re_uuid_to_string, line)) !== nothing - uuid = UUID(m.captures[1]) - end - end + for mfst in manifest_names + manifest_file = joinpath(dir, mfst) + isfile_casesensitive(manifest_file) && return manifest_file end - return uuid + return nothing end # given a directory (implicit env from LOAD_PATH) and a name, @@ -490,129 +393,113 @@ end # find project file root or deps `name => uuid` mapping # return `nothing` if `name` is not found -function explicit_project_deps_get(project_file::String, name::String)::Union{Nothing,UUID} - pkg_uuid = open(project_file) do io - root_name = nothing - root_uuid = dummy_uuid(project_file) - state = :top - for line in eachline(io) - if occursin(re_section, line) - state === :top && root_name == name && return root_uuid - state = occursin(re_section_deps, line) ? :deps : :other - elseif state === :top - if (m = match(re_name_to_string, line)) !== nothing - root_name = String(m.captures[1]) - elseif (m = match(re_uuid_to_string, line)) !== nothing - root_uuid = UUID(m.captures[1]) - end - elseif state === :deps - if (m = match(re_key_to_string, line)) !== nothing - m.captures[1] == name && return UUID(m.captures[2]) - end - end - end - return root_name == name ? root_uuid : nothing +function explicit_project_deps_get(project_file::String, name::String, cache::TOMLCache)::Union{Nothing,UUID} + d = parsed_toml(cache, project_file) + root_uuid = dummy_uuid(project_file) + if get(d, "name", nothing)::Union{String, Nothing} === name + uuid = get(d, "uuid", nothing)::Union{String, Nothing} + return uuid === nothing ? root_uuid : UUID(uuid) + end + deps = get(d, "deps", nothing)::Union{Dict{String, Any}, Nothing} + if deps !== nothing + uuid = get(deps, name, nothing)::Union{String, Nothing} + uuid === nothing || return UUID(uuid) end - return pkg_uuid + return nothing end # find `where` stanza and return the PkgId for `name` # return `nothing` if it did not find `where` (indicating caller should continue searching) -function explicit_manifest_deps_get(project_file::String, where::UUID, name::String)::Union{Nothing,PkgId} - manifest_file = project_file_manifest_path(project_file) +function explicit_manifest_deps_get(project_file::String, where::UUID, name::String, cache::TOMLCache)::Union{Nothing,PkgId} + manifest_file = project_file_manifest_path(project_file, cache) manifest_file === nothing && return nothing # manifest not found--keep searching LOAD_PATH - found_or_uuid = open(manifest_file) do io - uuid = deps = nothing - state = :other - # first search the manifest for the deps section associated with `where` (by uuid) - for line in eachline(io) - if occursin(re_array_of_tables, line) - uuid == where && break - uuid = deps = nothing - state = :stanza - elseif state === :stanza - if (m = match(re_uuid_to_string, line)) !== nothing - uuid = UUID(m.captures[1]) - elseif (m = match(re_deps_to_any, line)) !== nothing - deps = String(m.captures[1]) - elseif occursin(re_subsection_deps, line) - state = :deps - elseif occursin(re_section, line) - state = :other - end - elseif state === :deps && uuid == where - # [deps] section format gives both name and uuid - if (m = match(re_key_to_string, line)) !== nothing - m.captures[1] == name && return UUID(m.captures[2]) + d = parsed_toml(cache, manifest_file) + found_where = false + found_name = false + for (dep_name, entries) in d + entries::Vector{Any} + for entry in entries + entry::Dict{String, Any} + uuid = get(entry, "uuid", nothing)::Union{String, Nothing} + uuid === nothing && continue + if UUID(uuid) === where + found_where = true + # deps is either a list of names (deps = ["DepA", "DepB"]) or + # a table of entries (deps = {"DepA" = "6ea...", "DepB" = "55d..."} + deps = get(entry, "deps", nothing)::Union{Vector{String}, Dict{String, Any}, Nothing} + deps === nothing && continue + if deps isa Vector{String} + found_name = name in deps + break + else + deps::Dict{String, Any} + for (dep, uuid) in deps + uuid::String + if dep === name + return PkgId(UUID(uuid), name) + end + end end end end - # now search through `deps = []` string to see if we have an entry for `name` - uuid == where || return false - deps === nothing && return true - # TODO: handle inline table syntax - if deps[1] != '[' || deps[end] != ']' - @warn "Unexpected TOML deps format:\n$deps" - return false - end - occursin(repr(name), deps) || return true - seekstart(io) # rewind IO handle - # finally, find out the `uuid` associated with `name` - return something(manifest_file_name_uuid(io, name), false) end - found_or_uuid isa UUID && return PkgId(found_or_uuid, name) - found_or_uuid && return PkgId(name) - return nothing + found_where || return nothing + found_name || return PkgId(name) + # Only reach here if deps was not a dict which mean we have a unique name for the dep + name_deps = get(d, name, nothing)::Union{Nothing, Vector{Any}} + if name_deps === nothing || length(name_deps) != 1 + error("expected a single entry for $(repr(name)) in $(repr(project_file))") + end + entry = first(name_deps::Vector{Any})::Dict{String, Any} + uuid = get(entry, "uuid", nothing)::Union{String, Nothing} + uuid === nothing && return nothing + return PkgId(UUID(uuid), name) end # find `uuid` stanza, return the corresponding path -function explicit_manifest_uuid_path(project_file::String, pkg::PkgId)::Union{Nothing,String} - manifest_file = project_file_manifest_path(project_file) +function explicit_manifest_uuid_path(project_file::String, pkg::PkgId, cache::TOMLCache)::Union{Nothing,String} + manifest_file = project_file_manifest_path(project_file, cache) manifest_file === nothing && return nothing # no manifest, skip env - open(manifest_file) do io - uuid = name = path = hash = nothing - for line in eachline(io) - if (m = match(re_section_capture, line)) !== nothing - uuid == pkg.uuid && break - name = String(m.captures[1]) - path = hash = nothing - elseif (m = match(re_uuid_to_string, line)) !== nothing - uuid = UUID(m.captures[1]) - elseif (m = match(re_path_to_string, line)) !== nothing - path = String(m.captures[1]) - elseif (m = match(re_hash_to_string, line)) !== nothing - hash = SHA1(m.captures[1]) + + d = parsed_toml(cache, manifest_file) + entries = get(d, pkg.name, nothing)::Union{Nothing, Vector{Any}} + entries === nothing && return nothing # TODO: allow name to mismatch? + for entry in entries + entry::Dict{String, Any} + uuid = get(entry, "uuid", nothing)::Union{Nothing, String} + uuid === nothing && continue + if UUID(uuid) === pkg.uuid + path = get(entry, "path", nothing)::Union{Nothing, String} + if path !== nothing + path = normpath(abspath(dirname(manifest_file), path)) + return path end - end - uuid == pkg.uuid || return nothing - name == pkg.name || return nothing # TODO: allow a mismatch? - if path !== nothing - path = normpath(abspath(dirname(manifest_file), path)) - return path - end - hash === nothing && return nothing - # Keep the 4 since it used to be the default - for slug in (version_slug(uuid, hash, 4), version_slug(uuid, hash)) - for depot in DEPOT_PATH - path = abspath(depot, "packages", name, slug) - ispath(path) && return path + hash = get(entry, "git-tree-sha1", nothing)::Union{Nothing, String} + hash === nothing && return nothing + hash = SHA1(hash) + # Keep the 4 since it used to be the default + for slug in (version_slug(pkg.uuid, hash, 4), version_slug(pkg.uuid, hash)) + for depot in DEPOT_PATH + path = abspath(depot, "packages", pkg.name, slug) + ispath(path) && return path + end end end - return nothing end + return nothing end ## implicit project & manifest API ## # look for an entry point for `name` from a top-level package (no environment) # otherwise return `nothing` to indicate the caller should keep searching -function implicit_project_deps_get(dir::String, name::String)::Union{Nothing,PkgId} +function implicit_project_deps_get(dir::String, name::String, cache::TOMLCache)::Union{Nothing,PkgId} path, project_file = entry_point_and_project_file(dir, name) if project_file === nothing path === nothing && return nothing return PkgId(name) end - proj = project_file_name_uuid(project_file, name) + proj = project_file_name_uuid(project_file, name, cache) proj.name == name || return nothing return proj end @@ -620,25 +507,25 @@ end # look for an entry-point for `name`, check that UUID matches # if there's a project file, look up `name` in its deps and return that # otherwise return `nothing` to indicate the caller should keep searching -function implicit_manifest_deps_get(dir::String, where::PkgId, name::String)::Union{Nothing,PkgId} +function implicit_manifest_deps_get(dir::String, where::PkgId, name::String, cache::TOMLCache)::Union{Nothing,PkgId} @assert where.uuid !== nothing project_file = entry_point_and_project_file(dir, where.name)[2] project_file === nothing && return nothing # a project file is mandatory for a package with a uuid - proj = project_file_name_uuid(project_file, where.name) + proj = project_file_name_uuid(project_file, where.name, cache) proj == where || return nothing # verify that this is the correct project file # this is the correct project, so stop searching here - pkg_uuid = explicit_project_deps_get(project_file, name) + pkg_uuid = explicit_project_deps_get(project_file, name, cache) return PkgId(pkg_uuid, name) end # look for an entry-point for `pkg` and return its path if UUID matches -function implicit_manifest_uuid_path(dir::String, pkg::PkgId)::Union{Nothing,String} +function implicit_manifest_uuid_path(dir::String, pkg::PkgId, cache::TOMLCache)::Union{Nothing,String} path, project_file = entry_point_and_project_file(dir, pkg.name) if project_file === nothing pkg.uuid === nothing || return nothing return path end - proj = project_file_name_uuid(project_file, pkg.name) + proj = project_file_name_uuid(project_file, pkg.name, cache) proj == pkg || return nothing return path end @@ -698,7 +585,7 @@ function _include_from_serialized(path::String, depmods::Vector{Any}) return restored end -function _tryrequire_from_serialized(modkey::PkgId, build_id::UInt64, modpath::Union{Nothing, String}) +function _tryrequire_from_serialized(modkey::PkgId, build_id::UInt64, modpath::Union{Nothing, String}, cache::TOMLCache) if root_module_exists(modkey) M = root_module(modkey) if PkgId(M) == modkey && module_build_id(M) === build_id @@ -706,10 +593,10 @@ function _tryrequire_from_serialized(modkey::PkgId, build_id::UInt64, modpath::U end else if modpath === nothing - modpath = locate_package(modkey) + modpath = locate_package(modkey, cache) modpath === nothing && return nothing end - mod = _require_search_from_serialized(modkey, String(modpath)) + mod = _require_search_from_serialized(modkey, String(modpath), cache) if !isa(mod, Bool) for callback in package_callbacks invokelatest(callback, modkey) @@ -725,7 +612,7 @@ function _tryrequire_from_serialized(modkey::PkgId, build_id::UInt64, modpath::U return nothing end -function _require_from_serialized(path::String) +function _require_from_serialized(path::String, cache::TOMLCache) # loads a precompile cache file, ignoring stale_cachfile tests # load all of the dependent modules first local depmodnames @@ -741,7 +628,7 @@ function _require_from_serialized(path::String) depmods = Vector{Any}(undef, ndeps) for i in 1:ndeps modkey, build_id = depmodnames[i] - dep = _tryrequire_from_serialized(modkey, build_id, nothing) + dep = _tryrequire_from_serialized(modkey, build_id, nothing, cache) dep === nothing && return ErrorException("Required dependency $modkey failed to load from a cache file.") depmods[i] = dep::Module end @@ -752,10 +639,10 @@ end # returns `true` if require found a precompile cache for this sourcepath, but couldn't load it # returns `false` if the module isn't known to be precompilable # returns the set of modules restored if the cache load succeeded -function _require_search_from_serialized(pkg::PkgId, sourcepath::String) +function _require_search_from_serialized(pkg::PkgId, sourcepath::String, cache::TOMLCache) paths = find_all_in_cache_path(pkg) for path_to_try in paths::Vector{String} - staledeps = stale_cachefile(sourcepath, path_to_try) + staledeps = stale_cachefile(sourcepath, path_to_try, cache) if staledeps === true continue end @@ -768,7 +655,7 @@ function _require_search_from_serialized(pkg::PkgId, sourcepath::String) dep = staledeps[i] dep isa Module && continue modpath, modkey, build_id = dep::Tuple{String, PkgId, UInt64} - dep = _tryrequire_from_serialized(modkey, build_id, modpath) + dep = _tryrequire_from_serialized(modkey, build_id, modpath, cache) if dep === nothing @debug "Required dependency $modkey failed to load from cache file for $modpath." staledeps = true @@ -885,7 +772,8 @@ For more details regarding code loading, see the manual sections on [modules](@r [parallel computing](@ref code-availability). """ function require(into::Module, mod::Symbol) - uuidkey = identify_package(into, String(mod)) + cache = TOMLCache() + uuidkey = identify_package(into, String(mod), cache) # Core.println("require($(PkgId(into)), $mod) -> $uuidkey") if uuidkey === nothing where = PkgId(into) @@ -920,12 +808,12 @@ function require(into::Module, mod::Symbol) if _track_dependencies[] push!(_require_dependencies, (into, binpack(uuidkey), 0.0)) end - return require(uuidkey) + return require(uuidkey, cache) end -function require(uuidkey::PkgId) +function require(uuidkey::PkgId, cache::TOMLCache=TOMLCache()) if !root_module_exists(uuidkey) - _require(uuidkey) + _require(uuidkey, cache) # After successfully loading, notify downstream consumers for callback in package_callbacks invokelatest(callback, uuidkey) @@ -980,7 +868,7 @@ function unreference_module(key::PkgId) end end -function _require(pkg::PkgId) +function _require(pkg::PkgId, cache::TOMLCache) # handle recursive calls to require loading = get(package_locks, pkg, false) if loading !== false @@ -994,7 +882,7 @@ function _require(pkg::PkgId) try toplevel_load[] = false # perform the search operation to select the module file require intends to load - path = locate_package(pkg) + path = locate_package(pkg, cache) if path === nothing throw(ArgumentError(""" Package $pkg is required but does not seem to be installed: @@ -1004,7 +892,7 @@ function _require(pkg::PkgId) # attempt to load the module file via the precompile cache locations if JLOptions().use_compiled_modules != 0 - m = _require_search_from_serialized(pkg, path) + m = _require_search_from_serialized(pkg, path, cache) if !isa(m, Bool) return end @@ -1037,7 +925,7 @@ function _require(pkg::PkgId) end # fall-through to loading the file locally else - m = _require_from_serialized(cachefile) + m = _require_from_serialized(cachefile, cache) if isa(m, Exception) @warn "The call to compilecache failed to create a usable precompiled cache file for $pkg" exception=m else @@ -1290,8 +1178,8 @@ This can be used to reduce package load times. Cache files are stored in `DEPOT_PATH[1]/compiled`. See [Module initialization and precompilation](@ref) for important notes. """ -function compilecache(pkg::PkgId) - path = locate_package(pkg) +function compilecache(pkg::PkgId, cache::TOMLCache = TOMLCache()) + path = locate_package(pkg, cache) path === nothing && throw(ArgumentError("$pkg not found during precompilation")) return compilecache(pkg, path) end @@ -1469,7 +1357,8 @@ end # returns true if it "cachefile.ji" is stale relative to "modpath.jl" # otherwise returns the list of dependencies to also check -function stale_cachefile(modpath::String, cachefile::String) +stale_cachefile(modpath::String, cachefile::String) = stale_cachefile(modpath, cachefile, TOMLCache()) +function stale_cachefile(modpath::String, cachefile::String, cache::TOMLCache) io = open(cachefile, "r") try if !isvalid_cache_header(io) @@ -1494,7 +1383,7 @@ function stale_cachefile(modpath::String, cachefile::String) return true # Won't be able to fulfill dependency end else - path = locate_package(req_key) + path = locate_package(req_key, cache) if path === nothing @debug "Rejecting cache file $cachefile because dependency $req_key not found." return true # Won't be able to fulfill dependency @@ -1527,7 +1416,7 @@ function stale_cachefile(modpath::String, cachefile::String) end for (modkey, req_modkey) in requires # verify that `require(modkey, name(req_modkey))` ==> `req_modkey` - if identify_package(modkey, req_modkey.name) != req_modkey + if identify_package(modkey, req_modkey.name, cache) != req_modkey @debug "Rejecting cache file $cachefile because uuid mapping for $modkey => $req_modkey has changed" return true end diff --git a/base/pkgid.jl b/base/pkgid.jl new file mode 100644 index 0000000000000..6d588ffe6647d --- /dev/null +++ b/base/pkgid.jl @@ -0,0 +1,43 @@ +struct PkgId + uuid::Union{UUID,Nothing} + name::String + + PkgId(u::UUID, name::AbstractString) = new(UInt128(u) == 0 ? nothing : u, name) + PkgId(::Nothing, name::AbstractString) = new(nothing, name) +end +PkgId(name::AbstractString) = PkgId(nothing, name) + +function PkgId(m::Module, name::String = String(nameof(moduleroot(m)))) + uuid = UUID(ccall(:jl_module_uuid, NTuple{2, UInt64}, (Any,), m)) + UInt128(uuid) == 0 ? PkgId(name) : PkgId(uuid, name) +end + +==(a::PkgId, b::PkgId) = a.uuid == b.uuid && a.name == b.name + +function hash(pkg::PkgId, h::UInt) + h += 0xc9f248583a0ca36c % UInt + h = hash(pkg.uuid, h) + h = hash(pkg.name, h) + return h +end + +show(io::IO, pkg::PkgId) = + print(io, pkg.name, " [", pkg.uuid === nothing ? "top-level" : pkg.uuid, "]") + +function binpack(pkg::PkgId) + io = IOBuffer() + write(io, UInt8(0)) + uuid = pkg.uuid + write(io, uuid === nothing ? UInt128(0) : UInt128(uuid)) + write(io, pkg.name) + return String(take!(io)) +end + +function binunpack(s::String) + io = IOBuffer(s) + @assert read(io, UInt8) === 0x00 + uuid = read(io, UInt128) + name = read(io, String) + return PkgId(UUID(uuid), name) +end + diff --git a/stdlib/TOML/src/parser.jl b/base/toml_parser.jl similarity index 99% rename from stdlib/TOML/src/parser.jl rename to base/toml_parser.jl index 86de77fc0995d..95285b90e9025 100644 --- a/stdlib/TOML/src/parser.jl +++ b/base/toml_parser.jl @@ -1,3 +1,5 @@ +module TOML + using Base: IdSet # In case we do not have the Dates stdlib available @@ -96,7 +98,7 @@ function Parser(str::String; filepath=nothing) String[], # dotted_keys UnitRange{Int}[], # chunks IdSet{TOMLDict}(), # inline_tables - IdSet{Any}(), # static_arrays + IdSet{Any}(), # static_arrays IdSet{TOMLDict}(), # defined_tables root, filepath, @@ -1164,3 +1166,5 @@ function take_chunks(l::Parser, unescape::Bool)::String empty!(l.chunks) return unescape ? unescape_string(str) : str end + +end diff --git a/base/uuid.jl b/base/uuid.jl index 34f8c7f17e232..de6ea58dcc11e 100644 --- a/base/uuid.jl +++ b/base/uuid.jl @@ -8,6 +8,7 @@ struct UUID value::UInt128 end +UUID(u::UUID) = u UUID(u::NTuple{2, UInt64}) = UUID((UInt128(u[1]) << 64) | UInt128(u[2])) UUID(u::NTuple{4, UInt32}) = UUID((UInt128(u[1]) << 96) | (UInt128(u[2]) << 64) | (UInt128(u[3]) << 32) | UInt128(u[4])) diff --git a/stdlib/Profile/src/Profile.jl b/stdlib/Profile/src/Profile.jl index 29c28fc84f3d6..a0047f0f7767e 100644 --- a/stdlib/Profile/src/Profile.jl +++ b/stdlib/Profile/src/Profile.jl @@ -271,7 +271,7 @@ function short_path(spath::Symbol, filenamecache::Dict{Symbol, String}) for proj in Base.project_names project_file = joinpath(root, proj) if Base.isfile_casesensitive(project_file) - pkgid = Base.project_file_name_uuid(project_file, "") + pkgid = Base.project_file_name_uuid(project_file, "", Base.TOMLCache()) isempty(pkgid.name) && return path # bad Project file # return the joined the module name prefix and path suffix path = path[nextind(path, sizeof(root)):end] diff --git a/stdlib/REPL/src/REPLCompletions.jl b/stdlib/REPL/src/REPLCompletions.jl index bf82ece8c7562..e71769f5fbd09 100644 --- a/stdlib/REPL/src/REPLCompletions.jl +++ b/stdlib/REPL/src/REPLCompletions.jl @@ -556,23 +556,15 @@ end function project_deps_get_completion_candidates(pkgstarts::String, project_file::String)::Vector{Completion} loading_candidates = String[] - open(project_file) do io - state = :top - for line in eachline(io) - if occursin(Base.re_section, line) - state = occursin(Base.re_section_deps, line) ? :deps : :other - elseif state === :top - if (m = match(Base.re_name_to_string, line)) !== nothing - root_name = String(m.captures[1]) - startswith(root_name, pkgstarts) && push!(loading_candidates, root_name) - end - elseif state === :deps - if (m = match(Base.re_key_to_string, line)) !== nothing - dep_name = m.captures[1] - startswith(dep_name, pkgstarts) && push!(loading_candidates, dep_name) - end - end - end + p = Base.TOML.Parser() + Base.TOML.reinit!(p, read(project_file, String); filepath=project_file) + d = Base.TOML.parse(p) + pkg = get(d, "name", nothing) + if pkg !== nothing && startswith(pkg, pkgstarts) + push!(loading_candidates, pkg) + end + for (pkg, _) in get(d, "deps", []) + startswith(pkg, pkgstarts) && push!(loading_candidates, pkg) end return Completion[PackageCompletion(name) for name in loading_candidates] end diff --git a/stdlib/TOML/src/TOML.jl b/stdlib/TOML/src/TOML.jl index 70e497b9d779f..6c7001c741cea 100644 --- a/stdlib/TOML/src/TOML.jl +++ b/stdlib/TOML/src/TOML.jl @@ -1,7 +1,12 @@ module TOML module Internals - include("parser.jl") + # The parser is defined in Base + using Base.TOML: Parser, parse, tryparse, ParserError, isvalid_barekey_char + # Put the error instances in this module + for errtype in instances(Base.TOML.ErrorType) + @eval using Base.TOML: $(Symbol(errtype)) + end # We put the printing functionality in a separate module since It # defines a function `print` and we don't want that to collide with normal # usage of `(Base.)print` in other files diff --git a/test/loading.jl b/test/loading.jl index 6bca011830b04..d23d219276880 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -166,9 +166,10 @@ end d = findfirst(line -> line == "[deps]", p) t = findfirst(line -> startswith(line, "This"), p) # look up various packages by name - root = Base.explicit_project_deps_get(project_file, "Root") - this = Base.explicit_project_deps_get(project_file, "This") - that = Base.explicit_project_deps_get(project_file, "That") + cache = Base.TOMLCache() + root = Base.explicit_project_deps_get(project_file, "Root", cache) + this = Base.explicit_project_deps_get(project_file, "This", cache) + that = Base.explicit_project_deps_get(project_file, "That", cache) # test that the correct answers are given @test root == (something(n, N+1) ≥ something(d, N+1) ? nothing : something(u, N+1) < something(d, N+1) ? root_uuid : proj_uuid) @@ -191,6 +192,23 @@ Base.ACTIVE_PROJECT[] = nothing @test load_path() == [abspath("project","Project.toml")] + +# locate `tail(names)` package by following the search path graph through `names` starting from `where` +function recurse_package(where::PkgId, name::String, names::String...) + pkg = identify_package(where, name, Base.TOMLCache()) + pkg === nothing && return nothing + return recurse_package(pkg, names...) +end + +recurse_package(pkg::String) = identify_package(pkg) +recurse_package(where::PkgId, pkg::String) = identify_package(where, pkg, Base.TOMLCache()) + +function recurse_package(name::String, names::String...) + pkg = identify_package(name) + pkg === nothing && return nothing + return recurse_package(pkg, names...) +end + @testset "project & manifest identify_package & locate_package" begin local path for (names, uuid, path) in [ @@ -201,14 +219,14 @@ Base.ACTIVE_PROJECT[] = nothing ("Foo.Qux", "b5ec9b9c-e354-47fd-b367-a348bdc8f909", "project/deps/Qux.jl" ), ] n = map(String, split(names, '.')) - pkg = identify_package(n...) + pkg = recurse_package(n...) @test pkg == PkgId(UUID(uuid), n[end]) @test joinpath(@__DIR__, normpath(path)) == locate_package(pkg) end @test identify_package("Baz") == nothing @test identify_package("Qux") == nothing @testset "equivalent package names" begin - local classes = [ + classes = [ ["Foo"], ["Bar", "Foo.Bar"], ["Foo.Baz", "Bar.Baz", "Foo.Bar.Baz"], @@ -221,15 +239,15 @@ Base.ACTIVE_PROJECT[] = nothing for i = 1:length(classes) A = classes[i] for x in A - X = identify_package(map(String, split(x, '.'))...) + X = recurse_package(map(String, split(x, '.'))...) for y in A - Y = identify_package(map(String, split(y, '.'))...) + Y = recurse_package(map(String, split(y, '.'))...) @test X == Y end for j = i+1:length(classes) B = classes[j] for z in B - Z = identify_package(map(String, split(z, '.'))...) + Z = recurse_package(map(String, split(z, '.'))...) @test X != Z end end @@ -384,8 +402,6 @@ const envs = Dict{String,Any}() append!(empty!(DEPOT_PATH), depots) @testset "load code uniqueness" begin - @show UUIDS - @show depots @test allunique(UUIDS) @test allunique(depots) @test allunique(DEPOT_PATH) diff --git a/test/precompile.jl b/test/precompile.jl index 3f32fcb17c3ce..76a365b2dadf1 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -242,7 +242,7 @@ try # use _require_from_serialized to ensure that the test fails if # the module doesn't reload from the image: @test_logs (:warn, "Replacing module `$Foo_module`") begin - ms = Base._require_from_serialized(cachefile) + ms = Base._require_from_serialized(cachefile, Base.TOMLCache()) @test isa(ms, Array{Any,1}) end