From 3f9ecfdcaa260d2ba16e8af9a51818f50b452ce3 Mon Sep 17 00:00:00 2001 From: KristofferC Date: Fri, 29 May 2020 12:57:34 +0200 Subject: [PATCH] implement code loading on top of the Base TOML parser --- base/loading.jl | 388 ++++++++++++----------------- base/uuid.jl | 1 + stdlib/Profile/src/Profile.jl | 2 +- stdlib/REPL/src/REPLCompletions.jl | 26 +- test/loading.jl | 36 ++- 5 files changed, 191 insertions(+), 262 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index 69a7e5cc66d6e..e4660edc8b93d 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -151,12 +151,26 @@ function version_slug(uuid::UUID, sha1::SHA1, p::Int=5) return slug(crc, p) end +struct TOMLCache + p::TOML.Parser + d::Dict{String, Dict{String, Any}} +end +TOMLCache() = TOMLCache(TOML.Parser(), Dict()) + +function parsed_toml(cache::TOMLCache, project_file::String) + get!(cache.d, project_file) do + TOML.reinit!(cache.p, read(project_file, String); filepath=project_file) + TOML.parse(cache.p) + end +end + ## package identification: determine unique identity of package to be loaded ## -function find_package(args...) - pkg = identify_package(args...) +# Used by Pkg in but not used in loading itself +function find_package(arg) + pkg = identify_package(arg, TOMLCache()) pkg === nothing && return nothing - return locate_package(pkg) + return locate_package(pkg, cache) end struct PkgId @@ -203,16 +217,15 @@ function binunpack(s::String) end ## package identity: given a package name and a context, try to return its identity ## - -identify_package(where::Module, name::String) = identify_package(PkgId(where), name) +identify_package(where::Module, name::String, cache::TOMLCache = TOMLCache()) = identify_package(PkgId(where), name, cache) # identify_package computes the PkgId for `name` from the context of `where` # or return `nothing` if no mapping exists for it -function identify_package(where::PkgId, name::String)::Union{Nothing,PkgId} +function identify_package(where::PkgId, name::String, cache::TOMLCache=TOMLCache())::Union{Nothing,PkgId} where.name === name && return where - where.uuid === nothing && return identify_package(name) # ignore `where` + where.uuid === nothing && return identify_package(name, cache) # ignore `where` for env in load_path() - uuid = manifest_deps_get(env, where, name) + uuid = manifest_deps_get(env, where, name, cache) uuid === nothing && continue # not found--keep looking uuid.uuid === nothing || return uuid # found in explicit environment--use it return nothing # found in implicit environment--return "not found" @@ -222,47 +235,33 @@ end # identify_package computes the PkgId for `name` from toplevel context # by looking through the Project.toml files and directories -function identify_package(name::String)::Union{Nothing,PkgId} +function identify_package(name::String, cache::TOMLCache=TOMLCache())::Union{Nothing,PkgId} for env in load_path() - uuid = project_deps_get(env, name) + uuid = project_deps_get(env, name, cache) uuid === nothing || return uuid # found--return it end return nothing end -function identify_package(name::String, names::String...) - pkg = identify_package(name) - pkg === nothing && return nothing - return identify_package(pkg, names...) -end - -# locate `tail(names)` package by following the search path graph through `names` starting from `where` -function identify_package(where::PkgId, name::String, names::String...) - pkg = identify_package(where, name) - pkg === nothing && return nothing - return identify_package(pkg, names...) -end - ## package location: given a package identity, find file to load ## - -function locate_package(pkg::PkgId)::Union{Nothing,String} +function locate_package(pkg::PkgId, cache::TOMLCache=TOMLCache())::Union{Nothing,String} if pkg.uuid === nothing for env in load_path() # look for the toplevel pkg `pkg.name` in this entry - found = project_deps_get(env, pkg.name) + found = project_deps_get(env, pkg.name, cache) found === nothing && continue if pkg == found # pkg.name is present in this directory or project file, # return the path the entry point for the code, if it could be found # otherwise, signal failure - return implicit_manifest_uuid_path(env, pkg) + return implicit_manifest_uuid_path(env, pkg, cache) end @assert found.uuid !== nothing - return locate_package(found) # restart search now that we know the uuid for pkg + return locate_package(found, cache) # restart search now that we know the uuid for pkg end else for env in load_path() - path = manifest_uuid_path(env, pkg) + path = manifest_uuid_path(env, pkg, cache) path === nothing || return entry_path(path, pkg.name) end end @@ -319,136 +318,82 @@ function env_project_file(env::String)::Union{Bool,String} return false end -function project_deps_get(env::String, name::String)::Union{Nothing,PkgId} +function project_deps_get(env::String, name::String, cache::TOMLCache)::Union{Nothing,PkgId} project_file = env_project_file(env) if project_file isa String - pkg_uuid = explicit_project_deps_get(project_file, name) + pkg_uuid = explicit_project_deps_get(project_file, name, cache) pkg_uuid === nothing || return PkgId(pkg_uuid, name) elseif project_file - return implicit_project_deps_get(env, name) + return implicit_project_deps_get(env, name, cache) end return nothing end -function manifest_deps_get(env::String, where::PkgId, name::String)::Union{Nothing,PkgId} +function manifest_deps_get(env::String, where::PkgId, name::String, cache::TOMLCache)::Union{Nothing,PkgId} @assert where.uuid !== nothing project_file = env_project_file(env) if project_file isa String # first check if `where` names the Project itself - proj = project_file_name_uuid(project_file, where.name) + proj = project_file_name_uuid(project_file, where.name, cache) if proj == where # if `where` matches the project, use [deps] section as manifest, and stop searching - pkg_uuid = explicit_project_deps_get(project_file, name) + pkg_uuid = explicit_project_deps_get(project_file, name, cache) return PkgId(pkg_uuid, name) end # look for manifest file and `where` stanza - return explicit_manifest_deps_get(project_file, where.uuid, name) + return explicit_manifest_deps_get(project_file, where.uuid, name, cache) elseif project_file # if env names a directory, search it - return implicit_manifest_deps_get(env, where, name) + return implicit_manifest_deps_get(env, where, name, cache) end return nothing end -function manifest_uuid_path(env::String, pkg::PkgId)::Union{Nothing,String} +function manifest_uuid_path(env::String, pkg::PkgId, cache::TOMLCache)::Union{Nothing,String} project_file = env_project_file(env) if project_file isa String - proj = project_file_name_uuid(project_file, pkg.name) + proj = project_file_name_uuid(project_file, pkg.name, cache) if proj == pkg # if `pkg` matches the project, return the project itself - return project_file_path(project_file, pkg.name) + return project_file_path(project_file, pkg.name, cache) end # look for manifest file and `where` stanza - return explicit_manifest_uuid_path(project_file, pkg) + return explicit_manifest_uuid_path(project_file, pkg, cache) elseif project_file # if env names a directory, search it - return implicit_manifest_uuid_path(env, pkg) + return implicit_manifest_uuid_path(env, pkg, cache) end return nothing end -# regular expressions for scanning project & manifest files - -const re_section = r"^\s*\[" -const re_array_of_tables = r"^\s*\[\s*\[" -const re_section_deps = r"^\s*\[\s*\"?deps\"?\s*\]\s*(?:#|$)" -const re_section_capture = r"^\s*\[\s*\[\s*\"?(\w+)\"?\s*\]\s*\]\s*(?:#|$)" -const re_subsection_deps = r"^\s*\[\s*\"?(\w+)\"?\s*\.\s*\"?deps\"?\s*\]\s*(?:#|$)" -const re_key_to_string = r"^\s*(\w+)\s*=\s*\"(.*)\"\s*(?:#|$)" -const re_uuid_to_string = r"^\s*uuid\s*=\s*\"(.*)\"\s*(?:#|$)" -const re_name_to_string = r"^\s*name\s*=\s*\"(.*)\"\s*(?:#|$)" -const re_path_to_string = r"^\s*path\s*=\s*\"(.*)\"\s*(?:#|$)" -const re_hash_to_string = r"^\s*git-tree-sha1\s*=\s*\"(.*)\"\s*(?:#|$)" -const re_manifest_to_string = r"^\s*manifest\s*=\s*\"(.*)\"\s*(?:#|$)" -const re_deps_to_any = r"^\s*deps\s*=\s*(.*?)\s*(?:#|$)" - # find project file's top-level UUID entry (or nothing) -function project_file_name_uuid(project_file::String, name::String)::PkgId - pkg = open(project_file) do io - uuid = dummy_uuid(project_file) - for line in eachline(io) - occursin(re_section, line) && break - if (m = match(re_name_to_string, line)) !== nothing - name = String(m.captures[1]) - elseif (m = match(re_uuid_to_string, line)) !== nothing - uuid = UUID(m.captures[1]) - end - end - return PkgId(uuid, name) - end - return pkg +function project_file_name_uuid(project_file::String, name::String, cache::TOMLCache)::PkgId + uuid = dummy_uuid(project_file) + d = parsed_toml(cache, project_file) + uuid = get(d, "uuid", uuid) + name = get(d, "name", name) + return PkgId(UUID(uuid), name) end -function project_file_path(project_file::String, name::String)::String - path = open(project_file) do io - for line in eachline(io) - occursin(re_section, line) && break - if (m = match(re_path_to_string, line)) !== nothing - return String(m.captures[1]) - end - end - return "" - end - return joinpath(dirname(project_file), path) +function project_file_path(project_file::String, name::String, cache) + d = parsed_toml(cache, project_file) + joinpath(dirname(project_file), get(d, "path", "")) end - # find project file's corresponding manifest file -function project_file_manifest_path(project_file::String)::Union{Nothing,String} - open(project_file) do io - dir = abspath(dirname(project_file)) - for line in eachline(io) - occursin(re_section, line) && break - if (m = match(re_manifest_to_string, line)) !== nothing - manifest_file = normpath(joinpath(dir, m.captures[1])) - isfile_casesensitive(manifest_file) && return manifest_file - return nothing # silently stop if the explicitly listed manifest file is not present - end - end - for mfst in manifest_names - manifest_file = joinpath(dir, mfst) - isfile_casesensitive(manifest_file) && return manifest_file - end - return nothing +function project_file_manifest_path(project_file::String, cache::TOMLCache)::Union{Nothing,String} + dir = abspath(dirname(project_file)) + d = parsed_toml(cache, project_file) + explicit_manifest = get(d, "manifest", nothing) + if explicit_manifest !== nothing + manifest_file = normpath(joinpath(dir, explicit_manifest)) + isfile_casesensitive(manifest_file) && return manifest_file end -end - -# find `name` in a manifest file and return its UUID -# return `nothing` on failure -function manifest_file_name_uuid(manifest_file::IO, name::String)::Union{Nothing,UUID} - name_section = false - uuid = nothing - for line in eachline(manifest_file) - if (m = match(re_section_capture, line)) !== nothing - name_section && break - name_section = (m.captures[1] == name) - elseif name_section - if (m = match(re_uuid_to_string, line)) !== nothing - uuid = UUID(m.captures[1]) - end - end + for mfst in manifest_names + manifest_file = joinpath(dir, mfst) + isfile_casesensitive(manifest_file) && return manifest_file end - return uuid + return nothing end # given a directory (implicit env from LOAD_PATH) and a name, @@ -490,129 +435,102 @@ end # find project file root or deps `name => uuid` mapping # return `nothing` if `name` is not found -function explicit_project_deps_get(project_file::String, name::String)::Union{Nothing,UUID} - pkg_uuid = open(project_file) do io - root_name = nothing - root_uuid = dummy_uuid(project_file) - state = :top - for line in eachline(io) - if occursin(re_section, line) - state === :top && root_name == name && return root_uuid - state = occursin(re_section_deps, line) ? :deps : :other - elseif state === :top - if (m = match(re_name_to_string, line)) !== nothing - root_name = String(m.captures[1]) - elseif (m = match(re_uuid_to_string, line)) !== nothing - root_uuid = UUID(m.captures[1]) - end - elseif state === :deps - if (m = match(re_key_to_string, line)) !== nothing - m.captures[1] == name && return UUID(m.captures[2]) - end - end - end - return root_name == name ? root_uuid : nothing +function explicit_project_deps_get(project_file::String, name::String, cache::TOMLCache)::Union{Nothing,UUID} + root_name = nothing + root_uuid = dummy_uuid(project_file) + d = parsed_toml(cache, project_file) + if get(d, "name", nothing) == name + return UUID(get(d, "uuid", root_uuid)) + elseif haskey(d, "deps") + deps = d["deps"] + uuid = get(deps, name, nothing) + uuid === nothing || return UUID(uuid) end - return pkg_uuid + return nothing end # find `where` stanza and return the PkgId for `name` # return `nothing` if it did not find `where` (indicating caller should continue searching) -function explicit_manifest_deps_get(project_file::String, where::UUID, name::String)::Union{Nothing,PkgId} - manifest_file = project_file_manifest_path(project_file) +function explicit_manifest_deps_get(project_file::String, where::UUID, name::String, cache::TOMLCache)::Union{Nothing,PkgId} + manifest_file = project_file_manifest_path(project_file, cache) manifest_file === nothing && return nothing # manifest not found--keep searching LOAD_PATH - found_or_uuid = open(manifest_file) do io - uuid = deps = nothing - state = :other - # first search the manifest for the deps section associated with `where` (by uuid) - for line in eachline(io) - if occursin(re_array_of_tables, line) - uuid == where && break - uuid = deps = nothing - state = :stanza - elseif state === :stanza - if (m = match(re_uuid_to_string, line)) !== nothing - uuid = UUID(m.captures[1]) - elseif (m = match(re_deps_to_any, line)) !== nothing - deps = String(m.captures[1]) - elseif occursin(re_subsection_deps, line) - state = :deps - elseif occursin(re_section, line) - state = :other - end - elseif state === :deps && uuid == where - # [deps] section format gives both name and uuid - if (m = match(re_key_to_string, line)) !== nothing - m.captures[1] == name && return UUID(m.captures[2]) + d = parsed_toml(cache, manifest_file) + found_where = false + found_name = false + for (dep_name, entries) in d + for entry in entries + haskey(entry, "uuid") || continue + if UUID(entry["uuid"]) == where + found_where = true + # deps is either a list of names (deps = ["DepA", "DepB"]) or + # a table of entries (deps = {"DepA" = "6ea...", "DepB" = "55d..."} + deps = get(entry, "deps", nothing) + if deps isa Vector + found_name = name in deps + break + elseif deps isa Dict + for (dep, uuid) in deps + if dep === name + return PkgId(UUID(uuid), name) + end + end end end end - # now search through `deps = []` string to see if we have an entry for `name` - uuid == where || return false - deps === nothing && return true - # TODO: handle inline table syntax - if deps[1] != '[' || deps[end] != ']' - @warn "Unexpected TOML deps format:\n$deps" - return false - end - occursin(repr(name), deps) || return true - seekstart(io) # rewind IO handle - # finally, find out the `uuid` associated with `name` - return something(manifest_file_name_uuid(io, name), false) end - found_or_uuid isa UUID && return PkgId(found_or_uuid, name) - found_or_uuid && return PkgId(name) - return nothing + found_where || return nothing + found_name || return PkgId(name) + # Only here is deps was not a dict which mean we have a unique name for the dep + if !haskey(d, name) || length(d[name]) != 1 + error("expected a single entry for $(repr(name)) in $(repr(project_file))") + end + entry = first(d[name]) + if found_name + uuid = get(entry, "uuid", nothing) + return PkgId(UUID(uuid), name) + else + return PkgId(name) + end end # find `uuid` stanza, return the corresponding path -function explicit_manifest_uuid_path(project_file::String, pkg::PkgId)::Union{Nothing,String} - manifest_file = project_file_manifest_path(project_file) +function explicit_manifest_uuid_path(project_file::String, pkg::PkgId, cache::TOMLCache)::Union{Nothing,String} + manifest_file = project_file_manifest_path(project_file, cache) manifest_file === nothing && return nothing # no manifest, skip env - open(manifest_file) do io - uuid = name = path = hash = nothing - for line in eachline(io) - if (m = match(re_section_capture, line)) !== nothing - uuid == pkg.uuid && break - name = String(m.captures[1]) - path = hash = nothing - elseif (m = match(re_uuid_to_string, line)) !== nothing - uuid = UUID(m.captures[1]) - elseif (m = match(re_path_to_string, line)) !== nothing - path = String(m.captures[1]) - elseif (m = match(re_hash_to_string, line)) !== nothing - hash = SHA1(m.captures[1]) + + d = parsed_toml(cache, manifest_file) + entries = get(d, pkg.name, nothing) + entries === nothing && return nothing # TODO: allow name to mismatch? + for entry in entries + if UUID(get(entry, "uuid", nothing)) === pkg.uuid + path = get(entry, "path", nothing) + if path !== nothing + path = normpath(abspath(dirname(manifest_file), path)) + return path end - end - uuid == pkg.uuid || return nothing - name == pkg.name || return nothing # TODO: allow a mismatch? - if path !== nothing - path = normpath(abspath(dirname(manifest_file), path)) - return path - end - hash === nothing && return nothing - # Keep the 4 since it used to be the default - for slug in (version_slug(uuid, hash, 4), version_slug(uuid, hash)) + hash = get(entry, "git-tree-sha1", nothing) + hash === nothing && return nothing + slug = version_slug(pkg.uuid, SHA1(hash)) for depot in DEPOT_PATH - path = abspath(depot, "packages", name, slug) + path = abspath(depot, "packages", pkg.name, slug) ispath(path) && return path end end - return nothing end + return nothing end ## implicit project & manifest API ## # look for an entry point for `name` from a top-level package (no environment) # otherwise return `nothing` to indicate the caller should keep searching -function implicit_project_deps_get(dir::String, name::String)::Union{Nothing,PkgId} +function implicit_project_deps_get(dir::String, name::String, cache::TOMLCache)::Union{Nothing,PkgId} path, project_file = entry_point_and_project_file(dir, name) if project_file === nothing path === nothing && return nothing return PkgId(name) end - proj = project_file_name_uuid(project_file, name) + proj = project_file_name_uuid(project_file, name, cache) proj.name == name || return nothing return proj end @@ -620,25 +538,25 @@ end # look for an entry-point for `name`, check that UUID matches # if there's a project file, look up `name` in its deps and return that # otherwise return `nothing` to indicate the caller should keep searching -function implicit_manifest_deps_get(dir::String, where::PkgId, name::String)::Union{Nothing,PkgId} +function implicit_manifest_deps_get(dir::String, where::PkgId, name::String, cache::TOMLCache)::Union{Nothing,PkgId} @assert where.uuid !== nothing project_file = entry_point_and_project_file(dir, where.name)[2] project_file === nothing && return nothing # a project file is mandatory for a package with a uuid - proj = project_file_name_uuid(project_file, where.name) + proj = project_file_name_uuid(project_file, where.name, cache) proj == where || return nothing # verify that this is the correct project file # this is the correct project, so stop searching here - pkg_uuid = explicit_project_deps_get(project_file, name) + pkg_uuid = explicit_project_deps_get(project_file, name, cache) return PkgId(pkg_uuid, name) end # look for an entry-point for `pkg` and return its path if UUID matches -function implicit_manifest_uuid_path(dir::String, pkg::PkgId)::Union{Nothing,String} +function implicit_manifest_uuid_path(dir::String, pkg::PkgId, cache::TOMLCache)::Union{Nothing,String} path, project_file = entry_point_and_project_file(dir, pkg.name) if project_file === nothing pkg.uuid === nothing || return nothing return path end - proj = project_file_name_uuid(project_file, pkg.name) + proj = project_file_name_uuid(project_file, pkg.name, cache) proj == pkg || return nothing return path end @@ -698,7 +616,7 @@ function _include_from_serialized(path::String, depmods::Vector{Any}) return restored end -function _tryrequire_from_serialized(modkey::PkgId, build_id::UInt64, modpath::Union{Nothing, String}) +function _tryrequire_from_serialized(modkey::PkgId, build_id::UInt64, modpath::Union{Nothing, String}, cache::TOMLCache) if root_module_exists(modkey) M = root_module(modkey) if PkgId(M) == modkey && module_build_id(M) === build_id @@ -706,10 +624,10 @@ function _tryrequire_from_serialized(modkey::PkgId, build_id::UInt64, modpath::U end else if modpath === nothing - modpath = locate_package(modkey) + modpath = locate_package(modkey, cache) modpath === nothing && return nothing end - mod = _require_search_from_serialized(modkey, String(modpath)) + mod = _require_search_from_serialized(modkey, String(modpath), cache) if !isa(mod, Bool) for callback in package_callbacks invokelatest(callback, modkey) @@ -725,7 +643,7 @@ function _tryrequire_from_serialized(modkey::PkgId, build_id::UInt64, modpath::U return nothing end -function _require_from_serialized(path::String) +function _require_from_serialized(path::String, cache::TOMLCache) # loads a precompile cache file, ignoring stale_cachfile tests # load all of the dependent modules first local depmodnames @@ -741,7 +659,7 @@ function _require_from_serialized(path::String) depmods = Vector{Any}(undef, ndeps) for i in 1:ndeps modkey, build_id = depmodnames[i] - dep = _tryrequire_from_serialized(modkey, build_id, nothing) + dep = _tryrequire_from_serialized(modkey, build_id, nothing, cache) dep === nothing && return ErrorException("Required dependency $modkey failed to load from a cache file.") depmods[i] = dep::Module end @@ -752,10 +670,10 @@ end # returns `true` if require found a precompile cache for this sourcepath, but couldn't load it # returns `false` if the module isn't known to be precompilable # returns the set of modules restored if the cache load succeeded -function _require_search_from_serialized(pkg::PkgId, sourcepath::String) +function _require_search_from_serialized(pkg::PkgId, sourcepath::String, cache::TOMLCache) paths = find_all_in_cache_path(pkg) for path_to_try in paths::Vector{String} - staledeps = stale_cachefile(sourcepath, path_to_try) + staledeps = stale_cachefile(sourcepath, path_to_try, cache) if staledeps === true continue end @@ -768,7 +686,7 @@ function _require_search_from_serialized(pkg::PkgId, sourcepath::String) dep = staledeps[i] dep isa Module && continue modpath, modkey, build_id = dep::Tuple{String, PkgId, UInt64} - dep = _tryrequire_from_serialized(modkey, build_id, modpath) + dep = _tryrequire_from_serialized(modkey, build_id, modpath, cache) if dep === nothing @debug "Required dependency $modkey failed to load from cache file for $modpath." staledeps = true @@ -885,7 +803,8 @@ For more details regarding code loading, see the manual sections on [modules](@r [parallel computing](@ref code-availability). """ function require(into::Module, mod::Symbol) - uuidkey = identify_package(into, String(mod)) + cache = TOMLCache() + uuidkey = identify_package(into, String(mod), cache) # Core.println("require($(PkgId(into)), $mod) -> $uuidkey") if uuidkey === nothing where = PkgId(into) @@ -920,12 +839,12 @@ function require(into::Module, mod::Symbol) if _track_dependencies[] push!(_require_dependencies, (into, binpack(uuidkey), 0.0)) end - return require(uuidkey) + return require(uuidkey, cache) end -function require(uuidkey::PkgId) +function require(uuidkey::PkgId, cache::TOMLCache=TOMLCache()) if !root_module_exists(uuidkey) - _require(uuidkey) + _require(uuidkey, cache) # After successfully loading, notify downstream consumers for callback in package_callbacks invokelatest(callback, uuidkey) @@ -980,7 +899,7 @@ function unreference_module(key::PkgId) end end -function _require(pkg::PkgId) +function _require(pkg::PkgId, cache::TOMLCache) # handle recursive calls to require loading = get(package_locks, pkg, false) if loading !== false @@ -994,7 +913,7 @@ function _require(pkg::PkgId) try toplevel_load[] = false # perform the search operation to select the module file require intends to load - path = locate_package(pkg) + path = locate_package(pkg, cache) if path === nothing throw(ArgumentError(""" Package $pkg is required but does not seem to be installed: @@ -1004,7 +923,7 @@ function _require(pkg::PkgId) # attempt to load the module file via the precompile cache locations if JLOptions().use_compiled_modules != 0 - m = _require_search_from_serialized(pkg, path) + m = _require_search_from_serialized(pkg, path, cache) if !isa(m, Bool) return end @@ -1037,7 +956,7 @@ function _require(pkg::PkgId) end # fall-through to loading the file locally else - m = _require_from_serialized(cachefile) + m = _require_from_serialized(cachefile, cache) if isa(m, Exception) @warn "The call to compilecache failed to create a usable precompiled cache file for $pkg" exception=m else @@ -1283,8 +1202,8 @@ This can be used to reduce package load times. Cache files are stored in `DEPOT_PATH[1]/compiled`. See [Module initialization and precompilation](@ref) for important notes. """ -function compilecache(pkg::PkgId) - path = locate_package(pkg) +function compilecache(pkg::PkgId, cache::TOMLCache = TOMLCache()) + path = locate_package(pkg, cache) path === nothing && throw(ArgumentError("$pkg not found during precompilation")) return compilecache(pkg, path) end @@ -1448,7 +1367,8 @@ end # returns true if it "cachefile.ji" is stale relative to "modpath.jl" # otherwise returns the list of dependencies to also check -function stale_cachefile(modpath::String, cachefile::String) +stale_cachefile(modpath::String, cachefile::String) = stale_cachefile(modpath, cachefile, TOMLCache()) +function stale_cachefile(modpath::String, cachefile::String, cache::TOMLCache) io = open(cachefile, "r") try if !isvalid_cache_header(io) @@ -1473,7 +1393,7 @@ function stale_cachefile(modpath::String, cachefile::String) return true # Won't be able to fulfill dependency end else - path = locate_package(req_key) + path = locate_package(req_key, cache) if path === nothing @debug "Rejecting cache file $cachefile because dependency $req_key not found." return true # Won't be able to fulfill dependency @@ -1506,7 +1426,7 @@ function stale_cachefile(modpath::String, cachefile::String) end for (modkey, req_modkey) in requires # verify that `require(modkey, name(req_modkey))` ==> `req_modkey` - if identify_package(modkey, req_modkey.name) != req_modkey + if identify_package(modkey, req_modkey.name, cache) != req_modkey @debug "Rejecting cache file $cachefile because uuid mapping for $modkey => $req_modkey has changed" return true end diff --git a/base/uuid.jl b/base/uuid.jl index c72bfd9dff500..e42c952de5f25 100644 --- a/base/uuid.jl +++ b/base/uuid.jl @@ -8,6 +8,7 @@ struct UUID value::UInt128 end +UUID(u::UUID) = u UUID(u::NTuple{2, UInt64}) = UUID((UInt128(u[1]) << 64) | UInt128(u[2])) UUID(u::NTuple{4, UInt32}) = UUID((UInt128(u[1]) << 96) | (UInt128(u[2]) << 64) | (UInt128(u[3]) << 32) | UInt128(u[4])) diff --git a/stdlib/Profile/src/Profile.jl b/stdlib/Profile/src/Profile.jl index 1032429d96f5e..538e696ae6660 100644 --- a/stdlib/Profile/src/Profile.jl +++ b/stdlib/Profile/src/Profile.jl @@ -271,7 +271,7 @@ function short_path(spath::Symbol, filenamecache::Dict{Symbol, String}) for proj in Base.project_names project_file = joinpath(root, proj) if Base.isfile_casesensitive(project_file) - pkgid = Base.project_file_name_uuid(project_file, "") + pkgid = Base.project_file_name_uuid(project_file, "", Base.TOMLCache()) isempty(pkgid.name) && return path # bad Project file # return the joined the module name prefix and path suffix path = path[nextind(path, sizeof(root)):end] diff --git a/stdlib/REPL/src/REPLCompletions.jl b/stdlib/REPL/src/REPLCompletions.jl index a16565bba3751..b814268d06649 100644 --- a/stdlib/REPL/src/REPLCompletions.jl +++ b/stdlib/REPL/src/REPLCompletions.jl @@ -554,23 +554,15 @@ end function project_deps_get_completion_candidates(pkgstarts::String, project_file::String)::Vector{Completion} loading_candidates = String[] - open(project_file) do io - state = :top - for line in eachline(io) - if occursin(Base.re_section, line) - state = occursin(Base.re_section_deps, line) ? :deps : :other - elseif state === :top - if (m = match(Base.re_name_to_string, line)) !== nothing - root_name = String(m.captures[1]) - startswith(root_name, pkgstarts) && push!(loading_candidates, root_name) - end - elseif state === :deps - if (m = match(Base.re_key_to_string, line)) !== nothing - dep_name = m.captures[1] - startswith(dep_name, pkgstarts) && push!(loading_candidates, dep_name) - end - end - end + p = Base.TOML.Parser() + Base.TOML.reinit!(p, read(project_file, String); filepath=project_file) + d = Base.TOML.parse(p) + pkg = get(d, "name", nothing) + if pkg !== nothing && startswith(pkg, pkgstarts) + push!(loading_candidates, pkg) + end + for (pkg, _) in get(d, "deps", []) + startswith(pkg, pkgstarts) && push!(loading_candidates, pkg) end return Completion[PackageCompletion(name) for name in loading_candidates] end diff --git a/test/loading.jl b/test/loading.jl index 4b42019da9c63..5343fcc8327cb 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -164,9 +164,10 @@ end d = findfirst(line -> line == "[deps]", p) t = findfirst(line -> startswith(line, "This"), p) # look up various packages by name - root = Base.explicit_project_deps_get(project_file, "Root") - this = Base.explicit_project_deps_get(project_file, "This") - that = Base.explicit_project_deps_get(project_file, "That") + cache = Base.TOMLCache() + root = Base.explicit_project_deps_get(project_file, "Root", cache) + this = Base.explicit_project_deps_get(project_file, "This", cache) + that = Base.explicit_project_deps_get(project_file, "That", cache) # test that the correct answers are given @test root == (something(n, N+1) ≥ something(d, N+1) ? nothing : something(u, N+1) < something(d, N+1) ? root_uuid : proj_uuid) @@ -191,6 +192,23 @@ Base.ACTIVE_PROJECT[] = nothing @test load_path() == [abspath("project","Project.toml")] + +# locate `tail(names)` package by following the search path graph through `names` starting from `where` +function recurse_package(where::PkgId, name::String, names::String...) + pkg = identify_package(where, name, Base.TOMLCache()) + pkg === nothing && return nothing + return recurse_package(pkg, names...) +end + +recurse_package(pkg::String) = identify_package(pkg) +recurse_package(where::PkgId, pkg::String) = identify_package(where, pkg, Base.TOMLCache()) + +function recurse_package(name::String, names::String...) + pkg = identify_package(name) + pkg === nothing && return nothing + return recurse_package(pkg, names...) +end + @testset "project & manifest identify_package & locate_package" begin local path for (names, uuid, path) in [ @@ -201,14 +219,14 @@ Base.ACTIVE_PROJECT[] = nothing ("Foo.Qux", "b5ec9b9c-e354-47fd-b367-a348bdc8f909", "project/deps/Qux.jl" ), ] n = map(String, split(names, '.')) - pkg = identify_package(n...) + pkg = recurse_package(n...) @test pkg == PkgId(UUID(uuid), n[end]) @test joinpath(@__DIR__, normpath(path)) == locate_package(pkg) end @test identify_package("Baz") == nothing @test identify_package("Qux") == nothing @testset "equivalent package names" begin - local classes = [ + classes = [ ["Foo"], ["Bar", "Foo.Bar"], ["Foo.Baz", "Bar.Baz", "Foo.Bar.Baz"], @@ -221,15 +239,15 @@ Base.ACTIVE_PROJECT[] = nothing for i = 1:length(classes) A = classes[i] for x in A - X = identify_package(map(String, split(x, '.'))...) + X = recurse_package(map(String, split(x, '.'))...) for y in A - Y = identify_package(map(String, split(y, '.'))...) + Y = recurse_package(map(String, split(y, '.'))...) @test X == Y end for j = i+1:length(classes) B = classes[j] for z in B - Z = identify_package(map(String, split(z, '.'))...) + Z = recurse_package(map(String, split(z, '.'))...) @test X != Z end end @@ -384,8 +402,6 @@ const envs = Dict{String,Any}() append!(empty!(DEPOT_PATH), depots) @testset "load code uniqueness" begin - @show UUIDS - @show depots @test allunique(UUIDS) @test allunique(depots) @test allunique(DEPOT_PATH)