move the TOML parser to Base and implement code loading on top of the…

… Base TOML parser

base/Base.jl

@@ -340,6 +340,8 @@ include("threadcall.jl")
 
 # code loading
 include("uuid.jl")
+include("pkgid.jl")
+include("toml_parser.jl")
 include("loading.jl")
 
 # misc useful functions & macros

base/initdefs.jl

@@ -234,7 +234,7 @@ function load_path_expand(env::AbstractString)::Union{String, Nothing}
  # if you put a `@` in LOAD_PATH manually, it's expanded late
  env == "@" && return active_project(false)
  env == "@." && return current_project()
- env == "@stdlib" && return Sys.STDLIB
+ env == "@stdlib" && return Sys.STDLIB::String
  env = replace(env, '#' => VERSION.major, count=1)
  env = replace(env, '#' => VERSION.minor, count=1)
  env = replace(env, '#' => VERSION.patch, count=1)

base/pkgid.jl

@@ -0,0 +1,43 @@
+struct PkgId
+ uuid::Union{UUID,Nothing}
+ name::String
+
+ PkgId(u::UUID, name::AbstractString) = new(UInt128(u) == 0 ? nothing : u, name)
+ PkgId(::Nothing, name::AbstractString) = new(nothing, name)
+end
+PkgId(name::AbstractString) = PkgId(nothing, name)
+
+function PkgId(m::Module, name::String = String(nameof(moduleroot(m))))
+ uuid = UUID(ccall(:jl_module_uuid, NTuple{2, UInt64}, (Any,), m))
+ UInt128(uuid) == 0 ? PkgId(name) : PkgId(uuid, name)
+end
+
+==(a::PkgId, b::PkgId) = a.uuid == b.uuid && a.name == b.name
+
+function hash(pkg::PkgId, h::UInt)
+ h += 0xc9f248583a0ca36c % UInt
+ h = hash(pkg.uuid, h)
+ h = hash(pkg.name, h)
+ return h
+end
+
+show(io::IO, pkg::PkgId) =
+ print(io, pkg.name, " [", pkg.uuid === nothing ? "top-level" : pkg.uuid, "]")
+
+function binpack(pkg::PkgId)
+ io = IOBuffer()
+ write(io, UInt8(0))
+ uuid = pkg.uuid
+ write(io, uuid === nothing ? UInt128(0) : UInt128(uuid))
+ write(io, pkg.name)
+ return String(take!(io))
+end
+
+function binunpack(s::String)
+ io = IOBuffer(s)
+ @assert read(io, UInt8) === 0x00
+ uuid = read(io, UInt128)
+ name = read(io, String)
+ return PkgId(UUID(uuid), name)
+end
+

stdlib/TOML/src/parser.jl → base/toml_parser.jl

@@ -1,3 +1,5 @@
+module TOML
+
 using Base: IdSet
 
 # In case we do not have the Dates stdlib available
@@ -96,7 +98,7 @@ function Parser(str::String; filepath=nothing)
  String[], # dotted_keys
  UnitRange{Int}[], # chunks
  IdSet{TOMLDict}(), # inline_tables
- IdSet{Any}(), # static_arrays
+ IdSet{Any}(),  # static_arrays
  IdSet{TOMLDict}(), # defined_tables
  root,
  filepath,
@@ -1164,3 +1166,5 @@ function take_chunks(l::Parser, unescape::Bool)::String
  empty!(l.chunks)
  return unescape ? unescape_string(str) : str
 end
+
+end

base/uuid.jl

@@ -8,6 +8,7 @@
 struct UUID
  value::UInt128
 end
+UUID(u::UUID) = u
 UUID(u::NTuple{2, UInt64}) = UUID((UInt128(u[1]) << 64) | UInt128(u[2]))
 UUID(u::NTuple{4, UInt32}) = UUID((UInt128(u[1]) << 96) | (UInt128(u[2]) << 64) |
  (UInt128(u[3]) << 32) | UInt128(u[4]))

stdlib/Profile/src/Profile.jl

@@ -271,7 +271,7 @@ function short_path(spath::Symbol, filenamecache::Dict{Symbol, String})
  for proj in Base.project_names
  project_file = joinpath(root, proj)
  if Base.isfile_casesensitive(project_file)
- pkgid = Base.project_file_name_uuid(project_file, "")
+ pkgid = Base.project_file_name_uuid(project_file, "", Base.TOMLCache())
  isempty(pkgid.name) && return path # bad Project file
  # return the joined the module name prefix and path suffix
  path = path[nextind(path, sizeof(root)):end]

stdlib/REPL/src/REPLCompletions.jl

@@ -556,23 +556,15 @@ end
 
 function project_deps_get_completion_candidates(pkgstarts::String, project_file::String)::Vector{Completion}
  loading_candidates = String[]
- open(project_file) do io
- state = :top
- for line in eachline(io)
- if occursin(Base.re_section, line)
- state = occursin(Base.re_section_deps, line) ? :deps : :other
- elseif state === :top
- if (m = match(Base.re_name_to_string, line)) !== nothing
- root_name = String(m.captures[1])
- startswith(root_name, pkgstarts) && push!(loading_candidates, root_name)
- end
- elseif state === :deps
- if (m = match(Base.re_key_to_string, line)) !== nothing
- dep_name = m.captures[1]
- startswith(dep_name, pkgstarts) && push!(loading_candidates, dep_name)
- end
- end
- end
+ p = Base.TOML.Parser()
+ Base.TOML.reinit!(p, read(project_file, String); filepath=project_file)
+ d = Base.TOML.parse(p)
+ pkg = get(d, "name", nothing)
+ if pkg !== nothing && startswith(pkg, pkgstarts)
+ push!(loading_candidates, pkg)
+ end
+ for (pkg, _) in get(d, "deps", [])
+ startswith(pkg, pkgstarts) && push!(loading_candidates, pkg)
  end
  return Completion[PackageCompletion(name) for name in loading_candidates]
 end

stdlib/TOML/src/TOML.jl

@@ -1,7 +1,12 @@
 module TOML
 
 module Internals
- include("parser.jl")
+ # The parser is defined in Base
+ using Base.TOML: Parser, parse, tryparse, ParserError, isvalid_barekey_char, reinit!
+ # Put the error instances in this module
+ for errtype in instances(Base.TOML.ErrorType)
+ @eval using Base.TOML: $(Symbol(errtype))
+ end
  # We put the printing functionality in a separate module since It
  # defines a function `print` and we don't want that to collide with normal
  # usage of `(Base.)print` in other files

test/loading.jl

@@ -166,9 +166,10 @@ end
  d = findfirst(line -> line == "[deps]", p)
  t = findfirst(line -> startswith(line, "This"), p)
  # look up various packages by name
- root = Base.explicit_project_deps_get(project_file, "Root")
- this = Base.explicit_project_deps_get(project_file, "This")
- that = Base.explicit_project_deps_get(project_file, "That")
+ cache = Base.TOMLCache()
+ root = Base.explicit_project_deps_get(project_file, "Root", cache)
+ this = Base.explicit_project_deps_get(project_file, "This", cache)
+ that = Base.explicit_project_deps_get(project_file, "That", cache)
  # test that the correct answers are given
  @test root == (something(n, N+1) ≥ something(d, N+1) ? nothing :
  something(u, N+1) < something(d, N+1) ? root_uuid : proj_uuid)
@@ -191,6 +192,23 @@ Base.ACTIVE_PROJECT[] = nothing
 
 @test load_path() == [abspath("project","Project.toml")]
 
+
+# locate `tail(names)` package by following the search path graph through `names` starting from `where`
+function recurse_package(where::PkgId, name::String, names::String...)
+ pkg = identify_package(where, name, Base.TOMLCache())
+ pkg === nothing && return nothing
+ return recurse_package(pkg, names...)
+end
+
+recurse_package(pkg::String) = identify_package(pkg)
+recurse_package(where::PkgId, pkg::String) = identify_package(where, pkg, Base.TOMLCache())
+
+function recurse_package(name::String, names::String...)
+ pkg = identify_package(name)
+ pkg === nothing && return nothing
+ return recurse_package(pkg, names...)
+end
+
 @testset "project & manifest identify_package & locate_package" begin
  local path
  for (names, uuid, path) in [
@@ -201,14 +219,14 @@ Base.ACTIVE_PROJECT[] = nothing
  ("Foo.Qux", "b5ec9b9c-e354-47fd-b367-a348bdc8f909", "project/deps/Qux.jl" ),
  ]
  n = map(String, split(names, '.'))
- pkg = identify_package(n...)
+ pkg = recurse_package(n...)
  @test pkg == PkgId(UUID(uuid), n[end])
  @test joinpath(@__DIR__, normpath(path)) == locate_package(pkg)
  end
  @test identify_package("Baz") == nothing
  @test identify_package("Qux") == nothing
  @testset "equivalent package names" begin
- local classes = [
+ classes = [
  ["Foo"],
  ["Bar", "Foo.Bar"],
  ["Foo.Baz", "Bar.Baz", "Foo.Bar.Baz"],
@@ -221,15 +239,15 @@ Base.ACTIVE_PROJECT[] = nothing
  for i = 1:length(classes)
  A = classes[i]
  for x in A
- X = identify_package(map(String, split(x, '.'))...)
+ X = recurse_package(map(String, split(x, '.'))...)
  for y in A
- Y = identify_package(map(String, split(y, '.'))...)
+ Y = recurse_package(map(String, split(y, '.'))...)
  @test X == Y
  end
  for j = i+1:length(classes)
  B = classes[j]
  for z in B
- Z = identify_package(map(String, split(z, '.'))...)
+ Z = recurse_package(map(String, split(z, '.'))...)
  @test X != Z
  end
  end
@@ -384,8 +402,6 @@ const envs = Dict{String,Any}()
 append!(empty!(DEPOT_PATH), depots)
 
 @testset "load code uniqueness" begin
- @show UUIDS
- @show depots
  @test allunique(UUIDS)
  @test allunique(depots)
  @test allunique(DEPOT_PATH)

test/precompile.jl

@@ -242,7 +242,7 @@ try
  # use _require_from_serialized to ensure that the test fails if
  # the module doesn't reload from the image:
  @test_logs (:warn, "Replacing module `$Foo_module`") begin
- ms = Base._require_from_serialized(cachefile)
+ ms = Base._require_from_serialized(cachefile, Base.TOMLCache())
  @test isa(ms, Array{Any,1})
  end