Skip to content

Commit

Permalink
move the TOML parser to Base and implement code loading on top of the…
Browse files Browse the repository at this point in the history
… Base TOML parser (#36018)
  • Loading branch information
KristofferC authored Aug 28, 2020
1 parent 7c48091 commit c885514
Show file tree
Hide file tree
Showing 11 changed files with 262 additions and 309 deletions.
2 changes: 2 additions & 0 deletions base/Base.jl
Original file line number Diff line number Diff line change
Expand Up @@ -340,6 +340,8 @@ include("threadcall.jl")

# code loading
include("uuid.jl")
include("pkgid.jl")
include("toml_parser.jl")
include("loading.jl")

# misc useful functions & macros
Expand Down
2 changes: 1 addition & 1 deletion base/initdefs.jl
Original file line number Diff line number Diff line change
Expand Up @@ -234,7 +234,7 @@ function load_path_expand(env::AbstractString)::Union{String, Nothing}
# if you put a `@` in LOAD_PATH manually, it's expanded late
env == "@" && return active_project(false)
env == "@." && return current_project()
env == "@stdlib" && return Sys.STDLIB
env == "@stdlib" && return Sys.STDLIB::String
env = replace(env, '#' => VERSION.major, count=1)
env = replace(env, '#' => VERSION.minor, count=1)
env = replace(env, '#' => VERSION.patch, count=1)
Expand Down
444 changes: 167 additions & 277 deletions base/loading.jl

Large diffs are not rendered by default.

43 changes: 43 additions & 0 deletions base/pkgid.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
struct PkgId
uuid::Union{UUID,Nothing}
name::String

PkgId(u::UUID, name::AbstractString) = new(UInt128(u) == 0 ? nothing : u, name)
PkgId(::Nothing, name::AbstractString) = new(nothing, name)
end
PkgId(name::AbstractString) = PkgId(nothing, name)

function PkgId(m::Module, name::String = String(nameof(moduleroot(m))))
uuid = UUID(ccall(:jl_module_uuid, NTuple{2, UInt64}, (Any,), m))
UInt128(uuid) == 0 ? PkgId(name) : PkgId(uuid, name)
end

==(a::PkgId, b::PkgId) = a.uuid == b.uuid && a.name == b.name

function hash(pkg::PkgId, h::UInt)
h += 0xc9f248583a0ca36c % UInt
h = hash(pkg.uuid, h)
h = hash(pkg.name, h)
return h
end

show(io::IO, pkg::PkgId) =
print(io, pkg.name, " [", pkg.uuid === nothing ? "top-level" : pkg.uuid, "]")

function binpack(pkg::PkgId)
io = IOBuffer()
write(io, UInt8(0))
uuid = pkg.uuid
write(io, uuid === nothing ? UInt128(0) : UInt128(uuid))
write(io, pkg.name)
return String(take!(io))
end

function binunpack(s::String)
io = IOBuffer(s)
@assert read(io, UInt8) === 0x00
uuid = read(io, UInt128)
name = read(io, String)
return PkgId(UUID(uuid), name)
end

6 changes: 5 additions & 1 deletion stdlib/TOML/src/parser.jl → base/toml_parser.jl
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
module TOML

using Base: IdSet

# In case we do not have the Dates stdlib available
Expand Down Expand Up @@ -96,7 +98,7 @@ function Parser(str::String; filepath=nothing)
String[], # dotted_keys
UnitRange{Int}[], # chunks
IdSet{TOMLDict}(), # inline_tables
IdSet{Any}(), # static_arrays
IdSet{Any}(), # static_arrays
IdSet{TOMLDict}(), # defined_tables
root,
filepath,
Expand Down Expand Up @@ -1164,3 +1166,5 @@ function take_chunks(l::Parser, unescape::Bool)::String
empty!(l.chunks)
return unescape ? unescape_string(str) : str
end

end
1 change: 1 addition & 0 deletions base/uuid.jl
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
struct UUID
value::UInt128
end
UUID(u::UUID) = u
UUID(u::NTuple{2, UInt64}) = UUID((UInt128(u[1]) << 64) | UInt128(u[2]))
UUID(u::NTuple{4, UInt32}) = UUID((UInt128(u[1]) << 96) | (UInt128(u[2]) << 64) |
(UInt128(u[3]) << 32) | UInt128(u[4]))
Expand Down
2 changes: 1 addition & 1 deletion stdlib/Profile/src/Profile.jl
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,7 @@ function short_path(spath::Symbol, filenamecache::Dict{Symbol, String})
for proj in Base.project_names
project_file = joinpath(root, proj)
if Base.isfile_casesensitive(project_file)
pkgid = Base.project_file_name_uuid(project_file, "")
pkgid = Base.project_file_name_uuid(project_file, "", Base.TOMLCache())
isempty(pkgid.name) && return path # bad Project file
# return the joined the module name prefix and path suffix
path = path[nextind(path, sizeof(root)):end]
Expand Down
26 changes: 9 additions & 17 deletions stdlib/REPL/src/REPLCompletions.jl
Original file line number Diff line number Diff line change
Expand Up @@ -585,23 +585,15 @@ end

function project_deps_get_completion_candidates(pkgstarts::String, project_file::String)
loading_candidates = String[]
open(project_file) do io
state = :top
for line in eachline(io)
if occursin(Base.re_section, line)
state = occursin(Base.re_section_deps, line) ? :deps : :other
elseif state === :top
if (m = match(Base.re_name_to_string, line)) !== nothing
root_name = String(m.captures[1])
startswith(root_name, pkgstarts) && push!(loading_candidates, root_name)
end
elseif state === :deps
if (m = match(Base.re_key_to_string, line)) !== nothing
dep_name = m.captures[1]
startswith(dep_name, pkgstarts) && push!(loading_candidates, dep_name)
end
end
end
p = Base.TOML.Parser()
Base.TOML.reinit!(p, read(project_file, String); filepath=project_file)
d = Base.TOML.parse(p)
pkg = get(d, "name", nothing)
if pkg !== nothing && startswith(pkg, pkgstarts)
push!(loading_candidates, pkg)
end
for (pkg, _) in get(d, "deps", [])
startswith(pkg, pkgstarts) && push!(loading_candidates, pkg)
end
return Completion[PackageCompletion(name) for name in loading_candidates]
end
Expand Down
7 changes: 6 additions & 1 deletion stdlib/TOML/src/TOML.jl
Original file line number Diff line number Diff line change
@@ -1,7 +1,12 @@
module TOML

module Internals
include("parser.jl")
# The parser is defined in Base
using Base.TOML: Parser, parse, tryparse, ParserError, isvalid_barekey_char, reinit!
# Put the error instances in this module
for errtype in instances(Base.TOML.ErrorType)
@eval using Base.TOML: $(Symbol(errtype))
end
# We put the printing functionality in a separate module since It
# defines a function `print` and we don't want that to collide with normal
# usage of `(Base.)print` in other files
Expand Down
36 changes: 26 additions & 10 deletions test/loading.jl
Original file line number Diff line number Diff line change
Expand Up @@ -166,9 +166,10 @@ end
d = findfirst(line -> line == "[deps]", p)
t = findfirst(line -> startswith(line, "This"), p)
# look up various packages by name
root = Base.explicit_project_deps_get(project_file, "Root")
this = Base.explicit_project_deps_get(project_file, "This")
that = Base.explicit_project_deps_get(project_file, "That")
cache = Base.TOMLCache()
root = Base.explicit_project_deps_get(project_file, "Root", cache)
this = Base.explicit_project_deps_get(project_file, "This", cache)
that = Base.explicit_project_deps_get(project_file, "That", cache)
# test that the correct answers are given
@test root == (something(n, N+1) something(d, N+1) ? nothing :
something(u, N+1) < something(d, N+1) ? root_uuid : proj_uuid)
Expand All @@ -191,6 +192,23 @@ Base.ACTIVE_PROJECT[] = nothing

@test load_path() == [abspath("project","Project.toml")]


# locate `tail(names)` package by following the search path graph through `names` starting from `where`
function recurse_package(where::PkgId, name::String, names::String...)
pkg = identify_package(where, name, Base.TOMLCache())
pkg === nothing && return nothing
return recurse_package(pkg, names...)
end

recurse_package(pkg::String) = identify_package(pkg)
recurse_package(where::PkgId, pkg::String) = identify_package(where, pkg, Base.TOMLCache())

function recurse_package(name::String, names::String...)
pkg = identify_package(name)
pkg === nothing && return nothing
return recurse_package(pkg, names...)
end

@testset "project & manifest identify_package & locate_package" begin
local path
for (names, uuid, path) in [
Expand All @@ -201,14 +219,14 @@ Base.ACTIVE_PROJECT[] = nothing
("Foo.Qux", "b5ec9b9c-e354-47fd-b367-a348bdc8f909", "project/deps/Qux.jl" ),
]
n = map(String, split(names, '.'))
pkg = identify_package(n...)
pkg = recurse_package(n...)
@test pkg == PkgId(UUID(uuid), n[end])
@test joinpath(@__DIR__, normpath(path)) == locate_package(pkg)
end
@test identify_package("Baz") == nothing
@test identify_package("Qux") == nothing
@testset "equivalent package names" begin
local classes = [
classes = [
["Foo"],
["Bar", "Foo.Bar"],
["Foo.Baz", "Bar.Baz", "Foo.Bar.Baz"],
Expand All @@ -221,15 +239,15 @@ Base.ACTIVE_PROJECT[] = nothing
for i = 1:length(classes)
A = classes[i]
for x in A
X = identify_package(map(String, split(x, '.'))...)
X = recurse_package(map(String, split(x, '.'))...)
for y in A
Y = identify_package(map(String, split(y, '.'))...)
Y = recurse_package(map(String, split(y, '.'))...)
@test X == Y
end
for j = i+1:length(classes)
B = classes[j]
for z in B
Z = identify_package(map(String, split(z, '.'))...)
Z = recurse_package(map(String, split(z, '.'))...)
@test X != Z
end
end
Expand Down Expand Up @@ -384,8 +402,6 @@ const envs = Dict{String,Any}()
append!(empty!(DEPOT_PATH), depots)

@testset "load code uniqueness" begin
@show UUIDS
@show depots
@test allunique(UUIDS)
@test allunique(depots)
@test allunique(DEPOT_PATH)
Expand Down
2 changes: 1 addition & 1 deletion test/precompile.jl
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,7 @@ try
# use _require_from_serialized to ensure that the test fails if
# the module doesn't reload from the image:
@test_logs (:warn, "Replacing module `$Foo_module`") begin
ms = Base._require_from_serialized(cachefile)
ms = Base._require_from_serialized(cachefile, Base.TOMLCache())
@test isa(ms, Array{Any,1})
end

Expand Down

0 comments on commit c885514

Please sign in to comment.