sync Julia master with Pkg3 master

stdlib/Pkg3/.travis.yml

@@ -15,7 +15,8 @@ before_script:
 
 script:
  - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi
- - julia --check-bounds=yes -e 'Pkg.clone(pwd()); Pkg.build("Pkg3"); Pkg.test("Pkg3"; coverage=true)'
+ - julia --check-bounds=yes -e 'Pkg.clone(pwd()); Pkg.build("Pkg3"); rm(Pkg.dir("Pkg3", "Project.toml"));
+ withenv("JULIA_LOAD_PATH" => Pkg.dir()) do; Pkg.test("Pkg3"; coverage=true); end'
 
 after_success:
  - julia -e 'cd(Pkg.dir("Pkg3")); Pkg.add("Coverage"); using Coverage; Codecov.submit(Codecov.process_folder())'

stdlib/Pkg3/src/API.jl

@@ -301,7 +301,7 @@ end
 
 function _get_deps!(ctx::Context, pkgs::Vector{PackageSpec}, uuids::Vector{UUID})
  for pkg in pkgs
- pkg.uuid in ctx.stdlib_uuids && continue
+ pkg.uuid in keys(ctx.stdlibs) && continue
  info = manifest_info(ctx.env, pkg.uuid)
  pkg.uuid in uuids && continue
  push!(uuids, pkg.uuid)
@@ -334,12 +334,14 @@ function build(ctx::Context, pkgs::Vector{PackageSpec}; kwargs...)
  uuids = UUID[]
  _get_deps!(ctx, pkgs, uuids)
  length(uuids) == 0 && (@info("no packages to build"); return)
- Pkg3.Operations.build_versions(ctx, uuids)
+ Pkg3.Operations.build_versions(ctx, uuids; do_resolve=true)
 end
 
-function init(path::String)
+init() = init(Context())
+init(path::String) = init(Context(), path)
+function init(ctx::Context, path::String=pwd())
  print_first_command_header()
- ctx = Context(env = EnvCache(joinpath(path, "Project.toml")))
+ Context!(ctx; env = EnvCache(joinpath(path, "Project.toml")))
  Pkg3.Operations.init(ctx)
 end
 

stdlib/Pkg3/src/Operations.jl

@@ -164,7 +164,7 @@ function deps_graph(ctx::Context, uuid_to_name::Dict{UUID,String}, reqs::Require
  deps_u_allvers["julia"] = uuid_julia
  end
 
- if uuid in ctx.stdlib_uuids
+ if uuid in keys(ctx.stdlibs)
  push!(all_versions_u, VERSION)
  continue
  end
@@ -213,7 +213,9 @@ function resolve_versions!(ctx::Context, pkgs::Vector{PackageSpec})::Dict{UUID,V
  @info("Resolving package versions")
  # anything not mentioned is fixed
  uuids = UUID[pkg.uuid for pkg in pkgs]
- uuid_to_name = Dict{UUID,String}(uuid_julia => "julia")
+ uuid_to_name = Dict{UUID, String}(uuid => stdlib for (uuid, stdlib) in ctx.stdlibs)
+ uuid_to_name[uuid_julia] = "julia"
+
  for (name::String, uuidstr::String) in ctx.env.project["deps"]
  uuid = UUID(uuidstr)
  uuid_to_name[uuid] = name
@@ -262,7 +264,7 @@ function version_data(ctx::Context, pkgs::Vector{PackageSpec})
  hashes = Dict{UUID,SHA1}()
  clones = Dict{UUID,Vector{String}}()
  for pkg in pkgs
- pkg.uuid in ctx.stdlib_uuids && continue
+ pkg.uuid in keys(ctx.stdlibs) && continue
  pkg.path == nothing || continue
  uuid = pkg.uuid
  ver = pkg.version::VersionNumber
@@ -393,7 +395,7 @@ function apply_versions(ctx::Context, pkgs::Vector{PackageSpec})::Vector{UUID}
 
  pkgs_to_install = Tuple{PackageSpec, String}[]
  for pkg in pkgs
- pkg.uuid in ctx.stdlib_uuids && continue
+ pkg.uuid in keys(ctx.stdlibs) && continue
  pkg.path == nothing || continue
  path = find_installed(pkg.name, pkg.uuid, hashes[pkg.uuid])
  if !ispath(path)
@@ -467,14 +469,14 @@ function apply_versions(ctx::Context, pkgs::Vector{PackageSpec})::Vector{UUID}
  ##########################################
  for pkg in pkgs
  uuid = pkg.uuid
- uuid in ctx.stdlib_uuids && continue
+ uuid in keys(ctx.stdlibs) && continue
  if pkg.path == nothing
  pkg.name = names[uuid]
  hash = hashes[uuid]
  else
  hash = nothing
  end
- update_manifest(ctx.env, pkg, hash)
+ update_manifest(ctx, pkg, hash)
  end
 
  prune_manifest(ctx.env)
@@ -484,7 +486,25 @@ end
 ################################
 # Manifest update and pruning #
 ################################
-function update_manifest(env::EnvCache, pkg::PackageSpec, hash::Union{SHA1, Nothing})
+function find_stdlib_deps(ctx::Context, path::String)
+ stdlib_deps = Dict{UUID, String}()
+ regexps = [Regex("\\b(import|using)\\s+((\\w|\\.)+\\s*,\\s*)*$lib\\b") for lib in values(ctx.stdlibs)]
+ for (root, dirs, files) in walkdir(path)
+ for file in files
+ endswith(file, ".jl") || continue
+ filecontent = read(joinpath(root, file), String)
+ for ((uuid, stdlib), r) in zip(ctx.stdlibs, regexps)
+ if contains(filecontent, r)
+ stdlib_deps[uuid] = stdlib
+ end
+ end
+ end
+ end
+ return stdlib_deps
+end
+
+function update_manifest(ctx::Context, pkg::PackageSpec, hash::Union{SHA1, Nothing})
+ env = ctx.env
  uuid, name, version, path, special_action = pkg.uuid, pkg.name, pkg.version, pkg.path, pkg.special_action
  hash == nothing && @assert path != nothing
  infos = get!(env.manifest, name, Dict{String,Any}[])
@@ -509,15 +529,21 @@ function update_manifest(env::EnvCache, pkg::PackageSpec, hash::Union{SHA1, Noth
 
  delete!(info, "deps")
  if path != nothing
- reqfile = joinpath(path, "REQUIRE")
- !isfile(reqfile) && return
- deps = Dict{String,String}()
+ # Remove when packages uses Project files properly
  dep_pkgs = PackageSpec[]
- for r in filter!(r->r isa Pkg2.Reqs.Requirement, Pkg2.Reqs.read(reqfile))
- push!(dep_pkgs, PackageSpec(r.package))
+ stdlib_deps = find_stdlib_deps(ctx, path)
+ for (stdlib_uuid, stdlib) in stdlib_deps
+ push!(dep_pkgs, PackageSpec(stdlib, stdlib_uuid))
+ end
+ reqfile = joinpath(path, "REQUIRE")
+ if isfile(reqfile)
+ for r in filter!(r->r isa Pkg2.Reqs.Requirement, Pkg2.Reqs.read(reqfile))
+ push!(dep_pkgs, PackageSpec(r.package))
+ end
+ registry_resolve!(env, dep_pkgs)
  end
- registry_resolve!(env, dep_pkgs)
  ensure_resolved(env, dep_pkgs)
+ deps = Dict{String,String}()
  for dep_pkg in dep_pkgs
  dep_pkg.name == "julia" && continue
  deps[dep_pkg.name] = string(dep_pkg.uuid)
@@ -559,6 +585,43 @@ function prune_manifest(env::EnvCache)
  end
 end
 
+function with_local_project(f, ctx::Context, pkg::PackageSpec; allow_self_load=true, do_resolve=false)
+ localctx = deepcopy(ctx)
+ empty!(localctx.env.project["deps"])
+ info = manifest_info(localctx.env, pkg.uuid)
+ # If package or its dependencies are checked out, will need to resolve
+ # unless we already have resolved for the current environment
+ if allow_self_load
+ localctx.env.project["deps"][pkg.name] = string(pkg.uuid)
+ end
+ need_to_resolve = haskey(info, "path")
+ # Allow to load dependent packages at top level by putting them in the project
+ deps = PackageSpec[]
+ for (dpkg, duuid) in get(info, "deps", [])
+ dinfo = manifest_info(localctx.env, UUID(duuid))
+ dinfo === nothing || (need_to_resolve |= haskey(info, "path"))
+ localctx.env.project["deps"][dpkg] = string(duuid)
+ end
+ local new
+ will_resolve = do_resolve && need_to_resolve
+ if will_resolve
+ pkgs = PackageSpec[]
+ resolve_versions!(localctx, pkgs)
+ new = apply_versions(localctx, pkgs)
+ else
+ prune_manifest(localctx.env)
+ end
+ mktempdir() do tmpdir
+ localctx.env.project_file = joinpath(tmpdir, "Project.toml")
+ localctx.env.manifest_file = joinpath(tmpdir, "Manifest.toml")
+ write_env(localctx, no_output = true)
+ will_resolve && build_versions(localctx, new)
+ withenv("JULIA_LOAD_PATH" => joinpath(tmpdir)) do
+ f()
+ end
+ end
+end
+
 #########
 # Build #
 #########
@@ -567,7 +630,7 @@ function dependency_order_uuids(ctx::Context, uuids::Vector{UUID})::Dict{UUID,In
  seen = UUID[]
  k = 0
  function visit(uuid::UUID)
- uuid in ctx.stdlib_uuids && return
+ uuid in keys(ctx.stdlibs) && return
  uuid in seen &&
  return @warn("Dependency graph not a DAG, linearizing anyway")
  haskey(order, uuid) && return
@@ -583,12 +646,12 @@ function dependency_order_uuids(ctx::Context, uuids::Vector{UUID})::Dict{UUID,In
  return order
 end
 
-function build_versions(ctx::Context, uuids::Vector{UUID})
+function build_versions(ctx::Context, uuids::Vector{UUID}; do_resolve=false)
  # collect builds for UUIDs with `deps/build.jl` files
  ctx.preview && (@info "Skipping building in preview mode"; return)
  builds = Tuple{UUID,String,Union{String,SHA1},String}[]
  for uuid in uuids
- uuid in ctx.stdlib_uuids && continue
+ uuid in keys(ctx.stdlibs) && continue
  info = manifest_info(ctx.env, uuid)
  name = info["name"]
  if haskey(info, "git-tree-sha1")
@@ -617,27 +680,25 @@ function build_versions(ctx::Context, uuids::Vector{UUID})
  end
  @info " → $log_file"
  code = """
- empty!(Base.LOAD_PATH)
- append!(Base.LOAD_PATH, $(repr(Base.load_path())))
  empty!(Base.DEPOT_PATH)
  append!(Base.DEPOT_PATH, $(repr(map(abspath, DEPOT_PATH))))
  empty!(Base.DL_LOAD_PATH)
  append!(Base.DL_LOAD_PATH, $(repr(map(abspath, Base.DL_LOAD_PATH))))
- m = Base.require(Base.PkgId(Base.UUID($(repr(string(uuid)))), $(repr(name))))
- eval(m, :(module __build__ end))
  cd($(repr(dirname(build_file))))
- Base.include_relative(m.__build__, $(repr(build_file)))
+ include($(repr(build_file)))
  """
  cmd = ```
  $(Base.julia_cmd()) -O0 --color=no --history-file=no
  --startup-file=$(Base.JLOptions().startupfile != 2 ? "yes" : "no")
  --compiled-modules=$(Bool(Base.JLOptions().use_compiled_modules) ? "yes" : "no")
  --eval $code
  ```
- open(log_file, "w") do log
- success(pipeline(cmd, stdout=log, stderr=log))
- end ? Base.rm(log_file, force=true) :
- @error("Error building `$name`; see log file for further info")
+ with_local_project(ctx, PackageSpec(name, uuid); allow_self_load=false, do_resolve=do_resolve) do
+ open(log_file, "w") do log
+ success(pipeline(cmd, stdout=log, stderr=log))
+ end ? Base.rm(log_file, force=true) :
+ @error("Error building `$name`; see log file for further info")
+ end
  end
  return
 end
@@ -870,16 +931,12 @@ function test(ctx::Context, pkgs::Vector{PackageSpec}; coverage=false)
  continue
  end
  code = """
- empty!(Base.LOAD_PATH)
- append!(Base.LOAD_PATH, $(repr(Base.load_path())))
  empty!(Base.DEPOT_PATH)
  append!(Base.DEPOT_PATH, $(repr(map(abspath, DEPOT_PATH))))
  empty!(Base.DL_LOAD_PATH)
  append!(Base.DL_LOAD_PATH, $(repr(map(abspath, Base.DL_LOAD_PATH))))
- m = Base.require(Base.PkgId(Base.UUID($(repr(string(pkg.uuid)))), $(repr(pkg.name))))
- eval(m, :(module __test__ end))
  cd($(repr(dirname(testfile))))
- Base.include_relative(m.__test__, $(repr(testfile)))
+ include($(repr(testfile)))
  """
  cmd = ```
  $(Base.julia_cmd())
@@ -891,8 +948,10 @@ function test(ctx::Context, pkgs::Vector{PackageSpec}; coverage=false)
  --eval $code
  ```
  try
- run(cmd)
- @info("$(pkg.name) tests passed")
+ with_local_project(ctx, pkg; do_resolve=true) do
+ run(cmd)
+ @info("$(pkg.name) tests passed")
+ end
  catch err
  push!(pkgs_errored, pkg.name)
  end

stdlib/Pkg3/src/REPLMode.jl

@@ -233,7 +233,7 @@ function do_cmd!(tokens::Vector{Token}, repl)
  end
  # Using invokelatest to hide the functions from inference.
  # Otherwise it would try to infer everything here.
- cmd.kind == CMD_INIT ? Base.invokelatest( do_init!, tokens) :
+ cmd.kind == CMD_INIT ? Base.invokelatest( do_init!, ctx, tokens) :
  cmd.kind == CMD_HELP ? Base.invokelatest( do_help!, ctx, tokens, repl) :
  cmd.kind == CMD_RM ? Base.invokelatest( do_rm!, ctx, tokens) :
  cmd.kind == CMD_ADD ? Base.invokelatest( do_add!, ctx, tokens) :

stdlib/Pkg3/src/Types.jl

@@ -432,7 +432,7 @@ const default_envs = [
  "default",
 ]
 
-struct EnvCache
+mutable struct EnvCache
  # environment info:
  env::Union{Nothing,String,AbstractEnv}
  git::Union{Nothing,LibGit2.GitRepo}
@@ -898,18 +898,18 @@ end
 # Context #
 ###########
 function gather_stdlib_uuids()
- stdlib_uuids = UUID[]
+ stdlibs = Dict{UUID, String}()
  stdlib_dir = joinpath(Sys.BINDIR, "..", "share", "julia", "site", "v$(VERSION.major).$(VERSION.minor)")
  for stdlib in readdir(stdlib_dir)
  projfile = joinpath(stdlib_dir, stdlib, "Project.toml")
  if isfile(projfile)
  proj = TOML.parsefile(joinpath(stdlib_dir, stdlib, "Project.toml"))
  if haskey(proj, "uuid")
- push!(stdlib_uuids, UUID(proj["uuid"]))
+ stdlibs[UUID(proj["uuid"])] = stdlib
  end
  end
  end
- return stdlib_uuids
+ return stdlibs
 end
 
 # ENV variables to set some of these defaults?
@@ -919,7 +919,7 @@ Base.@kwdef mutable struct Context
  use_libgit2_for_all_downloads::Bool = false
  num_concurrent_downloads::Int = 8
  graph_verbose::Bool = false
- stdlib_uuids::Vector{UUID} = gather_stdlib_uuids()
+ stdlibs::Dict{UUID,String} = gather_stdlib_uuids()
 end
 
 function Context!(ctx::Context; kwargs...)
@@ -928,16 +928,16 @@ function Context!(ctx::Context; kwargs...)
  end
 end
 
-function write_env(ctx::Context)
+function write_env(ctx::Context; no_output=false)
  env = ctx.env
  # load old environment for comparison
  old_env = EnvCache(env.env)
  # update the project file
  project = deepcopy(env.project)
  isempty(project["deps"]) && delete!(project, "deps")
  if !isempty(project) || ispath(env.project_file)
- @info "Updating $(pathrepr(env, env.project_file))"
- Pkg3.Display.print_project_diff(old_env, env)
+ no_output || @info "Updating $(pathrepr(env, env.project_file))"
+ no_output || Pkg3.Display.print_project_diff(old_env, env)
  if !ctx.preview
  mkpath(dirname(env.project_file))
  open(env.project_file, "w") do io
@@ -947,8 +947,8 @@ function write_env(ctx::Context)
  end
  # update the manifest file
  if !isempty(env.manifest) || ispath(env.manifest_file)
- @info "Updating $(pathrepr(env, env.manifest_file))"
- Pkg3.Display.print_manifest_diff(old_env, env)
+ no_output || @info "Updating $(pathrepr(env, env.manifest_file))"
+ no_output || Pkg3.Display.print_manifest_diff(old_env, env)
  manifest = deepcopy(env.manifest)
  uniques = sort!(collect(keys(manifest)), by=lowercase)
  filter!(name->length(manifest[name]) == 1, uniques)

stdlib/Pkg3/test/pkg.jl

@@ -109,15 +109,29 @@ temp_pkg_dir() do project_path
  Pkg3.checkout(TEST_PKG.name; path = devdir)
  @test isinstalled(TEST_PKG)
  @test Pkg3.installed()[TEST_PKG.name] > old_v
- @test isfile(joinpath(devdir, TEST_PKG.name, "src", TEST_PKG.name * ".jl"))
+ test_pkg_main_file = joinpath(devdir, TEST_PKG.name, "src", TEST_PKG.name * ".jl")
+ @test isfile(test_pkg_main_file)
+ # Pkg3 #152
+ write(test_pkg_main_file,
+ """
+ module Example
+ export hello, domath
+ const example2path = joinpath(@__DIR__, "..", "deps", "deps.jl")
+ if !isfile(example2path)
+ error("Example is not installed correctly")
+ end
+ hello(who::String) = "Hello, \$who"
+ domath(x::Number) = x + 5
+ end
+ """)
  mkpath(joinpath(devdir, TEST_PKG.name, "deps"))
  write(joinpath(devdir, TEST_PKG.name, "deps", "build.jl"),
  """
- touch("I_got_built")
+ touch("deps.jl")
  """
  )
  Pkg3.build(TEST_PKG.name)
- @test isfile(joinpath(devdir, TEST_PKG.name, "deps", "I_got_built"))
+ @test isfile(joinpath(devdir, TEST_PKG.name, "deps", "deps.jl"))
  Pkg3.test(TEST_PKG.name)
  Pkg3.free(TEST_PKG.name)
  @test Pkg3.installed()[TEST_PKG.name] == old_v