add Downloads stdlib; replace Base.download

- Base.download is deprecated in favor of Downloads.download
- the former is now implemented by calling the latter

NEWS.md

@@ -58,8 +58,9 @@ Build system changes
 
 Library functions
 -----------------
-* The `Base.Grisu` code has been officially removed (float printing was switched to the ryu algorithm code in 1.4)
 
+* The `Base.download` function has been deprecated (silently, by default) in favor of the new `Downloads.download` standard library function ([#37340]).
+* The `Base.Grisu` code has been officially removed (float printing was switched to the ryu algorithm code in 1.4)
 
 New library functions
 ---------------------
@@ -81,6 +82,7 @@ New library features
 
 Standard library changes
 ------------------------
+
 * The `nextprod` function now accepts tuples and other array types for its first argument ([#35791]).
 * The `reverse(A; dims)` function for multidimensional `A` can now reverse multiple dimensions at once
  by passing a tuple for `dims`, and defaults to reversing all dimensions; there is also a multidimensional
@@ -99,6 +101,7 @@ Standard library changes
 * `RegexMatch` objects can now be probed for whether a named capture group exists within it through `haskey()` ([#36717]).
 * For consistency `haskey(r::RegexMatch, i::Integer)` has also been added and returns if the capture group for `i` exists ([#37300]).
 * A new standard library `TOML` has been added for parsing and printing [TOML files](https://toml.io) ([#37034]).
+* A new standard library `Downloads` has been added, which replaces the old `Base.download` function with `Downloads.download`, providing cross-platform, multi-protocol, in-process download functionality implemented with [libcurl](https://curl.haxx.se/libcurl/) ([#37340]).
 * The `Pkg.BinaryPlatforms` module has been moved into `Base` as `Base.BinaryPlatforms` and heavily reworked.
  Applications that want to be compatible with the old API should continue to import `Pkg.BinaryPlatforms`,
  however new users should use `Base.BinaryPlatforms` directly. ([#37320])
@@ -107,6 +110,7 @@ Standard library changes
  all of `Pkg` alongside. ([#37320])
 
 #### LinearAlgebra
+
 * New method `LinearAlgebra.issuccess(::CholeskyPivoted)` for checking whether pivoted Cholesky factorization was successful ([#36002]).
 * `UniformScaling` can now be indexed into using ranges to return dense matrices and vectors ([#24359]).
 * New function `LinearAlgebra.BLAS.get_num_threads()` for getting the number of BLAS threads. ([#36360])

base/download.jl

@@ -1,51 +1,5 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-# file downloading
-
-if Sys.iswindows()
- function download_powershell(url::AbstractString, filename::AbstractString)
- ps = joinpath(get(ENV, "SYSTEMROOT", "C:\\Windows"), "System32\\WindowsPowerShell\\v1.0\\powershell.exe")
- tls12 = "[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12"
- client = "New-Object System.Net.Webclient"
- # in the following we escape ' with '' (see https://ss64.com/ps/syntax-esc.html)
- downloadfile = "($client).DownloadFile('$(replace(url, "'" => "''"))', '$(replace(filename, "'" => "''"))')"
- # PowerShell v3 or later is required for Tls12
- proc = run(pipeline(`$ps -Version 3 -NoProfile -Command "$tls12; $downloadfile"`; stderr=stderr); wait=false)
- if !success(proc)
- if proc.exitcode % Int32 == -393216
- # appears to be "wrong version" exit code, based on
- # https://docs.microsoft.com/en-us/azure/cloud-services/cloud-services-startup-tasks-common
- @error "Downloading files requires Windows Management Framework 3.0 or later."
- end
- pipeline_error(proc)
- end
- return filename
- end
-end
-
-function find_curl()
- if Sys.isapple() && Sys.isexecutable("/usr/bin/curl")
- "/usr/bin/curl"
- elseif Sys.iswindows() && Sys.isexecutable(joinpath(get(ENV, "SYSTEMROOT", "C:\\Windows"), "System32\\curl.exe"))
- joinpath(get(ENV, "SYSTEMROOT", "C:\\Windows"), "System32\\curl.exe")
- elseif !Sys.iswindows() && Sys.which("curl") !== nothing
- "curl"
- else
- nothing
- end
-end
-
-function download_curl(curl_exe::AbstractString, url::AbstractString, filename::AbstractString)
- err = PipeBuffer()
- process = run(pipeline(`$curl_exe -s -S -g -L -f -o $filename $url`, stderr=err), wait=false)
- if !success(process)
- error_msg = readline(err)
- @error "Download failed: $error_msg"
- pipeline_error(process)
- end
- return filename
-end
-
 const DOWNLOAD_HOOKS = Callable[]
 
 function download_url(url::AbstractString)
@@ -55,50 +9,25 @@ function download_url(url::AbstractString)
  return url
 end
 
-function download(url::AbstractString, filename::AbstractString)
- url = download_url(url)
- curl_exe = find_curl()
- if curl_exe !== nothing
- return download_curl(curl_exe, url, filename)
- elseif Sys.iswindows()
- return download_powershell(url, filename)
- elseif Sys.which("wget") !== nothing
- try
- run(`wget -O $filename $url`)
- catch
- rm(filename, force=true) # wget always creates a file
- rethrow()
- end
- elseif Sys.which("busybox") !== nothing
- try
- run(`busybox wget -O $filename $url`)
- catch
- rm(filename, force=true) # wget always creates a file
- rethrow()
- end
- elseif Sys.which("fetch") !== nothing
- run(`fetch -f $filename $url`)
- else
- error("No download agent available; install curl, wget, busybox or fetch.")
- end
- return filename
-end
-
-function download(url::AbstractString)
- filename = tempname()
- download(url, filename)
-end
+Downloads() = require(PkgId(
+ UUID((0xf43a241f_c20a_4ad4, 0x852c_f6b1247861c6)),
+ "Downloads",
+ ))
 
 """
- download(url::AbstractString, [localfile::AbstractString])
+ download(url::AbstractString, [path::AbstractString = tempname()]) -> path
 
-Download a file from the given url, optionally renaming it to the given local file name. If
-no filename is given this will download into a randomly-named file in your temp directory.
-Note that this function relies on the availability of external tools such as `curl`, `wget`
-or `fetch` to download the file and is provided for convenience. For production use or
-situations in which more options are needed, please use a package that provides the desired
-functionality instead.
+Download a file from the given url, saving it to the location `path`, or if not
+specified, a temporary path. Returns the path of the downloaded file.
 
-Returns the filename of the downloaded file.
+!!! note
+ Since Julia 1.6, this function is deprecated and is just a thin wrapper
+ around `Downloads.download`. In new code, you should use that function
+ directly instead of calling this.
 """
-download(url, filename)
+function download(url::AbstractString, path::AbstractString)
+ invokelatest(Downloads().download, download_url(url), path)
+end
+function download(url::AbstractString)
+ invokelatest(Downloads().download, download_url(url))
+end

base/sysimg.jl

@@ -55,6 +55,7 @@ let
  :MozillaCACerts_jll,
  :LibCURL_jll,
  :LibCURL,
+ :Downloads,
  ]
 
  maxlen = reduce(max, textwidth.(string.(stdlibs)); init=0)

deps/checksums/Downloads-1a1d2e0a10209512f5b29e585bfd78e7a47f8f61.tar.gz/md5

@@ -0,0 +1 @@
+24a8b8fc2398d20c24a13ce73482a3d7

deps/checksums/Downloads-1a1d2e0a10209512f5b29e585bfd78e7a47f8f61.tar.gz/sha512

@@ -0,0 +1 @@
+4b652be535dce6a5cf36e546a31d3221f4c2534d1a3ac710f31f9ed2dfbeabd21b36c0186e4abe12cfee992ff557317aefe2bdeb9cb8b1c44da085b030719329

stdlib/.gitignore

@@ -5,3 +5,5 @@
 /Statistics
 /LibCURL-*
 /LibCURL
+/Downloads-*
+/Downloads

stdlib/Downloads.version

@@ -0,0 +1,2 @@
+DOWNLOADS_BRANCH = master
+DOWNLOADS_SHA1 = 1a1d2e0a10209512f5b29e585bfd78e7a47f8f61

stdlib/Makefile

@@ -20,13 +20,15 @@ STDLIBS = Artifacts Base64 CRC32c Dates DelimitedFiles Distributed FileWatching
  SharedArrays Sockets SparseArrays SuiteSparse Test TOML Unicode UUIDs \
  MozillaCACerts_jll LibCURL_jll
 
-STDLIBS_EXT = Pkg Statistics LibCURL
+STDLIBS_EXT = Pkg Statistics LibCURL Downloads
 PKG_GIT_URL := git:https://github.com/JuliaLang/Pkg.jl.git
 PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1
 STATISTICS_GIT_URL := git:https://github.com/JuliaLang/Statistics.jl.git
 STATISTICS_TAR_URL = https://api.github.com/repos/JuliaLang/Statistics.jl/tarball/$1
 LIBCURL_GIT_URL := git:https://github.com/JuliaWeb/LibCURL.jl.git
 LIBCURL_TAR_URL = https://api.github.com/repos/JuliaWeb/LibCURL.jl/tarball/$1
+DOWNLOADS_GIT_URL := git:https://github.com/JuliaLang/Downloads.jl.git
+DOWNLOADS_TAR_URL = https://api.github.com/repos/JuliaLang/Downloads.jl/tarball/$1
 
 $(foreach module, $(STDLIBS_EXT), $(eval $(call stdlib-external,$(module),$(shell echo $(module) | tr a-z A-Z))))
 

test/choosetests.jl

@@ -111,7 +111,7 @@ function choosetests(choices = [])
  filter!(x -> (x != "Profile"), tests)
  end
 
- net_required_for = ["Sockets", "LibGit2", "LibCURL"]
+ net_required_for = ["Sockets", "LibGit2", "LibCURL", "Downloads"]
  net_on = true
  try
  ipa = getipaddr()

test/download.jl

@@ -40,20 +40,11 @@ mktempdir() do temp_dir
 
  # Make sure that failed downloads do not leave files around
  missing_file = joinpath(temp_dir, "missing")
- @test_throws ProcessFailedException download("https://httpbin.julialang.org/status/404", missing_file)
+ @test_throws ErrorException download("https://httpbin.julialang.org/status/404", missing_file)
  @test !isfile(missing_file)
 
- # Make sure we properly handle metachar ' on windows with ^ escaping
- if Sys.iswindows()
- metachar_file = joinpath(temp_dir, "metachar")
- Base.download_powershell("https://httpbin.julialang.org/get?test='^'", metachar_file)
- metachar_string = read(metachar_file, String)
- m = match(r"\"test\"\s*:\s*\"(.*)\"", metachar_string)
- @test m.captures[1] == "'^'"
- end
-
  # Use a TEST-NET (192.0.2.0/24) address which shouldn't be bound
  invalid_host_file = joinpath(temp_dir, "invalid_host")
- @test_throws ProcessFailedException download("http:https://192.0.2.1", invalid_host_file)
+ @test_throws ErrorException download("http:https://192.0.2.1", invalid_host_file)
  @test !isfile(invalid_host_file)
 end

test/precompile.jl

@@ -292,7 +292,7 @@ try
  :Future, :Libdl, :LinearAlgebra, :Logging, :Mmap, :Printf,
  :Profile, :Random, :Serialization, :SharedArrays, :SparseArrays, :SuiteSparse, :Test,
  :Unicode, :REPL, :InteractiveUtils, :Pkg, :LibGit2, :SHA, :UUIDs, :Sockets,
- :Statistics, :TOML, :MozillaCACerts_jll, :LibCURL_jll, :LibCURL,]),
+ :Statistics, :TOML, :MozillaCACerts_jll, :LibCURL_jll, :LibCURL, :Downloads,]),
  # Plus precompilation module generated at build time
  let id = Base.PkgId("__PackagePrecompilationStatementModule")
  Dict(id => Base.module_build_id(Base.root_module(id)))