Merge branch 'master' into mb/fast-partitions

.gitattributes

@@ -1,2 +1,2 @@
-# treat patches as files that should not be modified
-*.patch -text
+# treat all files as files that should not be modified
+* -text

CONTRIBUTING.md

@@ -264,6 +264,12 @@ If you need to restart your Julia session, just start at step 2 above.
 built and incorporate them automatically. You only need to rebuild
 Julia if you made code-changes that Revise cannot handle.
 
+For convenience, there are also `test-revise-*` targets for every `test-*`
+target that use Revise to load any modifications to Base into the current
+process before running the corresponding test. This can be useful as a shortcut
+on the command line (since tests aren't always designed to be run outside the
+runtest harness).
+
 ### Code Formatting Guidelines
 
 #### General Formatting Guidelines for Julia code contributions

LICENSE.md

@@ -35,7 +35,7 @@ Julia includes code from the following projects, which have their own licenses:
 - [MUSL](https://git.musl-libc.org/cgit/musl/tree/COPYRIGHT) (for getopt implementation on Windows) [MIT]
 - [MINGW](https://sourceforge.net/p/mingw/mingw-org-wsl/ci/legacy/tree/mingwrt/mingwex/dirname.c) (for dirname implementation on Windows) [MIT]
 - [NetBSD](https://www.netbsd.org/about/redistribution.html) (for setjmp, longjmp, and strptime implementations on Windows) [BSD-3]
-- [Python](https://docs.python.org/2/license.html) (for strtod implementation on Windows) [BSD-3, effectively]
+- [Python](https://docs.python.org/2/license.html) (for strtod and joinpath implementation on Windows) [BSD-3, effectively]
 
 The following components included in Julia `Base` have their own separate licenses:
 

Makefile

@@ -291,16 +291,16 @@ endif
 ifeq ($(OS),WINNT)
  -$(INSTALL_M) $(filter-out $(build_bindir)/libjulia-debug.dll,$(wildcard $(build_bindir)/*.dll)) $(DESTDIR)$(bindir)/
  -$(INSTALL_M) $(build_libdir)/libjulia.dll.a $(DESTDIR)$(libdir)/
+
+ # We have a single exception; we want 7z.dll to live in libexec, not bin, so that 7z.exe can find it.
+ -mv $(DESTDIR)$(bindir)/7z.dll $(DESTDIR)$(libexecdir)/
 ifeq ($(BUNDLE_DEBUG_LIBS),1)
  -$(INSTALL_M) $(build_bindir)/libjulia-debug.dll $(DESTDIR)$(bindir)/
  -$(INSTALL_M) $(build_libdir)/libjulia-debug.dll.a $(DESTDIR)$(libdir)/
 endif
  -$(INSTALL_M) $(build_bindir)/libopenlibm.dll.a $(DESTDIR)$(libdir)/
 else
 
- # Install `7z` into libexec/
- $(INSTALL_M) $(build_bindir)/7z$(EXE) $(DESTDIR)$(libexecdir)/
-
 # Copy over .dSYM directories directly for Darwin
 ifneq ($(DARWIN_FRAMEWORK),1)
 ifeq ($(OS),Darwin)
@@ -330,6 +330,7 @@ ifeq ($(BUNDLE_DEBUG_LIBS),1)
  @$(DSYMUTIL) -o $(DESTDIR)$(prefix)/$(framework_resources)/sys-debug.dylib.dSYM $(build_private_libdir)/sys-debug.dylib
 endif
 endif
+
  for suffix in $(JL_PRIVATE_LIBS-0) ; do \
  for lib in $(build_libdir)/$${suffix}.*$(SHLIB_EXT)*; do \
  if [ "$${lib##*.}" != "dSYM" ]; then \
@@ -342,6 +343,8 @@ endif
  $(INSTALL_M) $$lib $(DESTDIR)$(private_libdir) ; \
  done
 endif
+ # Install `7z` into libexec/
+ $(INSTALL_M) $(build_bindir)/7z$(EXE) $(DESTDIR)$(libexecdir)/
 
  # Copy public headers
  cp -R -L $(build_includedir)/julia/* $(DESTDIR)$(includedir)/julia
@@ -577,8 +580,13 @@ testall1: check-whitespace $(JULIA_BUILD_MODE)
  @env JULIA_CPU_THREADS=1 $(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/test all JULIA_BUILD_MODE=$(JULIA_BUILD_MODE)
 
 test-%: check-whitespace $(JULIA_BUILD_MODE)
+ @([ $$(( $$(date +%s) - $$(date +%s -r $(build_private_libdir)/sys.$(SHLIB_EXT)) )) -le 100 ] && \
+ printf '\033[93m HINT The system image was recently rebuilt. Are you aware of the test-revise-* targets? See CONTRIBUTING.md. \033[0m\n') || true
  @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/test $* JULIA_BUILD_MODE=$(JULIA_BUILD_MODE)
 
+test-revise-%:
+ @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/test revise-$* JULIA_BUILD_MODE=$(JULIA_BUILD_MODE)
+
 # download target for some hardcoded windows dependencies
 .PHONY: win-extras wine_path
 win-extras:

NEWS.md

@@ -8,6 +8,9 @@ New language features
 
 * `import` now allows quoted symbols, e.g. `import Base.:+` ([#33158]).
 
+* Function composition now supports multiple functions: `∘(f, g, h) = f ∘ g ∘ h`
+and splatting `∘(fs...)` for composing an iterable collection of functions ([#33568]).
+
 Language changes
 ----------------
 
@@ -30,7 +33,9 @@ New library functions
 * The `tempname` function now takes an optional `parent::AbstractString` argument to give it a directory in which to attempt to produce a temporary path name ([#33090]).
 * The `tempname` function now takes a `cleanup::Bool` keyword argument defaulting to `true`, which causes the process to try to ensure that any file or directory at the path returned by `tempname` is deleted upon process exit ([#33090]).
 * The `readdir` function now takes a `join::Bool` keyword argument defaulting to `false`, which when set causes `readdir` to join its directory argument with each listed name ([#33113]).
+* `readdir` output is now guaranteed to be sorted. The `sort` keyword allows opting out of sorting to get names in OS-native order ([#33542]).
 * The new `only(x)` function returns the one-and-only element of a collection `x`, and throws an `ArgumentError` if `x` contains zero or multiple elements. ([#33129])
+* `takewhile` and `dropwhile` have been added to the Iterators submodule ([#33437]).
 
 
 Standard library changes
@@ -44,6 +49,8 @@ Standard library changes
 
 * `Iterators.partition` now uses views (or smartly re-computed ranges) for partitions of all `AbstractArray`s ([#33533]).
 
+* Sets are now displayed less compactly in the REPL, as a column of elements, like vectors
+ and dictionaries ([#33300]).
 
 #### Libdl
 
@@ -55,6 +62,8 @@ Standard library changes
 
 * `ldlt` and non-pivoted `lu` now throw a new `ZeroPivotException` type ([#33372]).
 
+* `cond(A, p)` with `p=1` or `p=Inf` now computes the exact condition number instead of an estimate ([#33547]).
+
 #### Random
 
 * `AbstractRNG`s now behave like scalars when used in broadcasting ([#33213]).

base/abstractdict.jl

@@ -77,7 +77,8 @@ function keys end
 
 Return an iterator over all keys in a dictionary.
 `collect(keys(a))` returns an array of keys.
-Since the keys are stored internally in a hash table,
+When the keys are stored internally in a hash table,
+as is the case for `Dict`,
 the order in which they are returned may vary.
 But `keys(a)` and `values(a)` both iterate `a` and
 return the elements in the same order.
@@ -102,7 +103,8 @@ keys(a::AbstractDict) = KeySet(a)
 
 Return an iterator over all values in a collection.
 `collect(values(a))` returns an array of values.
-Since the values are stored internally in a hash table,
+When the values are stored internally in a hash table,
+as is the case for `Dict`,
 the order in which they are returned may vary.
 But `keys(a)` and `values(a)` both iterate `a` and
 return the elements in the same order.

base/abstractset.jl

@@ -35,7 +35,10 @@ julia> union([4, 2], 1:2)
  1
 
 julia> union(Set([1, 2]), 2:3)
-Set([2, 3, 1])
+Set{Int64} with 3 elements:
+ 2
+ 3
+ 1
 ```
 """
 function union end
@@ -60,7 +63,12 @@ julia> a = Set([1, 3, 4, 5]);
 julia> union!(a, 1:2:8);
 
 julia> a
-Set([7, 4, 3, 5, 1])
+Set{Int64} with 5 elements:
+ 7
+ 4
+ 3
+ 5
+ 1
 ```
 """
 function union!(s::AbstractSet, sets...)
@@ -105,7 +113,8 @@ julia> intersect([1, 4, 4, 5, 6], [4, 6, 6, 7, 8])
  6
 
 julia> intersect(Set([1, 2]), BitSet([2, 3]))
-Set([2])
+Set{Int64} with 1 element:
+ 2
 ```
 """
 intersect(s::AbstractSet, itr, itrs...) = intersect!(intersect(s, itr), itrs...)
@@ -160,7 +169,8 @@ julia> a = Set([1, 3, 4, 5]);
 julia> setdiff!(a, 1:2:6);
 
 julia> a
-Set([4])
+Set{Int64} with 1 element:
+ 4
 ```
 """
 function setdiff!(s::AbstractSet, itrs...)

base/array.jl

@@ -1076,13 +1076,16 @@ julia> A
  2
 
 julia> S = Set([1, 2])
-Set([2, 1])
+Set{Int64} with 2 elements:
+ 2
+ 1
 
 julia> pop!(S)
 2
 
 julia> S
-Set([1])
+Set{Int64} with 1 element:
+ 1
 
 julia> pop!(Dict(1=>2))
 1 => 2

base/bitset.jl

@@ -340,10 +340,13 @@ filter!(f, s::BitSet) = unsafe_filter!(f, s)
 @inline in(n::Int, s::BitSet) = _bits_getindex(s.bits, n, s.offset)
 @inline in(n::Integer, s::BitSet) = _is_convertible_Int(n) ? in(Int(n), s) : false
 
-function iterate(s::BitSet, idx=0)
- idx = _bits_findnext(s.bits, idx)
- idx == -1 && return nothing
- (idx + intoffset(s), idx+1)
+function iterate(s::BitSet, (word, idx) = (CHK0, 0))
+ while word == 0
+ idx == length(s.bits) && return nothing
+ idx += 1
+ word = @inbounds s.bits[idx]
+ end
+ trailing_zeros(word) + (idx - 1 + s.offset) << 6, (_blsr(word), idx)
 end
 
 @noinline _throw_bitset_notempty_error() =

base/boot.jl

@@ -210,6 +210,7 @@ else
  const UInt = UInt32
 end
 
+function iterate end
 function Typeof end
 ccall(:jl_toplevel_eval_in, Any, (Any, Any),
  Core, quote

base/cmd.jl

@@ -102,6 +102,8 @@ shell_escape(cmd::Cmd; special::AbstractString="") =
  shell_escape(cmd.exec..., special=special)
 shell_escape_posixly(cmd::Cmd) =
  shell_escape_posixly(cmd.exec...)
+shell_escape_winsomely(cmd::Cmd) =
+ shell_escape_winsomely(cmd.exec...)
 
 function show(io::IO, cmd::Cmd)
  print_env = cmd.env !== nothing

base/compiler/abstractinterpretation.jl

@@ -427,7 +427,7 @@ end
 # refine its type to an array of element types.
 # Union of Tuples of the same length is converted to Tuple of Unions.
 # returns an array of types
-function precise_container_type(@nospecialize(typ), vtypes::VarTable, sv::InferenceState)
+function precise_container_type(@nospecialize(itft), @nospecialize(typ), vtypes::VarTable, sv::InferenceState)
  if isa(typ, PartialStruct) && typ.typ.name === Tuple.name
  return typ.fields
  end
@@ -489,17 +489,24 @@ function precise_container_type(@nospecialize(typ), vtypes::VarTable, sv::Infere
  elseif tti0 <: Array
  return Any[Vararg{eltype(tti0)}]
  else
- return abstract_iteration(typ, vtypes, sv)
+ return abstract_iteration(itft, typ, vtypes, sv)
  end
 end
 
 # simulate iteration protocol on container type up to fixpoint
-function abstract_iteration(@nospecialize(itertype), vtypes::VarTable, sv::InferenceState)
+function abstract_iteration(@nospecialize(itft), @nospecialize(itertype), vtypes::VarTable, sv::InferenceState)
  if !isdefined(Main, :Base) || !isdefined(Main.Base, :iterate) || !isconst(Main.Base, :iterate)
  return Any[Vararg{Any}]
  end
- iteratef = getfield(Main.Base, :iterate)
- stateordonet = abstract_call(iteratef, nothing, Any[Const(iteratef), itertype], vtypes, sv)
+ if itft === nothing
+ iteratef = getfield(Main.Base, :iterate)
+ itft = Const(iteratef)
+ elseif isa(itft, Const)
+ iteratef = itft.val
+ else
+ return Any[Vararg{Any}]
+ end
+ stateordonet = abstract_call(iteratef, nothing, Any[itft, itertype], vtypes, sv)
  # Return Bottom if this is not an iterator.
  # WARNING: Changes to the iteration protocol must be reflected here,
  # this is not just an optimization.
@@ -543,7 +550,7 @@ function abstract_iteration(@nospecialize(itertype), vtypes::VarTable, sv::Infer
 end
 
 # do apply(af, fargs...), where af is a function value
-function abstract_apply(@nospecialize(aft), aargtypes::Vector{Any}, vtypes::VarTable, sv::InferenceState,
+function abstract_apply(@nospecialize(itft), @nospecialize(aft), aargtypes::Vector{Any}, vtypes::VarTable, sv::InferenceState,
  max_methods = sv.params.MAX_METHODS)
  aftw = widenconst(aft)
  if !isa(aft, Const) && (!isType(aftw) || has_free_typevars(aftw))
@@ -561,7 +568,7 @@ function abstract_apply(@nospecialize(aft), aargtypes::Vector{Any}, vtypes::VarT
  for i = 1:nargs
  ctypes´ = []
  for ti in (splitunions ? uniontypes(aargtypes[i]) : Any[aargtypes[i]])
- cti = precise_container_type(ti, vtypes, sv)
+ cti = precise_container_type(itft, ti, vtypes, sv)
  if _any(t -> t === Bottom, cti)
  continue
  end
@@ -634,7 +641,9 @@ end
 
 function abstract_call(@nospecialize(f), fargs::Union{Nothing,Vector{Any}}, argtypes::Vector{Any}, vtypes::VarTable, sv::InferenceState, max_methods = sv.params.MAX_METHODS)
  if f === _apply
- return abstract_apply(argtypes[2], argtypes[3:end], vtypes, sv, max_methods)
+ return abstract_apply(nothing, argtypes[2], argtypes[3:end], vtypes, sv, max_methods)
+ elseif f === _apply_iterate
+ return abstract_apply(argtypes[2], argtypes[3], argtypes[4:end], vtypes, sv, max_methods)
  end
 
  la = length(argtypes)
@@ -662,7 +671,7 @@ function abstract_call(@nospecialize(f), fargs::Union{Nothing,Vector{Any}}, argt
  end
  rt = builtin_tfunction(f, argtypes[2:end], sv)
  if f === getfield && isa(fargs, Vector{Any}) && length(argtypes) == 3 && isa(argtypes[3], Const) && isa(argtypes[3].val, Int) && argtypes[2] ⊑ Tuple
- cti = precise_container_type(argtypes[2], vtypes, sv)
+ cti = precise_container_type(nothing, argtypes[2], vtypes, sv)
  idx = argtypes[3].val
  if 1 <= idx <= length(cti)
  rt = unwrapva(cti[idx])

base/compiler/compiler.jl

@@ -5,7 +5,7 @@ getfield(getfield(Main, :Core), :eval)(getfield(Main, :Core), :(baremodule Compi
 using Core.Intrinsics, Core.IR
 
 import Core: print, println, show, write, unsafe_write, stdout, stderr,
- _apply, svec, apply_type, Builtin, IntrinsicFunction, MethodInstance, CodeInstance
+ _apply, _apply_iterate, svec, apply_type, Builtin, IntrinsicFunction, MethodInstance, CodeInstance
 
 const getproperty = getfield
 const setproperty! = setfield!

base/compiler/ssair/inlining.jl

@@ -592,11 +592,11 @@ function spec_lambda(@nospecialize(atype), sv::OptimizationState, @nospecialize(
 end
 
 # This assumes the caller has verified that all arguments to the _apply call are Tuples.
-function rewrite_apply_exprargs!(ir::IRCode, idx::Int, argexprs::Vector{Any}, atypes::Vector{Any})
- new_argexprs = Any[argexprs[2]]
- new_atypes = Any[atypes[2]]
+function rewrite_apply_exprargs!(ir::IRCode, idx::Int, argexprs::Vector{Any}, atypes::Vector{Any}, arg_start::Int)
+ new_argexprs = Any[argexprs[arg_start]]
+ new_atypes = Any[atypes[arg_start]]
  # loop over original arguments and flatten any known iterators
- for i in 3:length(argexprs)
+ for i in (arg_start+1):length(argexprs)
  def = argexprs[i]
  def_type = atypes[i]
  if def_type isa PartialStruct
@@ -882,25 +882,26 @@ end
 
 function inline_apply!(ir::IRCode, idx::Int, sig::Signature, params::Params)
  stmt = ir.stmts[idx]
- while sig.f === Core._apply
+ while sig.f === Core._apply || sig.f === Core._apply_iterate
+ arg_start = sig.f === Core._apply ? 2 : 3
  atypes = sig.atypes
  # Try to figure out the signature of the function being called
  # and if rewrite_apply_exprargs can deal with this form
- for i = 3:length(atypes)
+ for i = (arg_start + 1):length(atypes)
  # TODO: We could basically run the iteration protocol here
  if !is_valid_type_for_apply_rewrite(atypes[i], params)
  return nothing
  end
  end
  # Independent of whether we can inline, the above analysis allows us to rewrite
  # this apply call to a regular call
- ft = atypes[2]
- if length(atypes) == 3 && ft isa Const && ft.val === Core.tuple && atypes[3] ⊑ Tuple
+ ft = atypes[arg_start]
+ if length(atypes) == arg_start+1 && ft isa Const && ft.val === Core.tuple && atypes[arg_start+1] ⊑ Tuple
  # rewrite `((t::Tuple)...,)` to `t`
- ir.stmts[idx] = stmt.args[3]
+ ir.stmts[idx] = stmt.args[arg_start+1]
  return nothing
  end
- stmt.args, atypes = rewrite_apply_exprargs!(ir, idx, stmt.args, atypes)
+ stmt.args, atypes = rewrite_apply_exprargs!(ir, idx, stmt.args, atypes, arg_start)
  has_free_typevars(ft) && return nothing
  f = singleton_type(ft)
  sig = Signature(f, ft, atypes)

base/dict.jl

@@ -631,7 +631,7 @@ end
 """
  delete!(collection, key)
 
-Delete the mapping for the given key in a collection, and return the collection.
+Delete the mapping for the given key in a collection, if any, and return the collection.
 
 # Examples
 ```jldoctest
@@ -641,6 +641,10 @@ Dict{String,Int64} with 2 entries:
  "a" => 1
 
 julia> delete!(d, "b")
+Dict{String,Int64} with 1 entry:
+ "a" => 1
+
+julia> delete!(d, "b") # d is left unchanged
 Dict{String,Int64} with 1 entry:
  "a" => 1
 ```