Change iteration protocol

This changes the iteration protocol from `start`/`next`/`done` to `iterate`.
The new lowering of a for loop is as follows:

```
for x in itr
    ...
end
```

becomes

```
next = iterate(itr)
while next !== nothing
    x, state = next::Tuple{Any, Any}
    ...
    next = iterate(itr, state)
end
```

The semantics are as apparent from the above lowering. `iterate` returns
either `nothing` or a tuple of value and state. The state is passed
to any subsequent operation. The first iteration is indicated, by not passing
the second, state argument to the `iterate` method.

Adaptors in both directions are provided to keep the legacy iteration
protocol working for now. However, performance of the legacy iteration
protocol will be severely pessimized.

As an optional add-on for mutable iterators, a new `isdone` function is
provided. This function is intended as an O(1) approximate query for
iterator completion, where such a calculation is possible without mutation
and/or is significantly faster than attempting to obtain the element itself.
The function makes use of 3-value logic. `missing` is always an acceptable
answer, in which case the caller should go ahead and attempt the iteration
to obtain a definite result. If the result is not `missing`, it must be
exact (i.e. if true, the next call to iterate must return `nothing`, if
false it must not return nothing).

base/abstractarray.jl

@@ -284,9 +284,9 @@ julia> first([1; 2; 3; 4])
 ```
 """
 function first(itr)
- state = start(itr)
- done(itr, state) && throw(ArgumentError("collection must be non-empty"))
- next(itr, state)[1]
+ x = iterate(itr)
+ x === nothing && throw(ArgumentError("collection must be non-empty"))
+ x[1]
 end
 
 """
@@ -635,10 +635,12 @@ emptymutable(itr, ::Type{U}) where {U} = Vector{U}()
 
 function copyto!(dest::AbstractArray, src)
  destiter = eachindex(dest)
- state = start(destiter)
+ y = iterate(destiter)
  for x in src
- i, state = next(destiter, state)
- dest[i] = x
+ y === nothing &&
+ throw(ArgumentError(string("source has fewer elements than required")))
+ dest[y[1]] = x
+ y = iterate(destiter, y[2])
  end
  return dest
 end
@@ -657,25 +659,24 @@ function copyto!(dest::AbstractArray, dstart::Integer, src, sstart::Integer)
  if (sstart < 1)
  throw(ArgumentError(string("source start offset (",sstart,") is < 1")))
  end
- st = start(src)
+ y = iterate(src)
  for j = 1:(sstart-1)
- if done(src, st)
+ if y === nothing
  throw(ArgumentError(string("source has fewer elements than required, ",
  "expected at least ",sstart,", got ",j-1)))
  end
- _, st = next(src, st)
+ y = iterate(src, y[2])
  end
- dn = done(src, st)
- if dn
+ if y === nothing
  throw(ArgumentError(string("source has fewer elements than required, ",
  "expected at least ",sstart,", got ",sstart-1)))
  end
  i = Int(dstart)
- while !dn
- val, st = next(src, st)
+ while y != nothing
+ val, st = y
  dest[i] = val
  i += 1
- dn = done(src, st)
+ y = iterate(src, st)
  end
  return dest
 end
@@ -690,18 +691,19 @@ function copyto!(dest::AbstractArray, dstart::Integer, src, sstart::Integer, n::
  sstart < 1 && throw(ArgumentError(string("source start offset (",sstart,") is < 1")))
  throw(BoundsError(dest, dstart:dmax))
  end
- st = start(src)
+ y = iterate(src)
  for j = 1:(sstart-1)
- if done(src, st)
+ if y === nothing
  throw(ArgumentError(string("source has fewer elements than required, ",
  "expected at least ",sstart,", got ",j-1)))
  end
- _, st = next(src, st)
+ y = iterate(src, y[2])
  end
  i = Int(dstart)
- while i <= dmax && !done(src, st)
- val, st = next(src, st)
+ while i <= dmax && y !== nothing
+ val, st = y
  @inbounds dest[i] = val
+ y = iterate(src, st)
  i += 1
  end
  i <= dmax && throw(BoundsError(dest, i))
@@ -823,9 +825,11 @@ zero(x::AbstractArray{T}) where {T} = fill!(similar(x), zero(T))
 # While the definitions for IndexLinear are all simple enough to inline on their
 # own, IndexCartesian's CartesianIndices is more complicated and requires explicit
 # inlining.
-start(A::AbstractArray) = (@_inline_meta; itr = eachindex(A); (itr, start(itr)))
-next(A::AbstractArray, i) = (@_propagate_inbounds_meta; (idx, s) = next(i[1], i[2]); (A[idx], (i[1], s)))
-done(A::AbstractArray, i) = (@_propagate_inbounds_meta; done(i[1], i[2]))
+function iterate(A::AbstractArray, state=(eachindex(A),))
+ y = iterate(state...)
+ y === nothing && return nothing
+ A[y[1]], (state[1], tail(y)...)
+end
 
 isempty(a::AbstractArray) = (_length(a) == 0)
 
@@ -2042,12 +2046,17 @@ function hash(a::AbstractArray{T}, h::UInt) where T
  h += hashaa_seed
  h += hash(size(a))
 
- state = start(a)
- done(a, state) && return h
- x1, state = next(a, state)
- done(a, state) && return hash(x1, h)
- x2, state = next(a, state)
- done(a, state) && return hash(x2, hash(x1, h))
+ y1 = iterate(a)
+ y1 === nothing && return h
+ y2 = iterate(a, y1[2])
+ y2 === nothing && return hash(y1[1], h)
+ y = iterate(a, y2[2])
+ y === nothing && return hash(y2[1], hash(y1[1], h))
+ x1, x2 = y1[1], y2[1]
+
+ # For the rest of the function, we keep three elements worth of state,
+ # x1, x2, y[1], with `y` potentially being `nothing` if there's only
+ # two elements remaining
 
  # Check whether the array is equal to a range, and hash the elements
  # at the beginning of the array as such as long as they match this assumption
@@ -2056,6 +2065,10 @@ function hash(a::AbstractArray{T}, h::UInt) where T
  if isa(a, AbstractVector) && applicable(-, x2, x1)
  n = 1
  local step, laststep, laststate
+
+ h = hash(x1, h)
+ h += hashr_seed
+
  while true
  # If overflow happens with entries of the same type, a cannot be equal
  # to a range with more than two elements because more extreme values
@@ -2077,47 +2090,48 @@ function hash(a::AbstractArray{T}, h::UInt) where T
  end
  n > 1 && !isequal(step, laststep) && break
  n += 1
- x1 = x2
  laststep = step
- laststate = state
- done(a, state) && break
- x2, state = next(a, state)
+ if y === nothing
+ # The array matches a range exactly
+ return hash(x2, hash(n, h))
+ end
+ x1, x2 = x2, y[1]
+ y = iterate(a, y[2])
  end
 
- h = hash(first(a), h)
- h += hashr_seed
  # Always hash at least the two first elements as a range (even in case of overflow)
  if n < 2
  h = hash(2, h)
- h = hash(x2, h)
- done(a, state) && return h
- x1 = x2
- x2, state = next(a, state)
+ h = hash(y2[1], h)
+ @assert y !== nothing
+ x1, x2 = x2, y[1]
+ y = iterate(a, y[2])
  else
  h = hash(n, h)
  h = hash(x1, h)
- done(a, laststate) && return h
  end
  end
 
- # Hash elements which do not correspond to a range (if any)
+ # Hash elements which do not correspond to a range
  while true
  if isequal(x2, x1)
  # For repeated elements, use run length encoding
  # This allows efficient hashing of sparse arrays
  runlength = 2
- while !done(a, state)
- x2, state = next(a, state)
+ while y !== nothing
+ # No need to update x1 (it's isequal x2)
+ x2 = y[1]
+ y = iterate(a, y[2])
  isequal(x1, x2) || break
  runlength += 1
  end
  h += hashrle_seed
  h = hash(runlength, h)
  end
  h = hash(x1, h)
- done(a, state) && break
- x1 = x2
- x2, state = next(a, state)
+ y === nothing && break
+ x1, x2 = x2, y[1]
+ y = iterate(a, y[2])
  end
  !isequal(x2, x1) && (h = hash(x2, h))
  return h

base/abstractdict.jl

@@ -55,17 +55,10 @@ isempty(v::Union{KeySet,ValueIterator}) = isempty(v.dict)
 _tt2(::Type{Pair{A,B}}) where {A,B} = B
 eltype(::Type{ValueIterator{D}}) where {D} = _tt2(eltype(D))
 
-start(v::Union{KeySet,ValueIterator}) = start(v.dict)
-done(v::Union{KeySet,ValueIterator}, state) = done(v.dict, state)
-
-function next(v::KeySet, state)
- n = next(v.dict, state)
- n[1][1], n[2]
-end
-
-function next(v::ValueIterator, state)
- n = next(v.dict, state)
- n[1][2], n[2]
+function iterate(v::Union{KeySet,ValueIterator}, state...)
+ y = iterate(v.dict, state...)
+ y === nothing && return nothing
+ return (y[1][isa(v, KeySet) ? 1 : 2], y[2])
 end
 
 in(k, v::KeySet) = get(v.dict, k, secret_table_token) !== secret_table_token
@@ -670,9 +663,11 @@ end
 
 _oidd_nextind(a, i) = reinterpret(Int, ccall(:jl_eqtable_nextind, Csize_t, (Any, Csize_t), a, i))
 
-start(d::IdDict) = _oidd_nextind(d.ht, 0)
-done(d::IdDict, i) = (i == -1)
-next(d::IdDict{K, V}, i) where {K, V} = (Pair{K, V}(d.ht[i + 1]::K, d.ht[i + 2]::V), _oidd_nextind(d.ht, i + 2))
+function iterate(d::IdDict{K,V}, idx=0) where {K, V}
+ idx = _oidd_nextind(d.ht, idx)
+ idx == -1 && return nothing
+ return (Pair{K, V}(d.ht[idx + 1]::K, d.ht[idx + 2]::V), idx + 2)
+end
 
 length(d::IdDict) = d.count
 
@@ -711,9 +706,9 @@ empty!(s::IdSet) = (empty!(s.dict); s)
 
 filter!(f, d::IdSet) = unsafe_filter!(f, d)
 
-start(s::IdSet) = start(s.dict)
-done(s::IdSet, state) = done(s.dict, state)
-function next(s::IdSet, state)
- ((k, _), i) = next(s.dict, state)
+function iterate(s::IdSet, state...)
+ y = iterate(s.dict, state...)
+ y === nothing && return nothing
+ ((k, _), i) = y
  return (k, i)
 end