Merge remote-tracking branch 'origin/master' into jn/static_compile_2

DISTRIBUTING.md

@@ -86,6 +86,30 @@ win-extras` in between `make` and `make dist`. After that process is
 completed, the `.zip` file created in the head Julia directory will
 hold a completely self-contained Julia.
 
+Notes on BLAS and LAPACK
+------------------------
+
+Julia builds OpenBLAS by default, which includes the BLAS and LAPACK
+libraries. On 32-bit architectures, Julia builds OpenBLAS to use
+32-bit integers, while on 64-bit architectuers, Julia builds OpenBLAS
+to use 64-bit integers. It is essential that all Julia functions that
+call BLAS and LAPACK API routines use integers of the correct width.
+
+Most BLAS and LAPACK distributions provided on linux distributions,
+and even commercial implementations ship libraries that use 32-bit
+APIs. In many cases, a 64-bit API is provided as a separate library.
+
+When using vendor provided or OS provided libraries, a `make` option
+called `USE_BLAS64` is available as part of the Julia build. When doing
+`make USE_BLAS64=0`, Julia will call BLAS and LAPACK assuming a 32-bit
+API, where all integers are 32-bit wide, even on a 64-bit architecture.
+
+Other libraries that Julia uses, such as ARPACK and SuiteSparse also
+use BLAS and LAPACK internally. The APIs need to be consistent across
+all libraries that depend on BLAS and LAPACK. The Julia build process
+will build all these libraries correctly, but when overriding defaults
+and using system provided libraries, this consistency must be ensured.
+
 
 Compilation scripts
 ===================

Make.inc

@@ -132,11 +132,11 @@ USE_COPY_STACKS = 1
 # Compiler specific stuff
 
 ifeq ($(CC), clang)
-USEGCC = 0
-USECLANG = 1
+USEGCC ?= 0
+USECLANG ?= 1
 else
-USEGCC = 1
-USECLANG = 0
+USEGCC ?= 1
+USECLANG ?= 0
 endif
 
 STDLIBCPP_FLAG = 
@@ -168,6 +168,9 @@ ifeq ($(USEGCC),1)
 ifeq ($(USE_LIBCPP),1)
 $(error USE_LIBCPP only supported with clang. Try setting USE_LIBCPP=0)
 endif
+ifeq ($(SANITIZE),1)
+$(error Address Sanitizer only supported with clang. Try setting SANITIZE=0)
+endif
 CC = $(CROSS_COMPILE)gcc
 CXX = $(CROSS_COMPILE)g++
 JCFLAGS = -std=gnu99 -pipe $(fPIC) -fno-strict-aliasing -D_FILE_OFFSET_BITS=64
@@ -210,6 +213,12 @@ ifeq (exists, $(shell [ -e $(JULIAHOME)/Make.user ] && echo exists ))
 include $(JULIAHOME)/Make.user
 endif
 
+ifeq ($(SANITIZE),1)
+JCXXFLAGS += -fsanitize=address -mllvm -asan-stack=0
+JCFLAGS += -fsanitize=address -mllvm -asan-stack=0
+LDFLAGS += -fsanitize=address -mllvm -asan-stack=0
+endif
+
 # ===========================================================================
 
 BUILD_MACHINE := $(shell $(HOSTCC) -dumpmachine)
@@ -389,7 +398,7 @@ else
 endif
 
 ifeq ($(OS), Linux)
-OSLIBS += -ldl -lrt -lpthread -Wl,--export-dynamic -Wl,--version-script=$(JULIAHOME)/src/julia.expmap -Wl,--no-whole-archive $(LIBUNWIND)
+OSLIBS += -ldl -ltinfo -lrt -lpthread -Wl,--export-dynamic -Wl,--version-script=$(JULIAHOME)/src/julia.expmap -Wl,--no-whole-archive $(LIBUNWIND)
 JLDFLAGS = -Wl,-Bdynamic
 endif
 

NEWS.md

@@ -42,6 +42,8 @@ Library improvements
 
  * Faster sparse `kron` ([#4958]).
 
+ * `rand` now supports arbitrary `Ranges` arguments ([#5059]).
+
 Deprecated or removed
 ---------------------
 

base/abstractarray.jl

@@ -2,6 +2,7 @@
 
 typealias AbstractVector{T} AbstractArray{T,1}
 typealias AbstractMatrix{T} AbstractArray{T,2}
+typealias AbstractVecOrMat{T} Union(AbstractVector{T}, AbstractMatrix{T})
 
 ## Basic functions ##
 
@@ -737,7 +738,7 @@ function vcat{T}(V::AbstractVector{T}...)
  a
 end
 
-function hcat{T}(A::Union(AbstractMatrix{T},AbstractVector{T})...)
+function hcat{T}(A::AbstractVecOrMat{T}...)
  nargs = length(A)
  nrows = size(A[1], 1)
  ncols = 0
@@ -1180,7 +1181,7 @@ function hvcat(rows::(Int...), as...)
  vcat(rs...)
 end
 
-function repmat(a::Union(AbstractVector,AbstractMatrix), m::Int, n::Int=1)
+function repmat(a::AbstractVecOrMat, m::Int, n::Int=1)
  o, p = size(a,1), size(a,2)
  b = similar(a, o*m, p*n)
  for j=1:n

base/datafmt.jl

@@ -202,7 +202,7 @@ readcsv(io, T::Type; opts...) = readdlm(io, ',', T; opts...)
 # todo: keyword argument for # of digits to print
 writedlm_cell(io::IO, elt::FloatingPoint) = print_shortest(io, elt)
 writedlm_cell(io::IO, elt) = print(io, elt)
-function writedlm(io::IO, a::Union(AbstractMatrix,AbstractVector), dlm::Char)
+function writedlm(io::IO, a::AbstractVecOrMat, dlm::Char)
  pb = PipeBuffer()
  nr = size(a,1)
  nc = size(a,2)
@@ -239,5 +239,5 @@ end
 writedlm(io, a) = writedlm(io, a, '\t')
 writecsv(io, a) = writedlm(io, a, ',')
 
-writemime(io::IO, ::MIME"text/csv", a::Union(AbstractVector,AbstractMatrix)) = writedlm(io, a, ',')
-writemime(io::IO, ::MIME"text/tab-separated-values", a::Union(AbstractVector,AbstractMatrix)) = writedlm(io, a, '\t')
+writemime(io::IO, ::MIME"text/csv", a::AbstractVecOrMat) = writedlm(io, a, ',')
+writemime(io::IO, ::MIME"text/tab-separated-values", a::AbstractVecOrMat) = writedlm(io, a, '\t')

base/exports.jl

@@ -16,6 +16,7 @@ export
  AbstractMatrix,
  AbstractSparseMatrix,
  AbstractVector,
+ AbstractVecOrMat,
  Array,
  Associative,
  Bidiagonal,
@@ -25,6 +26,7 @@ export
  BitMatrix,
  BitVector,
  BunchKaufman,
+ CFILE,
  Cholesky,
  CholeskyPivoted,
  Cmd,
@@ -695,6 +697,8 @@ export
  endof,
  filter!,
  filter,
+ foldl,
+ foldr,
  get,
  get!,
  getindex,

base/intfuncs.jl

@@ -88,27 +88,21 @@ function power_by_squaring(x, p::Integer)
  elseif p < 0
  throw(DomainError())
  end
- t = 1
- while t <= p
- t *= 2
+ t = trailing_zeros(p) + 1
+ p >>= t
+ while (t -= 1) > 0
+ x *= x
  end
- t = div(t,2)
- p -= t
- a = x
- while true
- t = div(t,2)
- if t > 0
- x = x*x
- else
- break
- end
-
- if p >= t
- x = x*a
- p -= t
+ y = x
+ while p > 0
+ t = trailing_zeros(p) + 1
+ p >>= t
+ while (t -= 1) >= 0
+ x *= x
  end
+ y *= x
  end
- return x
+ return y
 end
 
 ^{T<:Integer}(x::T, p::T) = power_by_squaring(x,p)

base/io.jl

@@ -267,9 +267,10 @@ fd(s::IOStream) = int(ccall(:jl_ios_fd, Clong, (Ptr{Void},), s.ios))
 close(s::IOStream) = ccall(:ios_close, Void, (Ptr{Void},), s.ios)
 isopen(s::IOStream) = bool(ccall(:ios_isopen, Cint, (Ptr{Void},), s.ios))
 flush(s::IOStream) = ccall(:ios_flush, Void, (Ptr{Void},), s.ios)
-isreadonly(s::IOStream) = bool(ccall(:ios_get_readonly, Cint, (Ptr{Void},), s.ios))
-iswritable(s::IOStream) = !isreadonly(s)
-isreadable(s::IOStream) = true
+iswritable(s::IOStream) = bool(ccall(:ios_get_writable, Cint, (Ptr{Void},), s.ios))
+isreadable(s::IOStream) = bool(ccall(:ios_get_readable, Cint, (Ptr{Void},), s.ios))
+modestr(s::IO) = modestr(isreadable(s), iswritable(s))
+modestr(r::Bool, w::Bool) = r ? (w ? "r+" : "r") : (w ? "w" : error("Neither readable nor writable"))
 
 function truncate(s::IOStream, n::Integer)
  ccall(:ios_trunc, Int32, (Ptr{Void}, Uint), s.ios, n) == 0 ||
@@ -301,6 +302,33 @@ position(s::IOStream) = ccall(:ios_pos, FileOffset, (Ptr{Void},), s.ios)
 
 eof(s::IOStream) = bool(ccall(:jl_ios_eof, Int32, (Ptr{Void},), s.ios))
 
+# For interfacing with C FILE* functions
+
+
+immutable CFILE
+ ptr::Ptr{Void}
+end
+
+function CFILE(s::IO)
+ @unix_only FILEp = ccall(:fdopen, Ptr{Void}, (Cint, Ptr{Uint8}), convert(Cint, fd(s)), modestr(s))
+ @windows_only FILEp = ccall(:_fdopen, Ptr{Void}, (Cint, Ptr{Uint8}), convert(Cint, fd(s)), modestr(s))
+ if FILEp == 0
+ error("fdopen failed")
+ end
+ seek(CFILE(FILEp), position(s))
+end
+
+convert(::Type{CFILE}, s::IO) = CFILE(s)
+
+function seek(h::CFILE, offset::Integer)
+ ccall(:fseek, Cint, (Ptr{Void}, Clong, Cint), 
+ h.ptr, convert(Clong, offset), int32(0)) == 0 ||
+ error("fseek failed")
+ h
+end
+
+position(h::CFILE) = ccall(:ftell, Clong, (Ptr{Void},), h.ptr)
+
 ## constructing and opening streams ##
 
 # "own" means the descriptor will be closed with the IOStream
@@ -351,7 +379,7 @@ write(s::IOStream, b::Uint8) = int(ccall(:jl_putc, Int32, (Uint8, Ptr{Void}), b,
 
 function write{T}(s::IOStream, a::Array{T})
  if isbits(T)
- if isreadonly(s)
+ if !iswritable(s)
  error("attempt to write to a read-only IOStream")
  end
  int(ccall(:ios_write, Uint, (Ptr{Void}, Ptr{Void}, Uint),
@@ -362,7 +390,7 @@ function write{T}(s::IOStream, a::Array{T})
 end
 
 function write(s::IOStream, p::Ptr, nb::Integer)
- if isreadonly(s)
+ if !iswritable(s)
  error("attempt to write to a read-only IOStream")
  end
  int(ccall(:ios_write, Uint, (Ptr{Void}, Ptr{Void}, Uint), s.ios, p, nb))
@@ -409,7 +437,7 @@ end
 ## text I/O ##
 
 function write(s::IOStream, c::Char)
- if isreadonly(s)
+ if !iswritable(s)
  error("attempt to write to a read-only IOStream")
  end
  int(ccall(:ios_pututf8, Int32, (Ptr{Void}, Char), s.ios, c))

base/linalg.jl

@@ -1,7 +1,7 @@
 module LinAlg
 
 importall Base
-import Base.USE_BLAS64, Base.size, Base.copy, Base.copy_transpose!, Base.power_by_squaring, Base.print_matrix
+import Base: USE_BLAS64, size, copy, copy_transpose!, power_by_squaring, print_matrix
 
 export 
 # Modules

base/linalg/dense.jl

@@ -74,16 +74,6 @@ end
 
 tril(M::Matrix, k::Integer) = tril!(copy(M), k)
 
-diff(a::Vector) = [ a[i+1] - a[i] for i=1:length(a)-1 ]
-
-function diff(A::Matrix, dim::Integer)
- if dim == 1
- [A[i+1,j] - A[i,j] for i=1:size(A,1)-1, j=1:size(A,2)]
- else
- [A[i,j+1] - A[i,j] for i=1:size(A,1), j=1:size(A,2)-1]
- end
-end
-
 function gradient(F::Vector, h::Vector)
  n = length(F)
  g = similar(F)
@@ -289,23 +279,21 @@ function expm!{T<:BlasFloat}(A::StridedMatrix{T})
  end
  end
  if ilo > 1 # apply lower permutations in reverse order
- for j in (ilo-1):1:-1 rcswap!(j, int(scale[j]), X) end
+ for j in (ilo-1):-1:1 rcswap!(j, int(scale[j]), X) end
  end
  if ihi < n # apply upper permutations in forward order
  for j in (ihi+1):n rcswap!(j, int(scale[j]), X) end
  end
  X
 end
 
-## Swap rows j and jp and columns j and jp in X
-function rcswap!{T<:Number}(j::Integer, jp::Integer, X::StridedMatrix{T})
- for k in 1:size(X, 2)
- tmp = X[k,j]
- X[k,j] = X[k,jp]
- X[k,jp] = tmp
- tmp = X[j,k]
- X[j,k] = X[jp,k]
- X[jp,k] = tmp
+## Swap rows i and j and columns i and j in X
+function rcswap!{T<:Number}(i::Integer, j::Integer, X::StridedMatrix{T})
+ for k = 1:size(X,1)
+ X[k,i], X[k,j] = X[k,j], X[k,i]
+ end
+ for k = 1:size(X,2)
+ X[i,k], X[j,k] = X[j,k], X[i,k]
  end
 end
 

base/linalg/diagonal.jl

@@ -13,11 +13,7 @@ convert{T}(::Type{SymTridiagonal{T}}, D::Diagonal{T}) = SymTridiagonal(D.diag,ze
 convert{T}(::Type{Tridiagonal{T}}, D::Diagonal{T}) = Tridiagonal(zeros(T,length(D.diag)-1),D.diag,zeros(T,length(D.diag)-1))
 
 full(D::Diagonal) = diagm(D.diag)
-function show(io::IO, D::Diagonal)
- println(io, summary(D), ":")
- print(io, "diag: ")
- print_matrix(io, (D.diag)')
-end
+getindex(D::Diagonal, i::Integer, j::Integer) = i == j ? D.diag[i] : zero(eltype(D.diag))
 
 ishermitian(D::Diagonal) = true
 issym(D::Diagonal) = true
@@ -89,6 +85,7 @@ function inv{T<:BlasFloat}(D::Diagonal{T})
 end
 inv(D::Diagonal) = inv(Diagonal(float(D.diag)))
 
+svdvals(D::Diagonal) = sort(D.diag, rev = true)
 eigvals(D::Diagonal) = sort(D.diag)
 
 expm(D::Diagonal) = Diagonal(exp(D.diag))

base/linalg/factorization.jl

@@ -10,8 +10,6 @@ macro assertnonsingular(A, info)
  :(($info)==0 ? $A : throw(SingularException($info)))
 end
 
-typealias AbstractVecOrMat Union(AbstractVector, AbstractMatrix)
-
 \(F::Factorization, B::AbstractVecOrMat) = A_ldiv_B!(F, copy(B))
 Ac_ldiv_B(F::Factorization, B::AbstractVecOrMat) = Ac_ldiv_B!(F, copy(B))
 At_ldiv_B(F::Factorization, B::AbstractVecOrMat) = At_ldiv_B!(F, copy(B))

base/linalg/generic.jl

@@ -23,6 +23,16 @@ tril!(M::AbstractMatrix) = tril!(M,0)
 #diff(a::AbstractVector)
 #diff(a::AbstractMatrix, dim::Integer)
 diff(a::AbstractMatrix) = diff(a, 1)
+diff(a::AbstractVector) = [ a[i+1] - a[i] for i=1:length(a)-1 ]
+
+function diff(A::AbstractMatrix, dim::Integer)
+ if dim == 1
+ [A[i+1,j] - A[i,j] for i=1:size(A,1)-1, j=1:size(A,2)]
+ else
+ [A[i,j+1] - A[i,j] for i=1:size(A,1), j=1:size(A,2)-1]
+ end
+end
+
 
 gradient(F::AbstractVector) = gradient(F, [1:length(F)])
 gradient(F::AbstractVector, h::Real) = gradient(F, [h*(1:length(F))])
@@ -31,7 +41,7 @@ gradient(F::AbstractVector, h::Real) = gradient(F, [h*(1:length(F))])
 diag(A::AbstractVector) = error("use diagm instead of diag to construct a diagonal matrix")
 #diag(A::AbstractMatrix)
 
-#diagm{T}(v::Union(AbstractVector{T},AbstractMatrix{T}))
+#diagm{T}(v::AbstractVecOrMat{T})
 
 function norm{T}(x::AbstractVector{T}, p::Number)
  if length(x) == 0
@@ -100,12 +110,12 @@ trace(x::Number) = x
 
 inv(a::AbstractVector) = error("argument must be a square matrix")
 
-function \{TA<:Number,TB<:Number}(A::AbstractMatrix{TA}, B::Union(AbstractVector{TB},AbstractMatrix{TB}))
+function \{TA<:Number,TB<:Number}(A::AbstractMatrix{TA}, B::AbstractVecOrMat{TB})
  TC = typeof(one(TA)/one(TB))
  return TB == TC ? A_ldiv_B!(A, copy(B)) : A_ldiv_B!(A, convert(Array{TC}, B))
 end
 \(a::AbstractVector, b::AbstractArray) = reshape(a, length(a), 1) \ b
-/(A::Union(AbstractVector,AbstractMatrix), B::Union(AbstractVector,AbstractMatrix)) = (B' \ A')'
+/(A::AbstractVecOrMat, B::AbstractVecOrMat) = (B' \ A')'
 
 cond(x::Number) = x == 0 ? Inf : 1.0
 cond(x::Number, p) = cond(x)