change lowering of ccall cconvert arguments

ccall(:a, B, (C,), d) is now lowered as:
  d#1 = cconvert_gcroot(C, d)
  Expr(:call, :ccall, :a, B, (C,), cconvert(C, d#1), d#1)

This allows implementation of the special-case ccall lowering logic in Julia
and still create a gcroot for it as needed

the Ref{T} type is the new supertype of Ptr{T}. the ccall emission
turns a Ref{T} into a Ptr{T}, as if the &addressOf operator had been
applied. the specific rules for this will be moved from the current
gist to the documentation at a later point.

.gitignore

@@ -14,6 +14,7 @@
 *.do
 *.o
 *.obj
+*.so
 *.dylib
 *.dSYM
 *.jl.cov

base/abstractarray.jl

@@ -47,6 +47,17 @@ else
  end
 end
 
+# convert Arrays to pointer arrays for ccall
+cconvert_gcroot{P<:Ptr}(::Type{Ptr{P}}, a::Array{P}) = a
+function cconvert_gcroot{P<:Ptr}(::Type{Ptr{P}}, a::Array)
+ ptrs = Array(P, length(a)+1)
+ for i = 1:length(a)
+ ptrs[i] = convert(P, a[i])
+ end
+ ptrs[length(a)+1] = C_NULL
+ return ptrs
+end
+
 size{T,n}(t::AbstractArray{T,n}, d) = d <= n ? size(t)[d] : 1
 size(x, d1::Integer, d2::Integer, dx::Integer...) = tuple(size(x, d1), size(x, d2, dx...)...)
 eltype{T}(::Type{AbstractArray{T}}) = T

base/base.jl

@@ -52,16 +52,12 @@ convert{T}(::Type{(T...)}, x::Tuple) = cnvt_all(T, x...)
 cnvt_all(T) = ()
 cnvt_all(T, x, rest...) = tuple(convert(T,x), cnvt_all(T, rest...)...)
 
-
-ptr_arg_convert{T}(::Type{Ptr{T}}, x) = convert(T, x)
-ptr_arg_convert(::Type{Ptr{Void}}, x) = x
-
-# conversion used by ccall
-cconvert(T, x) = convert(T, x)
-# use the code in ccall.cpp to safely allocate temporary pointer arrays
-cconvert{T}(::Type{Ptr{Ptr{T}}}, a::Array) = a
-# convert strings to ByteString to pass as pointers
-cconvert{P<:Union(Int8,UInt8)}(::Type{Ptr{P}}, s::AbstractString) = bytestring(s)
+# conversions used by ccall
+ptr_arg_cconvert_gcroot{T}(::Type{Ptr{T}}, x) = cconvert_gcroot(T, x)
+ptr_arg_cconvert{T}(::Type{Ptr{T}}, x) = cconvert(T, x)
+ptr_arg_cconvert(::Type{Ptr{Void}}, x) = x
+cconvert_gcroot(T, x) = x
+cconvert(T, x) = convert(T,x)
 
 reinterpret{T,S}(::Type{T}, x::S) = box(T,unbox(S,x))
 

base/boot.jl

@@ -73,7 +73,8 @@
 # contents::T
 #end
 
-#bitstype {32|64} Ptr{T}
+#abstract Ref{T}
+#bitstype {32|64} Ptr{T} <: Ref{T}
 
 # types for the front end
 
@@ -125,7 +126,7 @@ export
  Module, Symbol, Task, Array, GenSym,
  # numeric types
  Bool, FloatingPoint, Float16, Float32, Float64, Number, Integer, Int, Int8, Int16,
- Int32, Int64, Int128, Ptr, Real, Signed, UInt, UInt8, UInt16, UInt32,
+ Int32, Int64, Int128, Ref, Ptr, Real, Signed, UInt, UInt8, UInt16, UInt32,
  UInt64, UInt128, Unsigned,
  # string types
  Char, ASCIIString, ByteString, DirectIndexString, AbstractString, UTF8String,

base/inference.jl

@@ -112,7 +112,20 @@ t_func[fpiseq] = (2, 2, cmp_tfunc)
 t_func[fpislt] = (2, 2, cmp_tfunc)
 t_func[nan_dom_err] = (2, 2, (a, b)->a)
 t_func[eval(Core.Intrinsics,:ccall)] =
- (3, Inf, (fptr, rt, at, a...)->(isType(rt) ? rt.parameters[1] : Any))
+ (3, Inf, function(fptr, rt, at, a...)
+ if !isType(rt)
+ return Any
+ end
+ t = rt.parameters[1]
+ if isa(t,DataType) && is((t::DataType).name,Ref.name)
+ t = t.parameters[1]
+ if is(t,Any)
+ return Union() # a return type of Box{Any} is invalid
+ end
+ return t
+ end
+ return t
+ end)
 t_func[eval(Core.Intrinsics,:llvmcall)] =
  (3, Inf, (fptr, rt, at, a...)->(isType(rt) ? rt.parameters[1] :
  isa(rt,Tuple) ? map(x->x.parameters[1],rt) : Any))

base/pcre.jl

@@ -57,7 +57,7 @@ function info{T}(
  regex::Ptr{Void},
  extra::Ptr{Void}, what::Integer, ::Type{T}
 )
- buf = Array(UInt8,sizeof(T))
+ buf = zeros(UInt8,sizeof(T))
  ret = ccall((:pcre_fullinfo, :libpcre), Int32,
  (Ptr{Void}, Ptr{Void}, Int32, Ptr{UInt8}),
  regex, extra, what, buf)
@@ -71,7 +71,7 @@ function info{T}(
 end
 
 function config{T}(what::Integer, ::Type{T})
- buf = Array(UInt8, sizeof(T))
+ buf = zeros(UInt8, sizeof(T))
  ret = ccall((:pcre_config, :libpcre), Int32,
  (Int32, Ptr{UInt8}),
  what, buf)
@@ -84,7 +84,8 @@ end
 
 function compile(pattern::AbstractString, options::Integer)
  errstr = Array(Ptr{UInt8},1)
- erroff = Array(Int32,1)
+ errstr[1] = C_NULL
+ erroff = zeros(Int32,1)
  re_ptr = ccall((:pcre_compile, :libpcre), Ptr{Void},
  (Ptr{UInt8}, Int32, Ptr{Ptr{UInt8}}, Ptr{Int32}, Ptr{UInt8}),
  pattern, options, errstr, erroff, C_NULL)
@@ -100,6 +101,7 @@ end
 function study(regex::Ptr{Void}, options::Integer)
  # NOTE: options should always be zero in current PCRE
  errstr = Array(Ptr{UInt8},1)
+ errstr[1] = C_NULL
  extra = ccall((:pcre_study, :libpcre), Ptr{Void},
  (Ptr{Void}, Int32, Ptr{Ptr{UInt8}}),
  regex, options, errstr)

base/pointer.jl

@@ -19,16 +19,13 @@ convert(::Type{Ptr{UInt8}}, x::Symbol) = ccall(:jl_symbol_name, Ptr{UInt8}, (Any
 convert(::Type{Ptr{Int8}}, x::Symbol) = ccall(:jl_symbol_name, Ptr{Int8}, (Any,), x)
 convert(::Type{Ptr{UInt8}}, s::ByteString) = convert(Ptr{UInt8}, s.data)
 convert(::Type{Ptr{Int8}}, s::ByteString) = convert(Ptr{Int8}, s.data)
+# convert strings to ByteString to pass as pointers
+cconvert_gcroot(::Type{Ptr{UInt8}}, s::AbstractString) = bytestring(s)
+cconvert_gcroot(::Type{Ptr{Int8}}, s::AbstractString) = bytestring(s)
 
 convert{T}(::Type{Ptr{T}}, a::Array{T}) = ccall(:jl_array_ptr, Ptr{T}, (Any,), a)
 convert(::Type{Ptr{Void}}, a::Array) = ccall(:jl_array_ptr, Ptr{Void}, (Any,), a)
 
-# note: these definitions don't mean any AbstractArray is convertible to
-# pointer. they just map the array element type to the pointer type for
-# convenience in cases that work.
-pointer{T}(x::AbstractArray{T}) = convert(Ptr{T},x)
-pointer{T}(x::AbstractArray{T}, i::Integer) = convert(Ptr{T},x) + (i-1)*elsize(x)
-
 # unsafe pointer to array conversions
 pointer_to_array(p, d::Integer, own=false) = pointer_to_array(p, (d,), own)
 function pointer_to_array{T,N}(p::Ptr{T}, dims::NTuple{N,Int}, own::Bool=false)
@@ -53,7 +50,8 @@ unsafe_store!{T}(p::Ptr{T}, x) = pointerset(p, convert(T,x), 1)
 
 # convert a raw Ptr to an object reference, and vice-versa
 unsafe_pointer_to_objref(x::Ptr) = ccall(:jl_value_ptr, Any, (Ptr{Void},), x)
-pointer_from_objref(x::Any) = ccall(:jl_value_ptr, Ptr{Void}, (Any,), x)
+pointer_from_objref(x::ANY) = ccall(:jl_value_ptr, Ptr{Void}, (Any,), x)
+data_pointer_from_objref(x::ANY) = pointer_from_objref(x)::Ptr{Void}+Core.sizeof(Int)
 
 integer(x::Ptr) = convert(UInt, x)
 unsigned(x::Ptr) = convert(UInt, x)

base/refpointer.jl

@@ -0,0 +1,121 @@
+### General Methods for Ref{T} type
+
+Base.eltype{T}(x::Type{Ref{T}}) = T
+Base.convert{T}(::Type{Ref{T}}, x::Ref{T}) = x
+
+# create Ref objects for general object conversion
+@inline Base.cconvert_gcroot{T}(::Type{Ref{T}}, x) = convert(Ref{T}, x)
+@inline Base.cconvert{T}(::Type{Ref{T}}, x) = convert(Ptr{T}, x)
+
+### Methods for a Ref object that can store a single value
+
+type RefValue{T} <: Ref{T}
+ x::T
+ RefValue() = new()
+ RefValue(x) = new(x)
+end
+Base.convert{T}(::Type{Ref{T}}, x) = RefValue{T}(x)
+Base.call{T}(::Type{Ref{T}}) = RefValue{T}()
+
+Ref(x::Ref) = x
+Ref{T}(x::T) = RefValue{T}(x)
+Ref{T}(x::Ptr{T}, i::Integer=1) = x + (i-1)*Core.sizeof(T)
+Ref(x, i::Integer) = (i != 1 && error("Object only has one element"); Ref(x))
+
+@inline function Base.convert{T}(P::Type{Ptr{T}}, b::RefValue{T})
+ if isbits(T)
+ return convert(P, data_pointer_from_objref(b))
+ else
+ return convert(P, data_pointer_from_objref(b.x))
+ end
+end
+@inline function Base.convert(P::Type{Ptr{Any}}, b::RefValue{Any})
+ return convert(P, data_pointer_from_objref(b))
+end
+@inline Base.convert{T}(::Type{Ptr{Void}}, b::RefValue{T}) = Base.convert(Ptr{Void}, Base.convert(Ptr{T}, b))
+
+### Methods for a Ref object that is backed by an array at index i
+
+# note: the following type definitions don't mean any AbstractArray is convertible to
+# a data Ref. they just map the array element type to the pointer type for
+# convenience in cases that work.
+pointer{T}(x::AbstractArray{T}) = convert(Ptr{T},x)
+pointer{T}(x::AbstractArray{T}, i::Integer) = convert(Ptr{T},x) + (i-1)*elsize(x)
+
+immutable RefArray{T, A<:AbstractArray} <: Ref{T}
+ x::A
+ i::Int
+ RefArray(x,i) = (@assert(eltype(A) == T); new(x,i))
+end
+convert{T}(::Type{Ref{T}}, x::AbstractArray{T}) = RefArray{T,typeof(x)}(x, 1)
+Ref{T}(x::AbstractArray{T}, i::Integer=1) = RefArray{T,typeof(x)}(x, i)
+
+@inline function Base.convert{T}(P::Type{Ptr{T}}, b::RefArray{T})
+ if isbits(T)
+ convert(P, pointer(b.x, b.i))
+ else
+ convert(P, data_pointer_from_objref(b.x[b.i]))
+ end
+end
+@inline function Base.convert(P::Type{Ptr{Any}}, b::RefArray{Any})
+ return convert(P, pointer(b.x, b.i))
+end
+@inline Base.convert{T}(::Type{Ptr{Void}}, b::RefArray{T}) = Base.convert(Ptr{Void}, Base.convert(Ptr{T}, b))
+
+### Methods for a Ref object that is backed by an array at index (i...)
+
+immutable RefArrayND{T, A<:AbstractArray} <: Ref{T}
+ x::A
+ i::(Int...)
+ RefArrayND(x,i) = (@assert(eltype(A) == T); new(x,i))
+end
+Ref{A<:AbstractArray}(x::A, i::Integer...) = RefArrayND{eltype(A),A}(x, i)
+Ref{A<:AbstractArray}(x::A, i::(Integer...)) = RefArrayND{eltype(A),A}(x, i)
+
+@inline function Base.convert{T}(P::Type{Ptr{T}}, b::RefArrayND{T})
+ if isbits(T)
+ convert(P, pointer(b.x, b.i...))
+ else
+ convert(P, data_pointer_from_objref(b.x[b.i...]))
+ end
+end
+@inline function Base.convert(P::Type{Ptr{Any}}, b::RefArrayND{Any})
+ return convert(P, pointer(b.x, b.i...))
+end
+@inline Base.convert{T}(::Type{Ptr{Void}}, b::RefArrayND{T}) = Base.convert(Ptr{Void}, Base.convert(Ptr{T}, b))
+
+### Methods for a Ref object that is backed by an array at index (Any...)
+
+immutable RefArrayI{T} <: Ref{T}
+ x::AbstractArray{T}
+ i::Tuple
+ RefArrayI(x,i::ANY) = (@assert(eltype(A) == T); new(x,i))
+end
+Ref{T}(x::AbstractArray{T}, i...) = RefArrayI{T}(x, i)
+Ref{T}(x::AbstractArray{T}, i::Tuple) = RefArrayI{T}(x, i)
+
+@inline function Base.convert{T}(P::Type{Ptr{T}}, b::RefArrayI{T})
+ if isbits(T)
+ convert(P, pointer(b.x, b.i...))
+ else
+ convert(P, data_pointer_from_objref(b.x[b.i...]))
+ end
+end
+@inline function Base.convert(P::Type{Ptr{Any}}, b::RefArrayI{Any})
+ return convert(P, pointer(b.x, b.i...))
+end
+@inline Base.convert{T}(::Type{Ptr{Void}}, b::RefArrayI{T}) = Base.convert(Ptr{Void}, Base.convert(Ptr{T}, b))
+
+###
+
+Base.getindex(b::RefValue) = b.x
+Base.getindex(b::RefArray) = b.x[b.i]
+Base.getindex(b::RefArrayND) = b.x[b.i...]
+Base.getindex(b::RefArrayI) = b.x[b.i...]
+
+Base.setindex!(b::RefValue, x) = (b.x = x; b)
+Base.setindex!(b::RefArray, x) = (b.x[b.i] = x; b)
+Base.setindex!(b::RefArrayND, x) = (b.x[b.i...] = x; b)
+Base.setindex!(b::RefArrayI, x) = (b.x[b.i...] = x; b)
+
+###

base/socket.jl

@@ -593,7 +593,8 @@ const _sizeof_uv_interface_address = ccall(:jl_uv_sizeof_interface_address,Int32
 
 function getipaddr()
  addr = Array(Ptr{UInt8},1)
- count = Array(Int32,1)
+ addr[1] = C_NULL
+ count = zeros(Int32,1)
  lo_present = false
  err = ccall(:jl_uv_interface_addresses,Int32,(Ptr{Ptr{UInt8}},Ptr{Int32}),addr,count)
  addr, count = addr[1],count[1]

base/sysimg.jl

@@ -67,6 +67,7 @@ include("dict.jl")
 include("set.jl")
 include("hashing.jl")
 include("iterator.jl")
+include("refpointer.jl") #XXX: move this earlier when we don't need @inline annotations anymore
 
 # SIMD loops
 include("simdloop.jl")

src/alloc.c

@@ -54,6 +54,7 @@ jl_datatype_t *jl_typeerror_type;
 jl_datatype_t *jl_methoderror_type;
 jl_datatype_t *jl_loaderror_type;
 jl_datatype_t *jl_undefvarerror_type;
+jl_datatype_t *jl_ref_type;
 jl_datatype_t *jl_pointer_type;
 jl_datatype_t *jl_void_type;
 jl_datatype_t *jl_voidpointer_type;

src/builtins.c

@@ -1156,6 +1156,7 @@ void jl_init_primitives(void)
  add_builtin("IntrinsicFunction", (jl_value_t*)jl_intrinsic_type);
  add_builtin("Function", (jl_value_t*)jl_function_type);
  add_builtin("LambdaStaticData", (jl_value_t*)jl_lambda_info_type);
+ add_builtin("Ref", (jl_value_t*)jl_ref_type);
  add_builtin("Ptr", (jl_value_t*)jl_pointer_type);
  add_builtin("Box", (jl_value_t*)jl_box_type);
  add_builtin("Task", (jl_value_t*)jl_task_type);