layout: support pointers inlining into parents [NFCI] (JuliaLang#33886)

Includes codegen support for immutable objects that contain pointers
appearing on stack (well, in registers, since LLVM support of
non-integral addrspace pointers inside aggregates in memory is poor),
and includes layout support, so that most (non-self-referential)
immutable objects can be stored inline inside their parent allocation.

Currently fully disabled, aside from some optimizations and improvements
to object_id / isa tests.

Co-Authored-By: Oscar Blumberg <[email protected]>

base/array.jl

@@ -158,7 +158,7 @@ size(a::Array{<:Any,N}) where {N} = (@_inline_meta; ntuple(M -> size(a, M), Val(
 
 asize_from(a::Array, n) = n > ndims(a) ? () : (arraysize(a,n), asize_from(a, n+1)...)
 
-allocatedinline(::Type{T}) where {T} = (@_pure_meta; ccall(:jl_array_store_unboxed, Cint, (Any,), T) != Cint(0))
+allocatedinline(::Type{T}) where {T} = (@_pure_meta; ccall(:jl_stored_inline, Cint, (Any,), T) != Cint(0))
 
 """
  Base.isbitsunion(::Type{T})
@@ -177,14 +177,20 @@ false
 isbitsunion(u::Union) = allocatedinline(u)
 isbitsunion(x) = false
 
-function _unsetindex!(A::Array{T}, i::Int) where {T}
+@inbounds function _unsetindex!(A::Array{T}, i::Int) where {T}
  @boundscheck checkbounds(A, i)
+ t = @_gc_preserve_begin A
+ p = Ptr{Ptr{Cvoid}}(pointer(A, i))
  if !allocatedinline(T)
- t = @_gc_preserve_begin A
- p = Ptr{Ptr{Cvoid}}(pointer(A))
- unsafe_store!(p, C_NULL, i)
- @_gc_preserve_end t
+ unsafe_store!(p, C_NULL)
+ elseif T isa DataType
+ if !datatype_pointerfree(T)
+ for j = 1:(Core.sizeof(T) ÷ Core.sizeof(Ptr{Cvoid}))
+ unsafe_store!(p, C_NULL, j)
+ end
+ end
  end
+ @_gc_preserve_end t
  return A
 end
 
@@ -255,19 +261,41 @@ the same manner as C.
 function unsafe_copyto!(dest::Array{T}, doffs, src::Array{T}, soffs, n) where T
  t1 = @_gc_preserve_begin dest
  t2 = @_gc_preserve_begin src
- if isbitsunion(T)
+ destp = pointer(dest, doffs)
+ srcp = pointer(src, soffs)
+ if !allocatedinline(T)
+ ccall(:jl_array_ptr_copy, Cvoid, (Any, Ptr{Cvoid}, Any, Ptr{Cvoid}, Int),
+ dest, destp, src, srcp, n)
+ elseif isbitstype(T)
  ccall(:memmove, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, UInt),
- pointer(dest, doffs), pointer(src, soffs), n * aligned_sizeof(T))
+ destp, srcp, n * aligned_sizeof(T))
+ elseif isbitsunion(T)
+ ccall(:memmove, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, UInt),
+ destp, srcp, n * aligned_sizeof(T))
  # copy selector bytes
  ccall(:memmove, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, UInt),
  ccall(:jl_array_typetagdata, Ptr{UInt8}, (Any,), dest) + doffs - 1,
  ccall(:jl_array_typetagdata, Ptr{UInt8}, (Any,), src) + soffs - 1,
  n)
- elseif allocatedinline(T)
- unsafe_copyto!(pointer(dest, doffs), pointer(src, soffs), n)
  else
- ccall(:jl_array_ptr_copy, Cvoid, (Any, Ptr{Cvoid}, Any, Ptr{Cvoid}, Int),
- dest, pointer(dest, doffs), src, pointer(src, soffs), n)
+ # handle base-case: everything else above was just optimizations
+ @inbounds if destp < srcp || destp > srcp + n
+ for i = 1:n
+ if isassigned(src, soffs + i - 1)
+ dest[doffs + i - 1] = src[soffs + i - 1]
+ else
+ _unsetindex!(dest, doffs + i - 1)
+ end
+ end
+ else
+ for i = n:-1:1
+ if isassigned(src, soffs + i - 1)
+ dest[doffs + i - 1] = src[soffs + i - 1]
+ else
+ _unsetindex!(dest, doffs + i - 1)
+ end
+ end
+ end
  end
  @_gc_preserve_end t2
  @_gc_preserve_end t1
@@ -1566,32 +1594,13 @@ function vcat(arrays::Vector{T}...) where T
  n += length(a)
  end
  arr = Vector{T}(undef, n)
- ptr = pointer(arr)
- if isbitsunion(T)
- selptr = ccall(:jl_array_typetagdata, Ptr{UInt8}, (Any,), arr)
- end
- elsz = aligned_sizeof(T)
- t = @_gc_preserve_begin arr
+ nd = 1
  for a in arrays
  na = length(a)
- nba = na * elsz
- if isbitsunion(T)
- ccall(:memcpy, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, UInt),
- ptr, a, nba)
- # copy selector bytes
- ccall(:memcpy, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, UInt),
- selptr, ccall(:jl_array_typetagdata, Ptr{UInt8}, (Any,), a), na)
- selptr += na
- elseif allocatedinline(T)
- ccall(:memcpy, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, UInt),
- ptr, a, nba)
- else
- ccall(:jl_array_ptr_copy, Cvoid, (Any, Ptr{Cvoid}, Any, Ptr{Cvoid}, Int),
- arr, ptr, a, pointer(a), na)
- end
- ptr += nba
+ @assert nd + na <= 1 + length(arr) # Concurrent modification of arrays?
+ unsafe_copyto!(arr, nd, a, 1, na)
+ nd += na
  end
- @_gc_preserve_end t
  return arr
 end
 

base/reflection.jl

@@ -318,6 +318,7 @@ datatype_fieldtypes(x::DataType) = ccall(:jl_get_fieldtypes, Any, (Any,), x)
 struct DataTypeLayout
  nfields::UInt32
  npointers::UInt32
+ firstptr::Int32
  alignment::UInt32
  # alignment : 9;
  # haspadding : 1;

base/refpointer.jl

@@ -74,7 +74,7 @@ RefArray(x::AbstractArray{T}, i::Int=1, roots::Nothing=nothing) where {T} = RefA
 convert(::Type{Ref{T}}, x::AbstractArray{T}) where {T} = RefArray(x, 1)
 
 function unsafe_convert(P::Type{Ptr{T}}, b::RefArray{T}) where T
- if datatype_pointerfree(RefValue{T})
+ if allocatedinline(T)
  p = pointer(b.x, b.i)
  elseif isconcretetype(T) && T.mutable
  p = pointer_from_objref(b.x[b.i])

base/refvalue.jl

@@ -11,7 +11,7 @@ RefValue(x::T) where {T} = RefValue{T}(x)
 isassigned(x::RefValue) = isdefined(x, :x)
 
 function unsafe_convert(P::Type{Ptr{T}}, b::RefValue{T}) where T
- if datatype_pointerfree(RefValue{T})
+ if allocatedinline(T)
  p = pointer_from_objref(b)
  elseif isconcretetype(T) && T.mutable
  p = pointer_from_objref(b.x)

src/array.c

@@ -27,12 +27,6 @@ char *jl_array_typetagdata(jl_array_t *a) JL_NOTSAFEPOINT
  return ((char*)jl_array_data(a)) + ((jl_array_ndims(a) == 1 ? (a->maxsize - a->offset) : jl_array_len(a)) * a->elsize) + a->offset;
 }
 
-JL_DLLEXPORT int jl_array_store_unboxed(jl_value_t *eltype) JL_NOTSAFEPOINT
-{
- size_t fsz = 0, al = 0;
- return jl_islayout_inline(eltype, &fsz, &al);
-}
-
 STATIC_INLINE jl_value_t *jl_array_owner(jl_array_t *a JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT
 {
  if (a->flags.how == 3) {
@@ -53,7 +47,7 @@ size_t jl_arr_xtralloc_limit = 0;
 #define MAXINTVAL (((size_t)-1)>>1)
 
 static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims,
- int isunboxed, int isunion, int elsz)
+ int isunboxed, int hasptr, int isunion, int elsz)
 {
  jl_ptls_t ptls = jl_get_ptls_states();
  size_t i, tot, nel=1;
@@ -101,7 +95,7 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims,
  // No allocation or safepoint allowed after this
  a->flags.how = 0;
  data = (char*)a + doffs;
- if ((tot > 0 && !isunboxed) || isunion)
+ if (tot > 0 && (!isunboxed || hasptr || isunion)) // TODO: check for zeroinit
  memset(data, 0, tot);
  }
  else {
@@ -113,7 +107,7 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims,
  // No allocation or safepoint allowed after this
  a->flags.how = 2;
  jl_gc_track_malloced_array(ptls, a);
- if (!isunboxed || isunion)
+ if (tot > 0 && (!isunboxed || hasptr || isunion)) // TODO: check for zeroinit
  // need to zero out isbits union array selector bytes to ensure a valid type index
  memset(data, 0, tot);
  }
@@ -127,6 +121,7 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims,
 #endif
  a->flags.ndims = ndims;
  a->flags.ptrarray = !isunboxed;
+ a->flags.hasptr = hasptr;
  a->elsize = elsz;
  a->flags.isshared = 0;
  a->flags.isaligned = 1;
@@ -135,9 +130,12 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims,
  a->nrows = nel;
  a->maxsize = nel;
  }
+ else if (a->flags.ndims != ndims) {
+ jl_exceptionf(jl_argumenterror_type, "invalid Array dimensions");
+ }
  else {
  size_t *adims = &a->nrows;
- for(i=0; i < ndims; i++)
+ for (i = 0; i < ndims; i++)
  adims[i] = dims[i];
  }
 
@@ -152,6 +150,7 @@ static inline jl_array_t *_new_array(jl_value_t *atype, uint32_t ndims, size_t *
  jl_type_error_rt("Array", "element type", (jl_value_t*)jl_type_type, eltype);
  int isunboxed = jl_islayout_inline(eltype, &elsz, &al);
  int isunion = jl_is_uniontype(eltype);
+ int hasptr = isunboxed && (jl_is_datatype(eltype) && ((jl_datatype_t*)eltype)->layout->npointers > 0);
  if (!isunboxed) {
  elsz = sizeof(void*);
  al = elsz;
@@ -160,13 +159,13 @@ static inline jl_array_t *_new_array(jl_value_t *atype, uint32_t ndims, size_t *
  elsz = LLT_ALIGN(elsz, al);
  }
 
- return _new_array_(atype, ndims, dims, isunboxed, isunion, elsz);
+ return _new_array_(atype, ndims, dims, isunboxed, hasptr, isunion, elsz);
 }
 
 jl_array_t *jl_new_array_for_deserialization(jl_value_t *atype, uint32_t ndims, size_t *dims,
- int isunboxed, int isunion, int elsz)
+ int isunboxed, int hasptr, int isunion, int elsz)
 {
- return _new_array_(atype, ndims, dims, isunboxed, isunion, elsz);
+ return _new_array_(atype, ndims, dims, isunboxed, hasptr, isunion, elsz);
 }
 
 #ifndef JL_NDEBUG
@@ -224,10 +223,12 @@ JL_DLLEXPORT jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data,
  "reinterpret from alignment %d bytes to alignment %d bytes not allowed",
  (int) oldalign, (int) align);
  a->flags.ptrarray = 0;
+ a->flags.hasptr = data->flags.hasptr;
  }
  else {
  a->elsize = sizeof(void*);
  a->flags.ptrarray = 1;
+ a->flags.hasptr = 0;
  }
 
  // if data is itself a shared wrapper,
@@ -247,6 +248,9 @@ JL_DLLEXPORT jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data,
  a->nrows = l;
  a->maxsize = l;
  }
+ else if (a->flags.ndims != ndims) {
+ jl_exceptionf(jl_argumenterror_type, "invalid Array dimensions");
+ }
  else {
  size_t *adims = &a->nrows;
  size_t l = 1;
@@ -281,6 +285,7 @@ JL_DLLEXPORT jl_array_t *jl_string_to_array(jl_value_t *str)
  a->flags.isaligned = 0;
  a->elsize = 1;
  a->flags.ptrarray = 0;
+ a->flags.hasptr = 0;
  jl_array_data_owner(a) = str;
  a->flags.how = 3;
  a->flags.isshared = 1;
@@ -300,12 +305,12 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data,
  jl_array_t *a;
  jl_value_t *eltype = jl_tparam0(atype);
 
- int isunboxed = jl_array_store_unboxed(eltype);
- size_t elsz;
- unsigned align;
+ int isunboxed = jl_stored_inline(eltype);
  if (isunboxed && jl_is_uniontype(eltype))
  jl_exceptionf(jl_argumenterror_type,
  "unsafe_wrap: unspecified layout for union element type");
+ size_t elsz;
+ unsigned align;
  if (isunboxed) {
  elsz = jl_datatype_size(eltype);
  align = jl_datatype_align(eltype);
@@ -328,6 +333,7 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data,
 #endif
  a->elsize = LLT_ALIGN(elsz, align);
  a->flags.ptrarray = !isunboxed;
+ a->flags.hasptr = isunboxed && (jl_is_datatype(eltype) && ((jl_datatype_t*)eltype)->layout->npointers > 0);
  a->flags.ndims = 1;
  a->flags.isshared = 1;
  a->flags.isaligned = 0; // TODO: allow passing memalign'd buffers
@@ -366,12 +372,12 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data,
  return jl_ptr_to_array_1d(atype, data, nel, own_buffer);
  jl_value_t *eltype = jl_tparam0(atype);
 
- int isunboxed = jl_array_store_unboxed(eltype);
- size_t elsz;
- unsigned align;
+ int isunboxed = jl_stored_inline(eltype);
  if (isunboxed && jl_is_uniontype(eltype))
  jl_exceptionf(jl_argumenterror_type,
  "unsafe_wrap: unspecified layout for union element type");
+ size_t elsz;
+ unsigned align;
  if (isunboxed) {
  elsz = jl_datatype_size(eltype);
  align = jl_datatype_align(eltype);
@@ -394,6 +400,7 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data,
 #endif
  a->elsize = LLT_ALIGN(elsz, align);
  a->flags.ptrarray = !isunboxed;
+ a->flags.hasptr = isunboxed && (jl_is_datatype(eltype) && ((jl_datatype_t*)eltype)->layout->npointers > 0);
  a->flags.ndims = ndims;
  a->offset = 0;
  a->flags.isshared = 1;
@@ -408,6 +415,8 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data,
  }
 
  assert(ndims != 1); // handled above
+ if (a->flags.ndims != ndims)
+ jl_exceptionf(jl_argumenterror_type, "invalid Array dimensions");
  memcpy(&a->nrows, dims, ndims * sizeof(size_t));
  return a;
 }
@@ -559,8 +568,16 @@ JL_DLLEXPORT jl_value_t *jl_arrayref(jl_array_t *a, size_t i)
 
 JL_DLLEXPORT int jl_array_isassigned(jl_array_t *a, size_t i)
 {
- if (a->flags.ptrarray)
+ if (a->flags.ptrarray) {
  return ((jl_value_t**)jl_array_data(a))[i] != NULL;
+ }
+ else if (a->flags.hasptr) {
+ jl_datatype_t *eltype = (jl_datatype_t*)jl_tparam0(jl_typeof(a));
+ assert(eltype->layout->first_ptr >= 0);
+ jl_value_t **slot =
+ (jl_value_t**)(&((char*)a->data)[i*a->elsize] + eltype->layout->first_ptr);
+ return *slot != NULL;
+ }
  return 1;
 }
 
@@ -585,6 +602,8 @@ JL_DLLEXPORT void jl_arrayset(jl_array_t *a JL_ROOTING_ARGUMENT, jl_value_t *rhs
  return;
  }
  jl_assign_bits(&((char*)a->data)[i * a->elsize], rhs);
+ if (a->flags.hasptr)
+ jl_gc_multi_wb(jl_array_owner(a), rhs);
  }
  else {
  ((jl_value_t**)a->data)[i] = rhs;
@@ -598,6 +617,11 @@ JL_DLLEXPORT void jl_arrayunset(jl_array_t *a, size_t i)
  jl_bounds_error_int((jl_value_t*)a, i + 1);
  if (a->flags.ptrarray)
  ((jl_value_t**)a->data)[i] = NULL;
+ else if (a->flags.hasptr) {
+ size_t elsize = a->elsize;
+ jl_assume(elsize >= sizeof(void*) && elsize % sizeof(void*) == 0);
+ memset(&((jl_value_t**)a->data)[i], 0, elsize);
+ }
 }
 
 // at this size and bigger, allocate resized array data with malloc directly
@@ -809,7 +833,7 @@ STATIC_INLINE void jl_array_grow_at_beg(jl_array_t *a, size_t idx, size_t inc,
 #endif
  a->nrows = newnrows;
  a->data = newdata;
- if (a->flags.ptrarray) {
+ if (a->flags.ptrarray || a->flags.hasptr) { // TODO: check for zeroinit
  memset(newdata + idx * elsz, 0, nbinc);
  }
  else if (isbitsunion) {
@@ -890,7 +914,7 @@ STATIC_INLINE void jl_array_grow_at_end(jl_array_t *a, size_t idx,
  a->length = newnrows;
 #endif
  a->nrows = newnrows;
- if (a->flags.ptrarray) {
+ if (a->flags.ptrarray || a->flags.hasptr) { // TODO: check for zeroinit
  memset(data + idx * elsz, 0, inc * elsz);
  }
 }
@@ -1129,7 +1153,7 @@ JL_DLLEXPORT jl_array_t *jl_array_copy(jl_array_t *ary)
  int isunion = jl_is_uniontype(jl_tparam0(jl_typeof(ary)));
  jl_array_t *new_ary = _new_array_(jl_typeof(ary), jl_array_ndims(ary),
  &ary->nrows, !ary->flags.ptrarray,
- isunion, elsz);
+ ary->flags.hasptr, isunion, elsz);
  memcpy(new_ary->data, ary->data, len * elsz);
  // ensure isbits union arrays copy their selector bytes correctly
  if (jl_array_isbitsunion(ary))