Merge pull request JuliaLang#11253 from JuliaLang/ob/fixsawb

Fix write barrier for shared arrays.

src/array.c

@@ -505,7 +505,11 @@ void jl_arrayset(jl_array_t *a, jl_value_t *rhs, size_t i)
  }
  else {
  ((jl_value_t**)a->data)[i] = rhs;
- gc_wb(a, rhs);
+ jl_value_t *owner = (jl_value_t*)a;
+ if (a->how == 3) {
+ owner = jl_array_data_owner(a);
+ }
+ gc_wb(owner, rhs);
  }
 }
 

src/cgutils.cpp

@@ -1381,6 +1381,16 @@ static Value *emit_arraysize(Value *t, jl_value_t *ex, int dim, jl_codectx_t *ct
  return emit_arraysize(t, dim);
 }
 
+static Value *emit_arrayflags(Value *t, jl_codectx_t *ctx)
+{
+ Value *addr = builder.CreateStructGEP(
+#ifdef LLVM37
+ nullptr,
+#endif
+ builder.CreateBitCast(t,jl_parray_llvmt), 2);
+ return builder.CreateLoad(addr); // TODO tbaa
+}
+
 static void assign_arrayvar(jl_arrayvar_t &av, Value *ar)
 {
  tbaa_decorate(tbaa_arrayptr,builder.CreateStore(builder.CreateBitCast(emit_arrayptr(ar),

src/codegen.cpp

@@ -2181,16 +2181,38 @@ static Value *emit_known_call(jl_value_t *ff, jl_value_t **args, size_t nargs,
  Value *ary = emit_expr(args[1], ctx);
  size_t nd = jl_is_long(ndp) ? jl_unbox_long(ndp) : 1;
  Value *idx = emit_array_nd_index(ary, args[1], nd, &args[3], nargs-2, ctx);
- if (jl_array_store_unboxed(ety) &&
-   ((jl_datatype_t*)ety)->size == 0) {
+ bool isboxed = !jl_array_store_unboxed(ety);
+ if (!isboxed && ((jl_datatype_t*)ety)->size == 0) {
  // no-op, but emit expr for possible effects
  assert(jl_is_datatype(ety));
  emit_expr(args[2],ctx,false);
  }
  else {
  Value* v = ety==(jl_value_t*)jl_any_type ? emit_expr(args[2],ctx) : emit_unboxed(args[2],ctx);
+ PHINode* data_owner = NULL; // owner object against which the write barrier must check
+ if (isboxed) { // if not boxed we don't need a write barrier
+ Value *flags = emit_arrayflags(ary,ctx);
+ // the owner of the data is ary itself except if ary->how == 3
+ flags = builder.CreateAnd(flags, 3);
+ Value *is_owned = builder.CreateICmpEQ(flags, ConstantInt::get(T_int16, 3));
+ BasicBlock *curBB = builder.GetInsertBlock();
+ BasicBlock *ownedBB = BasicBlock::Create(getGlobalContext(), "array_owned", ctx->f);
+ BasicBlock *mergeBB = BasicBlock::Create(getGlobalContext(), "merge_own", ctx->f);
+ builder.CreateCondBr(is_owned, ownedBB, mergeBB);
+ builder.SetInsertPoint(ownedBB);
+ // load owner pointer
+ Value *own_ptr = builder.CreateLoad(
+ builder.CreateBitCast(builder.CreateConstGEP1_32(
+ builder.CreateBitCast(ary,T_pint8), jl_array_data_owner_offset(nd)),
+ jl_ppvalue_llvmt));
+ builder.CreateBr(mergeBB);
+ builder.SetInsertPoint(mergeBB);
+ data_owner = builder.CreatePHI(jl_pvalue_llvmt, 2);
+ data_owner->addIncoming(ary, curBB);
+ data_owner->addIncoming(own_ptr, ownedBB);
+ }
  typed_store(emit_arrayptr(ary,args[1],ctx), idx, v,
- ety, ctx, tbaa_user, ety == (jl_value_t*)jl_any_type ? ary : NULL);
+ ety, ctx, tbaa_user, data_owner);
  }
  JL_GC_POP();
  return ary;
@@ -4834,6 +4856,7 @@ static void init_julia_llvm_env(Module *m)
 #ifdef STORE_ARRAY_LEN
  , T_size
 #endif
+ , T_int16
  };
  Type* jl_array_llvmt =
  StructType::create(jl_LLVMContext,

src/gc.c

@@ -2036,7 +2036,7 @@ static void gc_verify(void)
  jl_printf(JL_STDERR, "val : ");
  jl_(jl_valueof(v));
  jl_printf(JL_STDERR, "Let's try to backtrack the missing write barrier :\n");
- lostval = v;
+ lostval = jl_valueof(v);
  break;
  }
  }

src/julia.h

@@ -145,23 +145,26 @@ typedef struct {
 #ifdef STORE_ARRAY_LEN
  size_t length;
 #endif
-
- unsigned short ndims:10;
- unsigned short pooled:1;
- unsigned short ptrarray:1; // representation is pointer array
- /*
- how - allocation style
- 0 = data is inlined, or a foreign pointer we don't manage
- 1 = julia-allocated buffer that needs to be marked
- 2 = malloc-allocated pointer this array object manages
- 3 = has a pointer to the Array that owns the data
- */
- unsigned short how:2;
- unsigned short isshared:1; // data is shared by multiple Arrays
- unsigned short isaligned:1; // data allocated with memalign
+ union {
+ struct {
+ /*
+ how - allocation style
+ 0 = data is inlined, or a foreign pointer we don't manage
+ 1 = julia-allocated buffer that needs to be marked
+ 2 = malloc-allocated pointer this array object manages
+ 3 = has a pointer to the Array that owns the data
+ */
+ unsigned short how:2;
+ unsigned short ndims:10;
+ unsigned short pooled:1;
+ unsigned short ptrarray:1; // representation is pointer array
+ unsigned short isshared:1; // data is shared by multiple Arrays
+ unsigned short isaligned:1; // data allocated with memalign
+ };
+ unsigned short flags;
+ };
  uint16_t elsize;
  uint32_t offset; // for 1-d only. does not need to get big.
-
  size_t nrows;
  union {
  // 1d
@@ -677,7 +680,8 @@ DLLEXPORT size_t jl_array_len_(jl_array_t *a);
 #define jl_array_dim0(a) (((jl_array_t*)(a))->nrows)
 #define jl_array_nrows(a) (((jl_array_t*)(a))->nrows)
 #define jl_array_ndims(a) ((int32_t)(((jl_array_t*)a)->ndims))
-#define jl_array_data_owner(a) (*((jl_value_t**)(&a->ncols+1+jl_array_ndimwords(jl_array_ndims(a)))))
+#define jl_array_data_owner_offset(ndims) (offsetof(jl_array_t,ncols) + sizeof(size_t)*(1+jl_array_ndimwords(ndims))) // in bytes
+#define jl_array_data_owner(a) (*((jl_value_t**)((char*)a + jl_array_data_owner_offset(jl_array_ndims(a)))))
 
 STATIC_INLINE jl_value_t *jl_cellref(void *a, size_t i)
 {
@@ -688,7 +692,12 @@ STATIC_INLINE jl_value_t *jl_cellset(void *a, size_t i, void *x)
 {
  assert(i < jl_array_len(a));
  ((jl_value_t**)(jl_array_data(a)))[i] = (jl_value_t*)x;
- if (x) gc_wb(a, x);
+ if (x) {
+ if (((jl_array_t*)a)->how == 3) {
+ a = jl_array_data_owner(a);
+ }
+ gc_wb(a, x);
+ }
  return (jl_value_t*)x;
 }