Generational behavior for the garbage collector.

base/base.jl

@@ -154,7 +154,7 @@ end
 
 finalize(o::ANY) = ccall(:jl_finalize, Void, (Any,), o)
 
-gc() = ccall(:jl_gc_collect, Void, ())
+gc(full = true) = ccall(:jl_gc_collect, Void, (Int,), full ? 1 : 0)
 gc_enable() = ccall(:jl_gc_enable, Void, ())
 gc_disable() = ccall(:jl_gc_disable, Void, ())
 

base/util.jl

@@ -12,6 +12,9 @@ gc_time_ns() = ccall(:jl_gc_total_hrtime, UInt64, ())
 # total number of bytes allocated so far
 gc_bytes() = ccall(:jl_gc_total_bytes, Int64, ())
 
+gc_num_pause() = ccall(:jl_gc_num_pause, Int64, ())
+gc_num_full_sweep() = ccall(:jl_gc_num_full_sweep, Int64, ())
+
 function tic()
  t0 = time_ns()
  task_local_storage(:TIMERS, (t0, get(task_local_storage(), :TIMERS, ())))
@@ -36,12 +39,17 @@ function toc()
 end
 
 # print elapsed time, return expression value
-
-function time_print(t, b, g)
+const _units = ["bytes", "kB", "MB"]
+function time_print(t, b, g, np, nfs)
+ i = 1
+ while b > 1024 && i < length(_units)
+ b = div(b, 1024)
+ i += 1
+ end
  if 0 < g
- @printf("elapsed time: %s seconds (%d bytes allocated, %.2f%% gc time)\n", t/1e9, b, 100*g/t)
+ @printf("elapsed time: %s seconds (%d %s allocated, %.2f%% gc time in %d pauses with %d full sweep)\n", t/1e9, b, _units[i], 100*g/t, np, nfs)
  else
- @printf("elapsed time: %s seconds (%d bytes allocated)\n", t/1e9, b)
+ @printf("elapsed time: %s seconds (%d %s allocated)\n", t/1e9, b, _units[i])
  end
 end
 
@@ -50,11 +58,15 @@ macro time(ex)
  local b0 = gc_bytes()
  local t0 = time_ns()
  local g0 = gc_time_ns()
+ local n0 = gc_num_pause()
+ local nfs0 = gc_num_full_sweep()
  local val = $(esc(ex))
+ local nfs1 = gc_num_full_sweep()
+ local n1 = gc_num_pause()
  local g1 = gc_time_ns()
  local t1 = time_ns()
  local b1 = gc_bytes()
- time_print(t1-t0, b1-b0, g1-g0)
+ time_print(t1-t0, b1-b0, g1-g0, n1-n0, nfs1-nfs0)
  val
  end
 end

doc/manual/embedding.rst

@@ -107,6 +107,23 @@ Several Julia values can be pushed at once using the ``JL_GC_PUSH2`` , ``JL_GC_P
  // Do something with args (e.g. call jl_... functions)
  JL_GC_POP();
 
+The garbage collector also operates under the assumption that it is aware of every old-generation object pointing to a young-generation one. Any time a pointer is updated breaking that assumption, it must be signaled to the collector with the ``gc_wb`` (write barrier) function like so::
+
+ jl_value_t *parent = some_old_value, *child = some_young_value;
+ ((some_specific_type*)parent)->field = child;
+ gc_wb(parent, child);
+
+It is in general impossible to predict which values will be old at runtime, so the write barrier must be inserted after all explicit stores. One notable exception is if the ``parent`` object was just allocated and garbage collection was not run since then. Remember that most ``jl_...`` functions can sometimes invoke garbage collection.
+
+The write barrier is also necessary for arrays of pointers when updating their data directly. For example::
+
+ jl_array_t *some_array = ...; // e.g. a Vector{Any}
+ void **data = (void**)jl_array_data(some_array);
+ jl_value_t *some_value = ...;
+ data[0] = some_value;
+ gc_wb(some_array, some_value);
+
+
 Manipulating the Garbage Collector
 ---------------------------------------------------
 

src/alloc.c

@@ -278,6 +278,7 @@ void jl_set_nth_field(jl_value_t *v, size_t i, jl_value_t *rhs)
  size_t offs = jl_field_offset(st,i) + sizeof(void*);
  if (st->fields[i].isptr) {
  *(jl_value_t**)((char*)v + offs) = rhs;
+ if(rhs != NULL) gc_wb(v, rhs);
  }
  else {
  jl_assign_bits((char*)v + offs, rhs);
@@ -521,17 +522,18 @@ static jl_sym_t *mk_symbol(const char *str)
 static void unmark_symbols_(jl_sym_t *root)
 {
  while (root != NULL) {
- root->type = (jl_value_t*)(((uptrint_t)root->type)&~1UL);
+ root->type = (jl_value_t*)(((uptrint_t)root->type)&~3UL);
  unmark_symbols_(root->left);
  root = root->right;
  }
 }
 
 void jl_unmark_symbols(void) { unmark_symbols_(symtab); }
 
-static jl_sym_t **symtab_lookup(jl_sym_t **ptree, const char *str)
+static jl_sym_t **symtab_lookup(jl_sym_t **ptree, const char *str, jl_sym_t **parent)
 {
  int x;
+ if (parent != NULL) *parent = NULL;
  uptrint_t h = hash_symbol(str, strlen(str));
 
  // Tree nodes sorted by major key of (int(hash)) and minor key o (str).
@@ -542,6 +544,7 @@ static jl_sym_t **symtab_lookup(jl_sym_t **ptree, const char *str)
  if (x == 0)
  return ptree;
  }
+ if (parent != NULL) *parent = *ptree;
  if (x < 0)
  ptree = &(*ptree)->left;
  else
@@ -553,16 +556,19 @@ static jl_sym_t **symtab_lookup(jl_sym_t **ptree, const char *str)
 jl_sym_t *jl_symbol(const char *str)
 {
  jl_sym_t **pnode;
-
- pnode = symtab_lookup(&symtab, str);
- if (*pnode == NULL)
+ jl_sym_t *parent;
+ pnode = symtab_lookup(&symtab, str, &parent);
+ if (*pnode == NULL) {
  *pnode = mk_symbol(str);
+ if (parent != NULL)
+ gc_wb(parent, *pnode);
+ }
  return *pnode;
 }
 
 jl_sym_t *jl_symbol_lookup(const char *str)
 {
- return *symtab_lookup(&symtab, str);
+ return *symtab_lookup(&symtab, str, NULL);
 }
 
 DLLEXPORT jl_sym_t *jl_symbol_n(const char *str, int32_t len)
@@ -696,12 +702,15 @@ jl_datatype_t *jl_new_datatype(jl_sym_t *name, jl_datatype_t *super,
  t = jl_new_uninitialized_datatype(jl_tuple_len(fnames));
  else
  tn = t->name;
-
  // init before possibly calling jl_new_typename
  t->super = super;
+ if(super != NULL) gc_wb(t, t->super);
  t->parameters = parameters;
+ gc_wb(t, t->parameters);
  t->names = fnames;
+ gc_wb(t, t->names);
  t->types = ftypes;
+ if(ftypes != NULL) gc_wb(t, t->types);
  t->abstract = abstract;
  t->mutabl = mutabl;
  t->pointerfree = 0;
@@ -718,10 +727,13 @@ jl_datatype_t *jl_new_datatype(jl_sym_t *name, jl_datatype_t *super,
  else
  tn = jl_new_typename((jl_sym_t*)name);
  t->name = tn;
+ gc_wb(t, t->name);
  }
 
- if (t->name->primary == NULL)
+ if (t->name->primary == NULL) {
  t->name->primary = (jl_value_t*)t;
+ gc_wb(t->name, t);
+ }
 
  if (abstract || jl_tuple_len(parameters) > 0) {
  t->uid = 0;

src/array.c

@@ -99,6 +99,7 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims,
  memset(data, 0, tot);
  JL_GC_POP();
  }
+ a->pooled = tsz <= 2048;
 
  a->data = data;
  if (elsz == 1) ((char*)data)[tot-1] = '\0';
@@ -147,8 +148,10 @@ jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data, jl_tuple_t *di
  size_t ndims = jl_tuple_len(dims);
 
  int ndimwords = jl_array_ndimwords(ndims);
- a = (jl_array_t*)allocobj((sizeof(jl_array_t) + sizeof(void*) + ndimwords*sizeof(size_t) + 15)&-16);
+ int tsz = (sizeof(jl_array_t) + sizeof(void*) + ndimwords*sizeof(size_t) + 15)&-16;
+ a = (jl_array_t*)allocobj(tsz);
  a->type = atype;
+ a->pooled = tsz <= 2048;
  a->ndims = ndims;
  a->offset = 0;
  a->data = NULL;
@@ -211,8 +214,9 @@ jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data, size_t nel,
  elsz = jl_datatype_size(el_type);
  else
  elsz = sizeof(void*);
-
- a = (jl_array_t*)allocobj((sizeof(jl_array_t)+jl_array_ndimwords(1)*sizeof(size_t)+15)&-16);
+ int tsz = (sizeof(jl_array_t)+jl_array_ndimwords(1)*sizeof(size_t)+15)&-16;
+ a = (jl_array_t*)allocobj(tsz);
+ a->pooled = tsz <= 2048;
  a->type = atype;
  a->data = data;
 #ifdef STORE_ARRAY_LEN
@@ -226,6 +230,7 @@ jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data, size_t nel,
  if (own_buffer) {
  a->how = 2;
  jl_gc_track_malloced_array(a);
+ jl_gc_count_allocd(nel*elsz + (elsz == 1 ? 1 : 0));
  }
  else {
  a->how = 0;
@@ -260,7 +265,9 @@ jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data, jl_tuple_t *dims,
  elsz = sizeof(void*);
 
  int ndimwords = jl_array_ndimwords(ndims);
- a = (jl_array_t*)allocobj((sizeof(jl_array_t) + ndimwords*sizeof(size_t)+15)&-16);
+ int tsz = (sizeof(jl_array_t) + ndimwords*sizeof(size_t)+15)&-16;
+ a = (jl_array_t*)allocobj(tsz);
+ a->pooled = tsz <= 2048;
  a->type = atype;
  a->data = data;
 #ifdef STORE_ARRAY_LEN
@@ -275,6 +282,7 @@ jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data, jl_tuple_t *dims,
  if (own_buffer) {
  a->how = 2;
  jl_gc_track_malloced_array(a);
+ jl_gc_count_allocd(nel*elsz + (elsz == 1 ? 1 : 0));
  }
  else {
  a->how = 0;
@@ -502,6 +510,7 @@ void jl_arrayset(jl_array_t *a, jl_value_t *rhs, size_t i)
  }
  else {
  ((jl_value_t**)a->data)[i] = rhs;
+ gc_wb(a, rhs);
  }
 }
 
@@ -545,7 +554,7 @@ static void array_resize_buffer(jl_array_t *a, size_t newlen, size_t oldlen, siz
  if (a->how == 2) {
  // already malloc'd - use realloc
  newdata = (char*)jl_gc_managed_realloc((char*)a->data - oldoffsnb, nbytes,
- oldnbytes+oldoffsnb, a->isaligned);
+ oldnbytes+oldoffsnb, a->isaligned, (jl_value_t*)a);
  if (offs != a->offset) {
  memmove(&newdata[offsnb], &newdata[oldoffsnb], oldnbytes);
  }
@@ -574,6 +583,8 @@ static void array_resize_buffer(jl_array_t *a, size_t newlen, size_t oldlen, siz
  a->isshared = 0;
  if (a->ptrarray || es==1)
  memset(newdata+offsnb+oldnbytes, 0, nbytes-oldnbytes-offsnb);
+ if (a->how == 1)
+ gc_wb_buf(a, newdata);
  a->maxsize = newlen;
 }
 

src/ast.c

@@ -731,6 +731,7 @@ static jl_value_t *copy_ast(jl_value_t *expr, jl_tuple_t *sp, int do_sp)
  // of a top-level thunk that gets type inferred.
  li->def = li;
  li->ast = jl_prepare_ast(li, li->sparams);
+ gc_wb(li, li->ast);
  JL_GC_POP();
  return (jl_value_t*)li;
  }
@@ -749,17 +750,18 @@ static jl_value_t *copy_ast(jl_value_t *expr, jl_tuple_t *sp, int do_sp)
  jl_expr_t *ne = jl_exprn(e->head, jl_array_len(e->args));
  JL_GC_PUSH1(&ne);
  if (e->head == lambda_sym) {
- jl_exprarg(ne, 0) = copy_ast(jl_exprarg(e,0), sp, 0);
- jl_exprarg(ne, 1) = copy_ast(jl_exprarg(e,1), sp, 0);
- jl_exprarg(ne, 2) = copy_ast(jl_exprarg(e,2), sp, 1);
+ jl_exprargset(ne, 0, copy_ast(jl_exprarg(e,0), sp, 0));
+ jl_exprargset(ne, 1, copy_ast(jl_exprarg(e,1), sp, 0));
+ jl_exprargset(ne, 2, copy_ast(jl_exprarg(e,2), sp, 1));
  }
  else if (e->head == assign_sym) {
- jl_exprarg(ne, 0) = copy_ast(jl_exprarg(e,0), sp, 0);
- jl_exprarg(ne, 1) = copy_ast(jl_exprarg(e,1), sp, 1);
+ jl_exprargset(ne, 0, copy_ast(jl_exprarg(e,0), sp, 0));
+ jl_exprargset(ne, 1, copy_ast(jl_exprarg(e,1), sp, 1));
  }
  else {
- for(size_t i=0; i < jl_array_len(e->args); i++)
- jl_exprarg(ne, i) = copy_ast(jl_exprarg(e,i), sp, 1);
+ for(size_t i=0; i < jl_array_len(e->args); i++) {
+ jl_exprargset(ne, i, copy_ast(jl_exprarg(e,i), sp, 1));
+ }
  }
  JL_GC_POP();
  return (jl_value_t*)ne;
@@ -780,10 +782,12 @@ DLLEXPORT jl_value_t *jl_copy_ast(jl_value_t *expr)
  ne = jl_exprn(e->head, l);
  if (l == 0) {
  ne->args = jl_alloc_cell_1d(0);
+ gc_wb(ne, ne->args);
  }
  else {
- for(i=0; i < l; i++)
- jl_exprarg(ne, i) = jl_copy_ast(jl_exprarg(e,i));
+ for(i=0; i < l; i++) {
+ jl_exprargset(ne, i, jl_copy_ast(jl_exprarg(e,i)));
+ }
  }
  JL_GC_POP();
  return (jl_value_t*)ne;
@@ -820,17 +824,18 @@ static jl_value_t *dont_copy_ast(jl_value_t *expr, jl_tuple_t *sp, int do_sp)
  else if (jl_is_expr(expr)) {
  jl_expr_t *e = (jl_expr_t*)expr;
  if (e->head == lambda_sym) {
- jl_exprarg(e, 0) = dont_copy_ast(jl_exprarg(e,0), sp, 0);
- jl_exprarg(e, 1) = dont_copy_ast(jl_exprarg(e,1), sp, 0);
- jl_exprarg(e, 2) = dont_copy_ast(jl_exprarg(e,2), sp, 1);
+ jl_exprargset(e, 0, dont_copy_ast(jl_exprarg(e,0), sp, 0));
+ jl_exprargset(e, 1, dont_copy_ast(jl_exprarg(e,1), sp, 0));
+ jl_exprargset(e, 2, dont_copy_ast(jl_exprarg(e,2), sp, 1));
  }
  else if (e->head == assign_sym) {
- jl_exprarg(e, 0) = dont_copy_ast(jl_exprarg(e,0), sp, 0);
- jl_exprarg(e, 1) = dont_copy_ast(jl_exprarg(e,1), sp, 1);
+ jl_exprargset(e, 0, dont_copy_ast(jl_exprarg(e,0), sp, 0));
+ jl_exprargset(e, 1, dont_copy_ast(jl_exprarg(e,1), sp, 1));
  }
  else {
- for(size_t i=0; i < jl_array_len(e->args); i++)
- jl_exprarg(e, i) = dont_copy_ast(jl_exprarg(e,i), sp, 1);
+ for(size_t i=0; i < jl_array_len(e->args); i++) {
+ jl_exprargset(e, i, dont_copy_ast(jl_exprarg(e,i), sp, 1));
+ }
  }
  return (jl_value_t*)e;
  }