Instrument jl_gc_big_alloc() when called from generated code for Allo…

…cations Profiler (#44043)

* Instrument jl_gc_big_alloc() when called from generated code for Allocations Profiler

Add _maybe_record_alloc_to_profile() call into jl_gc_big_alloc() to
allow us to instrument it in the Allocations Profiler.

Followup to #43868

Finishes fixing #43688 (gets the stacks, though not the types, of big allocs).

-----

- Rename jl_gc_big_alloc to jl_gc_big_alloc_inner, make it inlined
- add jl_gc_pool_alloc_noinline that calls jl_gc_big_alloc_inner
    - replace all internal calls to jl_gc_big_alloc to call this instead
- add a new jl_gc_pool_alloc that calls jl_gc_big_alloc_inner
    - This one is instrumented, and is meant to be called by generated
      code.

Co-authored-by: Pete Vilter <[email protected]>

src/array.c

@@ -507,7 +507,7 @@ JL_DLLEXPORT jl_value_t *jl_alloc_string(size_t len)
  else {
  if (allocsz < sz) // overflow in adding offs, size was "negative"
  jl_throw(jl_memory_exception);
- s = jl_gc_big_alloc(ptls, allocsz);
+ s = jl_gc_big_alloc_noinline(ptls, allocsz);
  }
  jl_set_typeof(s, jl_string_type);
  maybe_record_alloc_to_profile(s, len, jl_string_type);

src/gc.c

@@ -944,7 +944,7 @@ static void sweep_weak_refs(void)
 // big value list
 
 // Size includes the tag and the tag is not cleared!!
-JL_DLLEXPORT jl_value_t *jl_gc_big_alloc(jl_ptls_t ptls, size_t sz)
+static inline jl_value_t *jl_gc_big_alloc_inner(jl_ptls_t ptls, size_t sz)
 {
  maybe_collect(ptls);
  size_t offs = offsetof(bigval_t, header);
@@ -972,6 +972,22 @@ JL_DLLEXPORT jl_value_t *jl_gc_big_alloc(jl_ptls_t ptls, size_t sz)
  return jl_valueof(&v->header);
 }
 
+// Instrumented version of jl_gc_big_alloc_inner, called into by LLVM-generated code.
+JL_DLLEXPORT jl_value_t *jl_gc_big_alloc(jl_ptls_t ptls, size_t sz)
+{
+ jl_value_t *val = jl_gc_big_alloc_inner(ptls, sz);
+
+ maybe_record_alloc_to_profile(val, sz, jl_gc_unknown_type_tag);
+ return val;
+}
+
+// This wrapper exists only to prevent `jl_gc_big_alloc_inner` from being inlined into
+// its callers. We provide an external-facing interface for callers, and inline `jl_gc_big_alloc_inner`
+// into this. (See https://github.com/JuliaLang/julia/pull/43868 for more details.)
+jl_value_t *jl_gc_big_alloc_noinline(jl_ptls_t ptls, size_t sz) {
+ return jl_gc_big_alloc_inner(ptls, sz);
+}
+
 // Sweep list rooted at *pv, removing and freeing any unmarked objects.
 // Return pointer to last `next` field in the culled list.
 static bigval_t **sweep_big_list(int sweep_full, bigval_t **pv) JL_NOTSAFEPOINT

src/julia_internal.h

@@ -231,7 +231,7 @@ JL_DLLEXPORT extern const char *jl_filename;
 
 jl_value_t *jl_gc_pool_alloc_noinline(jl_ptls_t ptls, int pool_offset,
  int osize);
-JL_DLLEXPORT jl_value_t *jl_gc_big_alloc(jl_ptls_t ptls, size_t allocsz);
+jl_value_t *jl_gc_big_alloc_noinline(jl_ptls_t ptls, size_t allocsz);
 JL_DLLEXPORT int jl_gc_classify_pools(size_t sz, int *osize);
 extern uv_mutex_t gc_perm_lock;
 void *jl_gc_perm_alloc_nolock(size_t sz, int zero,
@@ -365,7 +365,7 @@ STATIC_INLINE jl_value_t *jl_gc_alloc_(jl_ptls_t ptls, size_t sz, void *ty)
  else {
  if (allocsz < sz) // overflow in adding offs, size was "negative"
  jl_throw(jl_memory_exception);
- v = jl_gc_big_alloc(ptls, allocsz);
+ v = jl_gc_big_alloc_noinline(ptls, allocsz);
  }
  jl_set_typeof(v, ty);
  maybe_record_alloc_to_profile(v, sz, (jl_datatype_t*)ty);