make pool_live_bytes metric more accurate (JuliaLang#52015)

`pool_live_bytes` was previously lazily updated during the GC, meaning
it was only accurate right after a GC.

Make this metric accurate if gathered after a GC has happened.

src/gc.c

@@ -758,7 +758,6 @@ int current_sweep_full = 0;
 int under_pressure = 0;
 
 // Full collection heuristics
-static int64_t pool_live_bytes = 0;
 static int64_t live_bytes = 0;
 static int64_t promoted_bytes = 0;
 static int64_t last_live_bytes = 0; // live_bytes at last collection
@@ -1333,6 +1332,8 @@ STATIC_INLINE jl_value_t *jl_gc_pool_alloc_inner(jl_ptls_t ptls, int pool_offset
  maybe_collect(ptls);
  jl_atomic_store_relaxed(&ptls->gc_num.allocd,
  jl_atomic_load_relaxed(&ptls->gc_num.allocd) + osize);
+ jl_atomic_store_relaxed(&ptls->gc_num.pool_live_bytes,
+ jl_atomic_load_relaxed(&ptls->gc_num.pool_live_bytes) + osize);
  jl_atomic_store_relaxed(&ptls->gc_num.poolalloc,
  jl_atomic_load_relaxed(&ptls->gc_num.poolalloc) + 1);
  // first try to use the freelist
@@ -1520,7 +1521,8 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag
  }
  }
  gc_time_count_page(freedall, pg_skpd);
- jl_atomic_fetch_add((_Atomic(int64_t) *)&pool_live_bytes, GC_PAGE_SZ - GC_PAGE_OFFSET - nfree * osize);
+ jl_ptls_t ptls = gc_all_tls_states[pg->thread_n];
+ jl_atomic_fetch_add(&ptls->gc_num.pool_live_bytes, GC_PAGE_SZ - GC_PAGE_OFFSET - nfree * osize);
  jl_atomic_fetch_add((_Atomic(int64_t) *)&gc_num.freed, (nfree - old_nfree) * osize);
 }
 
@@ -1642,6 +1644,7 @@ static void gc_sweep_pool(void)
  }
  continue;
  }
+ jl_atomic_store_relaxed(&ptls2->gc_num.pool_live_bytes, 0);
  for (int i = 0; i < JL_GC_N_POOLS; i++) {
  jl_gc_pool_t *p = &ptls2->heap.norm_pools[i];
  jl_taggedvalue_t *last = p->freelist;
@@ -3182,6 +3185,13 @@ JL_DLLEXPORT int64_t jl_gc_sync_total_bytes(int64_t offset) JL_NOTSAFEPOINT
 
 JL_DLLEXPORT int64_t jl_gc_pool_live_bytes(void)
 {
+ int64_t pool_live_bytes = 0;
+ for (int i = 0; i < gc_n_threads; i++) {
+ jl_ptls_t ptls2 = gc_all_tls_states[i];
+ if (ptls2 != NULL) {
+ pool_live_bytes += jl_atomic_load_relaxed(&ptls2->gc_num.pool_live_bytes);
+ }
+ }
  return pool_live_bytes;
 }
 
@@ -3346,7 +3356,6 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection)
  promoted_bytes = 0;
  }
  scanned_bytes = 0;
- pool_live_bytes = 0;
  // 6. start sweeping
  uint64_t start_sweep_time = jl_hrtime();
  JL_PROBE_GC_SWEEP_BEGIN(sweep_full);

src/julia_threads.h

@@ -130,6 +130,7 @@ typedef struct {
 
 typedef struct {
  _Atomic(int64_t) allocd;
+ _Atomic(int64_t) pool_live_bytes;
  _Atomic(uint64_t) malloc;
  _Atomic(uint64_t) realloc;
  _Atomic(uint64_t) poolalloc;