Merge pull request #24751 from JuliaLang/jn/runtime_intrinsics_less_a…

…lloc

 runtime-intrinsics: reduce allocations for common types

src/datatype.c

@@ -486,21 +486,24 @@ JL_DLLEXPORT jl_datatype_t *jl_new_primitivetype(jl_value_t *name, jl_module_t *
 
 JL_DLLEXPORT jl_value_t *jl_new_bits(jl_value_t *dt, void *data)
 {
- // data may not have the alignment required by the data type.
+ // data may not have the alignment required by the size
+ // but will always have the alignment required by the datatype
  jl_ptls_t ptls = jl_get_ptls_states();
  assert(jl_is_datatype(dt));
  jl_datatype_t *bt = (jl_datatype_t*)dt;
  size_t nb = jl_datatype_size(bt);
+ // some types have special pools to minimize allocations
  if (nb == 0) return jl_new_struct_uninit(bt); // returns bt->instance
+ if (bt == jl_bool_type) return (1 & *(int8_t*)data) ? jl_true : jl_false;
  if (bt == jl_uint8_type) return jl_box_uint8(*(uint8_t*)data);
- if (bt == jl_int64_type) return jl_box_int64(jl_load_unaligned_i64(data));
- if (bt == jl_bool_type)  return (*(int8_t*)data) ? jl_true : jl_false;
- if (bt == jl_int32_type) return jl_box_int32(jl_load_unaligned_i32(data));
- if (bt == jl_float64_type) {
-  double f;
-  memcpy(&f, data, 8);
-  return jl_box_float64(f);
- }
+ if (bt == jl_int64_type) return jl_box_int64(*(int64_t*)data);
+ if (bt == jl_int32_type) return jl_box_int32(*(int32_t*)data);
+ if (bt == jl_int8_type)  return jl_box_int8(*(int8_t*)data);
+ if (bt == jl_int16_type) return jl_box_int16(*(int16_t*)data);
+ if (bt == jl_uint64_type) return jl_box_uint64(*(uint64_t*)data);
+ if (bt == jl_uint32_type) return jl_box_uint32(*(uint32_t*)data);
+ if (bt == jl_uint16_type) return jl_box_uint16(*(uint16_t*)data);
+ if (bt == jl_char_type) return jl_box_char(*(uint32_t*)data);
 
  jl_value_t *v = jl_gc_alloc(ptls, nb, bt);
  switch (nb) {

src/runtime_intrinsics.c

@@ -349,27 +349,26 @@ jl_value_t *jl_iintrinsic_1(jl_value_t *ty, jl_value_t *a, const char *name,
 
 static inline jl_value_t *jl_intrinsiclambda_ty1(jl_value_t *ty, void *pa, unsigned osize, unsigned osize2, const void *voidlist)
 {
- jl_ptls_t ptls = jl_get_ptls_states();
- jl_value_t *newv = jl_gc_alloc(ptls, jl_datatype_size(ty), ty);
  intrinsic_1_t op = select_intrinsic_1(osize2, (const intrinsic_1_t*)voidlist);
- op(osize * host_char_bit, pa, jl_data_ptr(newv));
- return newv;
+ void *pr = alloca(osize2);
+ op(osize * host_char_bit, pa, pr);
+ return jl_new_bits(ty, pr);
 }
 
 static inline jl_value_t *jl_intrinsiclambda_u1(jl_value_t *ty, void *pa, unsigned osize, unsigned osize2, const void *voidlist)
 {
  jl_ptls_t ptls = jl_get_ptls_states();
- jl_value_t *newv = jl_gc_alloc(ptls, jl_datatype_size(ty), ty);
  intrinsic_u1_t op = select_intrinsic_u1(osize2, (const intrinsic_u1_t*)voidlist);
- unsigned cnt = op(osize * host_char_bit, pa);
- // TODO: the following memset/memcpy assumes little-endian
+ uint64_t cnt = op(osize * host_char_bit, pa);
+ // TODO: the following assume little-endian
  // for big-endian, need to copy from the other end of cnt
- if (osize > sizeof(unsigned)) {
- // perform zext, if needed
- memset((char*)jl_data_ptr(newv) + sizeof(unsigned), 0, osize - sizeof(unsigned));
- osize = sizeof(unsigned);
+ if (osize <= sizeof(cnt)) {
+ return jl_new_bits(ty, &cnt);
  }
- memcpy(jl_data_ptr(newv), &cnt, osize);
+ jl_value_t *newv = jl_gc_alloc(ptls, osize, ty);
+ // perform zext, if needed
+ memset((char*)jl_data_ptr(newv) + sizeof(cnt), 0, osize - sizeof(cnt));
+ memcpy(jl_data_ptr(newv), &cnt, sizeof(cnt));
  return newv;
 }
 
@@ -385,7 +384,6 @@ JL_DLLEXPORT jl_value_t *jl_##name(jl_value_t *ty, jl_value_t *a) \
 
 static inline jl_value_t *jl_intrinsic_cvt(jl_value_t *ty, jl_value_t *a, const char *name, intrinsic_cvt_t op)
 {
- jl_ptls_t ptls = jl_get_ptls_states();
  jl_value_t *aty = jl_typeof(a);
  if (!jl_is_primitivetype(aty))
  jl_errorf("%s: value is not a primitive type", name);
@@ -394,12 +392,13 @@ static inline jl_value_t *jl_intrinsic_cvt(jl_value_t *ty, jl_value_t *a, const
  void *pa = jl_data_ptr(a);
  unsigned isize = jl_datatype_size(aty);
  unsigned osize = jl_datatype_size(ty);
- jl_value_t *newv = jl_gc_alloc(ptls, jl_datatype_size(ty), ty);
- op(aty == (jl_value_t*)jl_bool_type ? 1 : isize * host_char_bit, pa,
- osize * host_char_bit, jl_data_ptr(newv));
- if (ty == (jl_value_t*)jl_bool_type)
- return *(uint8_t*)jl_data_ptr(newv) & 1 ? jl_true : jl_false;
- return newv;
+ void *pr = alloca(osize);
+ unsigned isize_bits = isize * host_char_bit;
+ unsigned osize_bits = osize * host_char_bit;
+ if (aty == (jl_value_t*)jl_bool_type)
+ isize_bits = 1;
+ op(isize_bits, pa, osize_bits, pr);
+ return jl_new_bits(ty, pr);
 }
 
 // floating point
@@ -544,7 +543,7 @@ jl_value_t *jl_iintrinsic_2(jl_value_t *a, jl_value_t *b, const char *name,
  void *pa = jl_data_ptr(a), *pb = jl_data_ptr(b);
  unsigned sz = jl_datatype_size(ty);
  unsigned sz2 = next_power_of_two(sz);
- unsigned szb = jl_datatype_size(tyb);
+ unsigned szb = cvtb ? jl_datatype_size(tyb) : sz;
  if (sz2 > sz) {
  /* round type up to the appropriate c-type and set/clear the unused bits */
  void *pa2 = alloca(sz2);
@@ -553,10 +552,12 @@ jl_value_t *jl_iintrinsic_2(jl_value_t *a, jl_value_t *b, const char *name,
  pa = pa2;
  }
  if (sz2 > szb) {
- /* round type up to the appropriate c-type and set/clear/truncate the unused bits */
+ /* round type up to the appropriate c-type and set/clear/truncate the unused bits
+ * (zero-extend if cvtb is set, since in that case b is unsigned while the sign of a comes from the op)
+ */
  void *pb2 = alloca(sz2);
  memcpy(pb2, pb, szb);
- memset((char*)pb2 + szb, getsign(pb, sz), sz2 - szb);
+ memset((char*)pb2 + szb, cvtb ? 0 : getsign(pb, szb), sz2 - szb);
  pb = pb2;
  }
  jl_value_t *newv = lambda2(ty, pa, pb, sz, sz2, list);
@@ -565,13 +566,10 @@ jl_value_t *jl_iintrinsic_2(jl_value_t *a, jl_value_t *b, const char *name,
 
 static inline jl_value_t *jl_intrinsiclambda_2(jl_value_t *ty, void *pa, void *pb, unsigned sz, unsigned sz2, const void *voidlist)
 {
- jl_ptls_t ptls = jl_get_ptls_states();
- jl_value_t *newv = jl_gc_alloc(ptls, jl_datatype_size(ty), ty);
+ void *pr = alloca(sz2);
  intrinsic_2_t op = select_intrinsic_2(sz2, (const intrinsic_2_t*)voidlist);
- op(sz * host_char_bit, pa, pb, jl_data_ptr(newv));
- if (ty == (jl_value_t*)jl_bool_type)
- return *(uint8_t*)jl_data_ptr(newv) & 1 ? jl_true : jl_false;
- return newv;
+ op(sz * host_char_bit, pa, pb, pr);
+ return jl_new_bits(ty, pr);
 }
 
 static inline jl_value_t *jl_intrinsiclambda_cmp(jl_value_t *ty, void *pa, void *pb, unsigned sz, unsigned sz2, const void *voidlist)
@@ -586,7 +584,7 @@ static inline jl_value_t *jl_intrinsiclambda_checked(jl_value_t *ty, void *pa, v
  jl_value_t *params[2];
  params[0] = ty;
  params[1] = (jl_value_t*)jl_bool_type;
- jl_datatype_t *tuptyp = jl_apply_tuple_type_v(params,2);
+ jl_datatype_t *tuptyp = jl_apply_tuple_type_v(params, 2);
  jl_ptls_t ptls = jl_get_ptls_states();
  jl_value_t *newv = jl_gc_alloc(ptls, ((jl_datatype_t*)tuptyp)->size, tuptyp);
 
@@ -599,16 +597,12 @@ static inline jl_value_t *jl_intrinsiclambda_checked(jl_value_t *ty, void *pa, v
 }
 static inline jl_value_t *jl_intrinsiclambda_checkeddiv(jl_value_t *ty, void *pa, void *pb, unsigned sz, unsigned sz2, const void *voidlist)
 {
- jl_ptls_t ptls = jl_get_ptls_states();
- jl_value_t *newv = jl_gc_alloc(ptls, jl_datatype_size(ty), ty);
+ void *pr = alloca(sz2);
  intrinsic_checked_t op = select_intrinsic_checked(sz2, (const intrinsic_checked_t*)voidlist);
- int ovflw = op(sz * host_char_bit, pa, pb, jl_data_ptr(newv));
+ int ovflw = op(sz * host_char_bit, pa, pb, pr);
  if (ovflw)
  jl_throw(jl_diverror_exception);
- if (ty == (jl_value_t*)jl_bool_type)
- return *(uint8_t*)jl_data_ptr(newv) & 1 ? jl_true : jl_false;
-
- return newv;
+ return jl_new_bits(ty, pr);
 }
 
 // floating point