fix a bug in circular type detection (#35275)

src/datatype.c

@@ -279,25 +279,25 @@ int jl_pointer_egal(jl_value_t *t)
  return 0;
 }
 
-static int references_name(jl_value_t *p, jl_typename_t *name) JL_NOTSAFEPOINT
+static int references_name(jl_value_t *p, jl_typename_t *name, int affects_layout) JL_NOTSAFEPOINT
 {
  if (jl_is_uniontype(p))
- return references_name(((jl_uniontype_t*)p)->a, name) ||
- references_name(((jl_uniontype_t*)p)->b, name);
+ return references_name(((jl_uniontype_t*)p)->a, name, affects_layout) ||
+ references_name(((jl_uniontype_t*)p)->b, name, affects_layout);
  if (jl_is_unionall(p))
- return references_name((jl_value_t*)((jl_unionall_t*)p)->var, name) ||
- references_name(((jl_unionall_t*)p)->body, name);
+ return references_name((jl_value_t*)((jl_unionall_t*)p)->var, name, 0) ||
+ references_name(((jl_unionall_t*)p)->body, name, affects_layout);
  if (jl_is_typevar(p))
- return references_name(((jl_tvar_t*)p)->ub, name) ||
- references_name(((jl_tvar_t*)p)->lb, name);
+ return references_name(((jl_tvar_t*)p)->ub, name, 0) ||
+ references_name(((jl_tvar_t*)p)->lb, name, 0);
  if (jl_is_datatype(p)) {
- if (((jl_datatype_t*)p)->name == name)
+ jl_datatype_t *dp = (jl_datatype_t*)p;
+ if (affects_layout && dp->name == name)
  return 1;
- if (((jl_datatype_t*)p)->layout && jl_datatype_nfields(p) == 0)
- return 0;
+ affects_layout = dp->types == NULL || jl_svec_len(dp->types) != 0;
  size_t i, l = jl_nparams(p);
  for (i = 0; i < l; i++) {
- if (references_name(jl_tparam(p, i), name))
+ if (references_name(jl_tparam(p, i), name, affects_layout))
  return 1;
  }
  }
@@ -390,7 +390,7 @@ void jl_compute_field_offsets(jl_datatype_t *st)
  size_t i, nf = jl_svec_len(w->types);
  for (i = 0; i < nf; i++) {
  jl_value_t *fld = jl_svecref(w->types, i);
- if (references_name(fld, w->name)) {
+ if (references_name(fld, w->name, 1)) {
  isinlinealloc = 0;
  isbitstype = 0;
  break;

test/core.jl

@@ -7191,3 +7191,10 @@ end
 @test_throws ErrorException f34482()
 @test_throws TypeError g34482()
 @test_throws TypeError h34482()
+
+struct NFANode34126
+ edges::Vector{Tuple{Nothing,NFANode34126}}
+ NFANode34126() = new(Tuple{Nothing,NFANode34126}[])
+end
+
+@test repr(NFANode34126()) == "$NFANode34126(Tuple{Nothing,$NFANode34126}[])"