move jl_get_unspecialized logic entirely into codegen

allows removing inCompile, which wasn't necessarily entirely accurate anyways
for complex recursive calls from type-inference, it theoretically could have
compiled part of a cycle, but not the whole cycle, and failed when it tried to
finalize it, without realizing part of the cycle was still being compiled

base/inference.jl

@@ -1477,51 +1477,50 @@ function typeinf_edge(method::Method, atypes::ANY, sparams::SimpleVector, needtr
  local frame = nothing
  if isa(caller, LambdaInfo)
  code = caller
- elseif cached
+ elseif cached && !is(method.specializations, nothing)
  # check cached specializations
  # for an existing result stored there
- if !is(method.specializations, nothing)
- code = ccall(:jl_specializations_lookup, Any, (Any, Any), method, atypes)
- if isa(code, Void)
- # something completely new
- elseif isa(code, LambdaInfo)
- # something existing
- if isdefined(code, :inferred)
- if code.jlcall_api == 2
- if needtree
- tree = ccall(:jl_new_source_info_uninit, Ref{SourceInfo}, ())
- tree.code = Any[ Expr(:return, QuoteNode(code.inferred)) ]
- tree.slotnames = Any[ compiler_temp_sym for i = 1:method.nargs ]
- tree.slotflags = UInt8[ 0 for i = 1:method.nargs ]
- tree.slottypes = nothing
- tree.ssavaluetypes = 0
- tree.inferred = true
- tree.pure = true
- tree.inlineable = true
- else
- tree = Const(code.inferred)
- end
- return (tree, code.rettype, true)
- elseif isa(code.inferred, SourceInfo)
- if code.inferred.inferred
- return (code.inferred, code.rettype, true)
- end
- elseif !needtree
- return (nothing, code.rettype, true)
- else
- cached = false # don't need to save the new result
- end
- end
+ code = ccall(:jl_specializations_lookup, Any, (Any, Any), method, atypes)
+ if isa(code, Void)
+ # something completely new
+ elseif isa(code, LambdaInfo)
+ # something existing
+ else
+ # sometimes just a return type is stored here. if a full AST
+ # is not needed, we can return it.
+ typeassert(code, Type)
+ if !needtree
+ return (nothing, code, true)
+ end
+ cached = false # don't need to save the new result
+ code = nothing
+ end
+ end
+
+ if isa(code, LambdaInfo) && isdefined(code, :inferred)
+ if code.jlcall_api == 2
+ if needtree
+ tree = ccall(:jl_new_source_info_uninit, Ref{SourceInfo}, ())
+ tree.code = Any[ Expr(:return, QuoteNode(code.inferred)) ]
+ tree.slotnames = Any[ compiler_temp_sym for i = 1:method.nargs ]
+ tree.slotflags = UInt8[ 0 for i = 1:method.nargs ]
+ tree.slottypes = nothing
+ tree.ssavaluetypes = 0
+ tree.inferred = true
+ tree.pure = true
+ tree.inlineable = true
  else
- # sometimes just a return type is stored here. if a full AST
- # is not needed, we can return it.
- typeassert(code, Type)
- if !needtree
- return (nothing, code, true)
- end
- cached = false # don't need to save the new result
- code = nothing
+ tree = Const(code.inferred)
  end
+ return (tree, code.rettype, true)
+ elseif isa(code.inferred, SourceInfo)
+ if code.inferred.inferred
+ return (code.inferred, code.rettype, true)
+ end
+ elseif !needtree
+ return (nothing, code.rettype, true)
+ else
+ cached = false # don't need to save the new result
  end
  end
 

doc/devdocs/locks.rst

@@ -52,6 +52,8 @@ The following is a level 4 lock, which can only recurse to acquire level 1, 2, o
 
  * MethodTable->writelock
 
+No Julia code may be called while holding a lock above this point.
+
 The following is a level 6 lock, which can only recurse to acquire locks at lower levels:
 
  * codegen
@@ -85,20 +87,6 @@ The following locks are broken:
 
  fix: create it
 
-* codegen
-
- recursive (through ``static_eval``), but caller might also be holding locks (due to staged functions and type_infer)
-
- other issues?
-
- fix: prohibit codegen while holding any other lock (possibly by checking ``ptls->current_task->locks.len != 0`` & explicitly check the locks that are OK to hold simultaneously)?
-
-* typeinf
-
- not certain of whether there are issues here or what they are. staging functions, of course, are a source of deadlocks here.
-
- fix: unknown
-
 
 Shared Global Data Structures
 -----------------------------
@@ -115,10 +103,54 @@ Type application : typecache lock
 
 Module serializer : toplevel lock
 
-JIT : codegen lock
+JIT & type-inference : codegen lock
+
+LambdaInfo updates : codegen lock
+
+ - These fields are generally lazy initialized, using the test-and-test-and-set pattern.
+
+ - These are set at construction and immutable:
+
+ + specTypes
+
+ + sparam_vals
+
+ + def
+
+ - These are set by ``jl_type_infer`` (while holding codegen lock):
+
+ + rettype
+
+ + inferred
+
+ + these can also be reset, see ``jl_set_lambda_rettype`` for that logic as it needs to keep ``functionObjectsDecls`` in sync
+
+ - ``inInference`` flag:
+
+ + optimization to quickly avoid recurring into ``jl_type_infer`` while it is already running
+
+ + actual state (of setting ``inferred``, then ``fptr``) is protected by codegen lock
+
+ - Function pointers (``jlcall_api`` and ``fptr``, ``unspecialized_ducttape``):
+
+ + these transition once, from ``NULL`` to a value, while the codegen lock is held
+
+ - Code-generator cache (the contents of ``functionObjectsDecls``):
+
+ + these can transition multiple times, but only while the codegen lock is held
+
+ + it is valid to use old version of this, or block for new versions of this,
+ so races are benign, as long as the code is careful not to reference other data in the method instance (such as ``rettype``)
+ and assume it is coordinated, unless also holding the codegen lock
+
+ - ``compile_traced`` flag:
+
+ + unknown
+
 
 LLVMContext : codegen lock
 
 Method : Method->writelock
+
  - roots array (serializer and codegen)
  - invoke / specializations / tfunc modifications

src/alloc.c

@@ -416,7 +416,6 @@ JL_DLLEXPORT jl_lambda_info_t *jl_new_lambda_info_uninit(void)
  li->functionObjectsDecls.specFunctionObject = NULL;
  li->specTypes = NULL;
  li->inInference = 0;
- li->inCompile = 0;
  li->def = NULL;
  return li;
 }

src/ccall.cpp

@@ -251,6 +251,7 @@ static Value *emit_plt(FunctionType *functype, const AttributeSet &attrs,
  Function *plt = Function::Create(functype,
  GlobalVariable::ExternalLinkage,
  fname, M);
+ jl_init_function(plt);
  plt->setAttributes(attrs);
  if (cc != CallingConv::C)
  plt->setCallingConv(cc);
@@ -1041,10 +1042,13 @@ static jl_cgval_t emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *c
  std::stringstream name;
  name << "jl_llvmcall" << llvmcallnumbering++;
  f->setName(name.str());
+ jl_init_function(f);
  f = cast<Function>(prepare_call(function_proto(f)));
  }
- else
+ else {
+ jl_init_function(f);
  f->setLinkage(GlobalValue::LinkOnceODRLinkage);
+ }
 
  // the actual call
  builder.CreateCall(prepare_call(gcroot_flush_func));