OpaqueClosure - Runtime & Codegen only (JuliaLang#39023)

This is JuliaLang#37849 modulo inference/optimization support.
This is self-contained and the functionality tests
pass, though the tests testing for various optimizations
of course don't (they're marked as test_broken).

base/compiler/ssair/ir.jl

@@ -375,7 +375,8 @@ function is_relevant_expr(e::Expr)
  :gc_preserve_begin, :gc_preserve_end,
  :foreigncall, :isdefined, :copyast,
  :undefcheck, :throw_undef_if_not,
- :cfunction, :method, :pop_exception)
+ :cfunction, :method, :pop_exception,
+ :new_opaque_closure)
 end
 
 function setindex!(x::UseRef, @nospecialize(v))

base/compiler/validation.jl

@@ -30,7 +30,8 @@ const VALID_EXPR_HEADS = IdDict{Any,Any}(
  :thunk => 1:1,
  :throw_undef_if_not => 2:2,
  :aliasscope => 0:0,
- :popaliasscope => 0:0
+ :popaliasscope => 0:0,
+ :new_opaque_closure => 4:typemax(Int)
 )
 
 # @enum isn't defined yet, otherwise I'd use it for this

base/experimental.jl

@@ -252,4 +252,7 @@ function show_error_hints(io, ex, args...)
  end
 end
 
+# OpaqueClosure
+include("opaque_closure.jl")
+
 end

base/opaque_closure.jl

@@ -0,0 +1,27 @@
+function show(io::IO, oc::Core.OpaqueClosure)
+ A, R = typeof(oc).parameters
+ show_tuple_as_call(io, Symbol(""), A; hasfirst=false)
+ print(io, "::", R)
+ print(io, "->◌")
+end
+
+function show(io::IO, ::MIME"text/plain", oc::Core.OpaqueClosure{A, R}) where {A, R}
+ show(io, oc)
+end
+
+"""
+ @opaque (args...) -> body
+
+Marks a given closure as "opaque". Opaque closures capture the
+world age of their creation (as opposed to their invocation).
+This allows for more aggressive optimization of the capture
+list, but trades off against the ability to inline opaque
+closures at the call site, if their creation is not statically
+visible.
+
+!!! warning
+ This interface is experimental and subject to change or removal without notice.
+"""
+macro opaque(ex)
+ esc(Expr(:opaque_closure, ex))
+end

base/reflection.jl

@@ -1140,6 +1140,9 @@ function code_typed(@nospecialize(f), @nospecialize(types=Tuple);
  if isa(f, Core.Builtin)
  throw(ArgumentError("argument is not a generic function"))
  end
+ if isa(f, Core.OpaqueClosure)
+ return code_typed_opaque_closure(f, types; optimize, debuginfo, interp)
+ end
  ft = Core.Typeof(f)
  if isa(types, Type)
  u = unwrap_unionall(types)
@@ -1186,6 +1189,20 @@ function code_typed_by_type(@nospecialize(tt::Type);
  return asts
 end
 
+function code_typed_opaque_closure(@nospecialize(closure::Core.OpaqueClosure), @nospecialize(types=Tuple);
+ optimize=true,
+ debuginfo::Symbol=:default,
+ interp = Core.Compiler.NativeInterpreter(closure.world))
+ ccall(:jl_is_in_pure_context, Bool, ()) && error("code reflection cannot be used from generated functions")
+ if isa(closure.source, Method)
+ code = _uncompressed_ir(closure.source, closure.source.source)
+ debuginfo === :none && remove_linenums!(code)
+ return Any[Pair{CodeInfo,Any}(code, code.rettype)]
+ else
+ error("encountered invalid Core.OpaqueClosure object")
+ end
+end
+
 function return_types(@nospecialize(f), @nospecialize(types=Tuple), interp=Core.Compiler.NativeInterpreter())
  ccall(:jl_is_in_pure_context, Bool, ()) && error("code reflection cannot be used from generated functions")
  if isa(f, Core.Builtin)

src/Makefile

@@ -45,7 +45,7 @@ RUNTIME_SRCS := \
  simplevector runtime_intrinsics precompile \
  threading partr stackwalk gc gc-debug gc-pages gc-stacks method \
  jlapi signal-handling safepoint timing subtype \
- crc32c APInt-C processor ircode
+ crc32c APInt-C processor ircode opaque_closure
 SRCS := jloptions runtime_ccall rtutils
 
 LLVMLINK :=

src/ast.c

@@ -43,6 +43,8 @@ jl_sym_t *pop_exception_sym;
 jl_sym_t *exc_sym; jl_sym_t *error_sym;
 jl_sym_t *new_sym; jl_sym_t *using_sym;
 jl_sym_t *splatnew_sym;
+jl_sym_t *new_opaque_closure_sym;
+jl_sym_t *opaque_closure_method_sym;
 jl_sym_t *const_sym; jl_sym_t *thunk_sym;
 jl_sym_t *foreigncall_sym; jl_sym_t *as_sym;
 jl_sym_t *global_sym; jl_sym_t *list_sym;
@@ -359,6 +361,8 @@ void jl_init_common_symbols(void)
  pop_exception_sym = jl_symbol("pop_exception");
  new_sym = jl_symbol("new");
  splatnew_sym = jl_symbol("splatnew");
+ new_opaque_closure_sym = jl_symbol("new_opaque_closure");
+ opaque_closure_method_sym = jl_symbol("opaque_closure_method");
  const_sym = jl_symbol("const");
  global_sym = jl_symbol("global");
  thunk_sym = jl_symbol("thunk");

src/builtins.c

@@ -1503,6 +1503,9 @@ void jl_init_intrinsic_functions(void) JL_GC_DISABLED
  inm->parent = jl_core_module;
  jl_set_const(jl_core_module, jl_symbol("Intrinsics"), (jl_value_t*)inm);
  jl_mk_builtin_func(jl_intrinsic_type, "IntrinsicFunction", jl_f_intrinsic_call);
+ jl_mk_builtin_func(
+ (jl_datatype_t*)jl_unwrap_unionall((jl_value_t*)jl_opaque_closure_type),
+ "OpaqueClosure", jl_f_opaque_closure_call);
 
 #define ADD_I(name, nargs) add_intrinsic(inm, #name, name);
 #define ADD_HIDDEN(name, nargs)
@@ -1618,6 +1621,7 @@ void jl_init_primitives(void) JL_GC_DISABLED
  add_builtin("Ptr", (jl_value_t*)jl_pointer_type);
  add_builtin("LLVMPtr", (jl_value_t*)jl_llvmpointer_type);
  add_builtin("Task", (jl_value_t*)jl_task_type);
+ add_builtin("OpaqueClosure", (jl_value_t*)jl_opaque_closure_type);
 
  add_builtin("AbstractArray", (jl_value_t*)jl_abstractarray_type);
  add_builtin("DenseArray", (jl_value_t*)jl_densearray_type);

src/clangsa/GCChecker.cpp

@@ -753,6 +753,7 @@ bool GCChecker::isGCTrackedType(QualType QT) {
  Name.endswith_lower("jl_uniontype_t") ||
  Name.endswith_lower("jl_method_match_t") ||
  Name.endswith_lower("jl_vararg_t") ||
+ Name.endswith_lower("jl_opaque_closure_t") ||
  // Probably not technically true for these, but let's allow
  // it
  Name.endswith_lower("typemap_intersection_env") ||