add explicit error paths for common misunderstandings in pure contexts

this still misses some paths, but a complete solution would require an enforcing pure interpreter
while this is only attempting to provide rapid feedback against doing something
that might cause the runtime to run into assertion failures if it were allowed to proceed

also make the Base.Workqueue switching more robust after encountering errors

base/event.jl

@@ -105,12 +105,39 @@ function wait()
  end
  else
  t = shift!(Workqueue)
- t.state == :queued || throw(AssertionError("shift!(Workqueue).state == :queued"))
+ if t.state != :queued
+ # assume this somehow got queued twice,
+ # probably broken now, but try discarding this switch and keep going
+ # can't throw here, because it's probably not the fault of the caller to wait
+ # and don't want to use print() here, because that may try to incur a task switch
+ ccall(:jl_safe_printf, Void, (Ptr{UInt8}, Vararg{Int32}),
+ "\nWARNING: Workqueue inconsistency detected: shift!(Workqueue).state != :queued\n")
+ continue
+ end
  arg = t.result
  t.result = nothing
  t.state = :runnable
- result = yieldto(t, arg)
- current_task().state == :runnable || throw(AssertionError("current_task().state == :runnable"))
+ local result
+ try
+ result = yieldto(t, arg)
+ current_task().state == :runnable || throw(AssertionError("current_task().state == :runnable"))
+ catch e
+ ct = current_task()
+ if ct.state == :queued
+ if t.state == :runnable
+ # assume we failed to queue t
+ # return it to the queue to be scheduled later
+ t.result = arg
+ t.state = :queued
+ push!(Workqueue, t)
+ end
+ # return ourself to the runnable state
+ i = findfirst(Workqueue, ct)
+ i == 0 || deleteat!(Workqueue, i)
+ ct.state = :runnable
+ end
+ rethrow(e)
+ end
  process_events(false)
  # return when we come out of the queue
  return result

base/reflection.jl

@@ -198,6 +198,7 @@ function _methods_by_ftype(t::ANY, lim)
  if 1 < nu <= 64
  return _methods(Any[tp...], length(tp), lim, [])
  end
+ # TODO: the following can return incorrect answers that the above branch would have corrected
  return ccall(:jl_matching_methods, Any, (Any,Int32), t, lim)
 end
 function _methods(t::Array,i,lim::Integer,matching::Array{Any,1})
@@ -277,6 +278,7 @@ uncompressed_ast(l::LambdaInfo) =
 
 # Printing code representations in IR and assembly
 function _dump_function(f, t::ANY, native, wrapper, strip_ir_metadata, dump_module)
+ ccall(:jl_is_in_pure_context, Bool, ()) && error("native reflection cannot be used from generated functions")
  t = tt_cons(Core.Typeof(f), to_tuple_type(t))
  llvmf = ccall(:jl_get_llvmf, Ptr{Void}, (Any, Bool, Bool), t, wrapper, native)
 
@@ -314,26 +316,30 @@ function func_for_method_checked(m::Method, types)
 end
 
 function code_typed(f::ANY, types::ANY=Tuple; optimize=true)
+ ccall(:jl_is_in_pure_context, Bool, ()) && error("code reflection cannot be used from generated functions")
  types = to_tuple_type(types)
  asts = []
  for x in _methods(f,types,-1)
  linfo = func_for_method_checked(x[3].func, types)
  if optimize
- (li, ty) = Core.Inference.typeinf(linfo, x[1], x[2], true)
+ (li, ty, inf) = Core.Inference.typeinf(linfo, x[1], x[2], true)
  else
- (li, ty) = Core.Inference.typeinf_uncached(linfo, x[1], x[2], optimize=false)
+ (li, ty, inf) = Core.Inference.typeinf_uncached(linfo, x[1], x[2], optimize=false)
  end
+ inf || error("inference not successful") # Inference disabled
  push!(asts, li)
  end
  asts
 end
 
 function return_types(f::ANY, types::ANY=Tuple)
+ ccall(:jl_is_in_pure_context, Bool, ()) && error("code reflection cannot be used from generated functions")
  types = to_tuple_type(types)
  rt = []
  for x in _methods(f,types,-1)
  linfo = func_for_method_checked(x[3].func,types)
- (_li, ty) = Core.Inference.typeinf(linfo, x[1], x[2])
+ (_li, ty, inf) = Core.Inference.typeinf(linfo, x[1], x[2])
+ inf || error("inference not successful") # Inference disabled
  push!(rt, ty)
  end
  rt

src/alloc.c

@@ -399,59 +399,64 @@ static jl_lambda_info_t *jl_instantiate_staged(jl_method_t *generator, jl_tuplet
  jl_svec_t *sparam_vals = env;
  jl_lambda_info_t *func = generator->lambda_template;
  JL_GC_PUSH4(&ex, &linenum, &sparam_vals, &func);
-
+ int last_in = in_pure_callback;
  assert(jl_svec_len(func->sparam_syms) == jl_svec_len(sparam_vals));
- //if (!generated->inferred)
- // jl_type_infer(func); // this doesn't help all that much
-
- ex = jl_exprn(lambda_sym, 2);
-
- int nargs = func->nargs;
- jl_array_t *argnames = jl_alloc_cell_1d(nargs);
- jl_cellset(ex->args, 0, argnames);
- for (i = 0; i < nargs; i++)
- jl_cellset(argnames, i, jl_cellref(func->slotnames, i));
-
- jl_expr_t *scopeblock = jl_exprn(jl_symbol("scope-block"), 1);
- jl_cellset(ex->args, 1, scopeblock);
- jl_expr_t *body = jl_exprn(jl_symbol("block"), 2);
- jl_cellset(((jl_expr_t*)jl_exprarg(ex,1))->args, 0, body);
- linenum = jl_box_long(generator->line);
- jl_value_t *linenode = jl_new_struct(jl_linenumbernode_type, generator->file, linenum);
- jl_cellset(body->args, 0, linenode);
-
- // invoke code generator
- assert(jl_nparams(tt) == jl_array_len(argnames) ||
- (func->isva && (jl_nparams(tt) >= jl_array_len(argnames) - 1)));
- jl_cellset(body->args, 1,
- jl_call_unspecialized(sparam_vals, func, jl_svec_data(tt->parameters), jl_nparams(tt)));
-
- if (func->sparam_syms != jl_emptysvec) {
- // mark this function as having the same static parameters as the generator
- size_t i, nsp = jl_svec_len(func->sparam_syms);
- jl_expr_t *newast = jl_exprn(jl_symbol("with-static-parameters"), nsp + 1);
- jl_exprarg(newast, 0) = (jl_value_t*)ex;
- // (with-static-parameters func_expr sp_1 sp_2 ...)
- for (i = 0; i < nsp; i++)
- jl_exprarg(newast, i+1) = jl_svecref(func->sparam_syms, i);
- ex = newast;
- }
-
- // need to eval macros in the right module, but not give a warning for the `eval` call unless that results in a call to `eval`
- func = (jl_lambda_info_t*)jl_toplevel_eval_in_warn(generator->module, (jl_value_t*)ex, 1);
-
- // finish marking this as a specialization of the generator
- func->isva = generator->lambda_template->isva;
- func->def = generator;
- jl_gc_wb(func, generator);
- func->sparam_vals = env;
- jl_gc_wb(func, env);
- func->specTypes = tt;
- jl_gc_wb(func, tt);
+ JL_TRY {
+ in_pure_callback = 1;
+ ex = jl_exprn(lambda_sym, 2);
+
+ int nargs = func->nargs;
+ jl_array_t *argnames = jl_alloc_cell_1d(nargs);
+ jl_cellset(ex->args, 0, argnames);
+ for (i = 0; i < nargs; i++)
+ jl_cellset(argnames, i, jl_cellref(func->slotnames, i));
+
+ jl_expr_t *scopeblock = jl_exprn(jl_symbol("scope-block"), 1);
+ jl_cellset(ex->args, 1, scopeblock);
+ jl_expr_t *body = jl_exprn(jl_symbol("block"), 2);
+ jl_cellset(((jl_expr_t*)jl_exprarg(ex,1))->args, 0, body);
+ linenum = jl_box_long(generator->line);
+ jl_value_t *linenode = jl_new_struct(jl_linenumbernode_type, generator->file, linenum);
+ jl_cellset(body->args, 0, linenode);
+
+ // invoke code generator
+ assert(jl_nparams(tt) == jl_array_len(argnames) ||
+ (func->isva && (jl_nparams(tt) >= jl_array_len(argnames) - 1)));
+ jl_cellset(body->args, 1,
+ jl_call_unspecialized(sparam_vals, func, jl_svec_data(tt->parameters), jl_nparams(tt)));
+
+ if (func->sparam_syms != jl_emptysvec) {
+ // mark this function as having the same static parameters as the generator
+ size_t i, nsp = jl_svec_len(func->sparam_syms);
+ jl_expr_t *newast = jl_exprn(jl_symbol("with-static-parameters"), nsp + 1);
+ jl_exprarg(newast, 0) = (jl_value_t*)ex;
+ // (with-static-parameters func_expr sp_1 sp_2 ...)
+ for (i = 0; i < nsp; i++)
+ jl_exprarg(newast, i+1) = jl_svecref(func->sparam_syms, i);
+ ex = newast;
+ }
 
- jl_array_t *stmts = func->code;
- for(i = 0, l = jl_array_len(stmts); i < l; i++) {
- jl_cellset(stmts, i, jl_resolve_globals(jl_cellref(stmts, i), func));
+ // need to eval macros in the right module, but not give a warning for the `eval` call unless that results in a call to `eval`
+ func = (jl_lambda_info_t*)jl_toplevel_eval_in_warn(generator->module, (jl_value_t*)ex, 1);
+
+ // finish marking this as a specialization of the generator
+ func->isva = generator->lambda_template->isva;
+ func->def = generator;
+ jl_gc_wb(func, generator);
+ func->sparam_vals = env;
+ jl_gc_wb(func, env);
+ func->specTypes = tt;
+ jl_gc_wb(func, tt);
+
+ jl_array_t *stmts = func->code;
+ for(i = 0, l = jl_array_len(stmts); i < l; i++) {
+ jl_cellset(stmts, i, jl_resolve_globals(jl_cellref(stmts, i), func));
+ }
+ in_pure_callback = last_in;
+ }
+ JL_CATCH {
+ in_pure_callback = last_in;
+ jl_rethrow();
  }
  JL_GC_POP();
  return func;

src/ast.c

@@ -745,6 +745,8 @@ JL_DLLEXPORT jl_value_t *jl_parse_string(const char *str, size_t len,
 jl_value_t *jl_parse_eval_all(const char *fname, size_t len,
  const char *content, size_t contentlen)
 {
+ if (in_pure_callback)
+ jl_error("cannot use include inside a generated function");
  jl_ast_context_t *ctx = jl_ast_ctx_enter();
  fl_context_t *fl_ctx = &ctx->fl;
  value_t f, ast;

src/builtins.c

@@ -514,7 +514,7 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_in(jl_module_t *m, jl_value_t *ex)
  return jl_toplevel_eval_in_warn(m, ex, 0);
 }
 
-JL_DLLEXPORT jl_value_t *jl_toplevel_eval_in_warn(jl_module_t *m, jl_value_t *ex, int delay_warn)
+jl_value_t *jl_toplevel_eval_in_warn(jl_module_t *m, jl_value_t *ex, int delay_warn)
 {
  static int jl_warn_on_eval = 0;
  int last_delay_warn = jl_warn_on_eval;
@@ -539,6 +539,8 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_in_warn(jl_module_t *m, jl_value_t *ex
  jl_printf(JL_STDERR, "\n ** incremental compilation may be broken for these modules **\n\n");
  }
  }
+ if (in_pure_callback && !delay_warn)
+ jl_error("eval cannot be used in a generated function");
  JL_TRY {
  jl_warn_on_eval = delay_warn && (jl_warn_on_eval || m != last_m); // compute whether a warning was suppressed
  jl_current_task->current_module = jl_current_module = m;

src/codegen.cpp

@@ -963,7 +963,7 @@ static void jl_finalize_module(std::unique_ptr<Module> uniquem)
 
 // this ensures that llvmf has been emitted to the execution engine,
 // returning the function pointer to it
-extern void callback_triggered_linfos();
+extern void jl_callback_triggered_linfos(void);
 static uint64_t getAddressForFunction(llvm::Function *llvmf)
 {
 #ifdef JL_DEBUG_BUILD
@@ -974,7 +974,7 @@ static uint64_t getAddressForFunction(llvm::Function *llvmf)
  uint64_t ret = jl_ExecutionEngine->getFunctionAddress(llvmf->getName());
  // delay executing trace callbacks until here to make sure there's no
  // recursive compilation.
- callback_triggered_linfos();
+ jl_callback_triggered_linfos();
  return ret;
 #else
  return (uint64_t)jl_ExecutionEngine->getPointerToFunction(

src/debuginfo.cpp

@@ -214,19 +214,16 @@ struct strrefcomp {
 };
 #endif
 
-extern "C" {
- extern void (*jl_linfo_tracer)(jl_lambda_info_t *tracee);
-}
-
-std::vector<jl_lambda_info_t*> triggered_linfos;
-void callback_triggered_linfos()
+extern "C" tracer_cb jl_linfo_tracer;
+static std::vector<jl_lambda_info_t*> triggered_linfos;
+void jl_callback_triggered_linfos(void)
 {
  if (triggered_linfos.empty())
  return;
  if (jl_linfo_tracer) {
  std::vector<jl_lambda_info_t*> to_process(std::move(triggered_linfos));
  for (jl_lambda_info_t *linfo : to_process)
- jl_linfo_tracer(linfo);
+ jl_call_tracer(jl_linfo_tracer, (jl_value_t*)linfo);
  }
 }
 

src/gf.c

@@ -24,6 +24,76 @@
 extern "C" {
 #endif
 
+/// ----- Handling for Julia callbacks ----- ///
+
+int in_pure_callback;
+
+JL_DLLEXPORT int8_t jl_is_in_pure_context(void)
+{
+ return in_pure_callback;
+}
+
+JL_DLLEXPORT void jl_trace_method(jl_method_t *m)
+{
+ assert(jl_is_method(m));
+ m->traced = 1;
+}
+
+JL_DLLEXPORT void jl_untrace_method(jl_method_t *m)
+{
+ assert(jl_is_method(m));
+ m->traced = 0;
+}
+
+JL_DLLEXPORT void jl_trace_linfo(jl_lambda_info_t *linfo)
+{
+ assert(jl_is_lambda_info(linfo));
+ linfo->compile_traced = 1;
+}
+
+JL_DLLEXPORT void jl_untrace_linfo(jl_lambda_info_t *linfo)
+{
+ assert(jl_is_lambda_info(linfo));
+ linfo->compile_traced = 0;
+}
+
+static tracer_cb jl_method_tracer = NULL;
+JL_DLLEXPORT void jl_register_method_tracer(void (*callback)(jl_lambda_info_t *tracee))
+{
+ jl_method_tracer = (tracer_cb)callback;
+}
+
+static tracer_cb jl_newmeth_tracer = NULL;
+JL_DLLEXPORT void jl_register_newmeth_tracer(void (*callback)(jl_method_t *tracee))
+{
+ jl_newmeth_tracer = (tracer_cb)callback;
+}
+
+tracer_cb jl_linfo_tracer = NULL;
+JL_DLLEXPORT void jl_register_linfo_tracer(void (*callback)(jl_lambda_info_t *tracee))
+{
+ jl_linfo_tracer = (tracer_cb)callback;
+}
+
+void jl_call_tracer(tracer_cb callback, jl_value_t *tracee)
+{
+ int last_in = in_pure_callback;
+ JL_TRY {
+ in_pure_callback = 1;
+ callback(tracee);
+ in_pure_callback = last_in;
+ }
+ JL_CATCH {
+ in_pure_callback = last_in;
+ jl_printf(JL_STDERR, "WARNING: tracer callback function threw an error:\n");
+ jl_static_show(JL_STDERR, jl_exception_in_transit);
+ jl_printf(JL_STDERR, "\n");
+ jlbacktrace();
+ }
+}
+
+/// ----- Definitions for various internal TypeMaps ----- ///
+
 const struct jl_typemap_info method_defs = {
  0, &jl_method_type
 };
@@ -287,8 +357,6 @@ static jl_tupletype_t *join_tsig(jl_tupletype_t *tt, jl_tupletype_t *sig)
 static jl_value_t *ml_matches(union jl_typemap_t ml, int offs,
  jl_tupletype_t *type, int lim);
 
-extern void (*jl_method_tracer)(jl_lambda_info_t *tracee);
-
 static jl_lambda_info_t *cache_method(jl_methtable_t *mt, union jl_typemap_t *cache, jl_value_t *parent,
  jl_tupletype_t *type, jl_tupletype_t *origtype,
  jl_typemap_entry_t *m, jl_svec_t *sparams)
@@ -589,7 +657,7 @@ static jl_lambda_info_t *cache_method(jl_methtable_t *mt, union jl_typemap_t *ca
  JL_GC_POP();
  JL_UNLOCK(&codegen_lock);
  if (definition->traced && jl_method_tracer)
- jl_method_tracer(newmeth);
+ jl_call_tracer(jl_method_tracer, (jl_value_t*)newmeth);
  return newmeth;
 }
 
@@ -786,7 +854,6 @@ static void update_max_args(jl_methtable_t *mt, jl_tupletype_t *type)
  mt->max_args = na;
 }
 
-extern void (*jl_newmeth_tracer)(jl_method_t *tracee);
 void jl_method_table_insert(jl_methtable_t *mt, jl_tupletype_t *type, jl_tupletype_t *simpletype,
  jl_method_t *method, jl_svec_t *tvars)
 {
@@ -806,7 +873,7 @@ void jl_method_table_insert(jl_methtable_t *mt, jl_tupletype_t *type, jl_tuplety
  invalidate_conflicting(&mt->cache, (jl_value_t*)type, (jl_value_t*)mt);
  update_max_args(mt, type);
  if (jl_newmeth_tracer)
- jl_newmeth_tracer(method);
+ jl_call_tracer(jl_newmeth_tracer, (jl_value_t*)method);
  JL_SIGATOMIC_END();
 }