fix #28990, improve top-level location info

This gives better location info inside `:toplevel` expressions and
for REPL inputs that aren't block constructs.
The low-level parser entry points are simplified and renamed to parse-one,
parse-all, and parse-file.

base/client.jl

@@ -146,15 +146,30 @@ function eval_user_input(@nospecialize(ast), show_value::Bool)
  nothing
 end
 
+function _parse_input_line_core(s::String, filename::String)
+ ex = ccall(:jl_parse_all, Any, (Ptr{UInt8}, Csize_t, Ptr{UInt8}, Csize_t),
+ s, sizeof(s), filename, sizeof(filename))
+ if ex isa Expr && ex.head === :toplevel
+ if isempty(ex.args)
+ return nothing
+ end
+ last = ex.args[end]
+ if last isa Expr && (last.head === :error || last.head === :incomplete)
+ # if a parse error happens in the middle of a multi-line input
+ # return only the error, so that none of the input is evaluated.
+ return last
+ end
+ end
+ return ex
+end
+
 function parse_input_line(s::String; filename::String="none", depwarn=true)
  # For now, assume all parser warnings are depwarns
  ex = if depwarn
- ccall(:jl_parse_input_line, Any, (Ptr{UInt8}, Csize_t, Ptr{UInt8}, Csize_t),
- s, sizeof(s), filename, sizeof(filename))
+ _parse_input_line_core(s, filename)
  else
  with_logger(NullLogger()) do
- ccall(:jl_parse_input_line, Any, (Ptr{UInt8}, Csize_t, Ptr{UInt8}, Csize_t),
- s, sizeof(s), filename, sizeof(filename))
+ _parse_input_line_core(s, filename)
  end
  end
  return ex

base/meta.jl

@@ -128,10 +128,6 @@ function parse(str::AbstractString, pos::Integer; greedy::Bool=true, raise::Bool
  if raise && isa(ex,Expr) && ex.head === :error
  throw(ParseError(ex.args[1]))
  end
- if ex === ()
- raise && throw(ParseError("end of input"))
- ex = Expr(:error, "end of input")
- end
  return ex, pos+1 # C is zero-based, Julia is 1-based
 end
 

src/ast.c

@@ -763,20 +763,26 @@ static value_t julia_to_scm_(fl_context_t *fl_ctx, jl_value_t *v)
  return julia_to_scm_noalloc2(fl_ctx, v);
 }
 
-// this is used to parse a line of repl input
-JL_DLLEXPORT jl_value_t *jl_parse_input_line(const char *str, size_t len, const char *filename, size_t filename_len)
+// parse an entire string like a file, reading multiple expressions
+JL_DLLEXPORT jl_value_t *jl_parse_all(const char *str, size_t len, const char *filename, size_t filename_len)
 {
  JL_TIMING(PARSING);
  jl_ast_context_t *ctx = jl_ast_ctx_enter();
  fl_context_t *fl_ctx = &ctx->fl;
  value_t s = cvalue_static_cstrn(fl_ctx, str, len);
  value_t files = cvalue_static_cstrn(fl_ctx, filename, filename_len);
- value_t e = fl_applyn(fl_ctx, 2, symbol_value(symbol(fl_ctx, "jl-parse-string")), s, files);
+ value_t e = fl_applyn(fl_ctx, 2, symbol_value(symbol(fl_ctx, "jl-parse-all")), s, files);
  jl_value_t *res = e == fl_ctx->FL_EOF ? jl_nothing : scm_to_julia(fl_ctx, e, NULL);
  jl_ast_ctx_leave(ctx);
  return res;
 }
 
+// for backwards compat
+JL_DLLEXPORT jl_value_t *jl_parse_input_line(const char *str, size_t len, const char *filename, size_t filename_len)
+{
+ return jl_parse_all(str, len, filename, filename_len);
+}
+
 // this is for parsing one expression out of a string, keeping track of
 // the current position.
 JL_DLLEXPORT jl_value_t *jl_parse_string(const char *str, size_t len,
@@ -792,7 +798,7 @@ JL_DLLEXPORT jl_value_t *jl_parse_string(const char *str, size_t len,
  jl_ast_context_t *ctx = jl_ast_ctx_enter();
  fl_context_t *fl_ctx = &ctx->fl;
  value_t s = cvalue_static_cstrn(fl_ctx, str, len);
- value_t p = fl_applyn(fl_ctx, 3, symbol_value(symbol(fl_ctx, "jl-parse-one-string")),
+ value_t p = fl_applyn(fl_ctx, 3, symbol_value(symbol(fl_ctx, "jl-parse-one")),
  s, fixnum(pos0), greedy?fl_ctx->T:fl_ctx->F);
  jl_value_t *expr=NULL, *pos1=NULL;
  JL_GC_PUSH2(&expr, &pos1);
@@ -828,7 +834,7 @@ jl_value_t *jl_parse_eval_all(const char *fname,
  JL_TIMING(PARSING);
  value_t t = cvalue_static_cstrn(fl_ctx, content, contentlen);
  fl_gc_handle(fl_ctx, &t);
- ast = fl_applyn(fl_ctx, 2, symbol_value(symbol(fl_ctx, "jl-parse-string-stream")), t, f);
+ ast = fl_applyn(fl_ctx, 2, symbol_value(symbol(fl_ctx, "jl-parse-all")), t, f);
  fl_free_gc_handles(fl_ctx, 1);
  }
  else {
@@ -869,9 +875,9 @@ jl_value_t *jl_parse_eval_all(const char *fname,
  }
  // expand non-final expressions in statement position (value unused)
  expression =
- fl_applyn(fl_ctx, 1,
+ fl_applyn(fl_ctx, 3,
  symbol_value(symbol(fl_ctx, iscons(cdr_(ast)) ? "jl-expand-to-thunk-stmt" : "jl-expand-to-thunk")),
- expression);
+ expression, symbol(fl_ctx, jl_filename), fixnum(jl_lineno));
  }
  jl_get_ptls_states()->world_age = jl_world_counter;
  form = scm_to_julia(fl_ctx, expression, inmodule);
@@ -929,6 +935,21 @@ jl_value_t *jl_call_scm_on_ast(const char *funcname, jl_value_t *expr, jl_module
  return result;
 }
 
+jl_value_t *jl_call_scm_on_ast_and_loc(const char *funcname, jl_value_t *expr, jl_module_t *inmodule,
+ const char *file, int line)
+{
+ jl_ast_context_t *ctx = jl_ast_ctx_enter();
+ fl_context_t *fl_ctx = &ctx->fl;
+ JL_AST_PRESERVE_PUSH(ctx, old_roots, inmodule);
+ value_t arg = julia_to_scm(fl_ctx, expr);
+ value_t e = fl_applyn(fl_ctx, 3, symbol_value(symbol(fl_ctx, funcname)), arg,
+ symbol(fl_ctx, file), fixnum(line));
+ jl_value_t *result = scm_to_julia(fl_ctx, e, inmodule);
+ JL_AST_PRESERVE_POP(ctx, old_roots);
+ jl_ast_ctx_leave(ctx);
+ return result;
+}
+
 // syntax tree accessors
 
 JL_DLLEXPORT jl_value_t *jl_copy_ast(jl_value_t *expr)
@@ -1165,29 +1186,40 @@ JL_DLLEXPORT jl_value_t *jl_macroexpand1(jl_value_t *expr, jl_module_t *inmodule
  return expr;
 }
 
-JL_DLLEXPORT jl_value_t *jl_expand(jl_value_t *expr, jl_module_t *inmodule)
+JL_DLLEXPORT jl_value_t *jl_expand_with_loc(jl_value_t *expr, jl_module_t *inmodule,
+ const char *file, int line)
 {
  JL_TIMING(LOWERING);
  JL_GC_PUSH1(&expr);
  expr = jl_copy_ast(expr);
  expr = jl_expand_macros(expr, inmodule, NULL, 0);
- expr = jl_call_scm_on_ast("jl-expand-to-thunk", expr, inmodule);
+ expr = jl_call_scm_on_ast_and_loc("jl-expand-to-thunk", expr, inmodule, file, line);
  JL_GC_POP();
  return expr;
 }
 
+JL_DLLEXPORT jl_value_t *jl_expand(jl_value_t *expr, jl_module_t *inmodule)
+{
+ return jl_expand_with_loc(expr, inmodule, "none", 0);
+}
+
 // expand in a context where the expression value is unused
-JL_DLLEXPORT jl_value_t *jl_expand_stmt(jl_value_t *expr, jl_module_t *inmodule)
+JL_DLLEXPORT jl_value_t *jl_expand_stmt_with_loc(jl_value_t *expr, jl_module_t *inmodule,
+ const char *file, int line)
 {
  JL_TIMING(LOWERING);
  JL_GC_PUSH1(&expr);
  expr = jl_copy_ast(expr);
  expr = jl_expand_macros(expr, inmodule, NULL, 0);
- expr = jl_call_scm_on_ast("jl-expand-to-thunk-stmt", expr, inmodule);
+ expr = jl_call_scm_on_ast_and_loc("jl-expand-to-thunk-stmt", expr, inmodule, file, line);
  JL_GC_POP();
  return expr;
 }
 
+JL_DLLEXPORT jl_value_t *jl_expand_stmt(jl_value_t *expr, jl_module_t *inmodule)
+{
+ return jl_expand_stmt_with_loc(expr, inmodule, "none", 0);
+}
 
 #ifdef __cplusplus
 }

src/jlapi.c

@@ -88,7 +88,7 @@ JL_DLLEXPORT jl_value_t *jl_eval_string(const char *str)
  jl_value_t *r;
  JL_TRY {
  const char filename[] = "none";
- jl_value_t *ast = jl_parse_input_line(str, strlen(str),
+ jl_value_t *ast = jl_parse_all(str, strlen(str),
  filename, strlen(filename));
  JL_GC_PUSH1(&ast);
  r = jl_toplevel_eval_in(jl_main_module, ast);

src/jlfrontend.scm

@@ -73,7 +73,7 @@
 ;; return a lambda expression representing a thunk for a top-level expression
 ;; note: expansion of stuff inside module is delayed, so the contents obey
 ;; toplevel expansion order (don't expand until stuff before is evaluated).
-(define (expand-toplevel-expr-- e)
+(define (expand-toplevel-expr-- e file line)
  (let ((ex0 (julia-expand-macroscope e)))
  (if (and (pair? ex0) (eq? (car ex0) 'toplevel))
  ex0
@@ -94,7 +94,8 @@
  (scope-block
  (block ,@(map (lambda (v) `(implicit-global ,v)) existing-gv)
  ,@(map (lambda (v) `(implicit-global ,v)) gv)
- ,ex))))))
+ ,ex)))
+ file line)))
  (if (and (null? (cdadr (caddr th)))
  (and (length= (lam:body th) 2)
  (let ((retval (cadadr (lam:body th))))
@@ -114,7 +115,7 @@
  (and (eq? (car e) 'global) (every symbol? (cdr e))
  (every (lambda (x) (not (memq x '(true false)))) (cdr e))))))
 
-(define (expand-toplevel-expr e)
+(define (expand-toplevel-expr e file line)
  (cond ((or (atom? e) (toplevel-only-expr? e))
  (if (underscore-symbol? e)
  (error "all-underscore identifier used as rvalue"))
@@ -124,7 +125,7 @@
  (if (not last)
  (begin (reset-gensyms)
  (set! *in-expand* #t)))
- (begin0 (expand-toplevel-expr-- e)
+ (begin0 (expand-toplevel-expr-- e file line)
  (set! *in-expand* last))))))
 
 ;; construct default definitions of `eval` for non-bare modules
@@ -146,10 +147,11 @@
  (= (call include ,x)
  (block
  ,@loc
- (call (top include) ,name ,x)))))))
+ (call (top include) ,name ,x)))))
+ 'none 0))
 
-;; parse only, returning end position, no expansion.
-(define (jl-parse-one-string s pos0 greedy)
+;; parse one expression (if greedy) or atom, returning end position
+(define (jl-parse-one s pos0 greedy)
  (let ((inp (open-input-string s)))
  (io.seek inp pos0)
  (let ((expr (error-wrap (lambda ()
@@ -158,25 +160,7 @@
  (julia-parse inp parse-atom))))))
  (cons expr (io.pos inp)))))
 
-(define (jl-parse-string s filename)
- (with-bindings ((current-filename (symbol filename)))
- (error-wrap (lambda ()
- (let ((inp (make-token-stream (open-input-string s))))
- ;; parse all exprs into a (toplevel ...) form
- (let loop ((exprs '()))
- ;; delay expansion so macros run in the Task executing
- ;; the input, not the task parsing it (issue #2378)
- ;; used to be (expand-toplevel-expr expr)
- (let ((expr (julia-parse inp)))
- (if (eof-object? expr)
- (cond ((null? exprs) expr)
- ((length= exprs 1) (car exprs))
- (else (cons 'toplevel (reverse! exprs))))
- (if (and (pair? expr) (eq? (car expr) 'toplevel))
- (loop (nreconc (cdr expr) exprs))
- (loop (cons expr exprs)))))))))))
-
-(define (jl-parse-all io filename)
+(define (parse-all- io filename)
  (unwind-protect
  (with-bindings ((current-filename (symbol filename)))
  (let ((stream (make-token-stream io)))
@@ -192,44 +176,47 @@
  (julia-parse stream)))))
  (if (eof-object? expr)
  (cons 'toplevel (reverse! exprs))
- (let* ((iserr (and (pair? expr) (eq? (car expr) 'error)))
- (next (list* expr
- ;; for error, get most recent line number (#16720)
- (if iserr
- `(line ,(input-port-line io))
- `(line ,lineno))
- exprs)))
+ (let* ((iserr (and (pair? expr) (eq? (car expr) 'error)))
+ ;; for error, get most recent line number (#16720)
+ (lineno (if iserr (input-port-line io) lineno))
+ (next (list* expr
+ ;; include filename in first line node
+ (if (null? exprs)
+ `(line ,lineno ,(symbol filename))
+ `(line ,lineno))
+ exprs)))
  (if iserr
  (cons 'toplevel (reverse! next))
  (loop next))))))))))
  (io.close io)))
 
-;; parse file-in-a-string
-(define (jl-parse-string-stream str filename)
- (jl-parse-all (open-input-string str) filename))
+;; parse all expressions in a string, the same way files are parsed
+(define (jl-parse-all str filename)
+ (parse-all- (open-input-string str) filename))
 
 (define (jl-parse-file filename)
  (trycatch
- (jl-parse-all (open-input-file filename) filename)
+ (parse-all- (open-input-file filename) filename)
  (lambda (e) #f)))
 
 ; expand a piece of raw surface syntax to an executable thunk
-(define (jl-expand-to-thunk expr)
+(define (jl-expand-to-thunk expr file line)
  (error-wrap (lambda ()
- (expand-toplevel-expr expr))))
+ (expand-toplevel-expr expr file line))))
 
-(define (jl-expand-to-thunk-stmt expr)
+(define (jl-expand-to-thunk-stmt expr file line)
  (jl-expand-to-thunk (if (toplevel-only-expr? expr)
  expr
- `(block ,expr (null)))))
+ `(block ,expr (null)))
+ file line))
 
 (define (jl-expand-macroscope expr)
  (error-wrap (lambda ()
  (julia-expand-macroscope expr))))
 
 ; run whole frontend on a string. useful for testing.
 (define (fe str)
- (expand-toplevel-expr (julia-parse str)))
+ (expand-toplevel-expr (julia-parse str) 'none 0))
 
 (define (profile-e s)
  (with-exception-catcher