support line number offset in parsing (JuliaLang#43876)

This adds an extra keyword argument `lineno` to `Meta.parseatom` and
`Meta.parseall`, which causes the emitted `LineNumberNode`s to start at
the given line number instead.

base/boot.jl

@@ -791,7 +791,7 @@ Integer(x::Union{Float16, Float32, Float64}) = Int(x)
 
 # Binding for the julia parser, called as
 #
-# Core._parse(text, filename, offset, options)
+# Core._parse(text, filename, lineno, offset, options)
 #
 # Parse Julia code from the buffer `text`, starting at `offset` and attributing
 # it to `filename`. `text` may be a `String` or `svec(ptr::Ptr{UInt8},

base/compiler/parsing.jl

@@ -3,17 +3,17 @@
 # Call Julia's builtin flisp-based parser. `offset` is 0-based offset into the
 # byte buffer or string.
 function fl_parse(text::Union{Core.SimpleVector,String},
- filename::String, offset, options)
+ filename::String, lineno, offset, options)
  if text isa Core.SimpleVector
  # Will be generated by C entry points jl_parse_string etc
  text, text_len = text
  else
  text_len = sizeof(text)
  end
- ccall(:jl_fl_parse, Any, (Ptr{UInt8}, Csize_t, Any, Csize_t, Any),
- text, text_len, filename, offset, options)
+ ccall(:jl_fl_parse, Any, (Ptr{UInt8}, Csize_t, Any, Csize_t, Csize_t, Any),
+ text, text_len, filename, lineno, offset, options)
 end
 
-function fl_parse(text::AbstractString, filename::AbstractString, offset, options)
- fl_parse(String(text), String(filename), offset, options)
+function fl_parse(text::AbstractString, filename::AbstractString, lineno, offset, options)
+ fl_parse(String(text), String(filename), lineno, offset, options)
 end

base/meta.jl

@@ -191,11 +191,11 @@ struct ParseError <: Exception
 end
 
 function _parse_string(text::AbstractString, filename::AbstractString,
- index::Integer, options)
+ lineno::Integer, index::Integer, options)
  if index < 1 || index > ncodeunits(text) + 1
  throw(BoundsError(text, index))
  end
- ex, offset::Int = Core._parse(text, filename, index-1, options)
+ ex, offset::Int = Core._parse(text, filename, lineno, index-1, options)
  ex, offset+1
 end
 
@@ -232,7 +232,7 @@ julia> Meta.parse("(α, β) = 3, 5", 11, greedy=false)
 """
 function parse(str::AbstractString, pos::Integer; greedy::Bool=true, raise::Bool=true,
  depwarn::Bool=true)
- ex, pos = _parse_string(str, "none", pos, greedy ? :statement : :atom)
+ ex, pos = _parse_string(str, "none", 1, pos, greedy ? :statement : :atom)
  if raise && isa(ex,Expr) && ex.head === :error
  throw(ParseError(ex.args[1]))
  end
@@ -276,12 +276,12 @@ function parse(str::AbstractString; raise::Bool=true, depwarn::Bool=true)
  return ex
 end
 
-function parseatom(text::AbstractString, pos::Integer; filename="none")
- return _parse_string(text, String(filename), pos, :atom)
+function parseatom(text::AbstractString, pos::Integer; filename="none", lineno=1)
+ return _parse_string(text, String(filename), lineno, pos, :atom)
 end
 
-function parseall(text::AbstractString; filename="none")
- ex,_ = _parse_string(text, String(filename), 1, :all)
+function parseall(text::AbstractString; filename="none", lineno=1)
+ ex,_ = _parse_string(text, String(filename), lineno, 1, :all)
  return ex
 end
 

src/ast.c

@@ -764,8 +764,8 @@ static value_t julia_to_scm_(fl_context_t *fl_ctx, jl_value_t *v, int check_vali
 // Parse `text` starting at 0-based `offset` and attributing the content to
 // `filename`. Return an svec of (parsed_expr, final_offset)
 JL_DLLEXPORT jl_value_t *jl_fl_parse(const char *text, size_t text_len,
- jl_value_t *filename, size_t offset,
- jl_value_t *options)
+ jl_value_t *filename, size_t lineno,
+ size_t offset, jl_value_t *options)
 {
  JL_TIMING(PARSING);
  if (offset > text_len) {
@@ -791,15 +791,15 @@ JL_DLLEXPORT jl_value_t *jl_fl_parse(const char *text, size_t text_len,
  value_t fl_expr;
  size_t offset1 = 0;
  if (rule == jl_all_sym) {
- value_t e = fl_applyn(fl_ctx, 2, symbol_value(symbol(fl_ctx, "jl-parse-all")),
- fl_text, fl_filename);
+ value_t e = fl_applyn(fl_ctx, 3, symbol_value(symbol(fl_ctx, "jl-parse-all")),
+ fl_text, fl_filename, fixnum(lineno));
  fl_expr = e;
  offset1 = e == fl_ctx->FL_EOF ? text_len : 0;
  }
  else {
  value_t greedy = rule == jl_statement_sym ? fl_ctx->T : fl_ctx->F;
- value_t p = fl_applyn(fl_ctx, 4, symbol_value(symbol(fl_ctx, "jl-parse-one")),
- fl_text, fl_filename, fixnum(offset), greedy);
+ value_t p = fl_applyn(fl_ctx, 5, symbol_value(symbol(fl_ctx, "jl-parse-one")),
+ fl_text, fl_filename, fixnum(offset), greedy, fixnum(lineno));
  fl_expr = car_(p);
  offset1 = tosize(fl_ctx, cdr_(p), "parse");
  }
@@ -1243,30 +1243,31 @@ JL_DLLEXPORT jl_value_t *jl_expand_stmt(jl_value_t *expr, jl_module_t *inmodule)
 // `text` is passed as a pointer to allow raw non-String buffers to be used
 // without copying.
 JL_DLLEXPORT jl_value_t *jl_parse(const char *text, size_t text_len, jl_value_t *filename,
- size_t offset, jl_value_t *options)
+ size_t lineno, size_t offset, jl_value_t *options)
 {
  jl_value_t *core_parse = NULL;
  if (jl_core_module) {
  core_parse = jl_get_global(jl_core_module, jl_symbol("_parse"));
  }
  if (!core_parse || core_parse == jl_nothing) {
  // In bootstrap, directly call the builtin parser.
- jl_value_t *result = jl_fl_parse(text, text_len, filename, offset, options);
+ jl_value_t *result = jl_fl_parse(text, text_len, filename, lineno, offset, options);
  return result;
  }
  jl_value_t **args;
- JL_GC_PUSHARGS(args, 5);
+ JL_GC_PUSHARGS(args, 6);
  args[0] = core_parse;
  args[1] = (jl_value_t*)jl_alloc_svec(2);
  jl_svecset(args[1], 0, jl_box_uint8pointer((uint8_t*)text));
  jl_svecset(args[1], 1, jl_box_long(text_len));
  args[2] = filename;
- args[3] = jl_box_ulong(offset);
- args[4] = options;
+ args[3] = jl_box_ulong(lineno);
+ args[4] = jl_box_ulong(offset);
+ args[5] = options;
  jl_task_t *ct = jl_current_task;
  size_t last_age = ct->world_age;
  ct->world_age = jl_atomic_load_acquire(&jl_world_counter);
- jl_value_t *result = jl_apply(args, 5);
+ jl_value_t *result = jl_apply(args, 6);
  ct->world_age = last_age;
  args[0] = result; // root during error checks below
  JL_TYPECHK(parse, simplevector, result);
@@ -1280,11 +1281,11 @@ JL_DLLEXPORT jl_value_t *jl_parse(const char *text, size_t text_len, jl_value_t
 
 // parse an entire string as a file, reading multiple expressions
 JL_DLLEXPORT jl_value_t *jl_parse_all(const char *text, size_t text_len,
- const char *filename, size_t filename_len)
+ const char *filename, size_t filename_len, size_t lineno)
 {
  jl_value_t *fname = jl_pchar_to_string(filename, filename_len);
  JL_GC_PUSH1(&fname);
- jl_value_t *p = jl_parse(text, text_len, fname, 0, (jl_value_t*)jl_all_sym);
+ jl_value_t *p = jl_parse(text, text_len, fname, lineno, 0, (jl_value_t*)jl_all_sym);
  JL_GC_POP();
  return jl_svecref(p, 0);
 }
@@ -1296,7 +1297,7 @@ JL_DLLEXPORT jl_value_t *jl_parse_string(const char *text, size_t text_len,
 {
  jl_value_t *fname = jl_cstr_to_string("none");
  JL_GC_PUSH1(&fname);
- jl_value_t *result = jl_parse(text, text_len, fname, offset,
+ jl_value_t *result = jl_parse(text, text_len, fname, 1, offset,
  (jl_value_t*)(greedy ? jl_statement_sym : jl_atom_sym));
  JL_GC_POP();
  return result;
@@ -1306,7 +1307,7 @@ JL_DLLEXPORT jl_value_t *jl_parse_string(const char *text, size_t text_len,
 JL_DLLEXPORT jl_value_t *jl_parse_input_line(const char *text, size_t text_len,
  const char *filename, size_t filename_len)
 {
- return jl_parse_all(text, text_len, filename, filename_len);
+ return jl_parse_all(text, text_len, filename, filename_len, 1);
 }
 
 #ifdef __cplusplus

src/flisp/iostream.c

@@ -203,6 +203,15 @@ value_t fl_iolineno(fl_context_t *fl_ctx, value_t *args, uint32_t nargs)
  return size_wrap(fl_ctx, s->lineno);
 }
 
+value_t fl_iosetlineno(fl_context_t *fl_ctx, value_t *args, uint32_t nargs)
+{
+ argcount(fl_ctx, "io.set-lineno!", nargs, 2);
+ ios_t *s = toiostream(fl_ctx, args[0], "io.set-lineno!");
+ size_t new_lineno = tosize(fl_ctx, args[1], "io.set-lineno!");
+ s->lineno = new_lineno;
+ return args[1];
+}
+
 value_t fl_iocolno(fl_context_t *fl_ctx, value_t *args, uint32_t nargs)
 {
  argcount(fl_ctx, "input-port-column", nargs, 1);
@@ -439,6 +448,7 @@ static const builtinspec_t iostreamfunc_info[] = {
  { "io.tostring!", fl_iotostring },
  { "input-port-line", fl_iolineno },
  { "input-port-column", fl_iocolno },
+ { "io.set-lineno!", fl_iosetlineno },
 
  { NULL, NULL }
 };

src/jlapi.c

@@ -110,7 +110,7 @@ JL_DLLEXPORT jl_value_t *jl_eval_string(const char *str)
  JL_TRY {
  const char filename[] = "none";
  jl_value_t *ast = jl_parse_all(str, strlen(str),
- filename, strlen(filename));
+ filename, strlen(filename), 1);
  JL_GC_PUSH1(&ast);
  r = jl_toplevel_eval_in(jl_main_module, ast);
  JL_GC_POP();

src/jlfrontend.scm

@@ -37,9 +37,10 @@
 ;; parser entry points
 
 ;; parse one expression (if greedy) or atom, returning end position
-(define (jl-parse-one str filename pos0 greedy)
+(define (jl-parse-one str filename pos0 greedy (lineno 1))
  (let ((inp (open-input-string str)))
  (io.seek inp pos0)
+ (io.set-lineno! inp lineno)
  (with-bindings ((current-filename (symbol filename)))
  (let ((expr (error-wrap (lambda ()
  (if greedy
@@ -78,13 +79,17 @@
  (io.close io)))
 
 ;; parse all expressions in a string, the same way files are parsed
-(define (jl-parse-all str filename)
- (parse-all- (open-input-string str) filename))
+(define (jl-parse-all str filename (lineno 1))
+ (let ((io (open-input-string str)))
+ (io.set-lineno! io lineno)
+ (parse-all- io filename)))
 
-(define (jl-parse-file filename)
+(define (jl-parse-file filename (lineno 1))
  (trycatch
- (parse-all- (open-input-file filename) filename)
- (lambda (e) #f)))
+ (let ((io (open-input-string str)))
+ (io.set-lineno! io lineno)
+ (parse-all- io filename))
+ (lambda (e) #f)))
 
 ;; lowering entry points
 

src/julia.h

@@ -1685,7 +1685,7 @@ JL_DLLEXPORT jl_value_t *jl_restore_incremental_from_buf(const char *buf, size_t
 
 // parsing
 JL_DLLEXPORT jl_value_t *jl_parse_all(const char *text, size_t text_len,
- const char *filename, size_t filename_len);
+ const char *filename, size_t filename_len, size_t lineno);
 JL_DLLEXPORT jl_value_t *jl_parse_string(const char *text, size_t text_len,
  int offset, int greedy);
 // lowering

src/julia_internal.h

@@ -862,7 +862,7 @@ jl_tupletype_t *arg_type_tuple(jl_value_t *arg1, jl_value_t **args, size_t nargs
 JL_DLLEXPORT int jl_has_meta(jl_array_t *body, jl_sym_t *sym) JL_NOTSAFEPOINT;
 
 jl_value_t *jl_parse(const char *text, size_t text_len, jl_value_t *filename,
- size_t offset, jl_value_t *options);
+ size_t lineno, size_t offset, jl_value_t *options);
 
 //--------------------------------------------------
 // Backtraces

src/toplevel.c

@@ -1004,7 +1004,7 @@ static jl_value_t *jl_parse_eval_all(jl_module_t *module, jl_value_t *text,
  JL_GC_PUSH3(&ast, &result, &expression);
 
  ast = jl_svecref(jl_parse(jl_string_data(text), jl_string_len(text),
- filename, 0, (jl_value_t*)jl_all_sym), 0);
+ filename, 1, 0, (jl_value_t*)jl_all_sym), 0);
  if (!jl_is_expr(ast) || ((jl_expr_t*)ast)->head != jl_toplevel_sym) {
  jl_errorf("jl_parse_all() must generate a top level expression");
  }

test/show.jl

@@ -1999,7 +1999,7 @@ eval(Meta._parse_string("""function my_fun28173(x)
  "three"
  end
  return y
-end""", "a"^80, 1, :statement)[1]) # use parse to control the line numbers
+end""", "a"^80, 1, 1, :statement)[1]) # use parse to control the line numbers
 let src = code_typed(my_fun28173, (Int,), debuginfo=:source)[1][1]
  ir = Core.Compiler.inflate_ir(src)
  fill!(src.codelocs, 0) # IRCode printing is only capable of printing partial line info

test/syntax.jl

@@ -3137,3 +3137,6 @@ end
  @test_throws ParseError Meta.parse("[;; ;]")
  @test_throws ParseError Meta.parse("[;\n;]")
 end
+
+@test Meta.parseatom("@foo", 1; filename="foo", lineno=7) == (Expr(:macrocall, :var"@foo", LineNumberNode(7, :foo)), 5)
+@test Meta.parseall("@foo"; filename="foo", lineno=3) == Expr(:toplevel, LineNumberNode(3, :foo), Expr(:macrocall, :var"@foo", LineNumberNode(3, :foo)))