Merge pull request JuliaLang#35243 from JuliaLang/cjf/parser-api

Replaceable single entry point for parser

base/Base.jl

@@ -365,31 +365,8 @@ for m in methods(include)
 end
 # These functions are duplicated in client.jl/include(::String) for
 # nicer stacktraces. Modifications here have to be backported there
-include(mod::Module, _path::AbstractString) = include(identity, mod, _path)
-function include(mapexpr::Function, mod::Module, _path::AbstractString)
- path, prev = _include_dependency(mod, _path)
- for callback in include_callbacks # to preserve order, must come before Core.include
- invokelatest(callback, mod, path)
- end
- tls = task_local_storage()
- tls[:SOURCE_PATH] = path
- local result
- try
- # result = Core.include(mod, path)
- if mapexpr === identity
- result = ccall(:jl_load, Any, (Any, Cstring), mod, path)
- else
- result = ccall(:jl_load_rewrite, Any, (Any, Cstring, Any), mod, path, mapexpr)
- end
- finally
- if prev === nothing
- delete!(tls, :SOURCE_PATH)
- else
- tls[:SOURCE_PATH] = prev
- end
- end
- return result
-end
+include(mod::Module, _path::AbstractString) = _include(identity, mod, _path)
+include(mapexpr::Function, mod::Module, _path::AbstractString) = _include(mapexpr, mod, _path)
 
 end_base_include = time_ns()
 

base/boot.jl

@@ -740,4 +740,19 @@ Unsigned(x::Union{Float32, Float64, Bool}) = UInt(x)
 Integer(x::Integer) = x
 Integer(x::Union{Float32, Float64}) = Int(x)
 
+# Binding for the julia parser, called as
+#
+# Core._parse(text, filename, offset, options)
+#
+# Parse Julia code from the buffer `text`, starting at `offset` and attributing
+# it to `filename`. `text` may be a `String` or `svec(ptr::Ptr{UInt8},
+# len::Int)` for a raw unmanaged buffer. `options` should be one of `:atom`,
+# `:statement` or `:all`, indicating how much the parser will consume.
+#
+# `_parse` must return an `svec` containing an `Expr` and the new offset as an
+# `Int`.
+#
+# The internal jl_parse which will call into Core._parse if not `nothing`.
+_parse = nothing
+
 ccall(:jl_set_istopmod, Cvoid, (Any, Bool), Core, true)

base/client.jl

@@ -154,8 +154,7 @@ function eval_user_input(errio, @nospecialize(ast), show_value::Bool)
 end
 
 function _parse_input_line_core(s::String, filename::String)
- ex = ccall(:jl_parse_all, Any, (Ptr{UInt8}, Csize_t, Ptr{UInt8}, Csize_t),
- s, sizeof(s), filename, sizeof(filename))
+ ex = Meta.parseall(s, filename=filename)
  if ex isa Expr && ex.head === :toplevel
  if isempty(ex.args)
  return nothing
@@ -439,30 +438,12 @@ end
 # MainInclude exists to hide Main.include and eval from `names(Main)`.
 baremodule MainInclude
 using ..Base
-include(mapexpr::Function, fname::AbstractString) = Base.include(mapexpr, Main, fname)
-# We inline the definition of include from loading.jl/include_relative to get one-frame stacktraces
-# for the common case of include(fname). Otherwise we would use:
-# include(fname::AbstractString) = Base.include(Main, fname)
+# These definitions calls Base._include rather than Base.include to get
+# one-frame stacktraces for the common case of using include(fname) in Main.
+include(mapexpr::Function, fname::AbstractString) = Base._include(mapexpr, Main, fname)
 function include(fname::AbstractString)
- mod = Main
  isa(fname, String) || (fname = Base.convert(String, fname)::String)
- path, prev = Base._include_dependency(mod, fname)
- for callback in Base.include_callbacks # to preserve order, must come before Core.include
- Base.invokelatest(callback, mod, path)
- end
- tls = Base.task_local_storage()
- tls[:SOURCE_PATH] = path
- local result
- try
- result = ccall(:jl_load, Any, (Any, Cstring), mod, path)
- finally
- if prev === nothing
- Base.delete!(tls, :SOURCE_PATH)
- else
- tls[:SOURCE_PATH] = prev
- end
- end
- return result
+ Base._include(identity, Main, fname)
 end
 eval(x) = Core.eval(Main, x)
 end

base/compiler/compiler.jl

@@ -113,5 +113,9 @@ include("compiler/optimize.jl") # TODO: break this up further + extract utilitie
 include("compiler/bootstrap.jl")
 ccall(:jl_set_typeinf_func, Cvoid, (Any,), typeinf_ext_toplevel)
 
+include("compiler/parsing.jl")
+Core.eval(Core, :(_parse = Compiler.fl_parse))
+
 end # baremodule Compiler
 ))
+

base/compiler/parsing.jl

@@ -0,0 +1,19 @@
+# This file is a part of Julia. License is MIT: https://julialang.org/license
+
+# Call Julia's builtin flisp-based parser. `offset` is 0-based offset into the
+# byte buffer or string.
+function fl_parse(text::Union{Core.SimpleVector,String},
+ filename::String, offset, options)
+ if text isa Core.SimpleVector
+ # Will be generated by C entry points jl_parse_string etc
+ text, text_len = text
+ else
+ text_len = sizeof(text)
+ end
+ ccall(:jl_fl_parse, Any, (Ptr{UInt8}, Csize_t, Any, Csize_t, Any),
+ text, text_len, filename, offset, options)
+end
+
+function fl_parse(text::AbstractString, filename::AbstractString, offset, options)
+ fl_parse(String(text), String(filename), offset, options)
+end

base/errorshow.jl

@@ -681,6 +681,41 @@ function is_kw_sorter_name(name::Symbol)
  return !startswith(sn, '#') && endswith(sn, "##kw")
 end
 
+# For improved user experience, filter out frames for include() implementation
+# - see #33065. See also #35371 for extended discussion of internal frames.
+function _simplify_include_frames(trace)
+ i = length(trace)
+ kept_frames = trues(i)
+ first_ignored = nothing
+ while i >= 1
+ frame, _ = trace[i]
+ mod = parentmodule(frame)
+ if isnothing(first_ignored)
+ if mod === Base && frame.func === :_include
+ # Hide include() machinery by default
+ first_ignored = i
+ end
+ else
+ # Hack: allow `mod==nothing` as a workaround for inlined functions.
+ # TODO: Fix this by improving debug info.
+ if mod in (Base,Core,nothing) && 1+first_ignored-i <= 5
+ if frame.func == :eval
+ kept_frames[i:first_ignored] .= false
+ first_ignored = nothing
+ end
+ else
+ # Bail out to avoid hiding frames in unexpected circumstances
+ first_ignored = nothing
+ end
+ end
+ i -= 1
+ end
+ if !isnothing(first_ignored)
+ kept_frames[i:first_ignored] .= false
+ end
+ return trace[kept_frames]
+end
+
 function process_backtrace(t::Vector, limit::Int=typemax(Int); skipC = true)
  n = 0
  last_frame = StackTraces.UNKNOWN
@@ -721,7 +756,7 @@ function process_backtrace(t::Vector, limit::Int=typemax(Int); skipC = true)
  if n > 0
  push!(ret, (last_frame, n))
  end
- return ret
+ return _simplify_include_frames(ret)
 end
 
 function show_exception_stack(io::IO, stack::Vector)

base/loading.jl

@@ -1082,14 +1082,31 @@ The optional first argument `mapexpr` can be used to transform the included code
 it is evaluated: for each parsed expression `expr` in `code`, the `include_string` function
 actually evaluates `mapexpr(expr)`. If it is omitted, `mapexpr` defaults to [`identity`](@ref).
 """
-function include_string(mapexpr::Function, m::Module, txt_::AbstractString, fname::AbstractString="string")
- txt = String(txt_)
- if mapexpr === identity
- ccall(:jl_load_file_string, Any, (Ptr{UInt8}, Csize_t, Cstring, Any),
- txt, sizeof(txt), String(fname), m)
- else
- ccall(:jl_load_rewrite_file_string, Any, (Ptr{UInt8}, Csize_t, Cstring, Any, Any),
- txt, sizeof(txt), String(fname), m, mapexpr)
+function include_string(mapexpr::Function, mod::Module, code::AbstractString,
+ filename::AbstractString="string")
+ loc = LineNumberNode(1, Symbol(filename))
+ try
+ ast = Meta.parseall(code, filename=filename)
+ @assert Meta.isexpr(ast, :toplevel)
+ result = nothing
+ line_and_ex = Expr(:toplevel, loc, nothing)
+ for ex in ast.args
+ if ex isa LineNumberNode
+ loc = ex
+ line_and_ex.args[1] = ex
+ continue
+ end
+ ex = mapexpr(ex)
+ # Wrap things to be eval'd in a :toplevel expr to carry line
+ # information as part of the expr.
+ line_and_ex.args[2] = ex
+ result = Core.eval(mod, line_and_ex)
+ end
+ return result
+ catch exc
+ # TODO: Now that stacktraces are more reliable we should remove
+ # LoadError and expose the real error type directly.
+ rethrow(LoadError(filename, loc.line, exc))
  end
 end
 
@@ -1124,7 +1141,28 @@ The optional first argument `mapexpr` can be used to transform the included code
 it is evaluated: for each parsed expression `expr` in `path`, the `include` function
 actually evaluates `mapexpr(expr)`. If it is omitted, `mapexpr` defaults to [`identity`](@ref).
 """
-Base.include # defined in sysimg.jl
+Base.include # defined in Base.jl
+
+# Full include() implementation which is used after bootstrap
+function _include(mapexpr::Function, mod::Module, _path::AbstractString)
+ @_noinline_meta # Workaround for module availability in _simplify_include_frames
+ path, prev = _include_dependency(mod, _path)
+ for callback in include_callbacks # to preserve order, must come before eval in include_string
+ invokelatest(callback, mod, path)
+ end
+ code = read(path, String)
+ tls = task_local_storage()
+ tls[:SOURCE_PATH] = path
+ try
+ return include_string(mapexpr, mod, code, path)
+ finally
+ if prev === nothing
+ delete!(tls, :SOURCE_PATH)
+ else
+ tls[:SOURCE_PATH] = prev
+ end
+ end
+end
 
 """
  evalfile(path::AbstractString, args::Vector{String}=String[])

base/meta.jl

@@ -149,6 +149,15 @@ struct ParseError <: Exception
  msg::AbstractString
 end
 
+function _parse_string(text::AbstractString, filename::AbstractString,
+ index::Integer, options)
+ if index < 1 || index > ncodeunits(text) + 1
+ throw(BoundsError(text, index))
+ end
+ ex, offset = Core._parse(text, filename, index-1, options)
+ ex, offset+1
+end
+
 """
  parse(str, start; greedy=true, raise=true, depwarn=true)
 
@@ -171,19 +180,11 @@ julia> Meta.parse("x = 3, y = 5", 5)
 """
 function parse(str::AbstractString, pos::Integer; greedy::Bool=true, raise::Bool=true,
  depwarn::Bool=true)
- # pos is one based byte offset.
- # returns (expr, end_pos). expr is () in case of parse error.
- bstr = String(str)
- # For now, assume all parser warnings are depwarns
- ex, pos = with_logger(depwarn ? current_logger() : NullLogger()) do
- ccall(:jl_parse_string, Any,
- (Ptr{UInt8}, Csize_t, Int32, Int32),
- bstr, sizeof(bstr), pos-1, greedy ? 1 : 0)
- end
+ ex, pos = _parse_string(str, "none", pos, greedy ? :statement : :atom)
  if raise && isa(ex,Expr) && ex.head === :error
  throw(ParseError(ex.args[1]))
  end
- return ex, pos+1 # C is zero-based, Julia is 1-based
+ return ex, pos
 end
 
 """
@@ -223,6 +224,15 @@ function parse(str::AbstractString; raise::Bool=true, depwarn::Bool=true)
  return ex
 end
 
+function parseatom(text::AbstractString, pos::Integer; filename="none")
+ return _parse_string(text, filename, pos, :atom)
+end
+
+function parseall(text::AbstractString; filename="none")
+ ex,_ = _parse_string(text, filename, 1, :all)
+ return ex
+end
+
 """
  partially_inline!(code::Vector{Any}, slot_replacements::Vector{Any},
  type_signature::Type{<:Tuple}, static_param_values::Vector{Any},

base/stacktraces.jl

@@ -268,22 +268,28 @@ function show(io::IO, frame::StackFrame; full_path::Bool=false)
  end
 end
 
+function Base.parentmodule(frame::StackFrame)
+ if frame.linfo isa MethodInstance
+ def = frame.linfo.def
+ if def isa Module
+ return def
+ else
+ return (def::Method).module
+ end
+ else
+ # The module is not always available (common reasons include inlined
+ # frames and frames arising from the interpreter)
+ nothing
+ end
+end
+
 """
  from(frame::StackFrame, filter_mod::Module) -> Bool
 
 Returns whether the `frame` is from the provided `Module`
 """
 function from(frame::StackFrame, m::Module)
- finfo = frame.linfo
- result = false
-
- if finfo isa MethodInstance
- frame_m = finfo.def
- isa(frame_m, Method) && (frame_m = frame_m.module)
- result = nameof(frame_m) === nameof(m)
- end
-
- return result
+ return parentmodule(frame) === m
 end
 
 end

src/.gitignore

@@ -20,3 +20,7 @@
 /julia_version.h
 /flisp/host
 /support/host
+
+# Clang compilation database
+/compile_commands*.json
+.clangd/