move macroexpand to C

cuts down on complexity a bit (it now never needs to transition from Julia -> flisp -> Julia)
and fixes backtraces from macros! (fix JuliaLang#15643)

src/codegen.cpp

@@ -3811,18 +3811,14 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr)
  true, jl_any_type);
  }
  else if (head == copyast_sym) {
- jl_value_t *arg = args[0];
- if (jl_is_quotenode(arg)) {
- jl_value_t *arg1 = jl_fieldref(arg, 0);
- if (!(jl_is_expr(arg1) || jl_typeis(arg1, jl_array_any_type) || jl_is_quotenode(arg1))) {
- // elide call to jl_copy_ast when possible
- return emit_expr(ctx, arg);
- }
+ jl_cgval_t ast = emit_expr(ctx, args[0]);
+ if (ast.typ != (jl_value_t*)jl_expr_type && ast.typ != (jl_value_t*)jl_any_type) {
+ // elide call to jl_copy_ast when possible
+ return ast;
  }
- jl_cgval_t ast = emit_expr(ctx, arg);
  return mark_julia_type(ctx,
  ctx.builder.CreateCall(prepare_call(jlcopyast_func),
- maybe_decay_untracked(boxed(ctx, ast))), true, ast.typ);
+ maybe_decay_untracked(boxed(ctx, ast))), true, jl_expr_type);
  }
  else if (head == simdloop_sym) {
  llvm::annotateSimdLoop(ctx.builder.GetInsertBlock());

src/jlfrontend.scm

@@ -65,7 +65,7 @@
 ;; note: expansion of stuff inside module is delayed, so the contents obey
 ;; toplevel expansion order (don't expand until stuff before is evaluated).
 (define (expand-toplevel-expr-- e)
- (let ((ex0 (julia-expand-macros e)))
+ (let ((ex0 (julia-expand-macroscope e)))
  (if (and (pair? ex0) (eq? (car ex0) 'toplevel))
  ex0
  (let* ((ex (julia-expand0 ex0))
@@ -219,17 +219,6 @@
  (parser-wrap (lambda ()
  (expand-toplevel-expr expr))))
 
-; macroexpand only
-(define (jl-macroexpand expr)
- (reset-gensyms)
- (parser-wrap (lambda ()
- (julia-expand-macros expr))))
-
-(define (jl-macroexpand-1 expr)
- (reset-gensyms)
- (parser-wrap (lambda ()
- (julia-expand-macros expr 1))))
-
 ; run whole frontend on a string. useful for testing.
 (define (fe str)
  (expand-toplevel-expr (julia-parse str)))

src/julia-syntax.scm

@@ -3245,7 +3245,7 @@ f(x) = yt(x)
  ,@top-stmts
  (block ,@sp-inits
  (method ,name ,(cl-convert sig fname lam namemap toplevel interp)
- ,(julia-expand-macros `(quote ,newlam))
+ ,(julia-bq-macro newlam)
  ,(last e)))))))
  ;; local case - lift to a new type at top level
  (let* ((exists (get namemap name #f))
@@ -4004,4 +4004,4 @@ f(x) = yt(x)
 (define (julia-expand ex)
  (julia-expand1
  (julia-expand0
- (julia-expand-macros ex))))
+  julia-expand-macroscope ex)))

src/macroexpand.scm

@@ -46,10 +46,6 @@
  (call (top append_any) ,@forms)))
  (loop (cdr p) (cons (julia-bq-bracket (car p) d) q)))))))
 
-(define (julia-bq-expand-hygienic x unhygienic)
- (let ((expanded (julia-bq-expand x 0)))
- (if unhygienic expanded `(escape ,expanded))))
-
 ;; hygiene
 
 ;; return the names of vars introduced by forms, instead of their transformations.
@@ -297,7 +293,7 @@
  (case (car e)
  ((ssavalue) e)
  ((escape) (if (null? parent-scope)
- (julia-expand-macroscopes (cadr e))
+ (julia-expand-macroscopes- (cadr e))
  (let* ((scope (car parent-scope))
  (env (car scope))
  (m (cadr scope))
@@ -476,6 +472,27 @@
  ;; and wrap globals in (globalref module var) for macro's home module
  (resolve-expansion-vars-with-new-env e '() m '() #f #t))
 
+(define (julia-expand-quotes e)
+ (cond ((not (pair? e)) e)
+ ((eq? (car e) 'inert) e)
+ ((eq? (car e) 'module) e)
+ ((eq? (car e) 'quote)
+ (julia-expand-quotes (julia-bq-macro (cadr e))))
+ ((not (contains (lambda (e) (and (pair? e) (eq? (car e) 'quote))) (cdr e))) e)
+ (else
+ (cons (car e) (map julia-expand-quotes (cdr e))))))
+
+(define (julia-expand-macroscopes- e)
+ (cond ((not (pair? e)) e)
+ ((eq? (car e) 'inert) e)
+ ((eq? (car e) 'module) e)
+ ((eq? (car e) 'hygienic-scope)
+ (let ((form (cadr e)) ;; form is the expression returned from expand-macros
+ (modu (caddr e))) ;; m is the macro's def module
+ (resolve-expansion-vars form modu)))
+ (else
+ (map julia-expand-macroscopes- e))))
+
 (define (rename-symbolic-labels- e relabels parent-scope)
  (cond
  ((or (not (pair? e)) (quoted? e)) e)
@@ -502,48 +519,15 @@
 
 ;; macro expander entry point
 
-(define (julia-expand-macros e (max-depth -1))
- (julia-expand-macroscopes
+;; TODO: delete this file and fold this operation into resolve-scopes
+(define (julia-expand-macroscope e)
+ (julia-expand-macroscopes-
  (rename-symbolic-labels
- (julia-expand-macros- '() e max-depth))))
+  (julia-expand-quotes e))))
 
-(define (julia-expand-macros- m e max-depth)
- (cond ((= max-depth 0) e)
- ((not (pair? e)) e)
- ((eq? (car e) 'quote)
- ;; backquote is essentially a built-in unhygienic macro at the moment
- (julia-expand-macros- m (julia-bq-expand-hygienic (cadr e) (null? m)) max-depth))
- ((eq? (car e) 'inert) e)
- ((eq? (car e) 'macrocall)
- ;; expand macro
- (let ((form (apply invoke-julia-macro (if (null? m) 'false (car m)) (cdr e))))
- (if (not form)
- (error (string "macro \"" (cadr e) "\" not defined")))
- (if (and (pair? form) (eq? (car form) 'error))
- (error (cadr form)))
- (let* ((modu (cdr form)) ;; modu is the macro's def module
- (form (car form)) ;; form is the expression returned from expand-macros
- (form (julia-expand-macros- (cons modu m) form (- max-depth 1))))
- (if (and (pair? form) (eq? (car form) 'escape))
- (cadr form) ; immediately fold away (hygienic-scope (escape ...))
- `(hygienic-scope ,form ,modu)))))
- ((eq? (car e) 'module) e)
- ((eq? (car e) 'escape)
- (let ((m (if (null? m) m (cdr m))))
- `(escape ,(julia-expand-macros- m (cadr e) max-depth))))
- (else
- (map (lambda (ex)
- (julia-expand-macros- m ex max-depth))
- e))))
+(define (contains-macrocall e)
+ (and (pair? e)
+ (contains (lambda (e) (and (pair? e) (eq? (car e) 'macrocall))) e)))
 
-;; TODO: delete this file and fold this operation into resolve-scopes
-(define (julia-expand-macroscopes e)
- (cond ((not (pair? e)) e)
- ((eq? (car e) 'inert) e)
- ((eq? (car e) 'module) e)
- ((eq? (car e) 'hygienic-scope)
- (let ((form (cadr e)) ;; form is the expression returned from expand-macros
- (modu (caddr e))) ;; m is the macro's def module
- (resolve-expansion-vars form modu)))
- (else
- (map julia-expand-macroscopes e))))
+(define (julia-bq-macro x)
+ (julia-bq-expand x 0))

test/core.jl

@@ -3491,8 +3491,16 @@ macro m8846(a, b=0)
  a, b
 end
 @test @m8846(a) === (:a, 0)
-@test @m8846(a,1) === (:a, 1)
-@test_throws MethodError @eval @m8846(a,b,c)
+@test @m8846(a, 1) === (:a, 1)
+let nometh = try; @eval @m8846(a, b, c); false; catch ex; ex; end
+ __source__ = LineNumberNode(@__LINE__() - 1, Symbol(@__FILE__))
+ nometh::LoadError
+ @test nometh.file === string(__source__.file)
+ @test nometh.line === __source__.line
+ e = nometh.error::MethodError
+ @test e.f === getfield(@__MODULE__, Symbol("@m8846"))
+ @test e.args === (__source__, @__MODULE__, :a, :b, :c)
+ end
 
 # a simple case of parametric dispatch with unions
 let foo(x::Union{T, Void}, y::Union{T, Void}) where {T} = 1
@@ -5130,8 +5138,13 @@ f_isdefined_unionvar(y, t) = (t > 0 && (x = (t == 1 ? 1 : y)); @isdefined x)
 @test !f_isdefined_unionvar(1, 0)
 f_isdefined_splat(x...) = @isdefined x
 @test f_isdefined_splat(1, 2, 3)
-@test let err = @macroexpand @isdefined :x
- isa(err, Expr) && err.head === :error && isa(err.args[1], MethodError)
+let err = try; @macroexpand @isdefined :x; false; catch ex; ex; end,
+ __source__ = LineNumberNode(@__LINE__() - 1, Symbol(@__FILE__))
+ @test err.file === string(__source__.file)
+ @test err.line === __source__.line
+ e = err.error::MethodError
+ @test e.f === getfield(@__MODULE__, Symbol("@isdefined"))
+ @test e.args === (__source__, @__MODULE__, :(:x))
 end
 f_isdefined_cl_1(y) = (local x; for i = 1:y; x = 2; end; () -> x; @isdefined x)
 f_isdefined_cl_2(y) = (local x; for i = 1:y; x = 2; end; () -> @isdefined x)

test/parse.jl

@@ -823,10 +823,16 @@ module B15838
 end
 @test A15838.@f() === nothing
 @test A15838.@f(1) === :b
-let nometh = expand(@__MODULE__, :(A15838.@f(1, 2))), __source__ = LineNumberNode(@__LINE__, Symbol(@__FILE__))
- @test (nometh::Expr).head === :error
- @test length(nometh.args) == 1
- e = nometh.args[1]::MethodError
+let ex = :(A15838.@f(1, 2)), __source__ = LineNumberNode(@__LINE__, Symbol(@__FILE__))
+ nometh = try
+ macroexpand(@__MODULE__, ex)
+ false
+ catch ex
+ ex
+ end::LoadError
+ @test nometh.file === string(__source__.file)
+ @test nometh.line === __source__.line
+ e = nometh.error::MethodError
  @test e.f === getfield(A15838, Symbol("@f"))
  @test e.args === (__source__, @__MODULE__, 1, 2)
 end

test/replutil.jl

@@ -486,7 +486,15 @@ let
  @test (@macroexpand @fastmath 1+2 ) == :(Base.FastMath.add_fast(1,2))
  @test (@macroexpand @fastmath + ) == :(Base.FastMath.add_fast)
  @test (@macroexpand @fastmath min(1) ) == :(Base.FastMath.min_fast(1))
- @test (@macroexpand @doc "" f() = @x) == Expr(:error, UndefVarError(Symbol("@x")))
+ let err = try; @macroexpand @doc "" f() = @x; catch ex; ex; end
+ file, line = @__FILE__, @__LINE__() - 1
+ err = err::LoadError
+ @test err.file == file && err.line == line
+ err = err.error::LoadError
+ @test err.file == file && err.line == line
+ err = err.error::UndefVarError
+ @test err == UndefVarError(Symbol("@x"))
+ end
  @test (@macroexpand @seven_dollar $bar) == 7
  x = 2
  @test (@macroexpand @seven_dollar 1+$x) == :(1 + $(Expr(:$, :x)))