remove the stateful parse_next API

doc/devdocs/init.rst

@@ -126,7 +126,7 @@ creates the global "Main" module and sets
 
 Note: _julia_init() `then sets <https://github.com/JuliaLang/julia/blob/master/src/init.c>`_ :code:`jl_root_task->current_module = jl_core_module`. :code:`jl_root_task` is an alias of :code:`jl_current_task` at this point, so the current_module set by :c:func:`jl_new_main_module` above is overwritten.
 
-`jl_load("boot.jl", sizeof("boot.jl")) <https://github.com/JuliaLang/julia/blob/master/src/init.c>`_ calls `jl_parse_eval_all("boot.jl") <https://github.com/JuliaLang/julia/blob/master/src/toplevel.c>`_ which repeatedly calls `jl_parse_next() <https://github.com/JuliaLang/julia/blob/master/src/ast.c>`_ and `jl_toplevel_eval_flex() <https://github.com/JuliaLang/julia/blob/master/src/toplevel.c>`_ to parse and execute `boot.jl <https://github.com/JuliaLang/julia/blob/master/base/boot.jl>`_. TODO -- drill down into eval?
+`jl_load("boot.jl", sizeof("boot.jl")) <https://github.com/JuliaLang/julia/blob/master/src/init.c>`_ calls `jl_parse_eval_all <https://github.com/JuliaLang/julia/blob/master/src/ast.c>`_ which repeatedly calls `jl_toplevel_eval_flex() <https://github.com/JuliaLang/julia/blob/master/src/toplevel.c>`_ to execute `boot.jl <https://github.com/JuliaLang/julia/blob/master/base/boot.jl>`_. TODO -- drill down into eval?
 
 `jl_get_builtin_hooks() <https://github.com/JuliaLang/julia/blob/master/src/init.c>`_ initialises global C pointers to Julia globals defined in ``boot.jl``.
 
@@ -168,7 +168,7 @@ true_main()
 
 `true_main() <https://github.com/JuliaLang/julia/blob/master/ui/repl.c>`_ loads the contents of :code:`argv[]` into :data:`Base.ARGS`.
 
-If a .jl "program" file was supplied on the command line, then `exec_program() <https://github.com/JuliaLang/julia/blob/master/ui/repl.c>`_ calls `jl_load(program,len) <https://github.com/JuliaLang/julia/blob/master/src/toplevel.c>`_ which calls `jl_parse_eval_all() <https://github.com/JuliaLang/julia/blob/master/src/toplevel.c>`_ which repeatedly calls `jl_parse_next() <https://github.com/JuliaLang/julia/blob/master/src/ast.c>`_ and `jl_toplevel_eval_flex() <https://github.com/JuliaLang/julia/blob/master/src/toplevel.c>`_ to parse and execute the program.
+If a .jl "program" file was supplied on the command line, then `exec_program() <https://github.com/JuliaLang/julia/blob/master/ui/repl.c>`_ calls `jl_load(program,len) <https://github.com/JuliaLang/julia/blob/master/src/toplevel.c>`_ which calls `jl_parse_eval_all <https://github.com/JuliaLang/julia/blob/master/src/ast.c>`_ which repeatedly calls `jl_toplevel_eval_flex() <https://github.com/JuliaLang/julia/blob/master/src/toplevel.c>`_ to execute the program.
 
 However, in our example (:code:`julia -e 'println("Hello World!")'`), `jl_get_global(jl_base_module, jl_symbol("_start")) <https://github.com/JuliaLang/julia/blob/master/src/module.c>`_ looks up `Base._start <https://github.com/JuliaLang/julia/blob/master/base/client.jl>`_ and `jl_apply() <https://github.com/JuliaLang/julia/blob/master/src/julia.h>`_ executes it.
 

src/ast.c

@@ -152,7 +152,7 @@ value_t fl_invoke_julia_macro(fl_context_t *fl_ctx, value_t *args, uint32_t narg
  // macro expansions, since it will be referenced only from scheme and
  // not julia.
  // all calls to invoke-julia-macro happen under `jl_macroexpand`,
- // `jl_expand` or `jl_parse_next` so the preserved array is rooted there.
+ // `jl_expand` or `jl_parse_eval_all` so the preserved array is rooted there.
  assert(result != NULL);
  jl_ast_preserve(fl_ctx, result);
  value_t scm = julia_to_scm(fl_ctx, result);
@@ -694,23 +694,85 @@ JL_DLLEXPORT jl_value_t *jl_parse_string(const char *str, size_t len,
  return result;
 }
 
-jl_ast_context_t *jl_start_parsing_file(const char *fname)
+// parse and eval a whole file, possibly reading from a string (`content`)
+jl_value_t *jl_parse_eval_all(const char *fname, size_t len,
+ const char *content, size_t contentlen)
 {
  jl_ast_context_t *ctx = jl_ast_ctx_enter();
  fl_context_t *fl_ctx = &ctx->fl;
- value_t s = cvalue_static_cstring(fl_ctx, fname);
- if (fl_applyn(fl_ctx, 1, symbol_value(symbol(fl_ctx, "jl-parse-file")), s) == fl_ctx->F) {
+ value_t f, ast;
+ f = cvalue_static_cstring(fl_ctx, fname);
+ fl_gc_handle(fl_ctx, &f);
+ if (content != NULL) {
+ value_t t = cvalue_static_cstrn(fl_ctx, content, contentlen);
+ fl_gc_handle(fl_ctx, &t);
+ ast = fl_applyn(fl_ctx, 2, symbol_value(symbol(fl_ctx, "jl-parse-string-stream")), t, f);
+ fl_free_gc_handles(fl_ctx, 1);
+ }
+ else {
+ ast = fl_applyn(fl_ctx, 1, symbol_value(symbol(fl_ctx, "jl-parse-file")), f);
+ }
+ fl_free_gc_handles(fl_ctx, 1);
+ if (ast == fl_ctx->F) {
  jl_ast_ctx_leave(ctx);
- return NULL;
+ jl_errorf("could not open file %s", fname);
  }
- return ctx;
+ fl_gc_handle(fl_ctx, &ast);
+
+ int last_lineno = jl_lineno;
+ const char *last_filename = jl_filename;
+ jl_lineno = 0;
+ jl_filename = fname;
+ jl_array_t *roots = NULL;
+ jl_array_t **old_roots = ctx->roots;
+ ctx->roots = &roots;
+ jl_value_t *form=NULL, *result=jl_nothing;
+ int err = 0;
+ JL_GC_PUSH3(&roots, &form, &result);
+ JL_TRY {
+ assert(iscons(ast) && car_(ast) == symbol(fl_ctx,"toplevel"));
+ ast = cdr_(ast);
+ while (iscons(ast)) {
+ value_t expansion = fl_applyn(fl_ctx, 1, symbol_value(symbol(fl_ctx, "jl-expand-to-thunk")), car_(ast));
+ form = scm_to_julia(fl_ctx, expansion, 0);
+ jl_sym_t *head = NULL;
+ if (jl_is_expr(form)) head = ((jl_expr_t*)form)->head;
+ if (head == jl_incomplete_sym)
+ jl_errorf("syntax: %s", jl_string_data(jl_exprarg(form,0)));
+ else if (head == error_sym)
+ jl_interpret_toplevel_expr(form);
+ else if (head == line_sym)
+ jl_lineno = jl_unbox_long(jl_exprarg(form,0));
+ else
+ result = jl_toplevel_eval_flex(form, 1);
+ ast = cdr_(ast);
+ }
+ }
+ JL_CATCH {
+ form = jl_pchar_to_string(fname, len);
+ result = jl_box_long(jl_lineno);
+ err = 1;
+ }
+ jl_lineno = last_lineno;
+ jl_filename = last_filename;
+ fl_free_gc_handles(fl_ctx, 1);
+ ctx->roots = old_roots;
+ jl_ast_ctx_leave(ctx);
+ if (err) {
+ if (jl_loaderror_type == NULL)
+ jl_rethrow();
+ else
+ jl_rethrow_other(jl_new_struct(jl_loaderror_type, form, result,
+ jl_exception_in_transit));
+ }
+ JL_GC_POP();
+ return result;
 }
 
-void jl_stop_parsing(jl_ast_context_t *ctx)
+JL_DLLEXPORT jl_value_t *jl_load_file_string(const char *text, size_t len,
+ char *filename, size_t namelen)
 {
- fl_context_t *fl_ctx = &ctx->fl;
- fl_applyn(fl_ctx, 0, symbol_value(symbol(fl_ctx, "jl-parser-close-stream")));
- jl_ast_ctx_leave(ctx);
+ return jl_parse_eval_all(filename, namelen, text, len);
 }
 
 JL_DLLEXPORT int jl_parse_depwarn(int warn)
@@ -735,49 +797,6 @@ static int jl_parse_deperror(fl_context_t *fl_ctx, int err)
  return prev == fl_ctx->T ? 1 : 0;
 }
 
-jl_value_t *jl_parse_next(jl_ast_context_t *ctx)
-{
- fl_context_t *fl_ctx = &ctx->fl;
- JL_AST_PRESERVE_PUSH(ctx, roots, old_roots);
- value_t c = fl_applyn(fl_ctx, 0, symbol_value(symbol(fl_ctx, "jl-parser-next")));
- if (c == fl_ctx->FL_EOF) {
- JL_AST_PRESERVE_POP(ctx, old_roots);
- return NULL;
- }
- if (iscons(c)) {
- if (cdr_(c) == fl_ctx->FL_EOF) {
- JL_AST_PRESERVE_POP(ctx, old_roots);
- return NULL;
- }
- value_t a = car_(c);
- if (isfixnum(a)) {
- jl_lineno = numval(a);
- //jl_printf(JL_STDERR, " on line %d\n", jl_lineno);
- c = cdr_(c);
- }
- }
- // for error, get most recent line number
- if (iscons(c) && car_(c) == jl_ast_ctx(fl_ctx)->error_sym)
- jl_lineno = numval(fl_applyn(fl_ctx, 0, symbol_value(symbol(fl_ctx, "jl-parser-current-lineno"))));
- jl_value_t *res = scm_to_julia(fl_ctx, c, 0);
- JL_AST_PRESERVE_POP(ctx, old_roots);
- return res;
-}
-
-JL_DLLEXPORT jl_value_t *jl_load_file_string(const char *text, size_t len,
- char *filename, size_t namelen)
-{
- jl_ast_context_t *ctx = jl_ast_ctx_enter();
- fl_context_t *fl_ctx = &ctx->fl;
- value_t t, f;
- t = cvalue_static_cstrn(fl_ctx, text, len);
- fl_gc_handle(fl_ctx, &t);
- f = cvalue_static_cstrn(fl_ctx, filename, namelen);
- fl_applyn(fl_ctx, 2, symbol_value(symbol(fl_ctx, "jl-parse-string-stream")), t, f);
- fl_free_gc_handles(fl_ctx, 1);
- return jl_parse_eval_all(filename, namelen, ctx);
-}
-
 // returns either an expression or a thunk
 JL_DLLEXPORT jl_value_t *jl_expand(jl_value_t *expr)
 {

src/jlfrontend.scm

@@ -134,11 +134,10 @@
 (define (jl-parse-one-string s pos0 greedy)
  (let ((inp (open-input-string s)))
  (io.seek inp pos0)
- (let ((expr
- (parser-wrap (lambda ()
- (if greedy
- (julia-parse inp)
- (julia-parse inp parse-atom))))))
+ (let ((expr (parser-wrap (lambda ()
+ (if greedy
+ (julia-parse inp)
+ (julia-parse inp parse-atom))))))
  (cons expr (io.pos inp)))))
 
 (define (jl-parse-string s)
@@ -158,39 +157,37 @@
  (loop (nreconc (cdr expr) exprs))
  (loop (cons expr exprs))))))))))
 
+(define (jl-parse-all io filename)
+ (unwind-protect
+ (with-bindings ((current-filename (symbol filename)))
+ (let ((stream (make-token-stream io)))
+ (let loop ((exprs '()))
+ (let ((lineno (parser-wrap
+ (lambda ()
+ (skip-ws-and-comments (ts:port stream))
+ (input-port-line (ts:port stream))))))
+ (if (pair? lineno)
+ (cons 'toplevel (reverse! (cons lineno exprs)))
+ (let ((expr (parser-wrap
+ (lambda ()
+ (julia-parse stream)))))
+ (if (eof-object? expr)
+ (cons 'toplevel (reverse! exprs))
+ (let ((next (list* expr `(line ,lineno) exprs)))
+ (if (and (pair? expr) (eq? (car expr) 'error))
+ (cons 'toplevel (reverse! next))
+ (loop next))))))))))
+ (io.close io)))
+
 ;; parse file-in-a-string
 (define (jl-parse-string-stream str filename)
- (jl-parser-set-stream filename (open-input-string str)))
+ (jl-parse-all (open-input-string str) filename))
 
-(define (jl-parse-file s)
+(define (jl-parse-file filename)
  (trycatch
- (let ((b (buffer))
- (f (open-input-file s)))
- ;; read whole file first to avoid problems with concurrent modification (issue #10497)
- (io.copy b f)
- (io.close f)
- (io.seek b 0)
- (begin (jl-parser-set-stream s b)
- #t))
+ (jl-parse-all (open-input-file filename) filename)
  (lambda (e) #f)))
 
-(define *filename-stack* '())
-(define *ts-stack* '())
-(define current-token-stream #())
-
-(define (jl-parser-set-stream name stream)
- (set! *filename-stack* (cons current-filename *filename-stack*))
- (set! *ts-stack* (cons current-token-stream *ts-stack*))
- (set! current-filename (symbol name))
- (set! current-token-stream (make-token-stream stream)))
-
-(define (jl-parser-close-stream)
- (io.close (ts:port current-token-stream))
- (set! current-filename (car *filename-stack*))
- (set! current-token-stream (car *ts-stack*))
- (set! *filename-stack* (cdr *filename-stack*))
- (set! *ts-stack* (cdr *ts-stack*)))
-
 (define *depwarn* #t)
 (define (jl-parser-depwarn w)
  (let ((prev *depwarn*))
@@ -203,27 +200,6 @@
  (set! *deperror* (eq? e #t))
  prev))
 
-(define (jl-parser-next)
- (let* ((err (parser-wrap
- (lambda ()
- (skip-ws-and-comments (ts:port current-token-stream)))))
- (lineno (input-port-line (ts:port current-token-stream))))
- (cons lineno
- (if (pair? err)
- err
- (parser-wrap
- (lambda ()
- (let ((e (julia-parse current-token-stream)))
- (if (eof-object? e)
- e
- (if (and (pair? e) (or (eq? (car e) 'error)
- (eq? (car e) 'continue)))
- e
- (expand-toplevel-expr e))))))))))
-
-(define (jl-parser-current-lineno)
- (input-port-line (ts:port current-token-stream)))
-
 ; expand a piece of raw surface syntax to an executable thunk
 (define (jl-expand-to-thunk expr)
  (parser-wrap (lambda ()

src/julia_internal.h

@@ -169,14 +169,13 @@ jl_function_t *jl_module_call_func(jl_module_t *m);
 int jl_is_submodule(jl_module_t *child, jl_module_t *parent);
 
 typedef struct _jl_ast_context_t jl_ast_context_t;
-jl_ast_context_t *jl_start_parsing_file(const char *fname);
-void jl_stop_parsing(jl_ast_context_t *ctx);
-jl_value_t *jl_parse_next(jl_ast_context_t *ctx);
+jl_value_t *jl_toplevel_eval_flex(jl_value_t *e, int fast);
 
 jl_lambda_info_t *jl_wrap_expr(jl_value_t *expr);
 void jl_compile_linfo(jl_lambda_info_t *li, void *cyclectx);
 jl_value_t *jl_eval_global_var(jl_module_t *m, jl_sym_t *e);
-jl_value_t *jl_parse_eval_all(const char *fname, size_t len, jl_ast_context_t *ctx);
+jl_value_t *jl_parse_eval_all(const char *fname, size_t len,
+ const char *content, size_t contentlen);
 jl_value_t *jl_interpret_toplevel_thunk(jl_lambda_info_t *lam);
 jl_value_t *jl_interpret_toplevel_thunk_with(jl_lambda_info_t *lam,
  jl_value_t **loc, size_t nl);

src/toplevel.c

@@ -33,8 +33,6 @@ jl_module_t *jl_old_base_module = NULL;
 // the Main we started with, in case it is switched
 jl_module_t *jl_internal_main_module = NULL;
 
-jl_value_t *jl_toplevel_eval_flex(jl_value_t *e, int fast);
-
 JL_DLLEXPORT void jl_add_standard_imports(jl_module_t *m)
 {
  assert(jl_base_module != NULL);
@@ -554,54 +552,6 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval(jl_value_t *v)
  return jl_toplevel_eval_flex(v, 1);
 }
 
-// repeatedly call jl_parse_next and eval everything
-jl_value_t *jl_parse_eval_all(const char *fname, size_t len, jl_ast_context_t *ctx)
-{
- //jl_printf(JL_STDERR, "***** loading %s\n", fname);
- int last_lineno = jl_lineno;
- const char *last_filename = jl_filename;
- jl_lineno = 0;
- jl_filename = fname;
- jl_value_t *fn=NULL, *ln=NULL, *form=NULL, *result=jl_nothing;
- JL_GC_PUSH4(&fn, &ln, &form, &result);
- JL_TRY {
- // handle syntax error
- while (1) {
- form = jl_parse_next(ctx);
- if (form == NULL)
- break;
- if (jl_is_expr(form)) {
- if (((jl_expr_t*)form)->head == jl_incomplete_sym) {
- jl_errorf("syntax: %s", jl_string_data(jl_exprarg(form,0)));
- }
- if (((jl_expr_t*)form)->head == error_sym) {
- jl_interpret_toplevel_expr(form);
- }
- }
- result = jl_toplevel_eval_flex(form, 1);
- }
- }
- JL_CATCH {
- jl_stop_parsing(ctx);
- fn = jl_pchar_to_string(fname, len);
- ln = jl_box_long(jl_lineno);
- jl_lineno = last_lineno;
- jl_filename = last_filename;
- if (jl_loaderror_type == NULL) {
- jl_rethrow();
- }
- else {
- jl_rethrow_other(jl_new_struct(jl_loaderror_type, fn, ln,
- jl_exception_in_transit));
- }
- }
- jl_stop_parsing(ctx);
- jl_lineno = last_lineno;
- jl_filename = last_filename;
- JL_GC_POP();
- return result;
-}
-
 JL_DLLEXPORT jl_value_t *jl_load(const char *fname, size_t len)
 {
  if (jl_current_module->istopmod) {
@@ -615,13 +565,7 @@ JL_DLLEXPORT jl_value_t *jl_load(const char *fname, size_t len)
  if (jl_stat(fpath, (char*)&stbuf) != 0 || (stbuf.st_mode & S_IFMT) != S_IFREG) {
  jl_errorf("could not open file %s", fpath);
  }
- jl_ast_context_t *ctx = jl_start_parsing_file(fpath);
- if (!ctx) {
- jl_errorf("could not open file %s", fpath);
- }
- jl_value_t *result = jl_parse_eval_all(fpath, len, ctx);
- if (fpath != fname) free(fpath);
- return result;
+ return jl_parse_eval_all(fpath, len, NULL, 0);
 }
 
 // load from filename given as a ByteString object