From 19a306120e8069635129598dd827da9294c9f41f Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 18 Aug 2023 22:28:53 +0000 Subject: [PATCH 1/6] Standardize the entry-point for Julia execution This is a bit of a straw-man proposal (though I think mergable if people agree) to standardize the execution entrypoint for Julia scripts. I think there's at least four different ways that people might run a script: - As `julia main.jl` - As a PkgCompiler sysimage, then calling the main entry point - As a PkgCompiler "app", with the magic `julia_main` function - As a StaticCompiler product with an explicit entrypoint specified on the API. The main problem I have with all of these variants is that they're all different and it's kind of a pain to move between them. Here I propose that we standardize on `Main.main(ARGS)` as the entrypoint for all scripts. Downstream from that proposal, this PR then makes the following changes: 1. If a system image has an existing `Main.main`, that is the entry point for `julia -Jsysimage.so`. 2. If not, and the sysimage has a REPL, we call REPL.main (we could handle this by defaulting `Main.main` to a weak import of `REPL.main`, but for the purpose of this PR, it's an explicit fallback. That said, I do want to emhpasize the direction of moving the REPL to be "just another app". 3. If the REPL code is called and passed a script file, the REPL executes any newly defined Main.main after loading the script file. As a result, `julia` behaves the same as if we had generated a new system image after loading `main.jl` and then running julia with that system image. The further downstream implication of this is that I'd like to get rid of the distinction between PkgCompiler apps and system images. An app is simply a system image with a `Main.main` function defined (note that currently PkgCompiler uses `julia_main` instead). --- NEWS.md | 6 +++ base/client.jl | 28 +++++++++++-- doc/src/manual/command-line-interface.md | 52 ++++++++++++++++++++++++ src/jlapi.c | 7 +++- stdlib/REPL/src/REPL.jl | 3 ++ test/cmdlineargs.jl | 12 ++++++ 6 files changed, 103 insertions(+), 5 deletions(-) diff --git a/NEWS.md b/NEWS.md index 16afb8c168443..636c7c53f5120 100644 --- a/NEWS.md +++ b/NEWS.md @@ -16,6 +16,12 @@ Compiler/Runtime improvements Command-line option changes --------------------------- +* The entry point for Julia has been standardized to `Main.main(ARGS)`. When julia is invoked to run a script or expression +(i.e. using `julia script.jl` or `julia -e expr`), julia will subsequently run the `Main.main` function automatically if +such a function has been defined. This is identended to unify script and compilation workflows, where code loading may happen +in the compiler and execution of `Main.main` may happen in the resulting executable. For interactive use, there is no semantic +difference between defining a main function and executing the code directly at the end of the script. ([50974]) + Multi-threading changes ----------------------- diff --git a/base/client.jl b/base/client.jl index 0b11a330cf179..e7f189826afdb 100644 --- a/base/client.jl +++ b/base/client.jl @@ -323,6 +323,10 @@ function exec_options(opts) end end end + + ret = 0 + isdefined(Main, :main) && (ret = invokelatest(Main.main, ARGS)) + if repl || is_interactive::Bool b = opts.banner auto = b == -1 @@ -331,7 +335,7 @@ function exec_options(opts) :short # b == 2 run_main_repl(interactiveinput, quiet, banner, history_file, color_set) end - nothing + return ret end function _global_julia_startup_file() @@ -548,13 +552,31 @@ function _start() append!(ARGS, Core.ARGS) # clear any postoutput hooks that were saved in the sysimage empty!(Base.postoutput_hooks) + local ret try - exec_options(JLOptions()) + if isdefined(Core, :Main) && isdefined(Core.Main, :main) + ret = Core.Main.main(ARGS) + elseif isassigned(REPL_MODULE_REF) + ret = REPL_MODULE_REF[].main(ARGS) + else + # TODO: This is the case for system image execution without main function + # and without REPL loaded. We fall back here to the pre-main behavior. + # However, we may instead want to adjust the sysimage build process to + # emit an explicit main function for this case. We should revisit this once + # the `main` story is fully worked out. + # + # error("No entry point defined and REPL not loaded.") + # + ret = exec_options(JLOptions()) + end + ret === nothing && (ret = 0) + ret = Cint(ret) catch + ret = Cint(1) invokelatest(display_error, scrub_repl_backtrace(current_exceptions())) - exit(1) end if is_interactive && get(stdout, :color, false) print(color_normal) end + return ret end diff --git a/doc/src/manual/command-line-interface.md b/doc/src/manual/command-line-interface.md index 3a522079409a6..69763d14a7668 100644 --- a/doc/src/manual/command-line-interface.md +++ b/doc/src/manual/command-line-interface.md @@ -39,6 +39,58 @@ $ julia --color=yes -O -- script.jl arg1 arg2.. See also [Scripting](@ref man-scripting) for more information on writing Julia scripts. +## The `Main.main` entry point + +At the conclusion of executing a script or expression, `julia` will attempt to execute the function +`Main.main(ARGS)` (if such a function has been defined). This feature is intended to aid in the unification +of compiled and interactive workflows. In compiled workflows, loading the code that defines the `main` +function may be spatially and temporally separated from the invocation. However, for interactive workflows, +the behavior is equivalent to explicitly calling `exit(main(ARGS))` at the end of the evaluated script or +expression. + +!!! compat "Julia 1.11" + The special entry point `Main.main` was added in Julia 1.11. For compatibility with prior julia versions, + add an explicit `VERSION < v"1.11" && exit(main(ARGS))` at the end of your scripts. + +To see this feature in action, consider the following definition, which will execute the print function despite there being no explicit call to `main`: + +``` +$ julia -e 'main(ARGS) = println("Hello World!")' +Hello World! +$ +``` + +Only the `main` binding in the `Main`, module has this special behavior. For example, using `hello` +instead of `main` will result in the `hello` function not executing: + +``` +$ julia -e 'hello(ARGS) = println("Hello World!")' +$ +``` + +The `main` binding may be imported from a package. A hello package defined as + +``` +module Hello + +export main +main(ARGS) = println("Hello from the package!") + +end +``` + +may be used as: + +``` +$ julia -e 'using Hello' +Hello from the package! +$ julia -e 'import Hello' # N.B.: Execution depends on the binding not whether the package is loaded +$ +``` + +However, note that the current best practice recommendation is to not mix application and reusable library +code in the same package. Helper applications may be distributed as separate pacakges or as scripts with +separate `main` entry points in a package's `bin` folder. ## Parallel mode diff --git a/src/jlapi.c b/src/jlapi.c index 0dffaac627288..eeef60ef0c24b 100644 --- a/src/jlapi.c +++ b/src/jlapi.c @@ -576,16 +576,19 @@ static NOINLINE int true_main(int argc, char *argv[]) if (start_client) { jl_task_t *ct = jl_current_task; + int ret = 1; JL_TRY { size_t last_age = ct->world_age; ct->world_age = jl_get_world_counter(); - jl_apply(&start_client, 1); + jl_value_t *r = jl_apply(&start_client, 1); + jl_typeassert(r, (jl_value_t *)jl_int32_type); + ret = jl_unbox_int32(r); ct->world_age = last_age; } JL_CATCH { jl_no_exc_handler(jl_current_exception(), ct); } - return 0; + return ret; } // run program if specified, otherwise enter REPL diff --git a/stdlib/REPL/src/REPL.jl b/stdlib/REPL/src/REPL.jl index fe543ad2d512f..5a64d372cb386 100644 --- a/stdlib/REPL/src/REPL.jl +++ b/stdlib/REPL/src/REPL.jl @@ -1506,4 +1506,7 @@ end import .Numbered.numbered_prompt! +# TODO: Move more of this implementation into REPL. +main(ARGS) = Base.exec_options(Base.JLOptions()) + end # module diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index b29904fb5eb6c..86c2994655295 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -977,3 +977,15 @@ end #heap-size-hint, we reserve 250 MB for non GC memory (llvm, etc.) @test readchomp(`$(Base.julia_cmd()) --startup-file=no --heap-size-hint=500M -e "println(@ccall jl_gc_get_max_memory()::UInt64)"`) == "$((500-250)*1024*1024)" end + +## `Main.main` entrypoint + +# Basic usage +@test readchomp(`$(Base.julia_cmd()) -e 'main(ARGS) = println("hello")'`) == "hello" + +# Test ARGS with -e +@test readchomp(`$(Base.julia_cmd()) -e 'main(ARGS) = println(ARGS)' a b`) == repr(["a", "b"]) + +# Test import from module +@test readchomp(`$(Base.julia_cmd()) -e 'module Hello; export main; main(ARGS) = println("hello"); end; using .Hello'`) == "hello" +@test readchomp(`$(Base.julia_cmd()) -e 'module Hello; export main; main(ARGS) = println("hello"); end; import .Hello'`) == "" From e6bf0182fcca581604947659e89c14a36d5cbe9b Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Tue, 29 Aug 2023 06:41:36 +0000 Subject: [PATCH 2/6] Silence analyzegc flase positive, incorporate formatting review --- NEWS.md | 2 +- src/jlapi.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index 636c7c53f5120..8171d41efb4e3 100644 --- a/NEWS.md +++ b/NEWS.md @@ -20,7 +20,7 @@ Command-line option changes (i.e. using `julia script.jl` or `julia -e expr`), julia will subsequently run the `Main.main` function automatically if such a function has been defined. This is identended to unify script and compilation workflows, where code loading may happen in the compiler and execution of `Main.main` may happen in the resulting executable. For interactive use, there is no semantic -difference between defining a main function and executing the code directly at the end of the script. ([50974]) +difference between defining a `main` function and executing the code directly at the end of the script. ([50974]) Multi-threading changes ----------------------- diff --git a/src/jlapi.c b/src/jlapi.c index eeef60ef0c24b..29be3b9e6179c 100644 --- a/src/jlapi.c +++ b/src/jlapi.c @@ -581,7 +581,8 @@ static NOINLINE int true_main(int argc, char *argv[]) size_t last_age = ct->world_age; ct->world_age = jl_get_world_counter(); jl_value_t *r = jl_apply(&start_client, 1); - jl_typeassert(r, (jl_value_t *)jl_int32_type); + if (jl_typeof(r) != (jl_value_t*)jl_int32_type) + jl_type_error("typeassert", (jl_value_t*)jl_int32_type, r); ret = jl_unbox_int32(r); ct->world_age = last_age; } From 61df2d973a6b64bbb9a64110561d3d212110a36c Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Wed, 30 Aug 2023 06:41:04 +0000 Subject: [PATCH 3/6] Do not execute Main.main while building sysimages --- base/client.jl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/base/client.jl b/base/client.jl index e7f189826afdb..70716d84f39a2 100644 --- a/base/client.jl +++ b/base/client.jl @@ -325,7 +325,13 @@ function exec_options(opts) end ret = 0 - isdefined(Main, :main) && (ret = invokelatest(Main.main, ARGS)) + if isdefined(Main, :main) + if Core.Compiler.generating_sysimg() + precompile(Main.main, (typeof(ARGS),)) + else + ret = invokelatest(Main.main, ARGS) + end + end if repl || is_interactive::Bool b = opts.banner From 1567a0f6a0d6cbbcf39c393d2858fb2014385c9f Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Wed, 30 Aug 2023 11:34:53 -0400 Subject: [PATCH 4/6] Update NEWS.md Co-authored-by: Martijn Visser --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 8171d41efb4e3..6fb80a21d219b 100644 --- a/NEWS.md +++ b/NEWS.md @@ -18,7 +18,7 @@ Command-line option changes * The entry point for Julia has been standardized to `Main.main(ARGS)`. When julia is invoked to run a script or expression (i.e. using `julia script.jl` or `julia -e expr`), julia will subsequently run the `Main.main` function automatically if -such a function has been defined. This is identended to unify script and compilation workflows, where code loading may happen +such a function has been defined. This is intended to unify script and compilation workflows, where code loading may happen in the compiler and execution of `Main.main` may happen in the resulting executable. For interactive use, there is no semantic difference between defining a `main` function and executing the code directly at the end of the script. ([50974]) From edb89c5bb0b4d6f37fa2cbc5aa83cd5eab70fa72 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 1 Sep 2023 05:53:38 +0000 Subject: [PATCH 5/6] Rearrange per triage review - Unconditionally run exec_options from the `julia` driver entrypoint - Don't run Main.main if the REPL is requested --- base/client.jl | 51 +++++++++++------------------------------ stdlib/REPL/src/REPL.jl | 15 +++++++++++- 2 files changed, 28 insertions(+), 38 deletions(-) diff --git a/base/client.jl b/base/client.jl index 70716d84f39a2..9cc02bb245e3b 100644 --- a/base/client.jl +++ b/base/client.jl @@ -228,11 +228,8 @@ incomplete_tag(exc::Meta.ParseError) = incomplete_tag(exc.detail) cmd_suppresses_program(cmd) = cmd in ('e', 'E') function exec_options(opts) - quiet = (opts.quiet != 0) startup = (opts.startupfile != 2) - history_file = (opts.historyfile != 0) - color_set = (opts.color != 0) # --color!=auto - global have_color = color_set ? (opts.color == 1) : nothing # --color=on + global have_color = (opts.color != 0) ? (opts.color == 1) : nothing # --color=on global is_interactive = (opts.isinteractive != 0) # pre-process command line argument list @@ -324,24 +321,7 @@ function exec_options(opts) end end - ret = 0 - if isdefined(Main, :main) - if Core.Compiler.generating_sysimg() - precompile(Main.main, (typeof(ARGS),)) - else - ret = invokelatest(Main.main, ARGS) - end - end - - if repl || is_interactive::Bool - b = opts.banner - auto = b == -1 - banner = b == 0 || (auto && !interactiveinput) ? :no : - b == 1 || (auto && interactiveinput) ? :yes : - :short # b == 2 - run_main_repl(interactiveinput, quiet, banner, history_file, color_set) - end - return ret + return repl || is_interactive::Bool end function _global_julia_startup_file() @@ -558,22 +538,19 @@ function _start() append!(ARGS, Core.ARGS) # clear any postoutput hooks that were saved in the sysimage empty!(Base.postoutput_hooks) - local ret + local ret = 0 try - if isdefined(Core, :Main) && isdefined(Core.Main, :main) - ret = Core.Main.main(ARGS) - elseif isassigned(REPL_MODULE_REF) - ret = REPL_MODULE_REF[].main(ARGS) - else - # TODO: This is the case for system image execution without main function - # and without REPL loaded. We fall back here to the pre-main behavior. - # However, we may instead want to adjust the sysimage build process to - # emit an explicit main function for this case. We should revisit this once - # the `main` story is fully worked out. - # - # error("No entry point defined and REPL not loaded.") - # - ret = exec_options(JLOptions()) + should_run_repl = exec_options(JLOptions()) + if should_run_repl + if isassigned(REPL_MODULE_REF) + ret = REPL_MODULE_REF[].main(ARGS) + end + elseif isdefined(Main, :main) + if Core.Compiler.generating_sysimg() + precompile(Main.main, (typeof(ARGS),)) + else + ret = invokelatest(Main.main, ARGS) + end end ret === nothing && (ret = 0) ret = Cint(ret) diff --git a/stdlib/REPL/src/REPL.jl b/stdlib/REPL/src/REPL.jl index 5a64d372cb386..518514a960638 100644 --- a/stdlib/REPL/src/REPL.jl +++ b/stdlib/REPL/src/REPL.jl @@ -1507,6 +1507,19 @@ end import .Numbered.numbered_prompt! # TODO: Move more of this implementation into REPL. -main(ARGS) = Base.exec_options(Base.JLOptions()) +function main(ARGS) + opts = Base.JLOptions() + interactiveinput = isa(stdin, Base.TTY) + b = opts.banner + auto = b == -1 + banner = b == 0 || (auto && !interactiveinput) ? :no : + b == 1 || (auto && interactiveinput) ? :yes : + :short # b == 2 + + quiet = (opts.quiet != 0) + history_file = (opts.historyfile != 0) + color_set = (opts.color != 0) # --color!=auto + Base.run_main_repl(interactiveinput, quiet, banner, history_file, color_set) +end end # module From cc4108b34894a527b4fcaf25096bbdb4308322dd Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Sun, 3 Sep 2023 04:15:03 +0000 Subject: [PATCH 6/6] Make -i override the repl mode --- base/client.jl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/base/client.jl b/base/client.jl index 9cc02bb245e3b..7baef9d0010f0 100644 --- a/base/client.jl +++ b/base/client.jl @@ -321,7 +321,7 @@ function exec_options(opts) end end - return repl || is_interactive::Bool + return repl end function _global_julia_startup_file() @@ -540,17 +540,17 @@ function _start() empty!(Base.postoutput_hooks) local ret = 0 try - should_run_repl = exec_options(JLOptions()) - if should_run_repl - if isassigned(REPL_MODULE_REF) - ret = REPL_MODULE_REF[].main(ARGS) - end - elseif isdefined(Main, :main) + repl_was_requested = exec_options(JLOptions()) + if isdefined(Main, :main) && !is_interactive if Core.Compiler.generating_sysimg() precompile(Main.main, (typeof(ARGS),)) else ret = invokelatest(Main.main, ARGS) end + elseif (repl_was_requested || is_interactive) + if isassigned(REPL_MODULE_REF) + ret = REPL_MODULE_REF[].main(ARGS) + end end ret === nothing && (ret = 0) ret = Cint(ret)