diff --git a/Makefile b/Makefile index d11f258e73626..83a972d8036c9 100644 --- a/Makefile +++ b/Makefile @@ -140,14 +140,14 @@ endif $(build_private_libdir)/sys0.o: @$(call PRINT_JULIA, cd base && \ - $(call spawn,$(JULIA_EXECUTABLE)) --build $(call cygpath_w,$(build_private_libdir)/sys0) sysimg.jl) + $(call spawn,$(JULIA_EXECUTABLE)) -C $(JULIA_CPU_TARGET) --build $(call cygpath_w,$(build_private_libdir)/sys0) sysimg.jl) BASE_SRCS := $(wildcard base/*.jl base/*/*.jl base/*/*/*.jl) ,:=, $(build_private_libdir)/sys.o: VERSION $(BASE_SRCS) $(build_docdir)/helpdb.jl $(build_private_libdir)/sys0.$(SHLIB_EXT) @$(call PRINT_JULIA, cd base && \ - $(call spawn,$(JULIA_EXECUTABLE)) --build $(call cygpath_w,$(build_private_libdir)/sys) \ + $(call spawn,$(JULIA_EXECUTABLE)) -C $(JULIA_CPU_TARGET) --build $(call cygpath_w,$(build_private_libdir)/sys) \ -J$(call cygpath_w,$(build_private_libdir))/$$([ -e $(build_private_libdir)/sys.ji ] && echo sys.ji || echo sys0.ji) -f sysimg.jl \ || { echo '*** This error is usually fixed by running `make clean`. If the error persists$(,) try `make cleanall`. ***' && false; } ) @@ -271,6 +271,8 @@ endif # Copy system image $(INSTALL_F) $(build_private_libdir)/sys.ji $(DESTDIR)$(private_libdir) $(INSTALL_M) $(build_private_libdir)/sys.$(SHLIB_EXT) $(DESTDIR)$(private_libdir) + # Copy in system image build script + $(INSTALL_M) contrib/build_sysimg.jl $(DESTDIR)$(datarootdir)/julia/ # Copy in all .jl sources as well cp -R -L $(build_datarootdir)/julia $(DESTDIR)$(datarootdir)/ # Copy documentation diff --git a/contrib/build_sysimg.jl b/contrib/build_sysimg.jl new file mode 100644 index 0000000000000..a73445a06564a --- /dev/null +++ b/contrib/build_sysimg.jl @@ -0,0 +1,157 @@ +#!/usr/bin/env julia + +# Build a system image binary at sysimg_path.dlext. By default, put the system image next to libjulia +# Allow insertion of a userimg via userimg_path. If sysimg_path.dlext is currently loaded into memory, +# don't continue unless force is set to true. Allow targeting of a CPU architecture via cpu_target +const default_sysimg_path = joinpath(dirname(Sys.dlpath("libjulia")),"sys") +function build_sysimg(sysimg_path=default_sysimg_path, cpu_target="native", userimg_path=nothing; force=false) + # Quit out if a sysimg is already loaded and is in the same spot as sysimg_path, unless forcing + sysimg = dlopen_e("sys") + if sysimg != C_NULL + if !force && Sys.dlpath(sysimg) == "$(sysimg_path).$(Sys.dlext)" + println("System image already loaded at $(Sys.dlpath(sysimg)), set force to override") + return + end + end + + # Canonicalize userimg_path before we enter the base_dir + if userimg_path != nothing + userimg_path = abspath(userimg_path) + end + + # Enter base/ and setup some useful paths + base_dir = dirname(Base.find_source_file("sysimg.jl")) + cd(base_dir) do + try + julia = joinpath(JULIA_HOME, "julia") + julia_libdir = dirname(Sys.dlpath("libjulia")) + ld = find_system_linker() + + # Ensure we have write-permissions to wherever we're trying to write to + try + touch("$sysimg_path.$(Sys.dlext)") + catch + err_msg = "Unable to modify $sysimg_path.$(Sys.dlext), ensure parent directory exists " + err_msg *= "and is writable. Absolute paths work best. Do you need to run this with sudo?)" + error( err_msg ) + end + + # Copy in userimg.jl if it exists... + if userimg_path != nothing + if !isreadable(userimg_path) + error("$userimg_path is not readable, ensure it is an absolute path!") + end + cp(userimg_path, "userimg.jl") + end + + # Start by building sys0.{ji,o} + sys0_path = joinpath(dirname(sysimg_path), "sys0") + println("Building sys0.o...") + println("$julia -C $cpu_target --build $sys0_path sysimg.jl") + run(`$julia -C $cpu_target --build $sys0_path sysimg.jl`) + + # Bootstrap off of that to create sys.{ji,o} + println("Building sys.o...") + println("$julia -C $cpu_target --build $sysimg_path -J $sys0_path.ji -f sysimg.jl") + run(`$julia -C $cpu_target --build $sysimg_path -J $sys0_path.ji -f sysimg.jl`) + + # Link sys.o into sys.$(dlext) + FLAGS = ["-L$julia_libdir"] + if OS_NAME == :Darwin + push!(FLAGS, "-dylib") + push!(FLAGS, "-undefined") + push!(FLAGS, "dynamic_lookup") + push!(FLAGS, "-macosx_version_min") + push!(FLAGS, "10.7") + else + if OS_NAME == :Linux + push!(FLAGS, "-shared") + end + push!(FLAGS, "--unresolved-symbols") + push!(FLAGS, "ignore-all") + end + @windows_only append!(FLAGS, ["-L$JULIA_HOME", "-ljulia", "-lssp"]) + + if ld != nothing + println("Linking sys.$(Sys.dlext)") + run(`$ld $FLAGS -o $sysimg_path.$(Sys.dlext) $sysimg_path.o`) + end + + println("System image successfully built at $sysimg_path.$(Sys.dlext)") + @windows_only begin + if convert(VersionNumber, Base.libllvm_version) < v"3.5.0" + LLVM_msg = "Building sys.dll on Windows against LLVM < 3.5.0 can cause incorrect backtraces!" + LLVM_msg *= " Delete generated sys.dll to avoid these problems" + warn( LLVM_msg ) + end + end + + if default_sysimg_path != sysimg_path + println("To run Julia with this image loaded, run: julia -J $sysimg_path.ji") + else + println("Julia will automatically load this system image at next startup") + end + finally + # Cleanup userimg.jl + if isfile("userimg.jl") + rm("userimg.jl") + end + end + end +end + +# Search for a linker to link sys.o into sys.dl_ext. Honor LD environment variable. +function find_system_linker() + if haskey( ENV, "LD" ) + if !success(`$(ENV["LD"]) -v`) + warn("Using linker override $(ENV["LD"]), but unable to run `$(ENV["LD"]) -v`") + end + return ENV["LD"] + end + + # On Windows, check to see if WinRPM is installed, and if so, see if binutils is installed + @windows_only try + using WinRPM + if WinRPM.installed("binutils") + ENV["PATH"] = "$(ENV["PATH"]):$(joinpath(WinRPM.installdir,"usr","$(Sys.ARCH)-w64-mingw32","sys-root","mingw","bin"))" + else + throw() + end + catch + warn("Install Binutils via WinRPM.install(\"binutils\") to generate sys.dll for faster startup times" ) + end + + + # See if `ld` exists + try + if success(`ld -v`) + return "ld" + end + end + + warn( "No supported linker found; startup times will be longer" ) +end + +# When running this file as a script, try to do so with default values. If arguments are passed +# in, use them as the arguments to build_sysimg above +if !isinteractive() + if length(ARGS) > 4 || ("--help" in ARGS || "-h" in ARGS) + println("Usage: build_sysimg.jl [--force] [--help]") + println(" is an absolute, extensionless path to store the system image at") + println(" is an LLVM cpu target to build the system image against") + println(" is the path to a user image to be baked into the system image") + println(" --force Set if you wish to overwrite the default system image") + println(" --help Print out this help text and exit") + println() + println(" Example:") + println(" build_sysimg.jl /usr/local/lib/julia/sys core2 ~/my_usrimg.jl true") + println() + println(" Running this script with no arguments is equivalent to calling it via") + println(" build_sysimg.jl $(default_sysimg_path) native") + return 0 + end + + force_flag = "--force" in ARGS + filter!(x -> x != "--force", ARGS) + build_sysimg(ARGS..., force=force_flag) +end diff --git a/doc/devdocs/julia.rst b/doc/devdocs/julia.rst index df37b96e0775f..e7b7f13fe6b5b 100644 --- a/doc/devdocs/julia.rst +++ b/doc/devdocs/julia.rst @@ -12,3 +12,4 @@ cartesian meta subarrays + sysimg diff --git a/doc/devdocs/sysimg.rst b/doc/devdocs/sysimg.rst new file mode 100644 index 0000000000000..32bc40e58a24c --- /dev/null +++ b/doc/devdocs/sysimg.rst @@ -0,0 +1,35 @@ +********************* +System Image Building +********************* + +Building the Julia system image +------------------------------- + +Julia ships with a preparsed system image containing the contents of the ``Base`` module, named ``sys.ji``. This file is also precompiled into a shared library called ``sys.{so,dll,dylib}`` on as many platforms as possible, so as to give vastly improved startup times. On systems that do not ship with a precompiled system image file, one can be generated from the source files shipped in Julia's ``DATAROOTDIR/julia/base`` folder. + +This operation is useful for multiple reasons. A user may: + +* Build a precompiled shared library system image on a platform that did not ship with one, thereby improving startup times. + +* Modify ``Base``, rebuild the system image and use the new ``Base`` next time Julia is started. + +* Include a ``userimg.jl`` file that includes packages into the system image, thereby creating a system image that has packages embedded into the startup environment. + +Julia now ships with a script that automates the tasks of building the system image, wittingly named ``build_sysimg.jl`` that lives in ``DATAROOTDIR/julia/``. That is, to include it into a current Julia session, type: +:: + + include(joinpath(JULIA_HOME, Base.DATAROOTDIR, "julia", "build_sysimg.jl")) + +This will include a ``build_sysimg()`` function: + +.. function:: build_sysimg(sysimg_path=default_sysimg_path, cpu_target="native", userimg_path=nothing; force=false) + Rebuild the system image. + Store it in ``sysimg_path``, which defaults to a file named ``sys.ji`` that sits in the same folder as ``libjulia.{so,dll,dylib}``. + Use the cpu instruction set given by ``cpu_target``. Valid CPU targets are the same as for the ``-C`` option to ``julia``, or the ``-march`` option to ``gcc``. Defaults to ``native``, which means to use all CPU instructions available on the current processor. + Include the user image file given by ``userimg_path``, which should contain directives such as ``using MyPackage`` to include that package in the new system image. + New system image will not replace an older image unless ``force`` is set to true. + +Note that this file can also be run as a script itself, with command line arguments taking the place of arguments passed to the ``build_sysimg`` function. For example, to build a system image in ``/tmp/sys.{so,dll,dylib}``, with the ``core2`` CPU instruction set, a user image of ``~/userimg.jl`` and ``force`` set to ``true``, one would execute: +:: + + julia build_sysimg.jl /tmp/sys core2 ~/userimg.jl --force \ No newline at end of file diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 09de8dea0096a..803307ae065d8 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -246,7 +246,7 @@ static void jl_gen_llvm_gv_array(llvm::Module *mod, SmallVectorgetType(), true, @@ -255,7 +255,7 @@ static void jl_gen_llvm_gv_array(llvm::Module *mod, SmallVectorhandle; @@ -1350,6 +1349,32 @@ extern void jl_get_builtin_hooks(void); extern void jl_get_system_hooks(void); extern void jl_get_uv_hooks(); +// Takes in a path of the form "usr/lib/julia/sys.ji", such as passed in to jl_restore_system_image() +DLLEXPORT +const char * jl_get_system_image_cpu_target(const char *fname) +{ + // If passed NULL, don't even bother + if (!fname) + return NULL; + + // First, get "sys" from "sys.ji" + char *fname_shlib = (char*)alloca(strlen(fname)); + strcpy(fname_shlib, fname); + char *fname_shlib_dot = strrchr(fname_shlib, '.'); + if (fname_shlib_dot != NULL) + *fname_shlib_dot = 0; + + // Get handle to sys.so + uv_lib_t * sysimg_handle = jl_load_dynamic_library_e(fname_shlib, JL_RTLD_DEFAULT | JL_RTLD_GLOBAL); + + // Return jl_sysimg_cpu_target if we can + if (sysimg_handle) + return (const char *)jl_dlsym(sysimg_handle, "jl_sysimg_cpu_target"); + + // If something goes wrong, return NULL + return NULL; +} + DLLEXPORT void jl_restore_system_image(const char *fname) { diff --git a/src/init.c b/src/init.c index 41c99e47b4a54..ff11e346e5b46 100644 --- a/src/init.c +++ b/src/init.c @@ -92,6 +92,7 @@ jl_compileropts_t jl_compileropts = { NULL, // julia_home NULL, // julia_bin NULL, // build_path system_image_path, // image_file + NULL, // cpu_target ("native", "core2", etc...) 0, // code_coverage 0, // malloc_log JL_COMPILEROPT_CHECK_BOUNDS_DEFAULT, @@ -869,6 +870,15 @@ void _julia_init(JL_IMAGE_SEARCH rel) jl_io_loop = uv_default_loop(); // this loop will internal events (spawning process etc.), // best to call this first, since it also initializes libuv jl_resolve_sysimg_location(rel); + + // If we are able to load the sysimg and get a cpu_target, use that unless user has overridden + if (jl_compileropts.cpu_target == NULL) { + const char * sysimg_cpu_target = jl_get_system_image_cpu_target(jl_compileropts.image_file); + + // If we can't load anything from the sysimg, default to native + jl_compileropts.cpu_target = sysimg_cpu_target ? sysimg_cpu_target : "native"; + } + jl_page_size = jl_getpagesize(); jl_arr_xtralloc_limit = uv_get_total_memory() / 100; // Extra allocation limited to 1% of total RAM jl_find_stack_bottom(); diff --git a/src/julia.h b/src/julia.h index cfff3af879c2c..86b122fab6081 100644 --- a/src/julia.h +++ b/src/julia.h @@ -868,6 +868,7 @@ DLLEXPORT int julia_trampoline(int argc, char *argv[], int (*pmain)(int ac,char DLLEXPORT void jl_atexit_hook(void); DLLEXPORT void NORETURN jl_exit(int status); +DLLEXPORT const char * jl_get_system_image_cpu_target(const char *fname); DLLEXPORT void jl_save_system_image(const char *fname); DLLEXPORT void jl_restore_system_image(const char *fname); DLLEXPORT int jl_save_new_module(const char *fname, jl_module_t *mod); @@ -1320,6 +1321,7 @@ typedef struct { const char *julia_bin; const char *build_path; const char *image_file; + const char *cpu_target; int8_t code_coverage; int8_t malloc_log; int8_t check_bounds; diff --git a/ui/repl.c b/ui/repl.c index 4cf6e3230ad55..37d4f88bd0ab4 100644 --- a/ui/repl.c +++ b/ui/repl.c @@ -53,7 +53,8 @@ static const char *opts = " -E, --print Evaluate and show \n" " -P, --post-boot Evaluate , but don't disable interactive mode\n" " -L, --load Load immediately on all processors\n" - " -J, --sysimage Start up with the given system image file\n\n" + " -J, --sysimage Start up with the given system image file\n" + " -C --cpu-target Limit usage of cpu features up to \n\n" " -p Run n local processes\n" " --machinefile Run processes on hosts listed in \n\n" @@ -77,15 +78,15 @@ static const char *opts = void parse_opts(int *argcp, char ***argvp) { - static char* shortopts = "+H:T:hJ:O"; + static char* shortopts = "+H:hJ:C:O"; static struct option longopts[] = { { "home", required_argument, 0, 'H' }, - { "tab", required_argument, 0, 'T' }, { "build", required_argument, 0, 'b' }, { "lisp", no_argument, &lisp_prompt, 1 }, { "help", no_argument, 0, 'h' }, { "sysimage", required_argument, 0, 'J' }, { "code-coverage", optional_argument, 0, 'c' }, + { "cpu-target", required_argument, 0, 'C' }, { "track-allocation",required_argument, 0, 'm' }, { "check-bounds", required_argument, 0, 300 }, { "optimize", no_argument, 0, 'O' }, @@ -118,6 +119,9 @@ void parse_opts(int *argcp, char ***argvp) jl_compileropts.image_file = strdup(optarg); imagepathspecified = 1; break; + case 'C': + jl_compileropts.cpu_target = strdup(optarg); + break; case 'h': ios_printf(ios_stdout, "%s%s", usage, opts); exit(0);