Support building of system image in binary builds

This commit adds a few new pieces of functionality:

* The `contrib/build_sysimg.jl` script which builds a new system image.  This method can save the system image wherever the user desires, e.g. it could be stored in `~/.julia`, to allow for per-user system images each customized with packages in their own `userimg.jl`.  Or on a restricted system, this allows for creation of a system image without root access.

* The removal of compile-time `JULIA_CPU_TARGET`, in favor of runtime `--cpu-target`/`-C` command-line flags which default to `"native"` but can be set to `"native"`, `"i386"`, `"core2"` or any other LLVM cpu target.  This allows the creation of a system image targeting user-supplied cpu features, e.g. `cd base; ../julia -C i386 --build /tmp/sys_i386 sysimg.jl`.

* I implemented runtime selection of the cpu target by adding a new member to the `jl_compileropts_t` structure called `cpu_target`.

* Because all julia executables are now created equal, (rather than before where a julia executable needed to have the same `JULIA_CPU_TARGET` set internally as the system image had when it was built) we need to know what CPU feature set the system image is targeting before we initialize code generation.  So a new function `jl_get_system_image_cpu_target()` is exported, which does exactly what it sounds like.

* I added newlines to the end of a few error messages.

* I found an old parser option `-T` which hadn't been removed yet, so I took the opportunity to do so.

* Documentation on this has been added to doc/devdocs/sysimg.rst

Makefile

@@ -140,14 +140,14 @@ endif
 
 $(build_private_libdir)/sys0.o:
  @$(call PRINT_JULIA, cd base && \
- $(call spawn,$(JULIA_EXECUTABLE)) --build $(call cygpath_w,$(build_private_libdir)/sys0) sysimg.jl)
+ $(call spawn,$(JULIA_EXECUTABLE)) -C $(JULIA_CPU_TARGET) --build $(call cygpath_w,$(build_private_libdir)/sys0) sysimg.jl)
 
 BASE_SRCS := $(wildcard base/*.jl base/*/*.jl base/*/*/*.jl)
 
 ,:=,
 $(build_private_libdir)/sys.o: VERSION $(BASE_SRCS) $(build_docdir)/helpdb.jl $(build_private_libdir)/sys0.$(SHLIB_EXT)
  @$(call PRINT_JULIA, cd base && \
- $(call spawn,$(JULIA_EXECUTABLE)) --build $(call cygpath_w,$(build_private_libdir)/sys) \
+ $(call spawn,$(JULIA_EXECUTABLE)) -C $(JULIA_CPU_TARGET) --build $(call cygpath_w,$(build_private_libdir)/sys) \
  -J$(call cygpath_w,$(build_private_libdir))/$$([ -e $(build_private_libdir)/sys.ji ] && echo sys.ji || echo sys0.ji) -f sysimg.jl \
  || { echo '*** This error is usually fixed by running `make clean`. If the error persists$(,) try `make cleanall`. ***' && false; } )
 
@@ -271,6 +271,8 @@ endif
  # Copy system image
  $(INSTALL_F) $(build_private_libdir)/sys.ji $(DESTDIR)$(private_libdir)
  $(INSTALL_M) $(build_private_libdir)/sys.$(SHLIB_EXT) $(DESTDIR)$(private_libdir)
+ # Copy in system image build script
+ $(INSTALL_M) contrib/build_sysimg.jl $(DESTDIR)$(datarootdir)/julia/
  # Copy in all .jl sources as well
  cp -R -L $(build_datarootdir)/julia $(DESTDIR)$(datarootdir)/
  # Copy documentation

contrib/build_sysimg.jl

@@ -0,0 +1,157 @@
+#!/usr/bin/env julia
+
+# Build a system image binary at sysimg_path.dlext. By default, put the system image next to libjulia
+# Allow insertion of a userimg via userimg_path. If sysimg_path.dlext is currently loaded into memory,
+# don't continue unless force is set to true. Allow targeting of a CPU architecture via cpu_target
+const default_sysimg_path = joinpath(dirname(Sys.dlpath("libjulia")),"sys")
+function build_sysimg(sysimg_path=default_sysimg_path, cpu_target="native", userimg_path=nothing; force=false)
+ # Quit out if a sysimg is already loaded and is in the same spot as sysimg_path, unless forcing
+ sysimg = dlopen_e("sys")
+ if sysimg != C_NULL
+ if !force && Sys.dlpath(sysimg) == "$(sysimg_path).$(Sys.dlext)"
+ println("System image already loaded at $(Sys.dlpath(sysimg)), set force to override")
+ return
+ end
+ end
+
+ # Canonicalize userimg_path before we enter the base_dir
+ if userimg_path != nothing
+ userimg_path = abspath(userimg_path)
+ end
+
+ # Enter base/ and setup some useful paths
+ base_dir = dirname(Base.find_source_file("sysimg.jl"))
+ cd(base_dir) do
+ try
+ julia = joinpath(JULIA_HOME, "julia")
+ julia_libdir = dirname(Sys.dlpath("libjulia"))
+ ld = find_system_linker()
+
+ # Ensure we have write-permissions to wherever we're trying to write to
+ try
+ touch("$sysimg_path.$(Sys.dlext)")
+ catch
+ err_msg = "Unable to modify $sysimg_path.$(Sys.dlext), ensure parent directory exists "
+ err_msg *= "and is writable. Absolute paths work best. Do you need to run this with sudo?)"
+ error( err_msg )
+ end
+
+ # Copy in userimg.jl if it exists...
+ if userimg_path != nothing
+ if !isreadable(userimg_path)
+ error("$userimg_path is not readable, ensure it is an absolute path!")
+ end
+ cp(userimg_path, "userimg.jl")
+ end
+
+ # Start by building sys0.{ji,o}
+ sys0_path = joinpath(dirname(sysimg_path), "sys0")
+ println("Building sys0.o...")
+ println("$julia -C $cpu_target --build $sys0_path sysimg.jl")
+ run(`$julia -C $cpu_target --build $sys0_path sysimg.jl`)
+
+ # Bootstrap off of that to create sys.{ji,o}
+ println("Building sys.o...")
+ println("$julia -C $cpu_target --build $sysimg_path -J $sys0_path.ji -f sysimg.jl")
+ run(`$julia -C $cpu_target --build $sysimg_path -J $sys0_path.ji -f sysimg.jl`)
+
+ # Link sys.o into sys.$(dlext)
+ FLAGS = ["-L$julia_libdir"]
+ if OS_NAME == :Darwin
+ push!(FLAGS, "-dylib")
+ push!(FLAGS, "-undefined")
+ push!(FLAGS, "dynamic_lookup")
+ push!(FLAGS, "-macosx_version_min")
+ push!(FLAGS, "10.7")
+ else
+ if OS_NAME == :Linux
+ push!(FLAGS, "-shared")
+ end
+ push!(FLAGS, "--unresolved-symbols")
+ push!(FLAGS, "ignore-all")
+ end
+ @windows_only append!(FLAGS, ["-L$JULIA_HOME", "-ljulia", "-lssp"])
+
+ if ld != nothing
+ println("Linking sys.$(Sys.dlext)")
+ run(`$ld $FLAGS -o $sysimg_path.$(Sys.dlext) $sysimg_path.o`)
+ end
+
+ println("System image successfully built at $sysimg_path.$(Sys.dlext)")
+ @windows_only begin
+ if convert(VersionNumber, Base.libllvm_version) < v"3.5.0"
+ LLVM_msg = "Building sys.dll on Windows against LLVM < 3.5.0 can cause incorrect backtraces!"
+ LLVM_msg *= " Delete generated sys.dll to avoid these problems"
+ warn( LLVM_msg )
+ end
+ end
+
+ if default_sysimg_path != sysimg_path
+ println("To run Julia with this image loaded, run: julia -J $sysimg_path.ji")
+ else
+ println("Julia will automatically load this system image at next startup")
+ end
+ finally
+ # Cleanup userimg.jl
+ if isfile("userimg.jl")
+ rm("userimg.jl")
+ end
+ end
+ end
+end
+
+# Search for a linker to link sys.o into sys.dl_ext. Honor LD environment variable.
+function find_system_linker()
+ if haskey( ENV, "LD" )
+ if !success(`$(ENV["LD"]) -v`)
+ warn("Using linker override $(ENV["LD"]), but unable to run `$(ENV["LD"]) -v`")
+ end
+ return ENV["LD"]
+ end
+
+ # On Windows, check to see if WinRPM is installed, and if so, see if binutils is installed
+ @windows_only try
+ using WinRPM
+ if WinRPM.installed("binutils")
+ ENV["PATH"] = "$(ENV["PATH"]):$(joinpath(WinRPM.installdir,"usr","$(Sys.ARCH)-w64-mingw32","sys-root","mingw","bin"))"
+ else
+ throw()
+ end
+ catch
+ warn("Install Binutils via WinRPM.install(\"binutils\") to generate sys.dll for faster startup times" )
+ end
+
+
+ # See if `ld` exists
+ try
+ if success(`ld -v`)
+ return "ld"
+ end
+ end
+
+ warn( "No supported linker found; startup times will be longer" )
+end
+
+# When running this file as a script, try to do so with default values. If arguments are passed
+# in, use them as the arguments to build_sysimg above
+if !isinteractive()
+ if length(ARGS) > 4 || ("--help" in ARGS || "-h" in ARGS)
+ println("Usage: build_sysimg.jl <sysimg_path> <cpu_target> <usrimg_path.jl> [--force] [--help]")
+ println(" <sysimg_path> is an absolute, extensionless path to store the system image at")
+ println(" <cpu_target> is an LLVM cpu target to build the system image against")
+ println(" <usrimg_path.lj> is the path to a user image to be baked into the system image")
+ println(" --force Set if you wish to overwrite the default system image")
+ println(" --help Print out this help text and exit")
+ println()
+ println(" Example:")
+ println(" build_sysimg.jl /usr/local/lib/julia/sys core2 ~/my_usrimg.jl true")
+ println()
+ println(" Running this script with no arguments is equivalent to calling it via")
+ println(" build_sysimg.jl $(default_sysimg_path) native")
+ return 0
+ end
+
+ force_flag = "--force" in ARGS
+ filter!(x -> x != "--force", ARGS)
+ build_sysimg(ARGS..., force=force_flag)
+end

doc/devdocs/julia.rst

@@ -12,3 +12,4 @@
  cartesian
  meta
  subarrays
+ sysimg

doc/devdocs/sysimg.rst

@@ -0,0 +1,35 @@
+*********************
+System Image Building
+*********************
+
+Building the Julia system image
+-------------------------------
+
+Julia ships with a preparsed system image containing the contents of the ``Base`` module, named ``sys.ji``. This file is also precompiled into a shared library called ``sys.{so,dll,dylib}`` on as many platforms as possible, so as to give vastly improved startup times. On systems that do not ship with a precompiled system image file, one can be generated from the source files shipped in Julia's ``DATAROOTDIR/julia/base`` folder.
+
+This operation is useful for multiple reasons. A user may:
+
+* Build a precompiled shared library system image on a platform that did not ship with one, thereby improving startup times.
+
+* Modify ``Base``, rebuild the system image and use the new ``Base`` next time Julia is started.
+
+* Include a ``userimg.jl`` file that includes packages into the system image, thereby creating a system image that has packages embedded into the startup environment.
+
+Julia now ships with a script that automates the tasks of building the system image, wittingly named ``build_sysimg.jl`` that lives in ``DATAROOTDIR/julia/``. That is, to include it into a current Julia session, type:
+::
+
+ include(joinpath(JULIA_HOME, Base.DATAROOTDIR, "julia", "build_sysimg.jl"))
+
+This will include a ``build_sysimg()`` function:
+
+.. function:: build_sysimg(sysimg_path=default_sysimg_path, cpu_target="native", userimg_path=nothing; force=false)
+ Rebuild the system image.
+ Store it in ``sysimg_path``, which defaults to a file named ``sys.ji`` that sits in the same folder as ``libjulia.{so,dll,dylib}``.
+ Use the cpu instruction set given by ``cpu_target``. Valid CPU targets are the same as for the ``-C`` option to ``julia``, or the ``-march`` option to ``gcc``. Defaults to ``native``, which means to use all CPU instructions available on the current processor.
+ Include the user image file given by ``userimg_path``, which should contain directives such as ``using MyPackage`` to include that package in the new system image.
+ New system image will not replace an older image unless ``force`` is set to true.
+
+Note that this file can also be run as a script itself, with command line arguments taking the place of arguments passed to the ``build_sysimg`` function. For example, to build a system image in ``/tmp/sys.{so,dll,dylib}``, with the ``core2`` CPU instruction set, a user image of ``~/userimg.jl`` and ``force`` set to ``true``, one would execute:
+::
+
+ julia build_sysimg.jl /tmp/sys core2 ~/userimg.jl --force

src/cgutils.cpp

@@ -246,7 +246,7 @@ static void jl_gen_llvm_gv_array(llvm::Module *mod, SmallVector<GlobalVariable*,
  ConstantInt::get(T_size,globalUnique+1),
  "jl_globalUnique"));
 
- Constant *feature_string = ConstantDataArray::getString(jl_LLVMContext, jl_cpu_string);
+ Constant *feature_string = ConstantDataArray::getString(jl_LLVMContext, jl_compileropts.cpu_target);
  globalvars.push_back(new GlobalVariable(*mod,
  feature_string->getType(),
  true,
@@ -255,7 +255,7 @@ static void jl_gen_llvm_gv_array(llvm::Module *mod, SmallVector<GlobalVariable*,
  "jl_sysimg_cpu_target"));
 
  // For native also store the cpuid
- if (strcmp(jl_cpu_string,"native") == 0) {
+ if (strcmp(jl_compileropts.cpu_target,"native") == 0) {
  uint32_t info[4];
 
  jl_cpuid((int32_t*)info, 1);

src/codegen.cpp

@@ -522,10 +522,7 @@ static Type *NoopType;
 
 // --- utilities ---
 
-#define XSTR(x) #x
-#define MSTR(x) XSTR(x)
 extern "C" {
- const char *jl_cpu_string = MSTR(JULIA_TARGET_ARCH);
  int globalUnique = 0;
 }
 
@@ -4865,7 +4862,6 @@ extern "C" void jl_init_codegen(void)
  jl_setup_module(engine_module,false);
 #endif
 
-
 #if !defined(LLVM_VERSION_MAJOR) || (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR == 0)
  jl_ExecutionEngine = EngineBuilder(m).setEngineKind(EngineKind::JIT).create();
 #ifdef JL_DEBUG_BUILD
@@ -4939,9 +4935,9 @@ extern "C" void jl_init_codegen(void)
  TheTriple,
  "",
 #if LLVM35
- strcmp(jl_cpu_string,"native") ? jl_cpu_string : sys::getHostCPUName().data(),
+ strcmp(jl_compileropts.cpu_target,"native") ? jl_compileropts.cpu_target : sys::getHostCPUName().data(),
 #else
- strcmp(jl_cpu_string,"native") ? jl_cpu_string : "",
+ strcmp(jl_compileropts.cpu_target,"native") ? jl_compileropts.cpu_target : "",
 #endif
  MAttrs);
  assert(jl_TargetMachine);

src/dump.c

@@ -165,7 +165,6 @@ jl_value_t ***sysimg_gvars = NULL;
 
 extern int globalUnique;
 extern void jl_cpuid(int32_t CPUInfo[4], int32_t InfoType);
-extern const char *jl_cpu_string;
 uv_lib_t *jl_sysimg_handle = NULL;
 uint64_t jl_sysimage_base = 0;
 #ifdef _OS_WINDOWS_
@@ -185,21 +184,21 @@ static void jl_load_sysimg_so(char *fname)
  sysimg_gvars = (jl_value_t***)jl_dlsym(jl_sysimg_handle, "jl_sysimg_gvars");
  globalUnique = *(size_t*)jl_dlsym(jl_sysimg_handle, "jl_globalUnique");
  const char *cpu_target = (const char*)jl_dlsym(jl_sysimg_handle, "jl_sysimg_cpu_target");
- if (strcmp(cpu_target,jl_cpu_string) != 0)
+ if (strcmp(cpu_target,jl_compileropts.cpu_target) != 0)
  jl_error("Julia and the system image were compiled for different architectures.\n"
  "Please delete or regenerate sys.{so,dll,dylib}.\n");
  uint32_t info[4];
  jl_cpuid((int32_t*)info, 1);
  if (strcmp(cpu_target, "native") == 0) {
  uint64_t saved_cpuid = *(uint64_t*)jl_dlsym(jl_sysimg_handle, "jl_sysimg_cpu_cpuid");
  if (saved_cpuid != (((uint64_t)info[2])|(((uint64_t)info[3])<<32)))
- jl_error("Target architecture mismatch. Please delete or regenerate sys.{so,dll,dylib}.");
+ jl_error("Target architecture mismatch. Please delete or regenerate sys.{so,dll,dylib}.\n");
  }
  else if (strcmp(cpu_target,"core2") == 0) {
  int HasSSSE3 = (info[2] & 1<<9);
  if (!HasSSSE3)
  jl_error("The current host does not support SSSE3, but the system image was compiled for Core2.\n"
- "Please delete or regenerate sys.{so,dll,dylib}.");
+ "Please delete or regenerate sys.{so,dll,dylib}.\n");
  }
 #ifdef _OS_WINDOWS_
  jl_sysimage_base = (intptr_t)jl_sysimg_handle->handle;
@@ -1350,6 +1349,32 @@ extern void jl_get_builtin_hooks(void);
 extern void jl_get_system_hooks(void);
 extern void jl_get_uv_hooks();
 
+// Takes in a path of the form "usr/lib/julia/sys.ji", such as passed in to jl_restore_system_image()
+DLLEXPORT
+const char * jl_get_system_image_cpu_target(const char *fname)
+{
+ // If passed NULL, don't even bother
+ if (!fname)
+ return NULL;
+
+ // First, get "sys" from "sys.ji"
+ char *fname_shlib = (char*)alloca(strlen(fname));
+ strcpy(fname_shlib, fname);
+ char *fname_shlib_dot = strrchr(fname_shlib, '.');
+ if (fname_shlib_dot != NULL)
+ *fname_shlib_dot = 0;
+
+ // Get handle to sys.so
+ uv_lib_t * sysimg_handle = jl_load_dynamic_library_e(fname_shlib, JL_RTLD_DEFAULT | JL_RTLD_GLOBAL);
+
+ // Return jl_sysimg_cpu_target if we can
+ if (sysimg_handle)
+ return (const char *)jl_dlsym(sysimg_handle, "jl_sysimg_cpu_target");
+
+ // If something goes wrong, return NULL
+ return NULL;
+}
+
 DLLEXPORT
 void jl_restore_system_image(const char *fname)
 {

src/init.c

@@ -92,6 +92,7 @@ jl_compileropts_t jl_compileropts = { NULL, // julia_home
  NULL, // julia_bin
  NULL, // build_path
  system_image_path, // image_file
+ NULL, // cpu_target ("native", "core2", etc...)
  0, // code_coverage
  0, // malloc_log
  JL_COMPILEROPT_CHECK_BOUNDS_DEFAULT,
@@ -869,6 +870,15 @@ void _julia_init(JL_IMAGE_SEARCH rel)
  jl_io_loop = uv_default_loop(); // this loop will internal events (spawning process etc.),
  // best to call this first, since it also initializes libuv
  jl_resolve_sysimg_location(rel);
+
+ // If we are able to load the sysimg and get a cpu_target, use that unless user has overridden
+ if (jl_compileropts.cpu_target == NULL) {
+ const char * sysimg_cpu_target = jl_get_system_image_cpu_target(jl_compileropts.image_file);
+
+ // If we can't load anything from the sysimg, default to native
+ jl_compileropts.cpu_target = sysimg_cpu_target ? sysimg_cpu_target : "native";
+ }
+
  jl_page_size = jl_getpagesize();
  jl_arr_xtralloc_limit = uv_get_total_memory() / 100; // Extra allocation limited to 1% of total RAM
  jl_find_stack_bottom();

src/julia.h

@@ -868,6 +868,7 @@ DLLEXPORT int julia_trampoline(int argc, char *argv[], int (*pmain)(int ac,char
 DLLEXPORT void jl_atexit_hook(void);
 DLLEXPORT void NORETURN jl_exit(int status);
 
+DLLEXPORT const char * jl_get_system_image_cpu_target(const char *fname);
 DLLEXPORT void jl_save_system_image(const char *fname);
 DLLEXPORT void jl_restore_system_image(const char *fname);
 DLLEXPORT int jl_save_new_module(const char *fname, jl_module_t *mod);
@@ -1320,6 +1321,7 @@ typedef struct {
  const char *julia_bin;
  const char *build_path;
  const char *image_file;
+ const char *cpu_target;
  int8_t code_coverage;
  int8_t malloc_log;
  int8_t check_bounds;