From 50691212bcebaf1e540fe0371b61be199e4ad761 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sun, 7 Dec 2014 02:56:16 -0500 Subject: [PATCH 01/40] cleanup julia_init options moves all configuration options into jl_compileropts compute the abspath/realpath of file paths in the config to avoid issues with chdir --- Makefile | 4 +- base/client.jl | 3 - base/sysimg.jl | 2 + src/codegen.cpp | 2 +- src/dlload.c | 2 +- src/dump.c | 18 +++-- src/init.c | 164 ++++++++++++++++++++++++++++++++++-------- src/jlapi.c | 38 +++------- src/julia.h | 17 ++--- src/support/dirpath.h | 11 ++- src/support/ios.c | 2 +- src/support/ios.h | 2 +- ui/repl.c | 61 ++-------------- 13 files changed, 178 insertions(+), 148 deletions(-) diff --git a/Makefile b/Makefile index 03d203c8987e7..a905fa679b379 100644 --- a/Makefile +++ b/Makefile @@ -300,8 +300,8 @@ else ifeq ($(OS), Linux) done endif - # Overwrite JL_SYSTEM_IMAGE_PATH in julia binaries - for julia in $(DESTDIR)$(bindir)/julia* ; do \ + # Overwrite JL_SYSTEM_IMAGE_PATH in julia library + for julia in $(DESTDIR)$(private_libdir)/libjulia*.$(SHLIB_EXT) ; do \ $(call spawn,$(build_bindir)/stringreplace $$(strings -t x - $$julia | grep "sys.ji$$" | awk '{print $$1;}' ) "$(private_libdir_rel)/sys.ji" 256 $(call cygpath_w,$$julia)); \ done endif diff --git a/base/client.jl b/base/client.jl index 3518498795e83..e8dc8b0b19009 100644 --- a/base/client.jl +++ b/base/client.jl @@ -372,13 +372,10 @@ import .Terminals import .REPL function _start() - early_init() - try init_parallel() init_bind_addr(ARGS) any(a->(a=="--worker"), ARGS) || init_head_sched() - init_load_path() (quiet,repl,startup,color_set,no_history_file) = process_options(copy(ARGS)) local term diff --git a/base/sysimg.jl b/base/sysimg.jl index 67aa85ba7dd77..a33c5a37a52e6 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -294,6 +294,8 @@ function __init__() reinit_stdio() Multimedia.reinit_displays() # since Multimedia.displays uses STDOUT as fallback fdwatcher_init() + early_init() + init_load_path() end include("precompile.jl") diff --git a/src/codegen.cpp b/src/codegen.cpp index 8fb7738559692..69082212b1dbc 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -869,7 +869,7 @@ static void coverageVisitLine(std::string filename, int line) void write_log_data(logdata_t logData, const char *extension) { - std::string base = std::string(julia_home); + std::string base = std::string(jl_compileropts.julia_home); base = base + "/../share/julia/base/"; logdata_t::iterator it = logData.begin(); for (; it != logData.end(); it++) { diff --git a/src/dlload.c b/src/dlload.c index ef5332696e176..33b6249e34ff2 100644 --- a/src/dlload.c +++ b/src/dlload.c @@ -116,7 +116,7 @@ static uv_lib_t *jl_load_dynamic_library_(char *modname, unsigned flags, int thr ext = extensions[i]; path[0] = '\0'; handle->handle = NULL; - if (dl_path[len-1] == PATHSEP) + if (dl_path[len-1] == PATHSEPSTRING[0]) snprintf(path, PATHBUF, "%s%s%s", dl_path, modname, ext); else snprintf(path, PATHBUF, "%s" PATHSEPSTRING "%s%s", dl_path, modname, ext); diff --git a/src/dump.c b/src/dump.c index b9ea91b53e8f4..01f0d502ec0f0 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1305,7 +1305,7 @@ void jl_deserialize_lambdas_from_mod(ios_t *s) extern jl_array_t *jl_module_init_order; -DLLEXPORT void jl_save_system_image(char *fname) +DLLEXPORT void jl_save_system_image(const char *fname) { jl_gc_collect(); jl_gc_collect(); @@ -1339,8 +1339,8 @@ DLLEXPORT void jl_save_system_image(char *fname) for(i=0; i < jl_array_len(jl_module_init_order); i++) { // NULL out any modules that weren't saved jl_value_t *mod = jl_cellref(jl_module_init_order, i); - if (ptrhash_get(&backref_table, mod) == HT_NOTFOUND) - jl_cellset(jl_module_init_order, i, NULL); + (void)mod; + assert(ptrhash_get(&backref_table, mod) != HT_NOTFOUND); } } jl_serialize_value(&f, jl_module_init_order); @@ -1360,11 +1360,10 @@ extern void jl_get_system_hooks(void); extern void jl_get_uv_hooks(); DLLEXPORT -void jl_restore_system_image(char *fname) +void jl_restore_system_image(const char *fname) { ios_t f; - char *fpath = fname; - if (ios_file(&f, fpath, 1, 0, 0, 0) == NULL) { + if (ios_file(&f, fname, 1, 0, 0, 0) == NULL) { JL_PRINTF(JL_STDERR, "System image file \"%s\" not found\n", fname); exit(1); } @@ -1432,14 +1431,13 @@ void jl_restore_system_image(char *fname) //ios_printf(ios_stderr, "backref_list.len = %d\n", backref_list.len); arraylist_free(&backref_list); ios_close(&f); - if (fpath != fname) free(fpath); #ifdef JL_GC_MARKSWEEP if (en) jl_gc_enable(); #endif // restore the value of our "magic" JULIA_HOME variable/constant jl_get_binding_wr(jl_core_module, jl_symbol("JULIA_HOME"))->value = - jl_cstr_to_string(julia_home); + jl_cstr_to_string(jl_compileropts.julia_home); mode = last_mode; jl_update_all_fptrs(); } @@ -1543,7 +1541,7 @@ jl_value_t *jl_uncompress_ast(jl_lambda_info_t *li, jl_value_t *data) } DLLEXPORT -int jl_save_new_module(char *fname, jl_module_t *mod) +int jl_save_new_module(const char *fname, jl_module_t *mod) { ios_t f; if (ios_file(&f, fname, 1, 1, 1, 1) == NULL) { @@ -1583,7 +1581,7 @@ jl_function_t *jl_method_cache_insert(jl_methtable_t *mt, jl_tuple_t *type, jl_function_t *method); DLLEXPORT -jl_module_t *jl_restore_new_module(char *fname) +jl_module_t *jl_restore_new_module(const char *fname) { ios_t f; if (ios_file(&f, fname, 1, 0, 0, 0) == NULL) { diff --git a/src/init.c b/src/init.c index 9519582e2c661..55b2709a9d1a9 100644 --- a/src/init.c +++ b/src/init.c @@ -6,6 +6,7 @@ #include #include +#include #include #include @@ -42,6 +43,12 @@ extern "C" { #endif +#ifdef _MSC_VER +DLLEXPORT char * dirname(char *); +#else +#include +#endif + #ifdef _OS_WINDOWS_ #define WIN32_LEAN_AND_MEAN // Copied from MINGW_FLOAT_H which may not be found due to a collision with the builtin gcc float.h @@ -78,8 +85,11 @@ DLLEXPORT void jlbacktrace(); DLLEXPORT void gdbbacktrace(); DLLEXPORT void gdblookup(ptrint_t ip); -char *julia_home = NULL; -jl_compileropts_t jl_compileropts = { NULL, // build_path +static const char system_image_path[256] = JL_SYSTEM_IMAGE_PATH; + +jl_compileropts_t jl_compileropts = { NULL, // julia_home + NULL, // build_path + system_image_path, // image_file 0, // code_coverage 0, // malloc_log JL_COMPILEROPT_CHECK_BOUNDS_DEFAULT, @@ -729,10 +739,106 @@ kern_return_t catch_exception_raise(mach_port_t exception_port, #endif -void julia_init(char *imageFile) +static char *abspath(const char *in) +{ // compute an absolute path location, so that chdir doesn't change the file reference +#ifndef _OS_WINDOWS_ + char *out = realpath(in, NULL); + if (!out) { + if (in[0] == PATHSEPSTRING[0]) { + out = strdup(in); + } + else { + size_t path_size = PATH_MAX; + size_t len = strlen(in); + char *path = (char*)malloc(PATH_MAX); + if (uv_cwd(path, &path_size)) { + ios_printf(ios_stderr, "fatal error: unexpected error while retrieving current working directory\n"); + exit(1); + } + if (path_size + len + 1 >= PATH_MAX) { + ios_printf(ios_stderr, "fatal error: current working directory path too long\n"); + exit(1); + } + path[path_size-1] = PATHSEPSTRING[0]; + memcpy(path+path_size, in, len+1); + out = strdup(path); + } + } +#else + DWORD n = GetFullPathName(in, 0, NULL, C_NULL); + if (n <= 0) { + ios_printf(ios_stderr, "fatal error: jl_compileropts.image_file path too long or GetFullPathName failed\n"); + exit(1); + } + char *out = (char*)malloc(n); + DWORD m = GetFullPathName(in, n, out, C_NULL); + if (n != m + 1) { + ios_printf(ios_stderr, "fatal error: jl_compileropts.image_file path too long or GetFullPathName failed\n"); + exit(1); + } +#endif + return out; +} + +static void jl_resolve_sysimg_location() +{ // note: if you care about lost memory, you should compare the + // pointers in jl_compileropts before and after calling julia_init() + // and call the appropriate free function on the originals for any that changed + char *free_path = NULL; + if (!jl_compileropts.julia_home) { + jl_compileropts.julia_home = getenv("JULIA_HOME"); + if (!jl_compileropts.julia_home) { + size_t path_size = PATH_MAX; + char *path = (char*)malloc(PATH_MAX); + if (uv_exepath(path, &path_size)) { + ios_printf(ios_stderr, "fatal error: unexpected error while retrieving exepath\n"); + exit(1); + } + if (path_size >= PATH_MAX) { + ios_printf(ios_stderr, "fatal error: jl_compileropts.image_file path too long\n"); + exit(1); + } + free_path = path; + jl_compileropts.julia_home = dirname(path); + } + } + if (jl_compileropts.julia_home) + jl_compileropts.julia_home = abspath(jl_compileropts.julia_home); + if (free_path) { + free(free_path); + free_path = NULL; + } + if (jl_compileropts.image_file) { + if (jl_compileropts.image_file[0] != PATHSEPSTRING[0]) { + if (jl_compileropts.image_file == system_image_path) { + // build time path, relative to JULIA_HOME + char *path = (char*)malloc(PATH_MAX); + int n = snprintf(path, PATH_MAX, "%s" PATHSEPSTRING "%s", + jl_compileropts.julia_home, jl_compileropts.image_file); + if (n >= PATH_MAX || n < 0) { + ios_printf(ios_stderr, "fatal error: jl_compileropts.image_file path too long\n"); + exit(1); + } + free_path = path; + jl_compileropts.image_file = path; + } + } + } + if (jl_compileropts.image_file) + jl_compileropts.image_file = abspath(jl_compileropts.image_file); + if (free_path) { + free(free_path); + free_path = NULL; + } + if (jl_compileropts.build_path) + jl_compileropts.build_path = abspath(jl_compileropts.build_path); +} + +void julia_init() { jl_io_loop = uv_default_loop(); // this loop will internal events (spawning process etc.), // best to call this first, since it also initializes libuv + jl_resolve_sysimg_location(); jl_page_size = jl_getpagesize(); jl_arr_xtralloc_limit = uv_get_total_memory() / 100; // Extra allocation limited to 1% of total RAM jl_find_stack_bottom(); @@ -755,11 +861,11 @@ void julia_init(char *imageFile) if (!DuplicateHandle(GetCurrentProcess(), GetCurrentThread(), GetCurrentProcess(), (PHANDLE)&hMainThread, 0, TRUE, DUPLICATE_SAME_ACCESS)) { - JL_PRINTF(JL_STDERR, "WARNING: failed to access handle to main thread\n"); + ios_printf(ios_stderr, "WARNING: failed to access handle to main thread\n"); } SymSetOptions(SYMOPT_UNDNAME | SYMOPT_DEFERRED_LOADS | SYMOPT_LOAD_LINES); if (!SymInitialize(GetCurrentProcess(), NULL, 1)) { - JL_PRINTF(JL_STDERR, "WARNING: failed to initalize stack walk info\n"); + ios_printf(ios_stderr, "WARNING: failed to initialize stack walk info\n"); } needsSymRefreshModuleList = 0; uv_lib_t jl_dbghelp; @@ -805,7 +911,7 @@ void julia_init(char *imageFile) jl_init_serializer(); - if (!imageFile) { + if (!jl_compileropts.image_file) { jl_core_module = jl_new_module(jl_symbol("Core")); jl_init_intrinsic_functions(); jl_init_primitives(); @@ -823,13 +929,13 @@ void julia_init(char *imageFile) // Core.JULIA_HOME is a "magic" constant, we set it at runtime here // since its value gets excluded from the system image jl_set_const(jl_core_module, jl_symbol("JULIA_HOME"), - jl_cstr_to_string(julia_home)); + jl_cstr_to_string(jl_compileropts.julia_home)); jl_module_export(jl_core_module, jl_symbol("JULIA_HOME")); } - if (imageFile) { + if (jl_compileropts.image_file) { JL_TRY { - jl_restore_system_image(imageFile); + jl_restore_system_image(jl_compileropts.image_file); } JL_CATCH { JL_PRINTF(JL_STDERR, "error during init:\n"); @@ -863,7 +969,6 @@ void julia_init(char *imageFile) jl_current_module = jl_main_module; jl_root_task->current_module = jl_current_module; - #ifndef _OS_WINDOWS_ signal_stack = malloc(sig_stack_size); struct sigaction actf; @@ -872,11 +977,11 @@ void julia_init(char *imageFile) actf.sa_handler = fpe_handler; actf.sa_flags = 0; if (sigaction(SIGFPE, &actf, NULL) < 0) { - JL_PRINTF(JL_STDERR, "sigaction: %s\n", strerror(errno)); + JL_PRINTF(JL_STDERR, "fatal error: sigaction: %s\n", strerror(errno)); jl_exit(1); } if (signal(SIGPIPE,SIG_IGN) == SIG_ERR) { - JL_PRINTF(JL_STDERR, "Couldn't set SIGPIPE\n"); + JL_PRINTF(JL_STDERR, "fatal error: Couldn't set SIGPIPE\n"); jl_exit(1); } #if defined (_OS_DARWIN_) @@ -909,7 +1014,7 @@ void julia_init(char *imageFile) ss.ss_size = sig_stack_size; ss.ss_sp = signal_stack; if (sigaltstack(&ss, NULL) < 0) { - JL_PRINTF(JL_STDERR, "sigaltstack: %s\n", strerror(errno)); + JL_PRINTF(JL_STDERR, "fatal error: sigaltstack: %s\n", strerror(errno)); jl_exit(1); } @@ -919,7 +1024,7 @@ void julia_init(char *imageFile) act.sa_sigaction = segv_handler; act.sa_flags = SA_ONSTACK | SA_SIGINFO; if (sigaction(SIGSEGV, &act, NULL) < 0) { - JL_PRINTF(JL_STDERR, "sigaction: %s\n", strerror(errno)); + JL_PRINTF(JL_STDERR, "fatal error: sigaction: %s\n", strerror(errno)); jl_exit(1); } #endif // defined(_OS_DARWIN_) @@ -929,49 +1034,50 @@ void julia_init(char *imageFile) act_die.sa_sigaction = sigdie_handler; act_die.sa_flags = SA_SIGINFO; if (sigaction(SIGINFO, &act_die, NULL) < 0) { - JL_PRINTF(JL_STDERR, "sigaction: %s\n", strerror(errno)); + JL_PRINTF(JL_STDERR, "fatal error: sigaction: %s\n", strerror(errno)); jl_exit(1); } if (sigaction(SIGBUS, &act_die, NULL) < 0) { - JL_PRINTF(JL_STDERR, "sigaction: %s\n", strerror(errno)); + JL_PRINTF(JL_STDERR, "fatal error: sigaction: %s\n", strerror(errno)); jl_exit(1); } if (sigaction(SIGILL, &act_die, NULL) < 0) { - JL_PRINTF(JL_STDERR, "sigaction: %s\n", strerror(errno)); + JL_PRINTF(JL_STDERR, "fatal error: sigaction: %s\n", strerror(errno)); jl_exit(1); } if (sigaction(SIGTERM, &act_die, NULL) < 0) { - JL_PRINTF(JL_STDERR, "sigaction: %s\n", strerror(errno)); + JL_PRINTF(JL_STDERR, "fatal error: sigaction: %s\n", strerror(errno)); jl_exit(1); } if (sigaction(SIGABRT, &act_die, NULL) < 0) { - JL_PRINTF(JL_STDERR, "sigaction: %s\n", strerror(errno)); + JL_PRINTF(JL_STDERR, "fatal error: sigaction: %s\n", strerror(errno)); jl_exit(1); } if (sigaction(SIGQUIT, &act_die, NULL) < 0) { - JL_PRINTF(JL_STDERR, "sigaction: %s\n", strerror(errno)); + JL_PRINTF(JL_STDERR, "fatal error: sigaction: %s\n", strerror(errno)); jl_exit(1); } if (sigaction(SIGSYS, &act_die, NULL) < 0) { - JL_PRINTF(JL_STDERR, "sigaction: %s\n", strerror(errno)); + JL_PRINTF(JL_STDERR, "fatal error: sigaction: %s\n", strerror(errno)); jl_exit(1); } if (sigaction(SIGPIPE, &act_die, NULL) < 0) { - JL_PRINTF(JL_STDERR, "sigaction: %s\n", strerror(errno)); + JL_PRINTF(JL_STDERR, "fatal error: sigaction: %s\n", strerror(errno)); jl_exit(1); } #else // defined(_OS_WINDOWS_) if (signal(SIGFPE, (void (__cdecl *)(int))fpe_handler) == SIG_ERR) { - JL_PRINTF(JL_STDERR, "Couldn't set SIGFPE\n"); + JL_PRINTF(JL_STDERR, "fatal error: Couldn't set SIGFPE\n"); jl_exit(1); } + SetUnhandledExceptionFilter(exception_handler); #endif #ifdef JL_GC_MARKSWEEP jl_gc_enable(); #endif - if (imageFile) + if (jl_compileropts.image_file) jl_init_restored_modules(); jl_install_sigint_handler(); @@ -988,11 +1094,10 @@ DLLEXPORT void jl_install_sigint_handler() act.sa_sigaction = sigint_handler; act.sa_flags = SA_SIGINFO; if (sigaction(SIGINT, &act, NULL) < 0) { - JL_PRINTF(JL_STDERR, "sigaction: %s\n", strerror(errno)); + JL_PRINTF(JL_STDERR, "fatal error: sigaction: %s\n", strerror(errno)); jl_exit(1); } #endif - //printf("sigint installed\n"); } extern int asprintf(char **str, const char *fmt, ...); @@ -1002,20 +1107,17 @@ void jl_compile_all(void); DLLEXPORT int julia_trampoline(int argc, char **argv, int (*pmain)(int ac,char *av[])) { -#if defined(_OS_WINDOWS_) - SetUnhandledExceptionFilter(exception_handler); -#endif unsigned char *p = (unsigned char *)&__stack_chk_guard; char a = p[sizeof(__stack_chk_guard)-1]; char b = p[sizeof(__stack_chk_guard)-2]; char c = p[0]; - /* If you have the ability to generate random numbers in your kernel then use them */ + /* If you have the ability to generate random numbers in your kernel then they should be used here */ p[sizeof(__stack_chk_guard)-1] = 255; p[sizeof(__stack_chk_guard)-2] = '\n'; p[0] = 0; JL_SET_STACK_BASE; int ret = pmain(argc, argv); - char *build_path = jl_compileropts.build_path; + const char *build_path = jl_compileropts.build_path; if (build_path) { if (jl_compileropts.compile_enabled == JL_COMPILEROPT_COMPILE_ALL) jl_compile_all(); diff --git a/src/jlapi.c b/src/jlapi.c index a3eb39b7e0000..f3d94b2d2bc6b 100644 --- a/src/jlapi.c +++ b/src/jlapi.c @@ -22,24 +22,6 @@ DLLEXPORT char * __cdecl basename(char *); #include #endif -DLLEXPORT char *jl_locate_sysimg(char *jlhome, char *imgpath) -{ - if (jlhome == NULL) { - char *julia_path = (char*)malloc(512); - size_t path_size = 512; - uv_exepath(julia_path, &path_size); - julia_home = strdup(dirname(julia_path)); - free(julia_path); - } - else { - julia_home = jlhome; - } - char path[512]; - snprintf(path, sizeof(path), "%s%s%s", - julia_home, PATHSEPSTRING, imgpath); - return strdup(path); -} - DLLEXPORT void *jl_eval_string(char *str); int jl_is_initialized(void) { return jl_main_module!=NULL; } @@ -54,22 +36,22 @@ DLLEXPORT void jl_init_with_image(char *julia_home_dir, char *image_relative_pat { if (jl_is_initialized()) return; libsupport_init(); - if (image_relative_path == NULL) - image_relative_path = JL_SYSTEM_IMAGE_PATH; - char *image_file = jl_locate_sysimg(julia_home_dir, image_relative_path); - julia_init(image_file); - jl_set_const(jl_core_module, jl_symbol("JULIA_HOME"), - jl_cstr_to_string(julia_home)); - jl_module_export(jl_core_module, jl_symbol("JULIA_HOME")); - jl_eval_string("Base.early_init()"); + jl_compileropts.julia_home = julia_home_dir; + if (image_relative_path != NULL) + jl_compileropts.image_file = image_relative_path; + julia_init(); + //TODO: these should be part of Multi.__init__() + //currently, we have them here since we may not want them + //getting unconditionally set from Base.__init__() + jl_eval_string("Base.init_parallel()"); + jl_eval_string("Base.init_bind_addr(ARGS)"); jl_eval_string("Base.init_head_sched()"); - jl_eval_string("Base.init_load_path()"); jl_exception_clear(); } DLLEXPORT void jl_init(char *julia_home_dir) { - jl_init_with_image(julia_home_dir, JL_SYSTEM_IMAGE_PATH); + jl_init_with_image(julia_home_dir, NULL); } DLLEXPORT void *jl_eval_string(char *str) diff --git a/src/julia.h b/src/julia.h index 277f0685cd877..f0432ffba4f3e 100644 --- a/src/julia.h +++ b/src/julia.h @@ -856,17 +856,16 @@ DLLEXPORT void jl_exception_clear(void); } // initialization functions -DLLEXPORT void julia_init(char *imageFile); -DLLEXPORT int julia_trampoline(int argc, char *argv[], int (*pmain)(int ac,char *av[])); +DLLEXPORT void julia_init(); DLLEXPORT void jl_init(char *julia_home_dir); DLLEXPORT void jl_init_with_image(char *julia_home_dir, char *image_relative_path); DLLEXPORT int jl_is_initialized(void); -DLLEXPORT extern char *julia_home; +DLLEXPORT int julia_trampoline(int argc, char *argv[], int (*pmain)(int ac,char *av[])); -DLLEXPORT void jl_save_system_image(char *fname); -DLLEXPORT void jl_restore_system_image(char *fname); -DLLEXPORT int jl_save_new_module(char *fname, jl_module_t *mod); -DLLEXPORT jl_module_t *jl_restore_new_module(char *fname); +DLLEXPORT void jl_save_system_image(const char *fname); +DLLEXPORT void jl_restore_system_image(const char *fname); +DLLEXPORT int jl_save_new_module(const char *fname, jl_module_t *mod); +DLLEXPORT jl_module_t *jl_restore_new_module(const char *fname); void jl_init_restored_modules(); // front end interface @@ -1328,7 +1327,9 @@ void show_execution_point(char *filename, int lno); // compiler options ----------------------------------------------------------- typedef struct { - char *build_path; + const char *julia_home; + const char *build_path; + const char *image_file; int8_t code_coverage; int8_t malloc_log; int8_t check_bounds; diff --git a/src/support/dirpath.h b/src/support/dirpath.h index 6cfd965ea0ea8..28fff4ab96def 100644 --- a/src/support/dirpath.h +++ b/src/support/dirpath.h @@ -2,18 +2,15 @@ #define DIRPATH_H #ifdef _OS_WINDOWS_ -#define PATHSEP '\\' #define PATHSEPSTRING "\\" -#define PATHLISTSEP ';' #define PATHLISTSEPSTRING ";" -#define ISPATHSEP(c) ((c)=='/' || (c)=='\\') -#define MAXPATHLEN 1024 +#ifdef _MSC_VER +#define PATH_MAX MAX_PATH +#endif #else -#define PATHSEP '/' #define PATHSEPSTRING "/" -#define PATHLISTSEP ':' #define PATHLISTSEPSTRING ":" -#define ISPATHSEP(c) ((c)=='/') +#define PATH_MAX 1024 #endif #endif diff --git a/src/support/ios.c b/src/support/ios.c index dbab4edbca747..d8bd66c7ede4f 100644 --- a/src/support/ios.c +++ b/src/support/ios.c @@ -836,7 +836,7 @@ static void _ios_init(ios_t *s) /* stream object initializers. we do no allocation. */ -ios_t *ios_file(ios_t *s, char *fname, int rd, int wr, int create, int trunc) +ios_t *ios_file(ios_t *s, const char *fname, int rd, int wr, int create, int trunc) { int flags; int fd; diff --git a/src/support/ios.h b/src/support/ios.h index d5a72fcc79da5..78c6518df92f1 100644 --- a/src/support/ios.h +++ b/src/support/ios.h @@ -100,7 +100,7 @@ DLLEXPORT size_t ios_readprep(ios_t *from, size_t n); /* stream creation */ DLLEXPORT -ios_t *ios_file(ios_t *s, char *fname, int rd, int wr, int create, int trunc); +ios_t *ios_file(ios_t *s, const char *fname, int rd, int wr, int create, int trunc); DLLEXPORT ios_t *ios_mem(ios_t *s, size_t initsize); ios_t *ios_str(ios_t *s, char *str); ios_t *ios_static_buffer(ios_t *s, char *buf, size_t sz); diff --git a/ui/repl.c b/ui/repl.c index 31634902aacbc..49e68a4bbc162 100644 --- a/ui/repl.c +++ b/ui/repl.c @@ -4,7 +4,6 @@ */ #include -#include #include #include #include @@ -20,7 +19,6 @@ #ifndef _MSC_VER #include -#include #include #else #include "getopt.h" @@ -34,27 +32,14 @@ #error "JL_SYSTEM_IMAGE_PATH not defined!" #endif -#ifdef _MSC_VER -#define PATH_MAX MAX_PATH -#endif - #ifdef __cplusplus extern "C" { #endif -#ifdef _MSC_VER -DLLEXPORT char * dirname(char *); -#endif - -extern DLLEXPORT char *julia_home; - -char system_image[256] = JL_SYSTEM_IMAGE_PATH; - static int lisp_prompt = 0; static int codecov = JL_LOG_NONE; static int malloclog= JL_LOG_NONE; static char *program = NULL; -char *image_file = NULL; static const char *usage = "julia [options] [program] [args...]\n"; static const char *opts = @@ -111,7 +96,6 @@ void parse_opts(int *argcp, char ***argvp) int c; opterr = 0; int imagepathspecified=0; - image_file = system_image; int skip = 0; int lastind = optind; while ((c = getopt_long(*argcp,*argvp,shortopts,longopts,0)) != -1) { @@ -123,19 +107,19 @@ void parse_opts(int *argcp, char ***argvp) lastind = optind; break; case 'H': - julia_home = strdup(optarg); + jl_compileropts.julia_home = strdup(optarg); break; case 'b': jl_compileropts.build_path = strdup(optarg); if (!imagepathspecified) - image_file = NULL; + jl_compileropts.image_file = NULL; break; case 'J': - image_file = strdup(optarg); + jl_compileropts.image_file = strdup(optarg); imagepathspecified = 1; break; case 'h': - printf("%s%s", usage, opts); + ios_printf(ios_stdout, "%s%s", usage, opts); exit(0); case 'O': jl_compileropts.opt_level = 1; @@ -206,47 +190,14 @@ void parse_opts(int *argcp, char ***argvp) } jl_compileropts.code_coverage = codecov; jl_compileropts.malloc_log = malloclog; - if (!julia_home) { - julia_home = getenv("JULIA_HOME"); - if (julia_home) { - julia_home = strdup(julia_home); - } - else { - char *julia_path = (char*)malloc(PATH_MAX); - size_t path_size = PATH_MAX; - uv_exepath(julia_path, &path_size); - julia_home = strdup(dirname(julia_path)); - free(julia_path); - } - } optind -= skip; *argvp += optind; *argcp -= optind; - if (image_file==NULL && *argcp > 0) { + if (jl_compileropts.image_file==NULL && *argcp > 0) { if (strcmp((*argvp)[0], "-")) { program = (*argvp)[0]; } } - if (image_file) { - if (image_file[0] != PATHSEP) { - uv_stat_t stbuf; - char path[512]; - if (!imagepathspecified) { - // build time path relative to JULIA_HOME - snprintf(path, sizeof(path), "%s%s%s", - julia_home, PATHSEPSTRING, system_image); - image_file = strdup(path); - } - else if (jl_stat(image_file, (char*)&stbuf) != 0) { - // otherwise try julia_home/../lib/julia/%s - snprintf(path, sizeof(path), "%s%s%s", - julia_home, - PATHSEPSTRING ".." PATHSEPSTRING "lib" PATHSEPSTRING "julia" PATHSEPSTRING, - image_file); - image_file = strdup(path); - } - } - } } static int exec_program(void) @@ -384,7 +335,7 @@ int wmain(int argc, wchar_t *argv[], wchar_t *envp[]) jl_lisp_prompt(); return 0; } - julia_init(lisp_prompt ? NULL : image_file); + julia_init(); return julia_trampoline(argc, (char**)argv, true_main); } From 8f7af0e2bbce613746e8d9966cb16475e33c058a Mon Sep 17 00:00:00 2001 From: Blake Johnson Date: Thu, 30 Oct 2014 22:20:50 -0400 Subject: [PATCH 02/40] Modal prefix history search. Attempt to deal with down arrow better at end of history. --- base/LineEdit.jl | 199 +++++++++++++++++++++++++++++++++++---------- base/REPL.jl | 38 ++++----- base/precompile.jl | 12 +-- 3 files changed, 183 insertions(+), 66 deletions(-) diff --git a/base/LineEdit.jl b/base/LineEdit.jl index ae777dcf86fb4..b5fa988425330 100644 --- a/base/LineEdit.jl +++ b/base/LineEdit.jl @@ -61,7 +61,7 @@ type PromptState indent::Int end -input_string(s::PromptState) = bytestring(pointer(s.input_buffer.data), s.input_buffer.size) +input_string(s::PromptState) = bytestring(s.input_buffer) input_string_newlines(s::PromptState) = count(c->(c == '\n'), input_string(s)) function input_string_newlines_aftercursor(s::PromptState) @@ -609,8 +609,8 @@ history_next(::EmptyHistoryProvider) = ("", false) history_search(::EmptyHistoryProvider, args...) = false add_history(::EmptyHistoryProvider, s) = nothing add_history(s::PromptState) = add_history(mode(s).hist, s) -history_next_prefix(s, hist) = false -history_prev_prefix(s, hist) = false +history_next_prefix(s, hist, prefix) = false +history_prev_prefix(s, hist, prefix) = false function history_prev(s, hist) l, ok = history_prev(mode(s).hist) @@ -639,12 +639,12 @@ refresh_line(s, termbuf) = refresh_multi_line(termbuf, s) default_completion_cb(::IOBuffer) = [] default_enter_cb(_) = true -write_prompt(terminal, s::PromptState) = write_prompt(terminal, s, s.p.prompt) -function write_prompt(terminal, s::PromptState, prompt) - prefix = isa(s.p.prompt_prefix,Function) ? s.p.prompt_prefix() : s.p.prompt_prefix - suffix = isa(s.p.prompt_suffix,Function) ? s.p.prompt_suffix() : s.p.prompt_suffix +write_prompt(terminal, s::PromptState) = write_prompt(terminal, s.p) +function write_prompt(terminal, p::Prompt) + prefix = isa(p.prompt_prefix,Function) ? p.prompt_prefix() : p.prompt_prefix + suffix = isa(p.prompt_suffix,Function) ? p.prompt_suffix() : p.prompt_suffix write(terminal, prefix) - write(terminal, prompt) + write(terminal, p.prompt) write(terminal, Base.text_colors[:normal]) write(terminal, suffix) end @@ -898,9 +898,81 @@ function history_set_backward(s::SearchState, backward) s.backward = backward end -input_string(s::SearchState) = bytestring(pointer(s.query_buffer.data), s.query_buffer.size) +input_string(s::SearchState) = bytestring(s.query_buffer) -refresh_multi_line(termbuf::TerminalBuffer, term, s::Union(SearchState,PromptState)) = (@assert term == terminal(s); refresh_multi_line(termbuf,s)) +function reset_state(s::SearchState) + if s.query_buffer.size != 0 + s.query_buffer.size = 0 + s.query_buffer.ptr = 1 + end + if s.response_buffer.size != 0 + s.response_buffer.size = 0 + s.response_buffer.ptr = 1 + end + reset_state(s.histprompt.hp) +end + +type HistoryPrompt{T<:HistoryProvider} <: TextInterface + hp::T + complete + keymap_func::Function + HistoryPrompt(hp) = new(hp, EmptyCompletionProvider()) +end + +HistoryPrompt{T<:HistoryProvider}(hp::T) = HistoryPrompt{T}(hp) +init_state(terminal, p::HistoryPrompt) = SearchState(terminal, p, true, IOBuffer(), IOBuffer()) + +type PrefixSearchState + terminal + histprompt + prefix::ByteString + response_buffer::IOBuffer + ias::InputAreaState + #The prompt whose input will be replaced by the matched history + parent + PrefixSearchState(terminal, histprompt, prefix, response_buffer) = + new(terminal, histprompt, prefix, response_buffer, InputAreaState(0,0)) +end + +input_string(s::PrefixSearchState) = bytestring(s.response_buffer) + +# a meta-prompt that presents itself as parent_prompt, but which has an independent keymap +# for prefix searching +type PrefixHistoryPrompt{T<:HistoryProvider} <: TextInterface + hp::T + parent_prompt::Prompt + complete + keymap_func::Function + PrefixHistoryPrompt(hp, parent_prompt) = new(hp, parent_prompt, EmptyCompletionProvider()) +end + +PrefixHistoryPrompt{T<:HistoryProvider}(hp::T, parent_prompt) = PrefixHistoryPrompt{T}(hp, parent_prompt) +init_state(terminal, p::PrefixHistoryPrompt) = PrefixSearchState(terminal, p, "", IOBuffer()) + +write_prompt(terminal, s::PrefixSearchState) = write_prompt(terminal, s.histprompt.parent_prompt) +prompt_string(s::PrefixSearchState) = s.histprompt.parent_prompt.prompt + +terminal(s::PrefixSearchState) = s.terminal + +function reset_state(s::PrefixSearchState) + if s.response_buffer.size != 0 + s.response_buffer.size = 0 + s.response_buffer.ptr = 1 + end +end + +function transition(s::PrefixSearchState, mode) + s.histprompt.parent_prompt = mode +end + +replace_line(s::PrefixSearchState, l::IOBuffer) = s.response_buffer = l +function replace_line(s::PrefixSearchState, l) + s.response_buffer.ptr = 1 + s.response_buffer.size = 0 + write(s.response_buffer, l) +end + +refresh_multi_line(termbuf::TerminalBuffer, term, s::Union(SearchState,PrefixSearchState,PromptState)) = (@assert term == terminal(s); refresh_multi_line(termbuf,s)) function refresh_multi_line(termbuf::TerminalBuffer, s::SearchState) buf = IOBuffer() write(buf, pointer(s.query_buffer.data), s.query_buffer.ptr-1) @@ -914,7 +986,10 @@ function refresh_multi_line(termbuf::TerminalBuffer, s::SearchState) s.ias = refresh_multi_line(termbuf, s.terminal, buf, s.ias, s.backward ? "(reverse-i-search)`" : "(forward-i-search)`") end -function refresh_multi_line(s::Union(SearchState,PromptState)) +refresh_multi_line(termbuf::TerminalBuffer, s::PrefixSearchState) = s.ias = + refresh_multi_line(termbuf, terminal(s), buffer(s), s.ias, s) + +function refresh_multi_line(s::Union(SearchState,PrefixSearchState,PromptState)) refresh_multi_line(terminal(s), s) end @@ -928,33 +1003,12 @@ function refresh_multi_line(terminal::UnixTerminal, args...; kwargs...) return ret end -function reset_state(s::SearchState) - if s.query_buffer.size != 0 - s.query_buffer.size = 0 - s.query_buffer.ptr = 1 - end - if s.response_buffer.size != 0 - s.response_buffer.size = 0 - s.response_buffer.ptr = 1 - end - reset_state(s.histprompt.hp) -end - -type HistoryPrompt{T<:HistoryProvider} <: TextInterface - hp::T - complete - keymap_func::Function - HistoryPrompt(hp) = new(hp, EmptyCompletionProvider()) -end - -HistoryPrompt{T<:HistoryProvider}(hp::T) = HistoryPrompt{T}(hp) -init_state(terminal, p::HistoryPrompt) = SearchState(terminal, p, true, IOBuffer(), IOBuffer()) - state(s::MIState, p) = s.mode_state[p] state(s::PromptState, p) = (@assert s.p == p; s) mode(s::MIState) = s.current_mode mode(s::PromptState) = s.p mode(s::SearchState) = @assert false +mode(s::PrefixSearchState) = s.histprompt.parent_prompt # Search Mode completions function complete_line(s::SearchState, repeats) @@ -995,6 +1049,21 @@ function enter_search(s::MIState, p::HistoryPrompt, backward::Bool) transition(s, p) end +function enter_prefix_search(s::MIState, p::PrefixHistoryPrompt, backward::Bool) + buf = buffer(s) + + pss = state(s, p) + pss.parent = mode(s) + pss.prefix = bytestring(pointer(buf.data), position(buf)) + copybuf!(pss.response_buffer, buf) + transition(s, p) + if backward + history_prev_prefix(pss, pss.histprompt.hp, pss.prefix) + else + history_next_prefix(pss, pss.histprompt.hp, pss.prefix) + end +end + function setup_search_keymap(hp) p = HistoryPrompt(hp) pkeymap = AnyDict( @@ -1075,8 +1144,8 @@ function setup_search_keymap(hp) (p, skeymap) end -keymap(state, p::HistoryPrompt) = p.keymap_func -keymap_data(state, ::HistoryPrompt) = state +keymap(state, p::Union(HistoryPrompt,PrefixHistoryPrompt)) = p.keymap_func +keymap_data(state, ::Union(HistoryPrompt, PrefixHistoryPrompt)) = state Base.isempty(s::PromptState) = s.input_buffer.size == 0 @@ -1225,33 +1294,80 @@ AnyDict( end edit_insert(s, input) end, - "^T" => (s,o...)->edit_transpose(s), + "^T" => (s,o...)->edit_transpose(s) ) const history_keymap = AnyDict( "^P" => (s,o...)->(history_prev(s, mode(s).hist)), "^N" => (s,o...)->(history_next(s, mode(s).hist)), # Up Arrow - "\e[A" => (s,o...)->(edit_move_up(s) || history_prev_prefix(s, mode(s).hist)), + "\e[A" => (s,o...)->(edit_move_up(s) || history_prev(s, mode(s).hist)), # Down Arrow - "\e[B" => (s,o...)->(edit_move_down(s) || history_next_prefix(s, mode(s).hist)), + "\e[B" => (s,o...)->(edit_move_down(s) || history_next(s, mode(s).hist)), # Page Up "\e[5~" => (s,o...)->(history_prev(s, mode(s).hist)), # Page Down "\e[6~" => (s,o...)->(history_next(s, mode(s).hist)) ) -function deactivate(p::Union(Prompt,HistoryPrompt), s::Union(SearchState,PromptState), termbuf) +const prefix_history_keymap = AnyDict( + # Up Arrow + "\e[A" => (s,data,c)->history_prev_prefix(data, data.histprompt.hp, data.prefix), + # Down Arrow + "\e[B" => (s,data,c)->history_next_prefix(data, data.histprompt.hp, data.prefix), + # # Right Arrow + "\e[C" => (s,data,c)->(accept_result(s, data.histprompt); edit_move_right(s)), + # # Left Arrow + "\e[D" => (s,data,c)->(accept_result(s, data.histprompt); edit_move_left(s)), + '\r' => (s,data,c)->begin + accept_result(s, data.histprompt) + if on_enter(s) || (eof(buffer(s)) && s.key_repeats > 1) + commit_line(s) + return :done + else + edit_insert(s, '\n') + end + end, + '\n' => '\r', + "^C" => (s,data,c)->begin + accept_result(s, data.histprompt) + try # raise the debugger if present + ccall(:jl_raise_debugger, Int, ()) + end + move_input_end(s) + refresh_line(s) + print(terminal(s), "^C\n\n") + transition(s, :reset) + refresh_line(s) + end, + # by default I want to return to the parent mode and pass thru the character... how do I do that? + # "*" => (s,data,c)->(accept_result(s, data.histprompt); write(terminal(s).in_stream, c)) + "*" => (s,data,c)->accept_result(s, data.histprompt) +) + +function setup_prefix_keymap(hp, parent_prompt) + p = PrefixHistoryPrompt(hp, parent_prompt) + p.keymap_func = keymap([prefix_history_keymap]) + pkeymap = AnyDict( + # Up Arrow + "\e[A" => (s,o...)->(edit_move_up(s) || enter_prefix_search(s, p, true)), + # Down Arrow + "\e[B" => (s,o...)->(edit_move_down(s) || enter_prefix_search(s, p, false)), + ) + (p, pkeymap) +end + +function deactivate(p::Union(Prompt,HistoryPrompt,PrefixHistoryPrompt), s::Union(PrefixSearchState,SearchState,PromptState), termbuf) clear_input_area(termbuf, s) s end -function activate(p::Union(Prompt,HistoryPrompt), s::Union(SearchState,PromptState), termbuf) +function activate(p::Union(Prompt,HistoryPrompt,PrefixHistoryPrompt), s::Union(PrefixSearchState,SearchState,PromptState), termbuf) s.ias = InputAreaState(0, 0) refresh_line(s, termbuf) end -function activate(p::Union(Prompt,HistoryPrompt), s::MIState, termbuf) +function activate(p::Union(Prompt,HistoryPrompt,PrefixHistoryPrompt), s::MIState, termbuf) @assert p == s.current_mode activate(p, s.mode_state[s.current_mode], termbuf) end @@ -1332,6 +1448,7 @@ end buffer(s::PromptState) = s.input_buffer buffer(s::SearchState) = s.query_buffer +buffer(s::PrefixSearchState) = s.response_buffer keymap(s::PromptState, prompt::Prompt) = prompt.keymap_func keymap_data(s::PromptState, prompt::Prompt) = prompt.keymap_func_data diff --git a/base/REPL.jl b/base/REPL.jl index a0b5d57146813..9bfb4b326650b 100644 --- a/base/REPL.jl +++ b/base/REPL.jl @@ -384,7 +384,7 @@ function add_history(hist::REPLHistoryProvider, s) flush(hist.history_file) end -function history_move(s::LineEdit.MIState, hist::REPLHistoryProvider, idx::Int, save_idx::Int = hist.cur_idx) +function history_move(s::Union(LineEdit.MIState,LineEdit.PrefixSearchState), hist::REPLHistoryProvider, idx::Int, save_idx::Int = hist.cur_idx) max_idx = length(hist.history) + 1 @assert 1 <= hist.cur_idx <= max_idx (1 <= idx <= max_idx) || return :none @@ -469,34 +469,33 @@ function history_next(s::LineEdit.MIState, hist::REPLHistoryProvider, end end -function history_move_prefix(s::LineEdit.MIState, +function history_move_prefix(s::LineEdit.PrefixSearchState, hist::REPLHistoryProvider, + prefix::String, backwards::Bool) - buf = LineEdit.buffer(s) - pos = position(buf) - prefix = bytestring_beforecursor(buf) - allbuf = bytestring(buf) + cur_response = bytestring(LineEdit.buffer(s)) cur_idx = hist.cur_idx # when searching forward, start at last_idx if !backwards && hist.last_idx > 0 cur_idx = hist.last_idx end hist.last_idx = -1 - idxs = backwards ? ((cur_idx-1):-1:1) : ((cur_idx+1):length(hist.history)) + max_idx = length(hist.history)+1 + idxs = backwards ? ((cur_idx-1):-1:1) : ((cur_idx+1):max_idx) for idx in idxs - if beginswith(hist.history[idx], prefix) && hist.history[idx] != allbuf + if (idx == max_idx) || (beginswith(hist.history[idx], prefix) && hist.history[idx] != cur_response) history_move(s, hist, idx) - seek(LineEdit.buffer(s), pos) + LineEdit.move_input_end(s) LineEdit.refresh_line(s) return :ok end end Terminals.beep(LineEdit.terminal(s)) end -history_next_prefix(s::LineEdit.MIState, hist::REPLHistoryProvider) = - history_move_prefix(s, hist, false) -history_prev_prefix(s::LineEdit.MIState, hist::REPLHistoryProvider) = - history_move_prefix(s, hist, true) +history_next_prefix(s::LineEdit.PrefixSearchState, hist::REPLHistoryProvider, prefix::String) = + history_move_prefix(s, hist, prefix, false) +history_prev_prefix(s::LineEdit.PrefixSearchState, hist::REPLHistoryProvider, prefix::String) = + history_move_prefix(s, hist, prefix, true) function history_search(hist::REPLHistoryProvider, query_buffer::IOBuffer, response_buffer::IOBuffer, backwards::Bool=false, skip_current::Bool=false) @@ -695,9 +694,8 @@ function setup_interface(repl::LineEditREPL; hascolor = repl.hascolor, extra_rep shell_mode.hist = hp help_mode.hist = hp - hkp, hkeymap = LineEdit.setup_search_keymap(hp) - - hkp.complete = LatexCompletions() + search_prompt, skeymap = LineEdit.setup_search_keymap(hp) + search_prompt.complete = LatexCompletions() # Canonicalize user keymap input if isa(extra_repl_keymap, Dict) @@ -777,7 +775,9 @@ function setup_interface(repl::LineEditREPL; hascolor = repl.hascolor, extra_rep end, ) - a = Dict{Any,Any}[hkeymap, repl_keymap, LineEdit.history_keymap, LineEdit.default_keymap, LineEdit.escape_defaults] + prefix_prompt, prefix_keymap = LineEdit.setup_prefix_keymap(hp, julia_prompt) + + a = Dict{Any,Any}[skeymap, repl_keymap, prefix_keymap, LineEdit.history_keymap, LineEdit.default_keymap, LineEdit.escape_defaults] prepend!(a, extra_repl_keymap) julia_prompt.keymap_func = LineEdit.keymap(a) @@ -803,12 +803,12 @@ function setup_interface(repl::LineEditREPL; hascolor = repl.hascolor, extra_rep end ) - b = Dict{Any,Any}[hkeymap, mode_keymap, LineEdit.history_keymap, LineEdit.default_keymap, LineEdit.escape_defaults] + b = Dict{Any,Any}[skeymap, mode_keymap, prefix_keymap, LineEdit.history_keymap, LineEdit.default_keymap, LineEdit.escape_defaults] prepend!(b, extra_repl_keymap) shell_mode.keymap_func = help_mode.keymap_func = LineEdit.keymap(b) - ModalInterface([julia_prompt, shell_mode, help_mode,hkp]) + ModalInterface([julia_prompt, shell_mode, help_mode, search_prompt, prefix_prompt]) end function run_frontend(repl::LineEditREPL, backend) diff --git a/base/precompile.jl b/base/precompile.jl index 918bfd704bf47..277adf6669946 100644 --- a/base/precompile.jl +++ b/base/precompile.jl @@ -72,11 +72,11 @@ precompile(Base.LineEdit.init_state, (Base.Terminals.TTYTerminal, Base.LineEdit. precompile(Base.LineEdit.input_string, (Base.LineEdit.PromptState,)) precompile(Base.LineEdit.keymap, (Base.LineEdit.PromptState, Base.LineEdit.Prompt)) precompile(Base.LineEdit.keymap_data, (Base.LineEdit.PromptState, Base.LineEdit.Prompt)) -precompile(Base.LineEdit.keymap_fcn, (Function, Base.LineEdit.MIState, Char)) -precompile(Base.LineEdit.keymap_fcn, (Function, Base.LineEdit.MIState,Char)) +precompile(Base.LineEdit.keymap_fcn, (Function, Base.LineEdit.MIState, ByteString)) +precompile(Base.LineEdit.keymap_fcn, (Function, Base.LineEdit.MIState, ByteString)) precompile(Base.LineEdit.match_input, (Dict{Char,Any},Base.LineEdit.MIState)) -precompile(Base.LineEdit.match_input, (Function, Base.LineEdit.MIState, Array{Char, 1})) -precompile(Base.LineEdit.match_input, (Function, Base.LineEdit.MIState, Array{Char,1})) +precompile(Base.LineEdit.match_input, (Dict{Char, Any}, Base.LineEdit.MIState, Base.Terminals.TTYTerminal)) +precompile(Base.LineEdit.match_input, (Function, Base.LineEdit.MIState, Base.Terminals.TTYTerminal, Array{Char,1})) precompile(Base.LineEdit.mode, (Base.LineEdit.MIState,)) precompile(Base.LineEdit.move_line_end, (Base.LineEdit.MIState,)) precompile(Base.LineEdit.on_enter, (Base.LineEdit.MIState,)) @@ -104,8 +104,8 @@ precompile(Base.LineEdit.transition, (Base.LineEdit.MIState, Base.LineEdit.Promp precompile(Base.LineEdit.transition, (Base.LineEdit.MIState, Symbol)) precompile(Base.LineEdit.update_key_repeats, (Base.LineEdit.MIState, Array{Char, 1})) precompile(Base.LineEdit.update_key_repeats, (Base.LineEdit.MIState, Array{Char,1})) -precompile(Base.LineEdit.write_prompt, (Base.Terminals.TTYTerminal, Base.LineEdit.PromptState, ASCIIString)) -precompile(Base.LineEdit.write_prompt, (Base.Terminals.TerminalBuffer, Base.LineEdit.PromptState, ASCIIString)) +precompile(Base.LineEdit.write_prompt, (Base.Terminals.TTYTerminal, Base.LineEdit.Prompt)) +precompile(Base.LineEdit.write_prompt, (Base.Terminals.TerminalBuffer, Base.LineEdit.PromptState)) precompile(Base.Multimedia.TextDisplay, (Base.TTY,)) precompile(Base.Multimedia.display, (Int,)) precompile(Base.ProcessGroup, (Int, Array{Any,1}, Array{Any,1})) From 551fdfe8669af5ae23b98691328c56ed2fa7066a Mon Sep 17 00:00:00 2001 From: Blake Johnson Date: Sun, 2 Nov 2014 15:24:50 -0500 Subject: [PATCH 03/40] Refine prefix matching condition to consider mode changes as different. --- base/REPL.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/REPL.jl b/base/REPL.jl index 9bfb4b326650b..411306ffd397a 100644 --- a/base/REPL.jl +++ b/base/REPL.jl @@ -483,7 +483,7 @@ function history_move_prefix(s::LineEdit.PrefixSearchState, max_idx = length(hist.history)+1 idxs = backwards ? ((cur_idx-1):-1:1) : ((cur_idx+1):max_idx) for idx in idxs - if (idx == max_idx) || (beginswith(hist.history[idx], prefix) && hist.history[idx] != cur_response) + if (idx == max_idx) || (beginswith(hist.history[idx], prefix) && (hist.history[idx] != cur_response || hist.modes[idx] != LineEdit.mode(s))) history_move(s, hist, idx) LineEdit.move_input_end(s) LineEdit.refresh_line(s) From 97f2a5d34aa67a265125fe21ec84d1d913c4ad7f Mon Sep 17 00:00:00 2001 From: "Blake R. Johnson" Date: Mon, 3 Nov 2014 12:08:04 -0500 Subject: [PATCH 04/40] Pass thru keypresses to parent mode. Update lineedit test for new usage. --- base/LineEdit.jl | 70 +++++++++++++++++------------------------------- test/lineedit.jl | 2 +- 2 files changed, 25 insertions(+), 47 deletions(-) diff --git a/base/LineEdit.jl b/base/LineEdit.jl index b5fa988425330..26d11bdc4c193 100644 --- a/base/LineEdit.jl +++ b/base/LineEdit.jl @@ -725,18 +725,18 @@ function normalize_keymap(keymap::Dict) ret end -match_input(k::Function, s, cs) = (update_key_repeats(s, cs); return keymap_fcn(k, s, last(cs))) -match_input(k::Void, s, cs) = (s,p) -> return :ok -function match_input(keymap::Dict, s, cs=Char[]) - c = read(terminal(s), Char) +match_input(k::Function, s, term, cs) = (update_key_repeats(s, cs); return keymap_fcn(k, s, convert(ByteString, cs))) +match_input(k::Void, s, term, cs) = (s,p) -> return :ok +function match_input(keymap::Dict, s, term, cs=Char[]) + c = read(term, Char) push!(cs, c) k = haskey(keymap, c) ? c : '\0' # if we don't match on the key, look for a default action then fallback on 'nothing' to ignore - return match_input(get(keymap, k, nothing), s, cs) + return match_input(get(keymap, k, nothing), s, term, cs) end keymap_fcn(f::Void, s, c) = (s, p) -> return :ok -function keymap_fcn(f::Function, s, c::Char) +function keymap_fcn(f::Function, s, c) return (s, p) -> begin r = f(s, p, c) if isa(r, Symbol) @@ -830,7 +830,7 @@ function keymap{D<:Dict}(keymaps::Array{D}) # keymaps is a vector of prioritized keymaps, with highest priority first dict = map(normalize_keys, keymaps) dict = keymap_prepare(merge(reverse(dict)...)) - return (s,p)->match_input(dict, s)(s,p) + return (term,s,p)->match_input(dict, s, term)(s,p) end const escape_defaults = merge!( @@ -1315,34 +1315,12 @@ const prefix_history_keymap = AnyDict( "\e[A" => (s,data,c)->history_prev_prefix(data, data.histprompt.hp, data.prefix), # Down Arrow "\e[B" => (s,data,c)->history_next_prefix(data, data.histprompt.hp, data.prefix), - # # Right Arrow - "\e[C" => (s,data,c)->(accept_result(s, data.histprompt); edit_move_right(s)), - # # Left Arrow - "\e[D" => (s,data,c)->(accept_result(s, data.histprompt); edit_move_left(s)), - '\r' => (s,data,c)->begin - accept_result(s, data.histprompt) - if on_enter(s) || (eof(buffer(s)) && s.key_repeats > 1) - commit_line(s) - return :done - else - edit_insert(s, '\n') - end - end, - '\n' => '\r', - "^C" => (s,data,c)->begin - accept_result(s, data.histprompt) - try # raise the debugger if present - ccall(:jl_raise_debugger, Int, ()) - end - move_input_end(s) - refresh_line(s) - print(terminal(s), "^C\n\n") - transition(s, :reset) - refresh_line(s) - end, - # by default I want to return to the parent mode and pass thru the character... how do I do that? - # "*" => (s,data,c)->(accept_result(s, data.histprompt); write(terminal(s).in_stream, c)) - "*" => (s,data,c)->accept_result(s, data.histprompt) + # by default, pass thru to the parent mode + "*" => (s,data,c)->begin + accept_result(s, data.histprompt); + ps = state(s, mode(s)) + keymap(ps, mode(s))(IOBuffer(c), s, keymap_data(ps, mode(s))) + end ) function setup_prefix_keymap(hp, parent_prompt) @@ -1455,24 +1433,24 @@ keymap_data(s::PromptState, prompt::Prompt) = prompt.keymap_func_data keymap(ms::MIState, m::ModalInterface) = keymap(ms.mode_state[ms.current_mode], ms.current_mode) keymap_data(ms::MIState, m::ModalInterface) = keymap_data(ms.mode_state[ms.current_mode], ms.current_mode) -function prompt!(terminal, prompt, s = init_state(terminal, prompt)) - Base.reseteof(terminal) - raw!(terminal, true) - enable_bracketed_paste(terminal) +function prompt!(term, prompt, s = init_state(term, prompt)) + Base.reseteof(term) + raw!(term, true) + enable_bracketed_paste(term) try - start_reading(terminal) - activate(prompt, s, terminal) + start_reading(term) + activate(prompt, s, term) while true - state = keymap(s, prompt)(s, keymap_data(s, prompt)) + state = keymap(s, prompt)(terminal(s), s, keymap_data(s, prompt)) if state == :abort - stop_reading(terminal) + stop_reading(term) return buffer(s), false, false elseif state == :done - stop_reading(terminal) + stop_reading(term) return buffer(s), true, false elseif state == :suspend @unix_only begin - stop_reading(terminal) + stop_reading(term) return buffer(s), true, true end else @@ -1480,7 +1458,7 @@ function prompt!(terminal, prompt, s = init_state(terminal, prompt)) end end finally - raw!(terminal, false) && disable_bracketed_paste(terminal) + raw!(term, false) && disable_bracketed_paste(term) end end diff --git a/test/lineedit.jl b/test/lineedit.jl index c5cf284f12be4..9ffc608448cef 100644 --- a/test/lineedit.jl +++ b/test/lineedit.jl @@ -27,7 +27,7 @@ function run_test(f,buf) global a_foo, a_bar, b_bar a_foo = a_bar = b_bar = 0 while !eof(buf) - f(buf,nothing) + f(buf,nothing,nothing) end end From 423fbaa3c3c3b4d24d5622c81b086bfe0d58ed53 Mon Sep 17 00:00:00 2001 From: "Blake R. Johnson" Date: Mon, 3 Nov 2014 12:28:50 -0500 Subject: [PATCH 05/40] Cleanup some mess with types in LineEdit.jl --- base/LineEdit.jl | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/base/LineEdit.jl b/base/LineEdit.jl index 26d11bdc4c193..3910cfe4b5cb5 100644 --- a/base/LineEdit.jl +++ b/base/LineEdit.jl @@ -8,6 +8,7 @@ import ..Terminals: raw!, width, height, cmove, getX, import Base: ensureroom, peek, show, AnyDict abstract TextInterface +abstract ModeState export run_interface, Prompt, ModalInterface, transition, reset_state, edit_insert, keymap @@ -26,9 +27,6 @@ type MIState end MIState(i, c, a, m) = MIState(i, c, a, m, "", Char[], 0) -type Mode <: TextInterface -end - type Prompt <: TextInterface prompt first_prompt @@ -53,7 +51,7 @@ immutable InputAreaState curs_row::Int64 end -type PromptState +type PromptState <: ModeState terminal::TextTerminal p::Prompt input_buffer::IOBuffer @@ -868,7 +866,7 @@ function write_response_buffer(s::PromptState, data) refresh_line(s) end -type SearchState +type SearchState <: ModeState terminal histprompt #rsearch (true) or ssearch (false) @@ -922,7 +920,7 @@ end HistoryPrompt{T<:HistoryProvider}(hp::T) = HistoryPrompt{T}(hp) init_state(terminal, p::HistoryPrompt) = SearchState(terminal, p, true, IOBuffer(), IOBuffer()) -type PrefixSearchState +type PrefixSearchState <: ModeState terminal histprompt prefix::ByteString @@ -972,7 +970,10 @@ function replace_line(s::PrefixSearchState, l) write(s.response_buffer, l) end -refresh_multi_line(termbuf::TerminalBuffer, term, s::Union(SearchState,PrefixSearchState,PromptState)) = (@assert term == terminal(s); refresh_multi_line(termbuf,s)) +function refresh_multi_line(s::ModeState) + refresh_multi_line(terminal(s), s) +end +refresh_multi_line(termbuf::TerminalBuffer, term, s::ModeState) = (@assert term == terminal(s); refresh_multi_line(termbuf,s)) function refresh_multi_line(termbuf::TerminalBuffer, s::SearchState) buf = IOBuffer() write(buf, pointer(s.query_buffer.data), s.query_buffer.ptr-1) @@ -989,10 +990,6 @@ end refresh_multi_line(termbuf::TerminalBuffer, s::PrefixSearchState) = s.ias = refresh_multi_line(termbuf, terminal(s), buffer(s), s.ias, s) -function refresh_multi_line(s::Union(SearchState,PrefixSearchState,PromptState)) - refresh_multi_line(terminal(s), s) -end - function refresh_multi_line(terminal::UnixTerminal, args...; kwargs...) outbuf = IOBuffer() termbuf = TerminalBuffer(outbuf) @@ -1335,17 +1332,17 @@ function setup_prefix_keymap(hp, parent_prompt) (p, pkeymap) end -function deactivate(p::Union(Prompt,HistoryPrompt,PrefixHistoryPrompt), s::Union(PrefixSearchState,SearchState,PromptState), termbuf) +function deactivate(p::TextInterface, s::ModeState, termbuf) clear_input_area(termbuf, s) s end -function activate(p::Union(Prompt,HistoryPrompt,PrefixHistoryPrompt), s::Union(PrefixSearchState,SearchState,PromptState), termbuf) +function activate(p::TextInterface, s::ModeState, termbuf) s.ias = InputAreaState(0, 0) refresh_line(s, termbuf) end -function activate(p::Union(Prompt,HistoryPrompt,PrefixHistoryPrompt), s::MIState, termbuf) +function activate(p::TextInterface, s::MIState, termbuf) @assert p == s.current_mode activate(p, s.mode_state[s.current_mode], termbuf) end From 5b354e1f845ba6b3256e8094870af5c14d7dcbe8 Mon Sep 17 00:00:00 2001 From: "Blake R. Johnson" Date: Mon, 3 Nov 2014 15:58:03 -0500 Subject: [PATCH 06/40] Use parent prompt indent. --- base/LineEdit.jl | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/base/LineEdit.jl b/base/LineEdit.jl index 3910cfe4b5cb5..e9dcef78cbdff 100644 --- a/base/LineEdit.jl +++ b/base/LineEdit.jl @@ -183,9 +183,10 @@ end prompt_string(s::PromptState) = s.p.prompt prompt_string(s::AbstractString) = s -refresh_multi_line(termbuf::TerminalBuffer, s::PromptState) = s.ias = +refresh_multi_line(s::ModeState) = refresh_multi_line(terminal(s), s) +refresh_multi_line(termbuf::TerminalBuffer, s::ModeState) = s.ias = refresh_multi_line(termbuf, terminal(s), buffer(s), s.ias, s, indent = s.indent) - +refresh_multi_line(termbuf::TerminalBuffer, term, s::ModeState) = (@assert term == terminal(s); refresh_multi_line(termbuf,s)) function refresh_multi_line(termbuf::TerminalBuffer, terminal::UnixTerminal, buf, state::InputAreaState, prompt = ""; indent = 0) cols = width(terminal) @@ -297,6 +298,16 @@ function refresh_multi_line(termbuf::TerminalBuffer, terminal::UnixTerminal, buf return InputAreaState(cur_row, curs_row) end +function refresh_multi_line(terminal::UnixTerminal, args...; kwargs...) + outbuf = IOBuffer() + termbuf = TerminalBuffer(outbuf) + ret = refresh_multi_line(termbuf, terminal, args...;kwargs...) + # Output the entire refresh at once + write(terminal, takebuf_array(outbuf)) + flush(terminal) + return ret +end + # Edit functionality is_non_word_char(c) = c in " \t\n\"\\'`@\$><=:;|&{}()[].,+-*/?%^~" @@ -926,10 +937,11 @@ type PrefixSearchState <: ModeState prefix::ByteString response_buffer::IOBuffer ias::InputAreaState + indent::Int #The prompt whose input will be replaced by the matched history parent PrefixSearchState(terminal, histprompt, prefix, response_buffer) = - new(terminal, histprompt, prefix, response_buffer, InputAreaState(0,0)) + new(terminal, histprompt, prefix, response_buffer, InputAreaState(0,0), 0) end input_string(s::PrefixSearchState) = bytestring(s.response_buffer) @@ -970,10 +982,6 @@ function replace_line(s::PrefixSearchState, l) write(s.response_buffer, l) end -function refresh_multi_line(s::ModeState) - refresh_multi_line(terminal(s), s) -end -refresh_multi_line(termbuf::TerminalBuffer, term, s::ModeState) = (@assert term == terminal(s); refresh_multi_line(termbuf,s)) function refresh_multi_line(termbuf::TerminalBuffer, s::SearchState) buf = IOBuffer() write(buf, pointer(s.query_buffer.data), s.query_buffer.ptr-1) @@ -987,19 +995,6 @@ function refresh_multi_line(termbuf::TerminalBuffer, s::SearchState) s.ias = refresh_multi_line(termbuf, s.terminal, buf, s.ias, s.backward ? "(reverse-i-search)`" : "(forward-i-search)`") end -refresh_multi_line(termbuf::TerminalBuffer, s::PrefixSearchState) = s.ias = - refresh_multi_line(termbuf, terminal(s), buffer(s), s.ias, s) - -function refresh_multi_line(terminal::UnixTerminal, args...; kwargs...) - outbuf = IOBuffer() - termbuf = TerminalBuffer(outbuf) - ret = refresh_multi_line(termbuf, terminal, args...;kwargs...) - # Output the entire refresh at once - write(terminal, takebuf_array(outbuf)) - flush(terminal) - return ret -end - state(s::MIState, p) = s.mode_state[p] state(s::PromptState, p) = (@assert s.p == p; s) mode(s::MIState) = s.current_mode @@ -1053,6 +1048,7 @@ function enter_prefix_search(s::MIState, p::PrefixHistoryPrompt, backward::Bool) pss.parent = mode(s) pss.prefix = bytestring(pointer(buf.data), position(buf)) copybuf!(pss.response_buffer, buf) + pss.indent = state(s, mode(s)).indent transition(s, p) if backward history_prev_prefix(pss, pss.histprompt.hp, pss.prefix) From e2a5647826693a080015438eacb4747b7f87e91b Mon Sep 17 00:00:00 2001 From: Blake Johnson Date: Mon, 3 Nov 2014 21:05:28 -0500 Subject: [PATCH 07/40] Remove one unnecessary layer of anonymous functions. --- base/LineEdit.jl | 31 +++++++++++++++---------------- base/REPL.jl | 10 +++++----- test/lineedit.jl | 16 ++++++++-------- 3 files changed, 28 insertions(+), 29 deletions(-) diff --git a/base/LineEdit.jl b/base/LineEdit.jl index e9dcef78cbdff..dbd03b87ca7dc 100644 --- a/base/LineEdit.jl +++ b/base/LineEdit.jl @@ -35,7 +35,7 @@ type Prompt <: TextInterface prompt_prefix # Same as prefix except after the prompt prompt_suffix - keymap_func + keymap_dict keymap_func_data complete on_enter @@ -734,9 +734,9 @@ function normalize_keymap(keymap::Dict) ret end -match_input(k::Function, s, term, cs) = (update_key_repeats(s, cs); return keymap_fcn(k, s, convert(ByteString, cs))) +match_input(k::Function, s, term, cs) = (update_key_repeats(s, cs); return keymap_fcn(k, s, ByteString(cs))) match_input(k::Void, s, term, cs) = (s,p) -> return :ok -function match_input(keymap::Dict, s, term, cs=Char[]) +function match_input(keymap::Dict, s, term=terminal(s), cs=Char[]) c = read(term, Char) push!(cs, c) k = haskey(keymap, c) ? c : '\0' @@ -838,8 +838,7 @@ end function keymap{D<:Dict}(keymaps::Array{D}) # keymaps is a vector of prioritized keymaps, with highest priority first dict = map(normalize_keys, keymaps) - dict = keymap_prepare(merge(reverse(dict)...)) - return (term,s,p)->match_input(dict, s, term)(s,p) + return keymap_prepare(merge(reverse(dict)...)) end const escape_defaults = merge!( @@ -924,7 +923,7 @@ end type HistoryPrompt{T<:HistoryProvider} <: TextInterface hp::T complete - keymap_func::Function + keymap_dict::Dict{Char,Any} HistoryPrompt(hp) = new(hp, EmptyCompletionProvider()) end @@ -952,7 +951,7 @@ type PrefixHistoryPrompt{T<:HistoryProvider} <: TextInterface hp::T parent_prompt::Prompt complete - keymap_func::Function + keymap_dict::Dict{Char,Any} PrefixHistoryPrompt(hp, parent_prompt) = new(hp, parent_prompt, EmptyCompletionProvider()) end @@ -1129,7 +1128,7 @@ function setup_search_keymap(hp) end, "*" => (s,data,c)->(edit_insert(data.query_buffer, c); update_display_buffer(s, data)) ) - p.keymap_func = keymap([pkeymap, escape_defaults]) + p.keymap_dict = keymap([pkeymap, escape_defaults]) skeymap = AnyDict( "^R" => (s,o...)->(enter_search(s, p, true)), "^S" => (s,o...)->(enter_search(s, p, false)), @@ -1137,7 +1136,7 @@ function setup_search_keymap(hp) (p, skeymap) end -keymap(state, p::Union(HistoryPrompt,PrefixHistoryPrompt)) = p.keymap_func +keymap(state, p::Union(HistoryPrompt,PrefixHistoryPrompt)) = p.keymap_dict keymap_data(state, ::Union(HistoryPrompt, PrefixHistoryPrompt)) = state Base.isempty(s::PromptState) = s.input_buffer.size == 0 @@ -1312,13 +1311,13 @@ const prefix_history_keymap = AnyDict( "*" => (s,data,c)->begin accept_result(s, data.histprompt); ps = state(s, mode(s)) - keymap(ps, mode(s))(IOBuffer(c), s, keymap_data(ps, mode(s))) + match_input(keymap(ps, mode(s)), s, IOBuffer(c))(s, keymap_data(ps, mode(s))) end ) function setup_prefix_keymap(hp, parent_prompt) p = PrefixHistoryPrompt(hp, parent_prompt) - p.keymap_func = keymap([prefix_history_keymap]) + p.keymap_dict = keymap([prefix_history_keymap]) pkeymap = AnyDict( # Up Arrow "\e[A" => (s,o...)->(edit_move_up(s) || enter_prefix_search(s, p, true)), @@ -1374,13 +1373,13 @@ function reset_state(s::MIState) end end -const default_keymap_func = keymap([default_keymap, escape_defaults]) +const default_keymap_dict = keymap([default_keymap, escape_defaults]) function Prompt(prompt; first_prompt = prompt, prompt_prefix = "", prompt_suffix = "", - keymap_func = default_keymap_func, + keymap_dict = default_keymap_dict, keymap_func_data = nothing, complete = EmptyCompletionProvider(), on_enter = default_enter_cb, @@ -1388,7 +1387,7 @@ function Prompt(prompt; hist = EmptyHistoryProvider(), sticky = false) - Prompt(prompt, first_prompt, prompt_prefix, prompt_suffix, keymap_func, keymap_func_data, + Prompt(prompt, first_prompt, prompt_prefix, prompt_suffix, keymap_dict, keymap_func_data, complete, on_enter, on_done, hist, sticky) end @@ -1421,7 +1420,7 @@ buffer(s::PromptState) = s.input_buffer buffer(s::SearchState) = s.query_buffer buffer(s::PrefixSearchState) = s.response_buffer -keymap(s::PromptState, prompt::Prompt) = prompt.keymap_func +keymap(s::PromptState, prompt::Prompt) = prompt.keymap_dict keymap_data(s::PromptState, prompt::Prompt) = prompt.keymap_func_data keymap(ms::MIState, m::ModalInterface) = keymap(ms.mode_state[ms.current_mode], ms.current_mode) keymap_data(ms::MIState, m::ModalInterface) = keymap_data(ms.mode_state[ms.current_mode], ms.current_mode) @@ -1434,7 +1433,7 @@ function prompt!(term, prompt, s = init_state(term, prompt)) start_reading(term) activate(prompt, s, term) while true - state = keymap(s, prompt)(terminal(s), s, keymap_data(s, prompt)) + state = match_input(keymap(s, prompt), s)(s, keymap_data(s, prompt)) if state == :abort stop_reading(term) return buffer(s), false, false diff --git a/base/REPL.jl b/base/REPL.jl index 411306ffd397a..ccd7bc1bcae18 100644 --- a/base/REPL.jl +++ b/base/REPL.jl @@ -471,7 +471,7 @@ end function history_move_prefix(s::LineEdit.PrefixSearchState, hist::REPLHistoryProvider, - prefix::String, + prefix::AbstractString, backwards::Bool) cur_response = bytestring(LineEdit.buffer(s)) cur_idx = hist.cur_idx @@ -492,9 +492,9 @@ function history_move_prefix(s::LineEdit.PrefixSearchState, end Terminals.beep(LineEdit.terminal(s)) end -history_next_prefix(s::LineEdit.PrefixSearchState, hist::REPLHistoryProvider, prefix::String) = +history_next_prefix(s::LineEdit.PrefixSearchState, hist::REPLHistoryProvider, prefix::AbstractString) = history_move_prefix(s, hist, prefix, false) -history_prev_prefix(s::LineEdit.PrefixSearchState, hist::REPLHistoryProvider, prefix::String) = +history_prev_prefix(s::LineEdit.PrefixSearchState, hist::REPLHistoryProvider, prefix::AbstractString) = history_move_prefix(s, hist, prefix, true) function history_search(hist::REPLHistoryProvider, query_buffer::IOBuffer, response_buffer::IOBuffer, @@ -780,7 +780,7 @@ function setup_interface(repl::LineEditREPL; hascolor = repl.hascolor, extra_rep a = Dict{Any,Any}[skeymap, repl_keymap, prefix_keymap, LineEdit.history_keymap, LineEdit.default_keymap, LineEdit.escape_defaults] prepend!(a, extra_repl_keymap) - julia_prompt.keymap_func = LineEdit.keymap(a) + julia_prompt.keymap_dict = LineEdit.keymap(a) const mode_keymap = AnyDict( '\b' => function (s,o...) @@ -806,7 +806,7 @@ function setup_interface(repl::LineEditREPL; hascolor = repl.hascolor, extra_rep b = Dict{Any,Any}[skeymap, mode_keymap, prefix_keymap, LineEdit.history_keymap, LineEdit.default_keymap, LineEdit.escape_defaults] prepend!(b, extra_repl_keymap) - shell_mode.keymap_func = help_mode.keymap_func = LineEdit.keymap(b) + shell_mode.keymap_dict = help_mode.keymap_dict = LineEdit.keymap(b) ModalInterface([julia_prompt, shell_mode, help_mode, search_prompt, prefix_prompt]) end diff --git a/test/lineedit.jl b/test/lineedit.jl index 9ffc608448cef..b23a5fa9b07cf 100644 --- a/test/lineedit.jl +++ b/test/lineedit.jl @@ -21,28 +21,28 @@ const bar_keymap = Dict( 'b' => (o...)->(global b_bar; b_bar += 1) ) -test1_func = LineEdit.keymap([foo_keymap]) +test1_dict = LineEdit.keymap([foo_keymap]) -function run_test(f,buf) +function run_test(d,buf) global a_foo, a_bar, b_bar a_foo = a_bar = b_bar = 0 while !eof(buf) - f(buf,nothing,nothing) + LineEdit.match_input(d, nothing, buf)(nothing,nothing) end end -run_test(test1_func,IOBuffer("aa")) +run_test(test1_dict,IOBuffer("aa")) @test a_foo == 2 -test2_func = LineEdit.keymap([foo2_keymap, foo_keymap]) +test2_dict = LineEdit.keymap([foo2_keymap, foo_keymap]) -run_test(test2_func,IOBuffer("aaabb")) +run_test(test2_dict,IOBuffer("aaabb")) @test a_foo == 3 @test b_foo == 2 -test3_func = LineEdit.keymap([bar_keymap, foo_keymap]) +test3_dict = LineEdit.keymap([bar_keymap, foo_keymap]) -run_test(test3_func,IOBuffer("aab")) +run_test(test3_dict,IOBuffer("aab")) @test a_bar == 2 @test b_bar == 1 From 7c7f491447efd33d2117226f84ec370cb684bf41 Mon Sep 17 00:00:00 2001 From: Blake Johnson Date: Mon, 3 Nov 2014 23:00:04 -0500 Subject: [PATCH 08/40] Passthru escape sequences. --- base/LineEdit.jl | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/base/LineEdit.jl b/base/LineEdit.jl index dbd03b87ca7dc..4cd8fe40663de 100644 --- a/base/LineEdit.jl +++ b/base/LineEdit.jl @@ -1312,7 +1312,18 @@ const prefix_history_keymap = AnyDict( accept_result(s, data.histprompt); ps = state(s, mode(s)) match_input(keymap(ps, mode(s)), s, IOBuffer(c))(s, keymap_data(ps, mode(s))) - end + end, + # match escape sequences for pass thru + "\e*" => "*", + "\e[*" => "*", + "\e[1~" => "*", + "\e[3~" => "*", + "\e[4~" => "*", + "\e[5~" => "*", + "\e[6~" => "*", + "\e[7~" => "*", + "\e[8~" => "*", + "\e[200~" => "*" ) function setup_prefix_keymap(hp, parent_prompt) From 7814fd265b3b7e53e22ce7b0b9607652019a6af4 Mon Sep 17 00:00:00 2001 From: "Blake R. Johnson" Date: Mon, 17 Nov 2014 10:20:54 -0500 Subject: [PATCH 09/40] Preserve mode when editing history. --- base/LineEdit.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/base/LineEdit.jl b/base/LineEdit.jl index 4cd8fe40663de..3cb9b8bdd7027 100644 --- a/base/LineEdit.jl +++ b/base/LineEdit.jl @@ -971,6 +971,7 @@ function reset_state(s::PrefixSearchState) end function transition(s::PrefixSearchState, mode) + s.parent = mode s.histprompt.parent_prompt = mode end From 0bfc341bd151b98de39ad3df91cf2ad4498f1bba Mon Sep 17 00:00:00 2001 From: Blake Johnson Date: Tue, 9 Dec 2014 07:46:19 -0500 Subject: [PATCH 10/40] Drop unknown codes at REPL. Better than producing an EOF error. --- base/LineEdit.jl | 3 +++ base/Terminals.jl | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/base/LineEdit.jl b/base/LineEdit.jl index 3cb9b8bdd7027..88972b7f00a0c 100644 --- a/base/LineEdit.jl +++ b/base/LineEdit.jl @@ -737,6 +737,9 @@ end match_input(k::Function, s, term, cs) = (update_key_repeats(s, cs); return keymap_fcn(k, s, ByteString(cs))) match_input(k::Void, s, term, cs) = (s,p) -> return :ok function match_input(keymap::Dict, s, term=terminal(s), cs=Char[]) + # if we run out of characters to match before resolving an action, + # return an empty keymap function + eof(term) && return keymap_fcn(nothing, s, "") c = read(term, Char) push!(cs, c) k = haskey(keymap, c) ? c : '\0' diff --git a/base/Terminals.jl b/base/Terminals.jl index d0a56c7cca152..9ac5db6ef1def 100644 --- a/base/Terminals.jl +++ b/base/Terminals.jl @@ -32,7 +32,8 @@ import Base: stop_reading, write, writemime, - reseteof + reseteof, + eof ## TextTerminal ## @@ -193,6 +194,7 @@ readuntil(t::UnixTerminal, s) = readuntil(t.in_stream, s) read(t::UnixTerminal, ::Type{UInt8}) = read(t.in_stream, UInt8) start_reading(t::UnixTerminal) = start_reading(t.in_stream) stop_reading(t::UnixTerminal) = stop_reading(t.in_stream) +eof(t::UnixTerminal) = eof(t.in_stream) @unix_only hascolor(t::TTYTerminal) = (beginswith(t.term_type, "xterm") || success(`tput setaf 0`)) @windows_only hascolor(t::TTYTerminal) = true From d8c89d23bb20ce60c290ef5555eb89826e27cbd1 Mon Sep 17 00:00:00 2001 From: Blake Johnson Date: Thu, 11 Dec 2014 00:12:05 -0500 Subject: [PATCH 11/40] Keep cursor at prefix position. --- base/REPL.jl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/base/REPL.jl b/base/REPL.jl index ccd7bc1bcae18..e244d493c3b51 100644 --- a/base/REPL.jl +++ b/base/REPL.jl @@ -485,7 +485,13 @@ function history_move_prefix(s::LineEdit.PrefixSearchState, for idx in idxs if (idx == max_idx) || (beginswith(hist.history[idx], prefix) && (hist.history[idx] != cur_response || hist.modes[idx] != LineEdit.mode(s))) history_move(s, hist, idx) - LineEdit.move_input_end(s) + if length(prefix) == 0 + # on empty prefix search, move cursor to the end + LineEdit.move_input_end(s) + else + # otherwise, keep cursor at the prefix position as a visual cue + seek(LineEdit.buffer(s), length(prefix)) + end LineEdit.refresh_line(s) return :ok end From 54affdb3f37e8d02c9d6e1806229c4933890a094 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 12 Dec 2014 22:23:21 -0500 Subject: [PATCH 12/40] simplify initialization / startup / embedded / task-switch api --- base/boot.jl | 2 +- base/client.jl | 11 +-- base/exports.jl | 1 + src/ast.c | 2 +- src/codegen.cpp | 5 +- src/dump.c | 12 +-- src/gc.c | 2 +- src/init.c | 77 ++++++---------- src/jl_uv.c | 6 +- src/jlapi.c | 26 +++--- src/julia.h | 49 ++++++----- src/julia_internal.h | 8 +- src/task.c | 204 ++++++++++++++++++++++++++----------------- ui/repl.c | 13 ++- 14 files changed, 221 insertions(+), 197 deletions(-) diff --git a/base/boot.jl b/base/boot.jl index 0ecec7f2fa14c..8c6f984be0770 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -147,7 +147,7 @@ export # method reflection applicable, invoke, method_exists, # constants - JULIA_HOME, nothing, Main, + nothing, Main, # intrinsics module Intrinsics #ccall, cglobal, llvmcall, abs_float, add_float, add_int, and_int, ashr_int, diff --git a/base/client.jl b/base/client.jl index e8dc8b0b19009..db9de86654d25 100644 --- a/base/client.jl +++ b/base/client.jl @@ -356,6 +356,7 @@ function load_machine_file(path::AbstractString) end function early_init() + global const JULIA_HOME = ccall(:jl_get_julia_home, Any, ()) Sys.init_sysinfo() if CPU_CORES > 8 && !("OPENBLAS_NUM_THREADS" in keys(ENV)) && !("OMP_NUM_THREADS" in keys(ENV)) # Prevent openblas from stating to many threads, unless/until specifically requested @@ -409,19 +410,16 @@ function _start() # note: currently IOStream is used for file STDIN if isa(STDIN,File) || isa(STDIN,IOStream) # reading from a file, behave like include - eval(parse_input_line(readall(STDIN))) + eval(Main,parse_input_line(readall(STDIN))) else # otherwise behave repl-like while !eof(STDIN) eval_user_input(parse_input_line(STDIN), true) end end - if have_color - print(color_normal) - end - quit() + else + REPL.run_repl(active_repl) end - REPL.run_repl(active_repl) end catch err display_error(err,catch_backtrace()) @@ -434,7 +432,6 @@ function _start() end println() end - ccall(:uv_atexit_hook, Void, ()) end const atexit_hooks = [] diff --git a/base/exports.jl b/base/exports.jl index 9760a55ad1a36..a17935dce9885 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -163,6 +163,7 @@ export Inf, Inf16, Inf32, + JULIA_HOME, LOAD_PATH, MS_ASYNC, MS_INVALIDATE, diff --git a/src/ast.c b/src/ast.c index f743641d1cf73..544dafd845d28 100644 --- a/src/ast.c +++ b/src/ast.c @@ -114,7 +114,7 @@ static builtinspec_t julia_flisp_ast_ext[] = { { NULL, NULL } }; -DLLEXPORT void jl_init_frontend(void) +void jl_init_frontend(void) { fl_init(4*1024*1024); value_t img = cvalue(iostreamtype, sizeof(ios_t)); diff --git a/src/codegen.cpp b/src/codegen.cpp index 69082212b1dbc..14c6194435d94 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -130,9 +130,8 @@ void __attribute__(()) __stack_chk_fail() #endif { /* put your panic function or similar in here */ - fprintf(stderr, "warning: stack corruption detected\n"); - //assert(0 && "stack corruption detected"); - //abort(); + fprintf(stderr, "fatal error: stack corruption detected\n"); + abort(); // end with abort, since the compiler destroyed the stack upon entry to this function } } diff --git a/src/dump.c b/src/dump.c index 01f0d502ec0f0..ac511abf1e3d7 100644 --- a/src/dump.c +++ b/src/dump.c @@ -452,8 +452,6 @@ static void jl_serialize_module(ios_t *s, jl_module_t *m) jl_serialize_value(s, m->parent); if (ref_only) return; - // set on every startup; don't save value - jl_sym_t *jhsym = jl_symbol("JULIA_HOME"); size_t i; void **table = m->bindings.table; for(i=1; i < m->bindings.size; i+=2) { @@ -461,12 +459,7 @@ static void jl_serialize_module(ios_t *s, jl_module_t *m) jl_binding_t *b = (jl_binding_t*)table[i]; if (b->owner == m || m != jl_main_module) { jl_serialize_value(s, b->name); - if (table[i-1] == jhsym && m == jl_core_module) { - jl_serialize_value(s, NULL); - } - else { - jl_serialize_value(s, b->value); - } + jl_serialize_value(s, b->value); jl_serialize_value(s, b->type); jl_serialize_value(s, b->owner); write_int8(s, (b->constp<<2) | (b->exportp<<1) | (b->imported)); @@ -1435,9 +1428,6 @@ void jl_restore_system_image(const char *fname) #ifdef JL_GC_MARKSWEEP if (en) jl_gc_enable(); #endif - // restore the value of our "magic" JULIA_HOME variable/constant - jl_get_binding_wr(jl_core_module, jl_symbol("JULIA_HOME"))->value = - jl_cstr_to_string(jl_compileropts.julia_home); mode = last_mode; jl_update_all_fptrs(); } diff --git a/src/gc.c b/src/gc.c index 758e5832ba4ec..2c871d09a6cba 100644 --- a/src/gc.c +++ b/src/gc.c @@ -712,7 +712,7 @@ static void gc_mark_task(jl_task_t *ta, int d) gc_mark_stack(jl_pgcstack, offset, d); } else { - offset = (char *)ta->stkbuf - ((char *)ta->stackbase - ta->ssize); + offset = (char *)ta->stkbuf - ((char *)jl_stackbase - ta->ssize); gc_mark_stack(ta->gcstack, offset, d); } #else diff --git a/src/init.c b/src/init.c index 55b2709a9d1a9..0010bf484e882 100644 --- a/src/init.c +++ b/src/init.c @@ -7,7 +7,6 @@ #include #include #include -#include #include #if defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__) @@ -88,6 +87,7 @@ DLLEXPORT void gdblookup(ptrint_t ip); static const char system_image_path[256] = JL_SYSTEM_IMAGE_PATH; jl_compileropts_t jl_compileropts = { NULL, // julia_home + NULL, // julia_bin NULL, // build_path system_image_path, // image_file 0, // code_coverage @@ -436,7 +436,7 @@ static void jl_uv_exitcleanup_walk(uv_handle_t *handle, void *arg) void jl_write_coverage_data(void); void jl_write_malloc_log(void); -DLLEXPORT void uv_atexit_hook() +DLLEXPORT void jl_atexit_hook() { #if defined(JL_GC_MARKSWEEP) && defined(GC_FINAL_STATS) jl_print_gc_stats(JL_STDERR); @@ -784,57 +784,53 @@ static void jl_resolve_sysimg_location() { // note: if you care about lost memory, you should compare the // pointers in jl_compileropts before and after calling julia_init() // and call the appropriate free function on the originals for any that changed - char *free_path = NULL; + char *free_path = (char*)malloc(PATH_MAX); + size_t path_size = PATH_MAX; + if (uv_exepath(free_path, &path_size)) { + ios_printf(ios_stderr, "fatal error: unexpected error while retrieving exepath\n"); + exit(1); + } + if (path_size >= PATH_MAX) { + ios_printf(ios_stderr, "fatal error: jl_compileropts.julia_bin path too long\n"); + exit(1); + } + jl_compileropts.julia_bin = strdup(free_path); if (!jl_compileropts.julia_home) { jl_compileropts.julia_home = getenv("JULIA_HOME"); if (!jl_compileropts.julia_home) { - size_t path_size = PATH_MAX; - char *path = (char*)malloc(PATH_MAX); - if (uv_exepath(path, &path_size)) { - ios_printf(ios_stderr, "fatal error: unexpected error while retrieving exepath\n"); - exit(1); - } - if (path_size >= PATH_MAX) { - ios_printf(ios_stderr, "fatal error: jl_compileropts.image_file path too long\n"); - exit(1); - } - free_path = path; - jl_compileropts.julia_home = dirname(path); + jl_compileropts.julia_home = dirname(free_path); } } if (jl_compileropts.julia_home) jl_compileropts.julia_home = abspath(jl_compileropts.julia_home); - if (free_path) { - free(free_path); - free_path = NULL; - } + free(free_path); + free_path = NULL; if (jl_compileropts.image_file) { if (jl_compileropts.image_file[0] != PATHSEPSTRING[0]) { if (jl_compileropts.image_file == system_image_path) { // build time path, relative to JULIA_HOME - char *path = (char*)malloc(PATH_MAX); - int n = snprintf(path, PATH_MAX, "%s" PATHSEPSTRING "%s", + free_path = (char*)malloc(PATH_MAX); + int n = snprintf(free_path, PATH_MAX, "%s" PATHSEPSTRING "%s", jl_compileropts.julia_home, jl_compileropts.image_file); if (n >= PATH_MAX || n < 0) { ios_printf(ios_stderr, "fatal error: jl_compileropts.image_file path too long\n"); exit(1); } - free_path = path; - jl_compileropts.image_file = path; + jl_compileropts.image_file = free_path; } } - } - if (jl_compileropts.image_file) - jl_compileropts.image_file = abspath(jl_compileropts.image_file); - if (free_path) { - free(free_path); - free_path = NULL; + if (jl_compileropts.image_file) + jl_compileropts.image_file = abspath(jl_compileropts.image_file); + if (free_path) { + free(free_path); + free_path = NULL; + } } if (jl_compileropts.build_path) jl_compileropts.build_path = abspath(jl_compileropts.build_path); } -void julia_init() +void _julia_init() { jl_io_loop = uv_default_loop(); // this loop will internal events (spawning process etc.), // best to call this first, since it also initializes libuv @@ -926,11 +922,6 @@ void julia_init() jl_get_builtin_hooks(); jl_boot_file_loaded = 1; jl_init_box_caches(); - // Core.JULIA_HOME is a "magic" constant, we set it at runtime here - // since its value gets excluded from the system image - jl_set_const(jl_core_module, jl_symbol("JULIA_HOME"), - jl_cstr_to_string(jl_compileropts.julia_home)); - jl_module_export(jl_core_module, jl_symbol("JULIA_HOME")); } if (jl_compileropts.image_file) { @@ -1105,18 +1096,8 @@ extern void *__stack_chk_guard; void jl_compile_all(void); -DLLEXPORT int julia_trampoline(int argc, char **argv, int (*pmain)(int ac,char *av[])) +DLLEXPORT int julia_save() { - unsigned char *p = (unsigned char *)&__stack_chk_guard; - char a = p[sizeof(__stack_chk_guard)-1]; - char b = p[sizeof(__stack_chk_guard)-2]; - char c = p[0]; - /* If you have the ability to generate random numbers in your kernel then they should be used here */ - p[sizeof(__stack_chk_guard)-1] = 255; - p[sizeof(__stack_chk_guard)-2] = '\n'; - p[0] = 0; - JL_SET_STACK_BASE; - int ret = pmain(argc, argv); const char *build_path = jl_compileropts.build_path; if (build_path) { if (jl_compileropts.compile_enabled == JL_COMPILEROPT_COMPILE_ALL) @@ -1148,10 +1129,6 @@ DLLEXPORT int julia_trampoline(int argc, char **argv, int (*pmain)(int ac,char * ios_printf(ios_stderr,"\nFATAL: failed to create string for .ji build path\n"); } } - p[sizeof(__stack_chk_guard)-1] = a; - p[sizeof(__stack_chk_guard)-2] = b; - p[0] = c; - return ret; } jl_function_t *jl_typeinf_func=NULL; diff --git a/src/jl_uv.c b/src/jl_uv.c index 37615cbd156c8..6b68a895765e5 100644 --- a/src/jl_uv.c +++ b/src/jl_uv.c @@ -686,14 +686,10 @@ char *jl_bufptr(ios_t *s) return s->buf; } -DLLEXPORT void uv_atexit_hook(); DLLEXPORT void jl_exit(int exitcode) { - /*if (jl_io_loop) { - jl_process_events(&jl_io_loop); - }*/ uv_tty_reset_mode(); - uv_atexit_hook(); + jl_atexit_hook(); exit(exitcode); } diff --git a/src/jlapi.c b/src/jlapi.c index f3d94b2d2bc6b..1f24440ec353b 100644 --- a/src/jlapi.c +++ b/src/jlapi.c @@ -56,12 +56,6 @@ DLLEXPORT void jl_init(char *julia_home_dir) DLLEXPORT void *jl_eval_string(char *str) { -#ifdef COPY_STACKS - int outside_task = (jl_root_task->stackbase == NULL); - if (outside_task) { - JL_SET_STACK_BASE; - } -#endif jl_value_t *r; JL_TRY { jl_value_t *ast = jl_parse_input_line(str); @@ -74,11 +68,6 @@ DLLEXPORT void *jl_eval_string(char *str) //jl_show(jl_stderr_obj(), jl_exception_in_transit); r = NULL; } -#ifdef COPY_STACKS - if (outside_task) { - jl_root_task->stackbase = NULL; - } -#endif return r; } @@ -255,6 +244,21 @@ DLLEXPORT int jl_is_debugbuild(void) #endif } +DLLEXPORT jl_value_t *jl_get_julia_home(void) +{ + return jl_cstr_to_string(jl_compileropts.julia_home); +} + +DLLEXPORT jl_value_t *jl_get_julia_bin(void) +{ + return jl_cstr_to_string(jl_compileropts.julia_bin); +} + +DLLEXPORT jl_value_t *jl_get_image_file(void) +{ + return jl_cstr_to_string(jl_compileropts.image_file); +} + #ifdef __cplusplus } #endif diff --git a/src/julia.h b/src/julia.h index f0432ffba4f3e..d5d0be658786f 100644 --- a/src/julia.h +++ b/src/julia.h @@ -814,7 +814,6 @@ DLLEXPORT jl_value_t *jl_eqtable_get(jl_array_t *h, void *key, jl_value_t *deflt DLLEXPORT int jl_errno(void); DLLEXPORT void jl_set_errno(int e); DLLEXPORT int32_t jl_stat(const char *path, char *statbuf); -DLLEXPORT void NORETURN jl_exit(int status); DLLEXPORT int jl_cpu_cores(void); DLLEXPORT long jl_getpagesize(void); DLLEXPORT long jl_getallocationgranularity(void); @@ -856,11 +855,13 @@ DLLEXPORT void jl_exception_clear(void); } // initialization functions -DLLEXPORT void julia_init(); +DLLEXPORT void julia_init(void); DLLEXPORT void jl_init(char *julia_home_dir); DLLEXPORT void jl_init_with_image(char *julia_home_dir, char *image_relative_path); DLLEXPORT int jl_is_initialized(void); DLLEXPORT int julia_trampoline(int argc, char *argv[], int (*pmain)(int ac,char *av[])); +DLLEXPORT void jl_atexit_hook(void); +DLLEXPORT void NORETURN jl_exit(int status); DLLEXPORT void jl_save_system_image(const char *fname); DLLEXPORT void jl_restore_system_image(const char *fname); @@ -989,22 +990,6 @@ DLLEXPORT jl_value_t *jl_call3(jl_function_t *f, jl_value_t *a, jl_value_t *b, j // interfacing with Task runtime DLLEXPORT void jl_yield(); -DLLEXPORT void jl_handle_stack_start(); - -#ifdef COPY_STACKS -// initialize base context of root task -extern DLLEXPORT jl_jmp_buf jl_base_ctx; -#define JL_SET_STACK_BASE \ - do { \ - int __stk; \ - jl_root_task->stackbase = (char*)&__stk; \ - if (jl_setjmp(jl_base_ctx, 1)) { \ - jl_handle_stack_start(); \ - } \ - } while (0) -#else -#define JL_SET_STACK_BASE -#endif // gc ------------------------------------------------------------------------- @@ -1148,10 +1133,9 @@ typedef struct _jl_task_t { jl_value_t *exception; jl_function_t *start; jl_jmp_buf ctx; - union { - void *stackbase; - void *stack; - }; +#ifndef COPY_STACKS + void *stack; +#endif size_t bufsz; void *stkbuf; size_t ssize; @@ -1328,6 +1312,7 @@ void show_execution_point(char *filename, int lno); typedef struct { const char *julia_home; + const char *julia_bin; const char *build_path; const char *image_file; int8_t code_coverage; @@ -1357,6 +1342,26 @@ extern DLLEXPORT jl_compileropts_t jl_compileropts; #define JL_COMPILEROPT_DUMPBITCODE_ON 1 #define JL_COMPILEROPT_DUMPBITCODE_OFF 2 +extern void *__stack_chk_guard; +#define SET_STACK_CHK_GUARD(a,b,c) do { \ + unsigned char *p = (unsigned char *)&__stack_chk_guard; \ + a = p[sizeof(__stack_chk_guard)-1]; \ + b = p[sizeof(__stack_chk_guard)-2]; \ + c = p[0]; \ + /* If you have the ability to generate random numbers + * in your kernel then they should be used here */ \ + p[sizeof(__stack_chk_guard)-1] = 255; \ + p[sizeof(__stack_chk_guard)-2] = '\n'; \ + p[0] = 0; \ + } while (0) + +#define CLR_STACK_CHK_GUARD(a,b,c) do { \ + unsigned char *p = (unsigned char *)&__stack_chk_guard; \ + p[sizeof(__stack_chk_guard)-1] = a; \ + p[sizeof(__stack_chk_guard)-2] = b; \ + p[0] = c; \ + } while (0) + #ifdef __cplusplus } #endif diff --git a/src/julia_internal.h b/src/julia_internal.h index 9df408979c9be..5b45ec65403cc 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -81,12 +81,16 @@ extern size_t jl_arr_xtralloc_limit; void jl_init_types(void); void jl_init_box_caches(void); -DLLEXPORT void jl_init_frontend(void); +void jl_init_frontend(void); void jl_init_primitives(void); void jl_init_codegen(void); void jl_init_intrinsic_functions(void); void jl_init_tasks(void *stack, size_t ssize); void jl_init_serializer(void); +void _julia_init(void); +#ifdef COPY_STACKS +extern void *jl_stackbase; +#endif void jl_dump_bitcode(char *fname); void jl_dump_objfile(char *fname, int jit_model); @@ -141,6 +145,8 @@ extern uv_lib_t *jl_crtdll_handle; extern uv_lib_t *jl_winsock_handle; #endif +DLLEXPORT void jl_atexit_hook(); + #ifdef __cplusplus } #endif diff --git a/src/task.c b/src/task.c index 4fc7a0764d088..e2945a32c5f4a 100644 --- a/src/task.c +++ b/src/task.c @@ -154,18 +154,24 @@ jl_value_t *jl_exception_in_transit; jl_gcframe_t *jl_pgcstack = NULL; #endif -static void start_task(jl_task_t *t); - #ifdef COPY_STACKS -jl_jmp_buf * volatile jl_jmp_target; -DLLEXPORT jl_jmp_buf jl_base_ctx; +static jl_jmp_buf * volatile jl_jmp_target; + +#if defined(_CPU_X86_64_) || defined(_CPU_X86_) +#define ASM_COPY_STACKS +#endif +void *jl_stackbase; + +#ifndef ASM_COPY_STACKS +static jl_jmp_buf jl_base_ctx; // base context of stack +#endif static void save_stack(jl_task_t *t) { if (t->state == done_sym || t->state == failed_sym) return; volatile int _x; - size_t nb = (char*)t->stackbase - (char*)&_x; + size_t nb = (char*)jl_stackbase - (char*)&_x; char *buf; if (t->stkbuf == NULL || t->bufsz < nb) { buf = (char*)allocb(nb); @@ -180,12 +186,13 @@ static void save_stack(jl_task_t *t) } #if defined(_OS_WINDOWS_) && !defined(_COMPILER_MINGW_) -void __declspec(noinline) restore_stack(jl_task_t *t, jl_jmp_buf *where, char *p) +void __declspec(noinline) #else -void __attribute__((noinline)) restore_stack(jl_task_t *t, jl_jmp_buf *where, char *p) +void __attribute__((noinline)) #endif +restore_stack(jl_task_t *t, jl_jmp_buf *where, char *p) { - char *_x = (char*)t->stackbase - t->ssize; + char *_x = (char*)jl_stackbase - t->ssize; if (!p) { p = _x; if ((char*)&_x > _x) { @@ -202,6 +209,94 @@ void __attribute__((noinline)) restore_stack(jl_task_t *t, jl_jmp_buf *where, ch } #endif +static jl_function_t *task_done_hook_func=NULL; + +static void NORETURN finish_task(jl_task_t *t, jl_value_t *resultval) +{ + if (t->exception != jl_nothing) + t->state = failed_sym; + else + t->state = done_sym; + t->result = resultval; + // TODO: early free of t->stkbuf +#ifdef COPY_STACKS + t->stkbuf = NULL; +#endif + if (task_done_hook_func == NULL) { + task_done_hook_func = (jl_function_t*)jl_get_global(jl_base_module, + jl_symbol("task_done_hook")); + } + if (task_done_hook_func != NULL) { + jl_apply(task_done_hook_func, (jl_value_t**)&t, 1); + } + abort(); +} + +#if defined(_OS_WINDOWS_) && !defined(_COMPILER_MINGW_) +void __declspec(noinline) +#else +void __attribute__((noinline)) +#endif +NORETURN start_task() +{ + // this runs the first time we switch to a task + jl_task_t *t = jl_current_task; + jl_value_t *arg = jl_task_arg_in_transit; + jl_value_t *res; + JL_GC_PUSH1(&arg); + res = jl_apply(t->start, NULL, 0); + JL_GC_POP(); + finish_task(t, res); + abort(); +} + +#ifndef ASM_COPY_STACKS +#if defined(_OS_WINDOWS_) && !defined(_COMPILER_MINGW_) +static void __declspec(noinline) +#else +static void __attribute__((noinline)) +#endif +set_base_ctx(char *__stk) +{ + if (jl_setjmp(jl_base_ctx, 1)) { + start_task(); + } +} +#else +void set_base_ctx(char *__stk) { } +#endif + + +DLLEXPORT void julia_init() +{ // keep this function small, since we want to keep the stack frame + // leading up to this also quite small + _julia_init(); +#ifdef COPY_STACKS + char __stk; + jl_stackbase = (char*)(((uptrint_t)&__stk + sizeof(__stk))&-16); // also ensures stackbase is 16-byte aligned + set_base_ctx(&__stk); // separate function, to record the size of a stack frame +#endif +} + +#ifndef COPY_STACKS +static void init_task(jl_task_t *t) +{ + if (jl_setjmp(t->ctx, 0)) { + start_task(); + } + // this runs when the task is created + ptrint_t local_sp = (ptrint_t)&t; + ptrint_t new_sp = (ptrint_t)t->stack + t->ssize - _frame_offset; +#ifdef _P64 + // SP must be 16-byte aligned + new_sp = new_sp&-16; + local_sp = local_sp&-16; +#endif + memcpy((void*)new_sp, (void*)local_sp, _frame_offset); + rebase_state(&t->ctx, local_sp, new_sp); +} +#endif + static void ctx_switch(jl_task_t *t, jl_jmp_buf *where) { if (t == jl_current_task) @@ -248,7 +343,25 @@ static void ctx_switch(jl_task_t *t, jl_jmp_buf *where) if (t->stkbuf) { restore_stack(t, where, NULL); } else { +#ifdef ASM_COPY_STACKS +#ifdef _CPU_X86_64_ + asm(" movq %0, %%rsp;\n" + " xorq %%rbp, %%rbp;\n" + " push %%rbp;\n" // instead of RSP + " jmp _start_task" // call stack_task with fake stack frame + : : "r"(jl_stackbase-0x10) : ); +#elif defined(_CPU_X86_) + asm(" movl %0, %%esp;\n" + " xorl %%ebp, %%ebp;\n" + " push %%ebp;\n" // instead of ESP + " jmp _start_task" // call stack_task with fake stack frame + : : "r"(jl_stackbase-0x10) : ); +#else +#error ASM_COPY_STACKS not supported on this platform +#endif +#else // ASM_COPY_STACKS jl_longjmp(jl_base_ctx, 1); +#endif } #else jl_longjmp(*where, 1); @@ -368,74 +481,6 @@ jl_value_t *jl_switchto(jl_task_t *t, jl_value_t *arg) return switchto(t); } -static jl_function_t *task_done_hook_func=NULL; - -static void finish_task(jl_task_t *t, jl_value_t *resultval) -{ - if (t->exception != jl_nothing) - t->state = failed_sym; - else - t->state = done_sym; - t->result = resultval; - // TODO: early free of t->stkbuf -#ifdef COPY_STACKS - t->stkbuf = NULL; -#endif - if (task_done_hook_func == NULL) { - task_done_hook_func = (jl_function_t*)jl_get_global(jl_base_module, - jl_symbol("task_done_hook")); - } - if (task_done_hook_func != NULL) { - jl_apply(task_done_hook_func, (jl_value_t**)&t, 1); - } - assert(0); -} - -static void start_task(jl_task_t *t) -{ - // this runs the first time we switch to t - jl_value_t *arg = jl_task_arg_in_transit; - jl_value_t *res; - JL_GC_PUSH1(&arg); - -#ifdef COPY_STACKS - ptrint_t local_sp = (ptrint_t)jl_pgcstack; - // here we attempt to figure out how big our stack frame is, since we - // might need to copy all of it later. this is a bit of a fuzzy guess. - local_sp += sizeof(jl_gcframe_t); - local_sp += 12*sizeof(void*); - t->stackbase = (void*)(local_sp + _frame_offset); -#endif - res = jl_apply(t->start, NULL, 0); - JL_GC_POP(); - finish_task(t, res); - assert(0); -} - -DLLEXPORT void jl_handle_stack_start() -{ - start_task(jl_current_task); -} - -#ifndef COPY_STACKS -static void init_task(jl_task_t *t) -{ - if (jl_setjmp(t->ctx, 0)) { - start_task(t); - } - // this runs when the task is created - ptrint_t local_sp = (ptrint_t)&t; - ptrint_t new_sp = (ptrint_t)t->stack + t->ssize - _frame_offset; -#ifdef _P64 - // SP must be 16-byte aligned - new_sp = new_sp&-16; - local_sp = local_sp&-16; -#endif - memcpy((void*)new_sp, (void*)local_sp, _frame_offset); - rebase_state(&t->ctx, local_sp, new_sp); -} -#endif - ptrint_t bt_data[MAX_BT_SIZE+1]; size_t bt_size = 0; @@ -581,9 +626,8 @@ DLLEXPORT size_t rec_backtrace_ctx(ptrint_t *data, size_t maxsize, unw_context_t do { if (n >= maxsize) break; - if (unw_get_reg(&cursor, UNW_REG_IP, &ip) < 0) { + if (unw_get_reg(&cursor, UNW_REG_IP, &ip) < 0) break; - } data[n++] = ip; } while (unw_step(&cursor) > 0); return n; @@ -602,9 +646,8 @@ size_t rec_backtrace_ctx_dwarf(ptrint_t *data, size_t maxsize, unw_context_t *uc do { if (n >= maxsize) break; - if (unw_get_reg(&cursor, UNW_REG_IP, &ip) < 0) { + if (unw_get_reg(&cursor, UNW_REG_IP, &ip) < 0) break; - } data[n++] = ip; } while (unw_step(&cursor) > 0); return n; @@ -849,7 +892,6 @@ void jl_init_tasks(void *stack, size_t ssize) jl_current_task = (jl_task_t*)allocobj(sizeof(jl_task_t)); jl_current_task->type = (jl_value_t*)jl_task_type; #ifdef COPY_STACKS - jl_current_task->stackbase = NULL; jl_current_task->ssize = 0; // size of saved piece jl_current_task->bufsz = 0; #else diff --git a/ui/repl.c b/ui/repl.c index 49e68a4bbc162..88aa0c479ec09 100644 --- a/ui/repl.c +++ b/ui/repl.c @@ -256,7 +256,7 @@ static void print_profile(void) } #endif -int true_main(int argc, char *argv[]) +static int true_main(int argc, char *argv[]) { if (jl_base_module != NULL) { jl_array_t *args = (jl_array_t*)jl_get_global(jl_base_module, jl_symbol("ARGS")); @@ -308,10 +308,11 @@ int true_main(int argc, char *argv[]) JL_PUTS("\n",JL_STDOUT); goto again; } - uv_tty_reset_mode(); return iserr; } +DLLEXPORT extern int julia_save(); + #ifndef _OS_WINDOWS_ int main(int argc, char *argv[]) { @@ -329,6 +330,8 @@ int wmain(int argc, wchar_t *argv[], wchar_t *envp[]) argv[i] = (wchar_t*)arg; } #endif + char a,b,c; + SET_STACK_CHK_GUARD(a,b,c); libsupport_init(); parse_opts(&argc, (char***)&argv); if (lisp_prompt) { @@ -336,7 +339,11 @@ int wmain(int argc, wchar_t *argv[], wchar_t *envp[]) return 0; } julia_init(); - return julia_trampoline(argc, (char**)argv, true_main); + int ret = true_main(argc, (char**)argv); + jl_atexit_hook(); + julia_save(); + CLR_STACK_CHK_GUARD(a,b,c); + return ret; } #ifdef __cplusplus From c37f371c1ea26b0e2a7614a8536d94330546ee18 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 12 Dec 2014 22:54:00 -0500 Subject: [PATCH 13/40] code cleanup --- base/client.jl | 7 ++----- src/codegen.cpp | 2 +- src/disasm.cpp | 2 +- src/dump.c | 6 ++---- src/init.c | 6 +++--- src/julia.expmap | 1 + src/julia.h | 2 +- src/support/dirpath.h | 2 ++ src/task.c | 16 +++++++++------- ui/repl.c | 2 +- 10 files changed, 23 insertions(+), 23 deletions(-) diff --git a/base/client.jl b/base/client.jl index db9de86654d25..27ca40edba1e1 100644 --- a/base/client.jl +++ b/base/client.jl @@ -426,11 +426,8 @@ function _start() println() exit(1) end - if is_interactive - if have_color - print(color_normal) - end - println() + if is_interactive && have_color + print(color_normal) end end diff --git a/src/codegen.cpp b/src/codegen.cpp index 14c6194435d94..fce9e1c848f44 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -121,7 +121,7 @@ extern "C" { #include "builtin_proto.h" -void *__stack_chk_guard = NULL; +DLLEXPORT void *__stack_chk_guard = NULL; #if defined(_OS_WINDOWS_) && !defined(_COMPILER_MINGW_) void __stack_chk_fail() diff --git a/src/disasm.cpp b/src/disasm.cpp index 528d33b88be68..fdd998f84b252 100644 --- a/src/disasm.cpp +++ b/src/disasm.cpp @@ -482,7 +482,7 @@ void jl_dump_function_asm(const char *Fptr, size_t Fsize, #ifdef LLVM35 if (MCIA->evaluateBranch(Inst, Index, insSize, addr)) #else - if ((addr = MCIA->evaluateBranch(Inst, Index, insSize)) != -1) + if ((addr = MCIA->evaluateBranch(Inst, Index, insSize)) != (uint64_t)-1) #endif DisInfo.insertAddress(addr); } diff --git a/src/dump.c b/src/dump.c index ac511abf1e3d7..3e4cb37701d0a 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1330,10 +1330,8 @@ DLLEXPORT void jl_save_system_image(const char *fname) // save module initialization order if (jl_module_init_order != NULL) { for(i=0; i < jl_array_len(jl_module_init_order); i++) { - // NULL out any modules that weren't saved - jl_value_t *mod = jl_cellref(jl_module_init_order, i); - (void)mod; - assert(ptrhash_get(&backref_table, mod) != HT_NOTFOUND); + // verify that all these modules were saved + assert(ptrhash_get(&backref_table, jl_cellref(jl_module_init_order, i)) != HT_NOTFOUND); } } jl_serialize_value(&f, jl_module_init_order); diff --git a/src/init.c b/src/init.c index 0010bf484e882..ec1181508645e 100644 --- a/src/init.c +++ b/src/init.c @@ -765,13 +765,13 @@ static char *abspath(const char *in) } } #else - DWORD n = GetFullPathName(in, 0, NULL, C_NULL); + DWORD n = GetFullPathName(in, 0, NULL, NULL); if (n <= 0) { ios_printf(ios_stderr, "fatal error: jl_compileropts.image_file path too long or GetFullPathName failed\n"); exit(1); } char *out = (char*)malloc(n); - DWORD m = GetFullPathName(in, n, out, C_NULL); + DWORD m = GetFullPathName(in, n, out, NULL); if (n != m + 1) { ios_printf(ios_stderr, "fatal error: jl_compileropts.image_file path too long or GetFullPathName failed\n"); exit(1); @@ -1096,7 +1096,7 @@ extern void *__stack_chk_guard; void jl_compile_all(void); -DLLEXPORT int julia_save() +DLLEXPORT void julia_save() { const char *build_path = jl_compileropts.build_path; if (build_path) { diff --git a/src/julia.expmap b/src/julia.expmap index 08f54c22f5a0c..40edbbba6a1dc 100644 --- a/src/julia.expmap +++ b/src/julia.expmap @@ -1,6 +1,7 @@ { global: __asan*; + __stack_chk_guard; alloc_*w; allocobj; asprintf; diff --git a/src/julia.h b/src/julia.h index d5d0be658786f..ada5811a7d900 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1342,7 +1342,7 @@ extern DLLEXPORT jl_compileropts_t jl_compileropts; #define JL_COMPILEROPT_DUMPBITCODE_ON 1 #define JL_COMPILEROPT_DUMPBITCODE_OFF 2 -extern void *__stack_chk_guard; +DLLEXPORT extern void *__stack_chk_guard; #define SET_STACK_CHK_GUARD(a,b,c) do { \ unsigned char *p = (unsigned char *)&__stack_chk_guard; \ a = p[sizeof(__stack_chk_guard)-1]; \ diff --git a/src/support/dirpath.h b/src/support/dirpath.h index 28fff4ab96def..e906ddd7521b2 100644 --- a/src/support/dirpath.h +++ b/src/support/dirpath.h @@ -10,7 +10,9 @@ #else #define PATHSEPSTRING "/" #define PATHLISTSEPSTRING ":" +#ifndef PATH_MAX // many platforms don't have a max path, we define one anyways #define PATH_MAX 1024 #endif +#endif #endif diff --git a/src/task.c b/src/task.c index e2945a32c5f4a..cc6f8024f4f79 100644 --- a/src/task.c +++ b/src/task.c @@ -233,9 +233,9 @@ static void NORETURN finish_task(jl_task_t *t, jl_value_t *resultval) } #if defined(_OS_WINDOWS_) && !defined(_COMPILER_MINGW_) -void __declspec(noinline) +static void __declspec(noinline) #else -void __attribute__((noinline)) +static void __attribute__((noinline)) #endif NORETURN start_task() { @@ -348,16 +348,18 @@ static void ctx_switch(jl_task_t *t, jl_jmp_buf *where) asm(" movq %0, %%rsp;\n" " xorq %%rbp, %%rbp;\n" " push %%rbp;\n" // instead of RSP - " jmp _start_task" // call stack_task with fake stack frame - : : "r"(jl_stackbase-0x10) : ); + " jmp %P1;\n" // call stack_task with fake stack frame + " ud2" + : : "r"(jl_stackbase-0x10), "i"(start_task) : "memory" ); #elif defined(_CPU_X86_) asm(" movl %0, %%esp;\n" " xorl %%ebp, %%ebp;\n" " push %%ebp;\n" // instead of ESP - " jmp _start_task" // call stack_task with fake stack frame - : : "r"(jl_stackbase-0x10) : ); + " jmp %P1;\n" // call stack_task with fake stack frame + " ud2" + : : "r"(jl_stackbase-0x10), "i"(start_task) : "memory" ); #else -#error ASM_COPY_STACKS not supported on this platform +#error ASM_COPY_STACKS not supported on this cpu architecture #endif #else // ASM_COPY_STACKS jl_longjmp(jl_base_ctx, 1); diff --git a/ui/repl.c b/ui/repl.c index 88aa0c479ec09..e758b250ba823 100644 --- a/ui/repl.c +++ b/ui/repl.c @@ -311,7 +311,7 @@ static int true_main(int argc, char *argv[]) return iserr; } -DLLEXPORT extern int julia_save(); +DLLEXPORT extern void julia_save(); #ifndef _OS_WINDOWS_ int main(int argc, char *argv[]) From cb8448f41345d835613f379034dbca12a68bf28d Mon Sep 17 00:00:00 2001 From: Rafael Fourquet Date: Tue, 9 Dec 2014 18:54:43 +0530 Subject: [PATCH 14/40] implement RandomDevice RNG --- base/dSFMT.jl | 4 +-- base/exports.jl | 1 + base/random.jl | 66 ++++++++++++++++++++++++++++++++------------- doc/stdlib/base.rst | 22 ++++++++++++--- test/random.jl | 54 ++++++++++++++++++++++--------------- 5 files changed, 101 insertions(+), 46 deletions(-) diff --git a/base/dSFMT.jl b/base/dSFMT.jl index a77791db58c90..274810dc27328 100644 --- a/base/dSFMT.jl +++ b/base/dSFMT.jl @@ -67,8 +67,8 @@ end ## Windows entropy @windows_only begin - function win32_SystemFunction036!(a::Array{UInt32}) - ccall((:SystemFunction036,:Advapi32),stdcall,UInt8,(Ptr{Void},UInt32),a,length(a)*sizeof(eltype(a))) + function win32_SystemFunction036!{T}(a::Array{T}) + ccall((:SystemFunction036,:Advapi32),stdcall,UInt8,(Ptr{Void},UInt32),a,sizeof(a)) end end diff --git a/base/exports.jl b/base/exports.jl index 07c846233877f..07a5965aa3717 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -901,6 +901,7 @@ export # random numbers AbstractRNG, MersenneTwister, + RandomDevice, rand!, rand, randbool, diff --git a/base/random.jl b/base/random.jl index 7dcd0f26ce1a8..036c1cff90377 100644 --- a/base/random.jl +++ b/base/random.jl @@ -8,11 +8,54 @@ export srand, randn, randn!, randexp, randexp!, randbool, - AbstractRNG, RNG, MersenneTwister + AbstractRNG, RNG, MersenneTwister, RandomDevice abstract AbstractRNG +abstract FloatInterval +type CloseOpen <: FloatInterval end +type Close1Open2 <: FloatInterval end + + +## RandomDevice + +@unix_only begin + + immutable RandomDevice <: AbstractRNG + file::IOStream + + RandomDevice(unlimited::Bool=true) = new(open(unlimited ? "/dev/urandom" : "/dev/random")) + end + + rand {T<:Union(Bool, Base.IntTypes...)}(rd::RandomDevice, ::Type{T}) = read( rd.file, T) + rand!{T<:Union(Bool, Base.IntTypes...)}(rd::RandomDevice, A::Array{T}) = read!(rd.file, A) +end + +@windows_only begin + + immutable RandomDevice <: AbstractRNG + buffer::Vector{UInt128} + + RandomDevice() = new(Array(UInt128, 1)) + end + + function rand{T<:Union(Bool, Base.IntTypes...)}(rd::RandomDevice, ::Type{T}) + win32_SystemFunction036!(rd.buffer) + @inbounds return rd.buffer[1] % T + end + + rand!{T<:Union(Bool, Base.IntTypes...)}(rd::RandomDevice, A::Array{T}) = (win32_SystemFunction036!(A); A) +end + +rand(rng::RandomDevice, ::Type{Close1Open2}) = + reinterpret(Float64, 0x3ff0000000000000 | rand(rng, UInt64) & 0x000fffffffffffff) + +rand(rng::RandomDevice, ::Type{CloseOpen}) = rand(rng, Close1Open2) - 1.0 + + +## MersenneTwister + const MTCacheLength = dsfmt_get_min_array_size() type MersenneTwister <: AbstractRNG @@ -44,10 +87,6 @@ end # precondition: n <= MTCacheLength @inline reserve(r::MersenneTwister, n::Int) = mt_avail(r) < n && gen_rand(r) -abstract FloatInterval -type CloseOpen <: FloatInterval end -type Close1Open2 <: FloatInterval end - # precondition: !mt_empty(r) @inline rand_inbounds(r::MersenneTwister, ::Type{Close1Open2}) = mt_pop!(r) @inline rand_inbounds(r::MersenneTwister, ::Type{CloseOpen}) = rand_inbounds(r, Close1Open2) - 1.0 @@ -78,10 +117,8 @@ __init__() = srand() # make_seed methods produce values of type Array{UInt32}, suitable for MersenneTwister seeding function make_seed() - -@unix_only begin try - return make_seed("/dev/urandom", 4) + return rand(RandomDevice(), UInt32, 4) catch println(STDERR, "Entropy pool not available to seed RNG; using ad-hoc entropy sources.") seed = reinterpret(UInt64, time()) @@ -93,13 +130,6 @@ function make_seed() end end -@windows_only begin - a = zeros(UInt32, 2) - win32_SystemFunction036!(a) - return a -end -end - function make_seed(n::Integer) n < 0 && throw(DomainError()) seed = UInt32[] @@ -174,9 +204,9 @@ rand(r::AbstractArray, dims::Integer...) = rand(GLOBAL_RNG, r, convert((Int...), @inline rand(r::AbstractRNG) = rand(r, CloseOpen) -# MersenneTwister -rand(r::MersenneTwister, ::Type{Float64}) = rand(r, CloseOpen) -rand{T<:Union(Float16, Float32)}(r::MersenneTwister, ::Type{T}) = convert(T, rand(r, Float64)) +# MersenneTwister & RandomDevice +@inline rand(r::Union(RandomDevice,MersenneTwister), ::Type{Float64}) = rand(r, CloseOpen) +rand{T<:Union(Float16, Float32)}(r::Union(RandomDevice,MersenneTwister), ::Type{T}) = convert(T, rand(r, Float64)) ## random integers diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index ee7c767593ee9..7f5ee13592bc2 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -4099,20 +4099,34 @@ The `BigFloat` type implements arbitrary-precision floating-point arithmetic usi Random Numbers -------------- -Random number generation in Julia uses the `Mersenne Twister library `_. Julia has a global RNG, which is used by default. Multiple RNGs can be plugged in using the ``AbstractRNG`` object, which can then be used to have multiple streams of random numbers. Currently, only ``MersenneTwister`` is supported. +Random number generation in Julia uses the `Mersenne Twister library `_ via ``MersenneTwister`` objects. +Julia has a global RNG, which is used by default. Other RNG types can be plugged in by inheriting the ``AbstractRNG`` type; +they can then be used to have multiple streams of random numbers. +Besides ``MersenneTwister``, Julia also provides the ``RandomDevice`` RNG type, which is a wrapper over the OS provided entropy. -Most functions related to random generation accept an optional ``AbstractRNG`` as the first argument, ``rng`` , which defaults to the global one if not provided. Morever, some of them accept optionally dimension specifications ``dims...`` (which can be given as a tuple) to generate arrays of random values. +Most functions related to random generation accept an optional ``AbstractRNG`` as the first argument, ``rng`` , which defaults to the global one if not provided. +Morever, some of them accept optionally dimension specifications ``dims...`` (which can be given as a tuple) to generate arrays of random values. -A ``MersenneTwister`` RNG can generate random numbers of the following types: ``Float16``, ``Float32``, ``Float64``, ``Bool``, ``Int8``, ``UInt8``, ``Int16``, ``UInt16``, ``Int32``, ``UInt32``, ``Int64``, ``UInt64``, ``Int128``, ``UInt128``, ``BigInt`` (or complex numbers of those types). Random floating point numbers are generated uniformly in [0,1). As ``BigInt`` represents unbounded integers, the interval must be specified (e.g. ``rand(big(1:6))``). +A ``MersenneTwister`` or ``RandomDevice`` RNG can generate random numbers of the following types: +``Float16``, ``Float32``, ``Float64``, ``Bool``, ``Int8``, ``UInt8``, ``Int16``, ``UInt16``, +``Int32``, ``UInt32``, ``Int64``, ``UInt64``, ``Int128``, ``UInt128``, ``BigInt`` +(or complex numbers of those types). Random floating point numbers are generated uniformly in [0,1). +As ``BigInt`` represents unbounded integers, the interval must be specified (e.g. ``rand(big(1:6))``). .. function:: srand([rng], [seed]) - Reseed the random number generator. If a ``seed`` is provided, the RNG will give a reproducible sequence of numbers, otherwise Julia will get entropy from the system. The ``seed`` may be a non-negative integer, a vector of ``UInt32`` integers or a filename, in which case the seed is read from a file. + Reseed the random number generator. If a ``seed`` is provided, the RNG will give a reproducible sequence of numbers, otherwise Julia will get entropy from the system. + For ``MersenneTwister``, the ``seed`` may be a non-negative integer, a vector of ``UInt32`` integers or a filename, in which case the seed is read from a file. + ``RandomDevice`` does not support seeding. .. function:: MersenneTwister([seed]) Create a ``MersenneTwister`` RNG object. Different RNG objects can have their own seeds, which may be useful for generating different streams of random numbers. +.. function:: RandomDevice() + + Create a ``RandomDevice`` RNG object. Two such objects will always generate different streams of random numbers. + .. function:: rand([rng], [S], [dims...]) Pick a random element or array of random elements from the set of values specified by ``S``; ``S`` can be diff --git a/test/random.jl b/test/random.jl index 221118dcfded3..a59d0edeca2b6 100644 --- a/test/random.jl +++ b/test/random.jl @@ -95,20 +95,6 @@ for T in [UInt32, UInt64, UInt128, Int128] @test rand(s) == r end -randn() -randn(100000) -randn!(Array(Float64, 100000)) -randn(MersenneTwister(10)) -randn(MersenneTwister(10), 100000) -randn!(MersenneTwister(10), Array(Float64, 100000)) - -randexp() -randexp(100000) -randexp!(Array(Float64, 100000)) -randexp(MersenneTwister(10)) -randexp(MersenneTwister(10), 100000) -randexp!(MersenneTwister(10), Array(Float64, 100000)) - # Test ziggurat tables ziggurat_table_size = 256 nmantissa = int64(2)^51 # one bit for the sign @@ -278,14 +264,38 @@ let mt = MersenneTwister() end end -# test rand! API: rand!([rng], A, [coll]) -let mt = MersenneTwister(0) - for T in [Base.IntTypes..., Float16, Float32, Float64] - for A in (Array(T, 5), Array(T, 2, 2)) - rand!(A) - rand!(mt, A) - rand!(A, T[1,2,3]) - rand!(mt, A, T[1,2,3]) +# test all rand APIs +for rng in ([], [MersenneTwister()], [RandomDevice()]) + for f in [rand, randn, randexp] + f(rng...) ::Float64 + f(rng..., 5) ::Vector{Float64} + f(rng..., 2, 3) ::Array{Float64, 2} + end + for f! in [randn!, randexp!] + f!(rng..., Array(Float64, 5)) ::Vector{Float64} + f!(rng..., Array(Float64, 2, 3)) ::Array{Float64, 2} + end + + randbool(rng...) ::Bool + randbool(rng..., 5) ::BitArray{1} + randbool(rng..., 2, 3) ::BitArray{2} + rand!(rng..., BitArray(5)) ::BitArray{1} + rand!(rng..., BitArray(2, 3)) ::BitArray{2} + + for T in [Base.IntTypes..., Bool, Float16, Float32, Float64] + a0 = rand(rng..., T) ::T + a1 = rand(rng..., T, 5) ::Vector{T} + a2 = rand(rng..., T, 2, 3) ::Array{T, 2} + if T <: FloatingPoint + for a in [a0, a1..., a2...] + @test 0.0 <= a < 1.0 + end + end + for A in (Array(T, 5), Array(T, 2, 3)) + rand!(rng..., A) ::typeof(A) + rand!(rng..., A, T[0,1,2]) ::typeof(A) + rand!(rng..., sparse(A)) ::typeof(sparse(A)) + rand!(rng..., sparse(A), T[0,1,2]) ::typeof(sparse(A)) end end end From d3bdc6cb54d0cab43874c0da1c2476d9595dc8b8 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Sat, 13 Dec 2014 02:48:49 -0800 Subject: [PATCH 15/40] Put LLDB-specific flags back under `ifeq ($(BUILD_LLDB),1)` where they were before #9263 --- deps/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deps/Makefile b/deps/Makefile index 513fb749da3b8..14984345887a3 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -279,6 +279,7 @@ else LLVM_FLAGS += --disable-threads endif # USE_INTEL_JITEVENTS +ifeq ($(BUILD_LLDB),1) ifeq ($(USECLANG),1) LLVM_FLAGS += --enable-cxx11 else @@ -287,6 +288,7 @@ endif # USECLANG ifeq ($(LLDB_DISABLE_PYTHON),1) LLVM_CXXFLAGS += -DLLDB_DISABLE_PYTHON endif # LLDB_DISABLE_PYTHON +endif # BUILD_LLDB ifeq ($(ARCH), ppc64) LLVM_CXXFLAGS += -mminimal-toc From bcd382bbc8555f12beebafcb1f02354c5f69dfb4 Mon Sep 17 00:00:00 2001 From: Isaiah Norton Date: Sat, 13 Dec 2014 12:39:45 -0500 Subject: [PATCH 16/40] deps/Makefile: add update-llvm target to pull llvm+clang,lldb,and compiler-rt (if applicable) --- deps/Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/deps/Makefile b/deps/Makefile index 14984345887a3..fc1384fbb8353 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -550,7 +550,11 @@ check-llvm: llvm-$(LLVM_VER)/build_$(LLVM_BUILDTYPE)/checked install-llvm: $(LLVM_OBJ_TARGET) #todo: LLVM make check target is broken on julia.mit.edu (and really slow elsewhere) - +update-llvm: + (cd llvm-$(LLVM_VER); git fetch origin; git rebase origin/master) + ([ -d llvm-$(LLVM_VER)/tools/clang ] && (cd llvm-$(LLVM_VER)/tools/clang; git pull --ff-only)) + ([ -d llvm-$(LLVM_VER)/projects/compiler-rt ] && (cd llvm-$(LLVM_VER)/projects/compiler-rt; git pull --ff-only)) + ([ -d llvm-$(LLVM_VER)/tools/lldb ] && (cd llvm-$(LLVM_VER)/tools/lldb; git pull --ff-only)) ## LIBUV ## UV_SRC_TARGET = libuv/.libs/libuv.a From 795d7e0ae5cccf40cd25d75c23a0b044f8f2d6ec Mon Sep 17 00:00:00 2001 From: Isaiah Norton Date: Sat, 13 Dec 2014 14:29:06 -0500 Subject: [PATCH 17/40] add update-llvm to .PHONY also use 'pull --ff-only' for llvm too, because some builds might use 3.5 rather than svn. --- deps/Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/deps/Makefile b/deps/Makefile index fc1384fbb8353..ec2002a99d922 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -551,10 +551,11 @@ install-llvm: $(LLVM_OBJ_TARGET) #todo: LLVM make check target is broken on julia.mit.edu (and really slow elsewhere) update-llvm: - (cd llvm-$(LLVM_VER); git fetch origin; git rebase origin/master) + (cd llvm-$(LLVM_VER); git pull --ff-only) ([ -d llvm-$(LLVM_VER)/tools/clang ] && (cd llvm-$(LLVM_VER)/tools/clang; git pull --ff-only)) ([ -d llvm-$(LLVM_VER)/projects/compiler-rt ] && (cd llvm-$(LLVM_VER)/projects/compiler-rt; git pull --ff-only)) ([ -d llvm-$(LLVM_VER)/tools/lldb ] && (cd llvm-$(LLVM_VER)/tools/lldb; git pull --ff-only)) + ## LIBUV ## UV_SRC_TARGET = libuv/.libs/libuv.a @@ -1820,4 +1821,4 @@ install-libgit2: $(LIBGIT2_OBJ_TARGET) .PHONY: default compile install cleanall distcleanall \ get-* configure-* compile-* check-* install-* \ - clean-* distclean-* reinstall-* + clean-* distclean-* reinstall-* update-llvm From 3e23dfdb975d652651acf5628410aa3db90cf191 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 13 Dec 2014 19:13:23 -0500 Subject: [PATCH 18/40] fix #8137 --- src/debuginfo.cpp | 83 ++++++++++++++++++++++++++--------------------- 1 file changed, 46 insertions(+), 37 deletions(-) diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index 621b900b2dd0c..4730d2a817a31 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -33,6 +33,10 @@ #include #endif +#if defined(USE_MCJIT) && !defined(LLVM36) && !defined(_OS_LINUX_) +#include "../deps/llvm-3.5.0/lib/ExecutionEngine/MCJIT/MCJIT.h" +#endif + #include "julia.h" #include "julia_internal.h" @@ -191,22 +195,21 @@ class JuliaJITEventListener: public JITEventListener virtual void NotifyObjectEmitted(const ObjectImage &obj) #endif { - uint64_t SectAddr, Addr; + uint64_t SectAddr, Addr, SectionAddr; uint64_t Size; object::SymbolRef::Type SymbolType; #ifndef _OS_LINUX_ + StringRef sName; #ifdef LLVM36 object::section_iterator Section = obj.section_begin(); object::section_iterator EndSection = obj.section_end(); - StringRef sName; #else object::section_iterator Section = obj.begin_sections(); object::section_iterator EndSection = obj.end_sections(); + bool isText; #endif #endif #ifdef _OS_WINDOWS_ - StringRef Name; - uint64_t SectionAddr; uint64_t SectionSize; #endif @@ -216,27 +219,35 @@ class JuliaJITEventListener: public JITEventListener if (SymbolType != object::SymbolRef::ST_Function) continue; sym_iter.getSize(Size); sym_iter.getAddress(SectAddr); -#ifndef _OS_LINUX_ +#ifdef _OS_LINUX_ + SectionAddr = 0; +#else sym_iter.getSection(Section); if (Section == EndSection) continue; Section = Section->getRelocatedSection(); if (Section == EndSection) continue; -#ifdef LLVM36 +#if defined(LLVM36) if (!Section->isText()) continue; Section->getName(sName); - Addr = SectAddr + L.getSectionLoadAddress(sName); -#else - Addr = SectAddr; -#endif + SectionAddr = L.getSectionLoadAddress(sName); #else - Addr = SectAddr; -#endif + if (Section->isText(isText) || !isText) continue; + sym_iter.getName(sName); + SectionAddr = ((MCJIT*)jl_ExecutionEngine)->getSymbolAddress(sName, true); + if (!SectionAddr && sName[0] == '_') + SectionAddr = ((MCJIT*)jl_ExecutionEngine)->getSymbolAddress(sName.substr(1), true); + if (!SectionAddr) continue; + SectionAddr -= SectAddr; +#endif +#endif // _OS_LINUX_ + Addr = SectAddr + SectionAddr; #ifdef _OS_WINDOWS_ - sym_iter.getName(Name); - Section->getAddress(SectionAddr); +#ifndef LLVM36 + sym_iter.getName(sName); +#endif Section->getSize(SectionSize); create_PRUNTIME_FUNCTION( - (uint8_t*)(intptr_t)Addr, (size_t)Size, Name, + (uint8_t*)(intptr_t)Addr, (size_t)Size, sName, (uint8_t*)(intptr_t)SectionAddr, (size_t)SectionSize); #endif const object::ObjectFile *objfile = @@ -602,22 +613,21 @@ void jl_getFunctionInfo(const char **name, size_t *line, const char **filename, std::map &objmap = jl_jit_events->getObjectMap(); std::map::iterator it = objmap.lower_bound(pointer); - if (it == objmap.end() || (pointer - it->first) > it->second.size) - return jl_getDylibFunctionInfo(name,line,filename,pointer,fromC,skipC); - + if (it != objmap.end() && (intptr_t)(*it).first + (*it).second.size > pointer) { #ifdef LLVM36 - DIContext *context = DIContext::getDWARFContext(*it->second.object); + DIContext *context = DIContext::getDWARFContext(*it->second.object); #else - DIContext *context = DIContext::getDWARFContext(const_cast(it->second.object)); + DIContext *context = DIContext::getDWARFContext(const_cast(it->second.object)); #endif - lookup_pointer(context, name, line, filename, pointer, 1, fromC); + lookup_pointer(context, name, line, filename, pointer, 1, fromC); + return; + } #else // !USE_MCJIT - // Without MCJIT we use the FuncInfo structure containing address maps std::map &info = jl_jit_events->getMap(); std::map::iterator it = info.lower_bound(pointer); - if (it != info.end() && (size_t)(*it).first + (*it).second.lengthAdr >= pointer) { + if (it != info.end() && (intptr_t)(*it).first + (*it).second.lengthAdr >= pointer) { // We do this to hide the jlcall wrappers when getting julia backtraces, // but it is still good to have them for regular lookup of C frames. if (skipC && (*it).second.lines.empty()) { @@ -665,11 +675,10 @@ void jl_getFunctionInfo(const char **name, size_t *line, const char **filename, if (*line == (size_t) -1) { *line = prev.Loc.getLine(); } - } - else { - jl_getDylibFunctionInfo(name,line,filename,pointer,fromC,skipC); + return; } #endif // USE_MCJIT + jl_getDylibFunctionInfo(name,line,filename,pointer,fromC,skipC); } int jl_get_llvmf_info(size_t fptr, uint64_t *symsize, @@ -683,22 +692,22 @@ int jl_get_llvmf_info(size_t fptr, uint64_t *symsize, std::map &fmap = jl_jit_events->getMap(); std::map::iterator fit = fmap.find(fptr); - if (fit == fmap.end()) { - return 0; + if (fit != fmap.end()) { + *symsize = fit->second.lengthAdr; + *lines = fit->second.lines; + return 1; } - *symsize = fit->second.lengthAdr; - *lines = fit->second.lines; - return 1; + return 0; #else // MCJIT version - std::map objmap = jl_jit_events->getObjectMap(); + std::map &objmap = jl_jit_events->getObjectMap(); std::map::iterator fit = objmap.find(fptr); - if (fit == objmap.end()) { - return 0; + if (fit != objmap.end()) { + *symsize = fit->second.size; + *object = fit->second.object; + return 1; } - *symsize = fit->second.size; - *object = fit->second.object; - return 1; + return 0; #endif } From 9975b704da6072f71395dd218d47a5fe9e335094 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 13 Dec 2014 22:50:20 -0500 Subject: [PATCH 19/40] fix #9206 and partially fix #7910 --- src/codegen.cpp | 6 +----- src/debuginfo.cpp | 44 +++++++++++++++++++++++++++++++------------- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 54814ac37e4ba..98299f5184c9b 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -3615,14 +3615,10 @@ static Function *emit_function(jl_lambda_info_t *lam, bool cstyle) else { m = shadow_module; } -#ifndef LLVM35 - funcName << ";"; -#endif #else m = jl_Module; - funcName << ";"; #endif - funcName << globalUnique++; + funcName << "_" << globalUnique++; if (specsig) { std::vector fsig(0); diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index 4730d2a817a31..b985d5f9e5ffe 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -70,6 +70,9 @@ struct FuncInfo { struct ObjectInfo { const object::ObjectFile* object; size_t size; +#ifdef _OS_DARWIN_ + const char *name; +#endif }; #endif @@ -234,16 +237,19 @@ class JuliaJITEventListener: public JITEventListener if (Section->isText(isText) || !isText) continue; sym_iter.getName(sName); SectionAddr = ((MCJIT*)jl_ExecutionEngine)->getSymbolAddress(sName, true); - if (!SectionAddr && sName[0] == '_') - SectionAddr = ((MCJIT*)jl_ExecutionEngine)->getSymbolAddress(sName.substr(1), true); + if (!SectionAddr && sName[0] == '_') { + sName = sName.substr(1); + SectionAddr = ((MCJIT*)jl_ExecutionEngine)->getSymbolAddress(sName, true); + } if (!SectionAddr) continue; SectionAddr -= SectAddr; #endif #endif // _OS_LINUX_ Addr = SectAddr + SectionAddr; #ifdef _OS_WINDOWS_ -#ifndef LLVM36 +#ifdef LLVM36 sym_iter.getName(sName); + if (sName[0] == '_') sName = sName.substr(1); #endif Section->getSize(SectionSize); create_PRUNTIME_FUNCTION( @@ -256,7 +262,11 @@ class JuliaJITEventListener: public JITEventListener #else obj.getObjectFile(); #endif - ObjectInfo tmp = {objfile, (size_t)Size}; + ObjectInfo tmp = {objfile, (size_t)Size +#ifdef _OS_DARWIN_ + ,strdup(sName.data()) +#endif + }; objectmap[Addr] = tmp; } #else //LLVM34 @@ -289,16 +299,19 @@ class JuliaJITEventListener: public JITEventListener extern "C" const char *jl_demangle(const char *name) { - const char *start = name; - const char *end = start; + const char *start = name + 6; + const char *end = name + strlen(name); char *ret; - while ((*start++ != '_') && (*start != '\0')); - if (*name == '\0') goto done; - while ((*end++ != ';') && (*end != '\0')); - if (*name == '\0') goto done; - ret = (char*)malloc(end-start); - memcpy(ret,start,end-start-1); - ret[end-start-1] = '\0'; + if (strncmp(name, "julia_", 6)) goto done; + if (*start == '\0') goto done; + while (*(--end) != '_') { + char c = *end; + if (c < '0' || c > '9') goto done; + } + if (end <= start) goto done; + ret = (char*)malloc(end-start+1); + memcpy(ret,start,end-start); + ret[end-start] = '\0'; return ret; done: return strdup(name); @@ -614,10 +627,15 @@ void jl_getFunctionInfo(const char **name, size_t *line, const char **filename, std::map::iterator it = objmap.lower_bound(pointer); if (it != objmap.end() && (intptr_t)(*it).first + (*it).second.size > pointer) { +#if defined(_OS_DARWIN_) + *name = jl_demangle((*it).second.name); + DIContext *context = NULL; // current versions of MCJIT can't handle MachO relocations +#else #ifdef LLVM36 DIContext *context = DIContext::getDWARFContext(*it->second.object); #else DIContext *context = DIContext::getDWARFContext(const_cast(it->second.object)); +#endif #endif lookup_pointer(context, name, line, filename, pointer, 1, fromC); return; From 8203538c7f621000cf8fc8fa1509c50c2b83a8dd Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 13 Dec 2014 23:12:19 -0500 Subject: [PATCH 20/40] remove extraneous gc root in start_task --- src/task.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/task.c b/src/task.c index cc6f8024f4f79..6c10904839264 100644 --- a/src/task.c +++ b/src/task.c @@ -241,11 +241,8 @@ NORETURN start_task() { // this runs the first time we switch to a task jl_task_t *t = jl_current_task; - jl_value_t *arg = jl_task_arg_in_transit; jl_value_t *res; - JL_GC_PUSH1(&arg); res = jl_apply(t->start, NULL, 0); - JL_GC_POP(); finish_task(t, res); abort(); } From e2cb463f89ed52aabfc7cb487b5daf2d13dd3e4b Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 13 Dec 2014 23:50:30 -0500 Subject: [PATCH 21/40] allow the user to specify whether the image_file should be interpreted relative to JULIA_HOME or CWD --- src/init.c | 42 +++++++++++++++++++++++++++++------------- src/jlapi.c | 2 +- src/julia.h | 7 ++++++- src/julia_internal.h | 2 +- src/task.c | 4 ++-- ui/repl.c | 4 ++-- 6 files changed, 41 insertions(+), 20 deletions(-) diff --git a/src/init.c b/src/init.c index ec1181508645e..cd2288df6d196 100644 --- a/src/init.c +++ b/src/init.c @@ -780,7 +780,7 @@ static char *abspath(const char *in) return out; } -static void jl_resolve_sysimg_location() +static void jl_resolve_sysimg_location(JL_IMAGE_SEARCH rel) { // note: if you care about lost memory, you should compare the // pointers in jl_compileropts before and after calling julia_init() // and call the appropriate free function on the originals for any that changed @@ -806,18 +806,34 @@ static void jl_resolve_sysimg_location() free(free_path); free_path = NULL; if (jl_compileropts.image_file) { - if (jl_compileropts.image_file[0] != PATHSEPSTRING[0]) { - if (jl_compileropts.image_file == system_image_path) { - // build time path, relative to JULIA_HOME - free_path = (char*)malloc(PATH_MAX); - int n = snprintf(free_path, PATH_MAX, "%s" PATHSEPSTRING "%s", - jl_compileropts.julia_home, jl_compileropts.image_file); - if (n >= PATH_MAX || n < 0) { - ios_printf(ios_stderr, "fatal error: jl_compileropts.image_file path too long\n"); - exit(1); + if (rel == JL_IMAGE_JULIA_HOME) { +#ifdef _OS_WINDOWS_ + char c0 = jl_compileropts.image_file[0]; + if (c0 == '/' || c0 == '\\') { + rel = 0; // absolute path relative to %CD% (current drive) + } + else { + int s = strlen(jl_compileropts.image_file); + if (s > 2) { + char c1 = jl_compileropts.image_file[1]; + char c2 = jl_compileropts.image_file[2]; + if (c1 == ':' && (c2 == '/' || c2 == '\\')) rel = 0; // absolute path } - jl_compileropts.image_file = free_path; } +#else + if (jl_compileropts.image_file[0] == '/') rel = 0; // absolute path +#endif + } + if (rel == JL_IMAGE_JULIA_HOME) { + // build time path, relative to JULIA_HOME + free_path = (char*)malloc(PATH_MAX); + int n = snprintf(free_path, PATH_MAX, "%s" PATHSEPSTRING "%s", + jl_compileropts.julia_home, jl_compileropts.image_file); + if (n >= PATH_MAX || n < 0) { + ios_printf(ios_stderr, "fatal error: jl_compileropts.image_file path too long\n"); + exit(1); + } + jl_compileropts.image_file = free_path; } if (jl_compileropts.image_file) jl_compileropts.image_file = abspath(jl_compileropts.image_file); @@ -830,11 +846,11 @@ static void jl_resolve_sysimg_location() jl_compileropts.build_path = abspath(jl_compileropts.build_path); } -void _julia_init() +void _julia_init(JL_IMAGE_SEARCH rel) { jl_io_loop = uv_default_loop(); // this loop will internal events (spawning process etc.), // best to call this first, since it also initializes libuv - jl_resolve_sysimg_location(); + jl_resolve_sysimg_location(rel); jl_page_size = jl_getpagesize(); jl_arr_xtralloc_limit = uv_get_total_memory() / 100; // Extra allocation limited to 1% of total RAM jl_find_stack_bottom(); diff --git a/src/jlapi.c b/src/jlapi.c index 1f24440ec353b..006482e50d921 100644 --- a/src/jlapi.c +++ b/src/jlapi.c @@ -39,7 +39,7 @@ DLLEXPORT void jl_init_with_image(char *julia_home_dir, char *image_relative_pat jl_compileropts.julia_home = julia_home_dir; if (image_relative_path != NULL) jl_compileropts.image_file = image_relative_path; - julia_init(); + julia_init(JL_IMAGE_JULIA_HOME); //TODO: these should be part of Multi.__init__() //currently, we have them here since we may not want them //getting unconditionally set from Base.__init__() diff --git a/src/julia.h b/src/julia.h index ada5811a7d900..f3ad38c84f7a3 100644 --- a/src/julia.h +++ b/src/julia.h @@ -855,7 +855,12 @@ DLLEXPORT void jl_exception_clear(void); } // initialization functions -DLLEXPORT void julia_init(void); +typedef enum { + JL_IMAGE_CWD = 0, + JL_IMAGE_JULIA_HOME = 1, + //JL_IMAGE_LIBJULIA = 2, +} JL_IMAGE_SEARCH; +DLLEXPORT void julia_init(JL_IMAGE_SEARCH rel); DLLEXPORT void jl_init(char *julia_home_dir); DLLEXPORT void jl_init_with_image(char *julia_home_dir, char *image_relative_path); DLLEXPORT int jl_is_initialized(void); diff --git a/src/julia_internal.h b/src/julia_internal.h index 5b45ec65403cc..25a44e873680a 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -87,7 +87,7 @@ void jl_init_codegen(void); void jl_init_intrinsic_functions(void); void jl_init_tasks(void *stack, size_t ssize); void jl_init_serializer(void); -void _julia_init(void); +void _julia_init(JL_IMAGE_SEARCH rel); #ifdef COPY_STACKS extern void *jl_stackbase; #endif diff --git a/src/task.c b/src/task.c index 6c10904839264..01d551509d564 100644 --- a/src/task.c +++ b/src/task.c @@ -264,10 +264,10 @@ void set_base_ctx(char *__stk) { } #endif -DLLEXPORT void julia_init() +DLLEXPORT void julia_init(JL_IMAGE_SEARCH rel) { // keep this function small, since we want to keep the stack frame // leading up to this also quite small - _julia_init(); + _julia_init(rel); #ifdef COPY_STACKS char __stk; jl_stackbase = (char*)(((uptrint_t)&__stk + sizeof(__stk))&-16); // also ensures stackbase is 16-byte aligned diff --git a/ui/repl.c b/ui/repl.c index e758b250ba823..0c18ec88b1144 100644 --- a/ui/repl.c +++ b/ui/repl.c @@ -40,6 +40,7 @@ static int lisp_prompt = 0; static int codecov = JL_LOG_NONE; static int malloclog= JL_LOG_NONE; static char *program = NULL; +static int imagepathspecified = 0; static const char *usage = "julia [options] [program] [args...]\n"; static const char *opts = @@ -95,7 +96,6 @@ void parse_opts(int *argcp, char ***argvp) }; int c; opterr = 0; - int imagepathspecified=0; int skip = 0; int lastind = optind; while ((c = getopt_long(*argcp,*argvp,shortopts,longopts,0)) != -1) { @@ -338,7 +338,7 @@ int wmain(int argc, wchar_t *argv[], wchar_t *envp[]) jl_lisp_prompt(); return 0; } - julia_init(); + julia_init(imagepathspecified ? JL_IMAGE_CWD : JL_IMAGE_JULIA_HOME); int ret = true_main(argc, (char**)argv); jl_atexit_hook(); julia_save(); From 9da3b57d9f10affa6039d58860e426f2b96be33d Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sun, 14 Dec 2014 00:09:40 -0500 Subject: [PATCH 22/40] update embedding doc to be consistent with changes to julia_init --- doc/manual/embedding.rst | 18 ++++++++++++++++-- examples/embedding.c | 5 ++++- src/init.c | 1 + 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/doc/manual/embedding.rst b/doc/manual/embedding.rst index a5a0cdb1a73c3..230d3c5b953ad 100644 --- a/doc/manual/embedding.rst +++ b/doc/manual/embedding.rst @@ -18,11 +18,25 @@ We start with a simple C program that initializes Julia and calls some Julia cod int main(int argc, char *argv[]) { + /* optional: randomize the stack guard */ + char a, b, c; + SET_STACK_CHK_GUARD(a,b,c); + + /* required: setup the julia context */ jl_init(NULL); - JL_SET_STACK_BASE; + /* run julia commands */ jl_eval_string("print(sqrt(2.0))"); + /* strongly recommended: notify julia that the + program is about to terminate. this allows + julia time to cleanup pending write requests + and run all finalizers + */ + jl_atexit_hook(); + + /* if the stack guard is set: reset the stack guard */ + CLR_STACK_CHK_GUARD(a,b,c); return 0; } @@ -30,7 +44,7 @@ In order to build this program you have to put the path to the Julia header into gcc -o test -I$JULIA_DIR/include/julia -L$JULIA_DIR/usr/lib -ljulia test.c -Alternatively, look at the ``embedding.c`` program in the julia source tree in the ``examples/`` folder. +Alternatively, look at the ``embedding.c`` program in the julia source tree in the ``examples/`` folder. The file ``ui/repl.c`` program is another simple example of how to set ``jl_compileropts`` options while linking against libjulia. The first thing that has to be done before calling any other Julia C function is to initialize Julia. This is done by calling ``jl_init``, which takes as argument a C string (``const char*``) to the location where Julia is installed. When the argument is ``NULL``, Julia tries to determine the install location automatically. diff --git a/examples/embedding.c b/examples/embedding.c index f9ab53ae9e9e9..fadfd70c997de 100644 --- a/examples/embedding.c +++ b/examples/embedding.c @@ -9,8 +9,9 @@ double my_c_sqrt(double x) int main() { + char a, b, c; + SET_STACK_CHK_GUARD(a,b,c); jl_init(NULL); - JL_SET_STACK_BASE; { // Simple running Julia code @@ -94,5 +95,7 @@ int main() } } + jl_atexit_hook(); + CLR_STACK_CHK_GUARD(a,b,c); return 0; } diff --git a/src/init.c b/src/init.c index cd2288df6d196..335d82d152d5b 100644 --- a/src/init.c +++ b/src/init.c @@ -848,6 +848,7 @@ static void jl_resolve_sysimg_location(JL_IMAGE_SEARCH rel) void _julia_init(JL_IMAGE_SEARCH rel) { + libsupport_init(); jl_io_loop = uv_default_loop(); // this loop will internal events (spawning process etc.), // best to call this first, since it also initializes libuv jl_resolve_sysimg_location(rel); From dc8c12addcae4a162b319119295820a14842fb5b Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sun, 14 Dec 2014 00:42:05 -0500 Subject: [PATCH 23/40] some more init cleanup, and comments --- src/init.c | 51 +++++++++++++++++++++++++++++---------------------- ui/Makefile | 1 - 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/src/init.c b/src/init.c index 335d82d152d5b..8ec4fb1138549 100644 --- a/src/init.c +++ b/src/init.c @@ -739,6 +739,27 @@ kern_return_t catch_exception_raise(mach_port_t exception_port, #endif +static int isabspath(const char *in) +{ +#ifdef _OS_WINDOWS_ + char c0 = in[0]; + if (c0 == '/' || c0 == '\\') { + return 1; // absolute path relative to %CD% (current drive), or UNC + } + else { + int s = strlen(in); + if (s > 2) { + char c1 = in[1]; + char c2 = in[2]; + if (c1 == ':' && (c2 == '/' || c2 == '\\')) return 1; // absolute path + } + } +#else + if (jl_compileropts.image_file[0] == '/') return 1; // absolute path +#endif + return 0; // relative path +} + static char *abspath(const char *in) { // compute an absolute path location, so that chdir doesn't change the file reference #ifndef _OS_WINDOWS_ @@ -781,9 +802,13 @@ static char *abspath(const char *in) } static void jl_resolve_sysimg_location(JL_IMAGE_SEARCH rel) -{ // note: if you care about lost memory, you should compare the - // pointers in jl_compileropts before and after calling julia_init() - // and call the appropriate free function on the originals for any that changed +{ // this function resolves the paths in jl_compileropts to absolute file locations as needed + // and it replaces the pointers to `julia_home`, `julia_bin`, `image_file`, and `build_path` + // it may fail, print an error, and exit(1) if any of these paths are longer than PATH_MAX + // + // note: if you care about lost memory, you should call the appropriate `free()` function + // on the original pointer for each `char*` you've inserted into `jl_compileropts`, after + // calling `julia_init()` char *free_path = (char*)malloc(PATH_MAX); size_t path_size = PATH_MAX; if (uv_exepath(free_path, &path_size)) { @@ -806,25 +831,7 @@ static void jl_resolve_sysimg_location(JL_IMAGE_SEARCH rel) free(free_path); free_path = NULL; if (jl_compileropts.image_file) { - if (rel == JL_IMAGE_JULIA_HOME) { -#ifdef _OS_WINDOWS_ - char c0 = jl_compileropts.image_file[0]; - if (c0 == '/' || c0 == '\\') { - rel = 0; // absolute path relative to %CD% (current drive) - } - else { - int s = strlen(jl_compileropts.image_file); - if (s > 2) { - char c1 = jl_compileropts.image_file[1]; - char c2 = jl_compileropts.image_file[2]; - if (c1 == ':' && (c2 == '/' || c2 == '\\')) rel = 0; // absolute path - } - } -#else - if (jl_compileropts.image_file[0] == '/') rel = 0; // absolute path -#endif - } - if (rel == JL_IMAGE_JULIA_HOME) { + if (rel == JL_IMAGE_JULIA_HOME && !isabspath(jl_compileropts.image_file)) { // build time path, relative to JULIA_HOME free_path = (char*)malloc(PATH_MAX); int n = snprintf(free_path, PATH_MAX, "%s" PATHSEPSTRING "%s", diff --git a/ui/Makefile b/ui/Makefile index 181e79fa9ad75..d877f1027c09e 100644 --- a/ui/Makefile +++ b/ui/Makefile @@ -58,7 +58,6 @@ julia_res.o: $(JULIAHOME)/contrib/windows/julia.rc $(CROSS_COMPILE)windres $< -O coff -o $@ -DJLVER=$$JLVERi -DJLVER_STR=\\\"$$JLVER\\\" $(build_bindir)/julia$(EXE): julia_res.o $(build_bindir)/julia-debug$(EXE): julia_res.o -JLDFLAGS += julia_res.o endif endif From ebdaed77e7c5027e6662796295f75b9356991460 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sun, 14 Dec 2014 02:57:20 -0500 Subject: [PATCH 24/40] unbreak debuginfo on win64 mcjit (llvm35) --- src/debuginfo.cpp | 44 ++++++++++++++++++++------------------------ 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index b985d5f9e5ffe..00437431db3be 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -33,7 +33,7 @@ #include #endif -#if defined(USE_MCJIT) && !defined(LLVM36) && !defined(_OS_LINUX_) +#if defined(USE_MCJIT) && !defined(LLVM36) && defined(_OS_DARWIN_) #include "../deps/llvm-3.5.0/lib/ExecutionEngine/MCJIT/MCJIT.h" #endif @@ -198,11 +198,9 @@ class JuliaJITEventListener: public JITEventListener virtual void NotifyObjectEmitted(const ObjectImage &obj) #endif { - uint64_t SectAddr, Addr, SectionAddr; + uint64_t Addr; uint64_t Size; object::SymbolRef::Type SymbolType; -#ifndef _OS_LINUX_ - StringRef sName; #ifdef LLVM36 object::section_iterator Section = obj.section_begin(); object::section_iterator EndSection = obj.section_end(); @@ -211,8 +209,11 @@ class JuliaJITEventListener: public JITEventListener object::section_iterator EndSection = obj.end_sections(); bool isText; #endif +#ifndef _OS_LINUX_ + StringRef sName; #endif #ifdef _OS_WINDOWS_ + uint64_t SectionAddr; uint64_t SectionSize; #endif @@ -221,37 +222,34 @@ class JuliaJITEventListener: public JITEventListener sym_iter.getType(SymbolType); if (SymbolType != object::SymbolRef::ST_Function) continue; sym_iter.getSize(Size); - sym_iter.getAddress(SectAddr); -#ifdef _OS_LINUX_ - SectionAddr = 0; -#else + sym_iter.getAddress(Addr); sym_iter.getSection(Section); if (Section == EndSection) continue; - Section = Section->getRelocatedSection(); - if (Section == EndSection) continue; #if defined(LLVM36) if (!Section->isText()) continue; - Section->getName(sName); - SectionAddr = L.getSectionLoadAddress(sName); #else if (Section->isText(isText) || !isText) continue; +#endif +#ifdef _OS_DARWIN_ +#if defined(LLVM36) + Section->getName(sName); + Addr += L.getSectionLoadAddress(sName); +#else sym_iter.getName(sName); - SectionAddr = ((MCJIT*)jl_ExecutionEngine)->getSymbolAddress(sName, true); - if (!SectionAddr && sName[0] == '_') { + Addr = ((MCJIT*)jl_ExecutionEngine)->getSymbolAddress(sName, true); + if (!Addr && sName[0] == '_') { sName = sName.substr(1); - SectionAddr = ((MCJIT*)jl_ExecutionEngine)->getSymbolAddress(sName, true); + Addr = ((MCJIT*)jl_ExecutionEngine)->getSymbolAddress(sName, true); } - if (!SectionAddr) continue; - SectionAddr -= SectAddr; + if (!Addr) continue; #endif -#endif // _OS_LINUX_ - Addr = SectAddr + SectionAddr; -#ifdef _OS_WINDOWS_ -#ifdef LLVM36 +#elif defined(_OS_WINDOWS_) + Section->getAddress(SectionAddr); + Section->getSize(SectionSize); sym_iter.getName(sName); +#ifndef _CPU_X86_ if (sName[0] == '_') sName = sName.substr(1); #endif - Section->getSize(SectionSize); create_PRUNTIME_FUNCTION( (uint8_t*)(intptr_t)Addr, (size_t)Size, sName, (uint8_t*)(intptr_t)SectionAddr, (size_t)SectionSize); @@ -457,8 +455,6 @@ void jl_getDylibFunctionInfo(const char **name, size_t *line, const char **filen } else if (*fromC) { // No debug info, use dll name instead *filename = fname; - } else { - *filename = ""; } jl_in_stackwalk = 0; #else // ifdef _OS_WINDOWS_ From 5cff519069c4c9653dae25621ee082386b327391 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sun, 14 Dec 2014 14:47:06 -0500 Subject: [PATCH 25/40] unbreak win64 COPY_STACK --- Make.inc | 2 +- doc/manual/embedding.rst | 6 +++--- examples/Makefile | 3 --- examples/embedding.c | 11 ++++++----- src/codegen.cpp | 5 ++--- src/init.c | 4 ++++ src/julia.h | 33 +++++++++++++++------------------ src/task.c | 8 ++++++-- ui/repl.c | 6 +++--- 9 files changed, 40 insertions(+), 38 deletions(-) diff --git a/Make.inc b/Make.inc index db89e34b2cde5..9be382c5424bb 100644 --- a/Make.inc +++ b/Make.inc @@ -679,7 +679,7 @@ endif ifeq ($(OS), WINNT) ifneq ($(USEMSVC), 1) OSLIBS += -Wl,--export-all-symbols -Wl,--version-script=$(JULIAHOME)/src/julia.expmap \ - $(NO_WHOLE_ARCHIVE) -lpsapi -lkernel32 -lws2_32 -liphlpapi -lwinmm -ldbghelp -lssp + $(NO_WHOLE_ARCHIVE) -lpsapi -lkernel32 -lws2_32 -liphlpapi -lwinmm -ldbghelp JLDFLAGS = -Wl,--stack,8388608 ifeq ($(ARCH),i686) JLDFLAGS += -Wl,--large-address-aware diff --git a/doc/manual/embedding.rst b/doc/manual/embedding.rst index 230d3c5b953ad..c8bd5900025c7 100644 --- a/doc/manual/embedding.rst +++ b/doc/manual/embedding.rst @@ -19,8 +19,8 @@ We start with a simple C program that initializes Julia and calls some Julia cod int main(int argc, char *argv[]) { /* optional: randomize the stack guard */ - char a, b, c; - SET_STACK_CHK_GUARD(a,b,c); + char a=255, b='\n', c=0; + SWAP_STACK_CHK_GUARD(a,b,c); /* required: setup the julia context */ jl_init(NULL); @@ -36,7 +36,7 @@ We start with a simple C program that initializes Julia and calls some Julia cod jl_atexit_hook(); /* if the stack guard is set: reset the stack guard */ - CLR_STACK_CHK_GUARD(a,b,c); + SWAP_STACK_CHK_GUARD(a,b,c); return 0; } diff --git a/examples/Makefile b/examples/Makefile index c481347082774..66d7210adfc91 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -1,9 +1,6 @@ JULIAHOME = $(abspath ..) include $(JULIAHOME)/Make.inc -override CFLAGS += $(JCFLAGS) -override CXXFLAGS += $(JCXXFLAGS) - FLAGS = -Wall -Wno-strict-aliasing -fno-omit-frame-pointer \ -I$(JULIAHOME)/src -I$(JULIAHOME)/src/support -I$(build_includedir) $(CFLAGS) diff --git a/examples/embedding.c b/examples/embedding.c index fadfd70c997de..4d7f3d29cf90e 100644 --- a/examples/embedding.c +++ b/examples/embedding.c @@ -9,8 +9,8 @@ double my_c_sqrt(double x) int main() { - char a, b, c; - SET_STACK_CHK_GUARD(a,b,c); + char a=255, b='\n', c=0; + SWAP_STACK_CHK_GUARD(a,b,c); jl_init(NULL); { @@ -52,14 +52,15 @@ int main() double* xData = jl_array_data(x); - for(size_t i=0; isetDoesNotReturn(); + jl__stack_chk_fail->setDoesNotReturn(); add_named_global(jl__stack_chk_fail, (void*)&__stack_chk_fail); jltrue_var = global_to_llvm("jl_true", (void*)&jl_true, m); diff --git a/src/init.c b/src/init.c index 41c99e47b4a54..54a0618c653d6 100644 --- a/src/init.c +++ b/src/init.c @@ -104,6 +104,10 @@ jl_compileropts_t jl_compileropts = { NULL, // julia_home int jl_boot_file_loaded = 0; int exit_on_sigint = 0; +void *__stack_chk_guard = NULL; +DLLEXPORT unsigned char *jl_stack_chk_guard = + (unsigned char *)&__stack_chk_guard; + char *jl_stack_lo; char *jl_stack_hi; size_t jl_page_size; diff --git a/src/julia.h b/src/julia.h index 095c6782b077f..2423e327d4754 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1347,24 +1347,21 @@ extern DLLEXPORT jl_compileropts_t jl_compileropts; #define JL_COMPILEROPT_DUMPBITCODE_ON 1 #define JL_COMPILEROPT_DUMPBITCODE_OFF 2 -DLLEXPORT extern void *__stack_chk_guard; -#define SET_STACK_CHK_GUARD(a,b,c) do { \ - unsigned char *p = (unsigned char *)&__stack_chk_guard; \ - a = p[sizeof(__stack_chk_guard)-1]; \ - b = p[sizeof(__stack_chk_guard)-2]; \ - c = p[0]; \ - /* If you have the ability to generate random numbers - * in your kernel then they should be used here */ \ - p[sizeof(__stack_chk_guard)-1] = 255; \ - p[sizeof(__stack_chk_guard)-2] = '\n'; \ - p[0] = 0; \ - } while (0) - -#define CLR_STACK_CHK_GUARD(a,b,c) do { \ - unsigned char *p = (unsigned char *)&__stack_chk_guard; \ - p[sizeof(__stack_chk_guard)-1] = a; \ - p[sizeof(__stack_chk_guard)-2] = b; \ - p[0] = c; \ +/* If you have the ability to generate random numbers +* in your kernel then they should be used here */ +DLLEXPORT extern unsigned char *jl_stack_chk_guard; +#define SWAP_STACK_CHK_GUARD(a,b,c) do { \ + a ^= jl_stack_chk_guard[sizeof(void*)-1]; \ + jl_stack_chk_guard[sizeof(void*)-1] ^= a; \ + a ^= jl_stack_chk_guard[sizeof(void*)-1]; \ + \ + b ^= jl_stack_chk_guard[sizeof(void*)-2]; \ + jl_stack_chk_guard[sizeof(void*)-2] ^= b; \ + b ^= jl_stack_chk_guard[sizeof(void*)-2]; \ + \ + c ^= jl_stack_chk_guard[0]; \ + jl_stack_chk_guard[0] ^= c; \ + c ^= jl_stack_chk_guard[0]; \ } while (0) #ifdef __cplusplus diff --git a/src/task.c b/src/task.c index 01d551509d564..d15e355a897a2 100644 --- a/src/task.c +++ b/src/task.c @@ -341,20 +341,24 @@ static void ctx_switch(jl_task_t *t, jl_jmp_buf *where) restore_stack(t, where, NULL); } else { #ifdef ASM_COPY_STACKS + void *stackbase = jl_stackbase - 0x10; #ifdef _CPU_X86_64_ +#ifdef _OS_WINDOWS_ + stackbase -= 0x20; +#endif asm(" movq %0, %%rsp;\n" " xorq %%rbp, %%rbp;\n" " push %%rbp;\n" // instead of RSP " jmp %P1;\n" // call stack_task with fake stack frame " ud2" - : : "r"(jl_stackbase-0x10), "i"(start_task) : "memory" ); + : : "r"(stackbase), "i"(start_task) : "memory" ); #elif defined(_CPU_X86_) asm(" movl %0, %%esp;\n" " xorl %%ebp, %%ebp;\n" " push %%ebp;\n" // instead of ESP " jmp %P1;\n" // call stack_task with fake stack frame " ud2" - : : "r"(jl_stackbase-0x10), "i"(start_task) : "memory" ); + : : "r"(stackbase), "i"(start_task) : "memory" ); #else #error ASM_COPY_STACKS not supported on this cpu architecture #endif diff --git a/ui/repl.c b/ui/repl.c index 0c18ec88b1144..298f891742f84 100644 --- a/ui/repl.c +++ b/ui/repl.c @@ -330,8 +330,8 @@ int wmain(int argc, wchar_t *argv[], wchar_t *envp[]) argv[i] = (wchar_t*)arg; } #endif - char a,b,c; - SET_STACK_CHK_GUARD(a,b,c); + char a=255,b='\n',c=0; + SWAP_STACK_CHK_GUARD(a,b,c); libsupport_init(); parse_opts(&argc, (char***)&argv); if (lisp_prompt) { @@ -342,7 +342,7 @@ int wmain(int argc, wchar_t *argv[], wchar_t *envp[]) int ret = true_main(argc, (char**)argv); jl_atexit_hook(); julia_save(); - CLR_STACK_CHK_GUARD(a,b,c); + SWAP_STACK_CHK_GUARD(a,b,c); return ret; } From 7e3fc7cda88e0153d925dd51ccac3ea6b46f5774 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Wed, 20 Aug 2014 19:13:43 -0400 Subject: [PATCH 26/40] Support building of system image in binary builds This commit adds a few new pieces of functionality: * The `contrib/build_sysimg.jl` script which builds a new system image. This method can save the system image wherever the user desires, e.g. it could be stored in `~/.julia`, to allow for per-user system images each customized with packages in their own `userimg.jl`, for instance. Or on a restricted system, this allows for creation of a system image without root access. * The removal of compile-time `JULIA_CPU_TARGET`, in favor of runtime `--cpu-target`/`-C` command-line flags which default to `"native"` but can be set to `"native"`, `"i386"` or `"core2"`. This allows the creation of a system image targeting user-supplied cpu features, e.g. `cd base; ../julia -C i386 --build /tmp/sys_i386 sysimg.jl`. * I implemented runtime selection of the cpu target by adding a new member to the `jl_compileropts_t` structure called `cpu_target`. * Because all julia executables are now created equal, (rather than before where a julia executable needed to have the same `JULIA_CPU_TARGET` set internally as the system image had when it was built) we need to know what CPU feature set the system image is targeting before we initialize code generation. So a new function `jl_get_system_image_cpu_target()` is exported, which does exactly what it sounds like. * I added newlines to the end of a few error messages. * I found an old parser option `-T` which hadn't been removed yet, so I took the opportunity to do so. When testing this change out, I found [this gist](https://gist.github.com/staticfloat/93d7050a08ff7bb52373) helpful to put into my `~/.juliarc.jl` --- Makefile | 4 +-- contrib/build_sysimg.jl | 77 +++++++++++++++++++++++++++++++++++++++++ src/cgutils.cpp | 4 +-- src/codegen.cpp | 8 ++--- src/dump.c | 33 +++++++++++++++--- src/init.c | 9 +++++ src/julia.h | 2 ++ ui/repl.c | 16 +++++++-- 8 files changed, 136 insertions(+), 17 deletions(-) create mode 100644 contrib/build_sysimg.jl diff --git a/Makefile b/Makefile index d11f258e73626..44d8aa2a75af0 100644 --- a/Makefile +++ b/Makefile @@ -140,14 +140,14 @@ endif $(build_private_libdir)/sys0.o: @$(call PRINT_JULIA, cd base && \ - $(call spawn,$(JULIA_EXECUTABLE)) --build $(call cygpath_w,$(build_private_libdir)/sys0) sysimg.jl) + $(call spawn,$(JULIA_EXECUTABLE)) -C $(JULIA_CPU_TARGET) --build $(call cygpath_w,$(build_private_libdir)/sys0) sysimg.jl) BASE_SRCS := $(wildcard base/*.jl base/*/*.jl base/*/*/*.jl) ,:=, $(build_private_libdir)/sys.o: VERSION $(BASE_SRCS) $(build_docdir)/helpdb.jl $(build_private_libdir)/sys0.$(SHLIB_EXT) @$(call PRINT_JULIA, cd base && \ - $(call spawn,$(JULIA_EXECUTABLE)) --build $(call cygpath_w,$(build_private_libdir)/sys) \ + $(call spawn,$(JULIA_EXECUTABLE)) -C $(JULIA_CPU_TARGET) --build $(call cygpath_w,$(build_private_libdir)/sys) \ -J$(call cygpath_w,$(build_private_libdir))/$$([ -e $(build_private_libdir)/sys.ji ] && echo sys.ji || echo sys0.ji) -f sysimg.jl \ || { echo '*** This error is usually fixed by running `make clean`. If the error persists$(,) try `make cleanall`. ***' && false; } ) diff --git a/contrib/build_sysimg.jl b/contrib/build_sysimg.jl new file mode 100644 index 0000000000000..9bfb20710daff --- /dev/null +++ b/contrib/build_sysimg.jl @@ -0,0 +1,77 @@ +# By default, put the system image next to libjulia +build_sysimg(;force=false, cpu_target="native") = build_sysimg(joinpath(dirname(Sys.dlpath("libjulia")),"sys"), force=force, cpu_target=cpu_target) + +# Build a system image binary at sysimg_path.dlext. If a system image is already loaded, error out, or continue if force = true +function build_sysimg(sysimg_path; force=false, cpu_target="native") + # Unless force == true, quit out if a sysimg is already loadable + sysimg = dlopen_e("sys") + if !force && sysimg != C_NULL + println("System image already loaded at $(Sys.dlpath(sysimg)), pass \"force=true\" to override") + return; + end + + # Enter base/ and setup some useful paths + base_dir = dirname(Base.find_source_file("sysimg.jl")) + cd(base_dir) do + julia = joinpath(JULIA_HOME, "julia") + julia_libdir = dirname(Sys.dlpath("libjulia")) + ld = find_system_linker() + + # Ensure we have write-permissions to wherever we're trying to write to + if !success(`touch $sysimg_path.$(Sys.dlext)`) + error("$sysimg_path unwritable, ensure parent directory exists and is writable! (Do you need to run this with sudo?)") + end + + # Start by building sys0.{ji,o} + sys0_path = joinpath(dirname(sysimg_path), "sys0") + println("Building sys0.o...") + println("$julia -C $cpu_target --build $sys0_path sysimg.jl") + run(`$julia -C $cpu_target --build $sys0_path sysimg.jl`) + + # Bootstrap off of that to create sys.{ji,o} + println("Building sys.o...") + println("$julia -C $cpu_target --build $sysimg_path -J $sys0_path.ji -f sysimg.jl") + run(`$julia -C $cpu_target --build $sysimg_path -J $sys0_path.ji -f sysimg.jl`) + + # Link sys.o into sys.$(dlext) + FLAGS = ["-L$julia_libdir"] + if OS_NAME == :Darwin + push!(FLAGS, "-dylib") + push!(FLAGS, "-undefined") + push!(FLAGS, "dynamic_lookup") + else + push!(FLAGS, "--unresolved-symbols") + push!(FLAGS, "ignore-all") + end + @windows_only append!(FLAGS, ["-L$JULIA_HOME", "-ljulia", "-lssp"]) + + println("Linking sys.$(Sys.dlext)") + run(`$ld $FLAGS -o $sysimg_path.$(Sys.dlext) $sysimg_path.o`) + end +end + +# Search for a linker to link sys.o into sys.dl_ext. Honor LD environment variable, otherwise search for something we know works +function find_system_linker() + if haskey( ENV, "LD" ) + if !success(`which $(ENV["LD"])`) + warn("Using linker override $(ENV["LD"]), but unable to find `$(ENV["LD"])`") + end + return ENV["LD"] + end + + poss_linkers = ["ld", "link"] + + for linker in poss_linkers + try + if success(`which $linker`) + return linker + end + end + end + + error( "No supported linker found (tried $(join(poss_linkers, ", "))), override with LD environment variable!" ) +end + +if !isinteractive() + build_sysimg() +end diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 09de8dea0096a..803307ae065d8 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -246,7 +246,7 @@ static void jl_gen_llvm_gv_array(llvm::Module *mod, SmallVectorgetType(), true, @@ -255,7 +255,7 @@ static void jl_gen_llvm_gv_array(llvm::Module *mod, SmallVectorhandle; @@ -1350,6 +1349,32 @@ extern void jl_get_builtin_hooks(void); extern void jl_get_system_hooks(void); extern void jl_get_uv_hooks(); +// Takes in a path of the form "usr/lib/julia/sys.ji", such as passed in to jl_restore_system_image() +DLLEXPORT +const char * jl_get_system_image_cpu_target(char *fname) +{ + // If passed NULL, don't even bother + if (!fname) + return NULL; + + // First, get "sys" from "sys.ji" + char *fname_shlib = (char*)alloca(strlen(fname)); + strcpy(fname_shlib, fname); + char *fname_shlib_dot = strrchr(fname_shlib, '.'); + if (fname_shlib_dot != NULL) + *fname_shlib_dot = 0; + + // Get handle to sys.so + uv_lib_t * sysimg_handle = jl_load_dynamic_library_e(fname_shlib, JL_RTLD_DEFAULT | JL_RTLD_GLOBAL); + + // Return jl_sysimg_cpu_target if we can + if (sysimg_handle) + return (const char *)jl_dlsym(sysimg_handle, "jl_sysimg_cpu_target"); + + // If something goes wrong, return NULL + return NULL; +} + DLLEXPORT void jl_restore_system_image(const char *fname) { diff --git a/src/init.c b/src/init.c index 54a0618c653d6..2a11c995350a9 100644 --- a/src/init.c +++ b/src/init.c @@ -92,6 +92,7 @@ jl_compileropts_t jl_compileropts = { NULL, // julia_home NULL, // julia_bin NULL, // build_path system_image_path, // image_file + NULL, // cpu_target ("native", "core2", etc...) 0, // code_coverage 0, // malloc_log JL_COMPILEROPT_CHECK_BOUNDS_DEFAULT, @@ -940,6 +941,14 @@ void _julia_init(JL_IMAGE_SEARCH rel) jl_init_frontend(); jl_init_types(); jl_init_tasks(jl_stack_lo, jl_stack_hi-jl_stack_lo); + + // If we are able to load the imageFile and get a cpu_target, use that unless user has overridden + if (jl_compileropts.cpu_target == NULL) { + const char * sysimg_cpu_target = jl_get_system_image_cpu_target(imageFile); + + // If we can't load anything from the sysimg, default to native + jl_compileropts.cpu_target = sysimg_cpu_target ? sysimg_cpu_target : "native"; + } jl_init_codegen(); jl_an_empty_cell = (jl_value_t*)jl_alloc_cell_1d(0); diff --git a/src/julia.h b/src/julia.h index 2423e327d4754..acc32853d64c5 100644 --- a/src/julia.h +++ b/src/julia.h @@ -868,6 +868,7 @@ DLLEXPORT int julia_trampoline(int argc, char *argv[], int (*pmain)(int ac,char DLLEXPORT void jl_atexit_hook(void); DLLEXPORT void NORETURN jl_exit(int status); +DLLEXPORT const char * jl_get_system_image_cpu_target(const char *fname); DLLEXPORT void jl_save_system_image(const char *fname); DLLEXPORT void jl_restore_system_image(const char *fname); DLLEXPORT int jl_save_new_module(const char *fname, jl_module_t *mod); @@ -1320,6 +1321,7 @@ typedef struct { const char *julia_bin; const char *build_path; const char *image_file; + const char *cpu_target; int8_t code_coverage; int8_t malloc_log; int8_t check_bounds; diff --git a/ui/repl.c b/ui/repl.c index 298f891742f84..befd1e32c1af0 100644 --- a/ui/repl.c +++ b/ui/repl.c @@ -53,7 +53,8 @@ static const char *opts = " -E, --print Evaluate and show \n" " -P, --post-boot Evaluate , but don't disable interactive mode\n" " -L, --load Load immediately on all processors\n" - " -J, --sysimage Start up with the given system image file\n\n" + " -J, --sysimage Start up with the given system image file\n" + " -C --cpu-target Limit usage of cpu features up to \n\n" " -p Run n local processes\n" " --machinefile Run processes on hosts listed in \n\n" @@ -77,15 +78,15 @@ static const char *opts = void parse_opts(int *argcp, char ***argvp) { - static char* shortopts = "+H:T:hJ:O"; + static char* shortopts = "+H:hJ:C:O"; static struct option longopts[] = { { "home", required_argument, 0, 'H' }, - { "tab", required_argument, 0, 'T' }, { "build", required_argument, 0, 'b' }, { "lisp", no_argument, &lisp_prompt, 1 }, { "help", no_argument, 0, 'h' }, { "sysimage", required_argument, 0, 'J' }, { "code-coverage", optional_argument, 0, 'c' }, + { "cpu-target", required_argument, 0, 'C' }, { "track-allocation",required_argument, 0, 'm' }, { "check-bounds", required_argument, 0, 300 }, { "optimize", no_argument, 0, 'O' }, @@ -118,6 +119,15 @@ void parse_opts(int *argcp, char ***argvp) jl_compileropts.image_file = strdup(optarg); imagepathspecified = 1; break; + case 'C': + if( strcmp(optarg, "native") == 0 || strcmp(optarg, "i386") == 0 || strcmp(optarg, "core2") == 0 ) { + jl_compileropts.cpu_target = strdup(optarg); + } + else { + ios_printf(ios_stderr, "julia: invalid cpu-target \"%s\" specified\n", optarg ); + exit(1); + } + break; case 'h': ios_printf(ios_stdout, "%s%s", usage, opts); exit(0); From 67c3060bb3944130b615a93efcace6e2e97af0df Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Thu, 21 Aug 2014 01:40:54 -0400 Subject: [PATCH 27/40] Don't try to check for valid cpu targets, we support a lot now --- ui/repl.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/ui/repl.c b/ui/repl.c index befd1e32c1af0..4be3f55e87643 100644 --- a/ui/repl.c +++ b/ui/repl.c @@ -119,14 +119,8 @@ void parse_opts(int *argcp, char ***argvp) jl_compileropts.image_file = strdup(optarg); imagepathspecified = 1; break; - case 'C': - if( strcmp(optarg, "native") == 0 || strcmp(optarg, "i386") == 0 || strcmp(optarg, "core2") == 0 ) { - jl_compileropts.cpu_target = strdup(optarg); - } - else { - ios_printf(ios_stderr, "julia: invalid cpu-target \"%s\" specified\n", optarg ); - exit(1); - } + case 'C': + jl_compileropts.cpu_target = strdup(optarg); break; case 'h': ios_printf(ios_stdout, "%s%s", usage, opts); From 64b0f220bebc9de05c643eba5d69369104da9dda Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Thu, 21 Aug 2014 01:41:34 -0400 Subject: [PATCH 28/40] Use iswritable() for maximum portability --- contrib/build_sysimg.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/build_sysimg.jl b/contrib/build_sysimg.jl index 9bfb20710daff..e70799f699bdf 100644 --- a/contrib/build_sysimg.jl +++ b/contrib/build_sysimg.jl @@ -18,7 +18,7 @@ function build_sysimg(sysimg_path; force=false, cpu_target="native") ld = find_system_linker() # Ensure we have write-permissions to wherever we're trying to write to - if !success(`touch $sysimg_path.$(Sys.dlext)`) + if !iswritable("$sysimg_path.$(Sys.dlext)") error("$sysimg_path unwritable, ensure parent directory exists and is writable! (Do you need to run this with sudo?)") end From 73fbc1327a38230ecfe9207a21a6cee66001adec Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Sat, 25 Oct 2014 11:41:34 -0700 Subject: [PATCH 29/40] Attempt windows compatibility, don't require linking, allow for inclusion of usrimg.jl --- contrib/build_sysimg.jl | 54 ++++++++++++++++++++++++++++++++--------- 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/contrib/build_sysimg.jl b/contrib/build_sysimg.jl index e70799f699bdf..2ab68605fd1a0 100644 --- a/contrib/build_sysimg.jl +++ b/contrib/build_sysimg.jl @@ -1,8 +1,8 @@ # By default, put the system image next to libjulia -build_sysimg(;force=false, cpu_target="native") = build_sysimg(joinpath(dirname(Sys.dlpath("libjulia")),"sys"), force=force, cpu_target=cpu_target) +build_sysimg(; userimg_path=nothing, force=false, cpu_target="native") = build_sysimg(joinpath(dirname(Sys.dlpath("libjulia")),"sys"), userimg_path=userimg_path, force=force, cpu_target=cpu_target) # Build a system image binary at sysimg_path.dlext. If a system image is already loaded, error out, or continue if force = true -function build_sysimg(sysimg_path; force=false, cpu_target="native") +function build_sysimg(sysimg_path; userimg_path=nothing, force=false, cpu_target="native") # Unless force == true, quit out if a sysimg is already loadable sysimg = dlopen_e("sys") if !force && sysimg != C_NULL @@ -10,6 +10,9 @@ function build_sysimg(sysimg_path; force=false, cpu_target="native") return; end + # Canonicalize userimg_path before we enter the base_dir + userimg_path = abspath(userimg_path) + # Enter base/ and setup some useful paths base_dir = dirname(Base.find_source_file("sysimg.jl")) cd(base_dir) do @@ -18,10 +21,18 @@ function build_sysimg(sysimg_path; force=false, cpu_target="native") ld = find_system_linker() # Ensure we have write-permissions to wherever we're trying to write to - if !iswritable("$sysimg_path.$(Sys.dlext)") + if !success(`touch $sysimg_path.$(Sys.dlext)`) error("$sysimg_path unwritable, ensure parent directory exists and is writable! (Do you need to run this with sudo?)") end + # Copy in userimg.jl if it exists... + if userimg_path != nothing + if !isreadable(userimg_path) + error("$userimg_path is not readable, ensure it is an absolute path!") + end + cp(userimg_path, "userimg.jl") + end + # Start by building sys0.{ji,o} sys0_path = joinpath(dirname(sysimg_path), "sys0") println("Building sys0.o...") @@ -45,9 +56,18 @@ function build_sysimg(sysimg_path; force=false, cpu_target="native") end @windows_only append!(FLAGS, ["-L$JULIA_HOME", "-ljulia", "-lssp"]) - println("Linking sys.$(Sys.dlext)") - run(`$ld $FLAGS -o $sysimg_path.$(Sys.dlext) $sysimg_path.o`) + if ld != nothing + println("Linking sys.$(Sys.dlext)") + run(`$ld $FLAGS -o $sysimg_path.$(Sys.dlext) $sysimg_path.o`) + end + + # Cleanup userimg.jl + try + rm("userimg.jl") + end end + + println("System image built; run julia -J $sysimg_path.ji") end # Search for a linker to link sys.o into sys.dl_ext. Honor LD environment variable, otherwise search for something we know works @@ -59,17 +79,27 @@ function find_system_linker() return ENV["LD"] end - poss_linkers = ["ld", "link"] + # On Windows, check to see if WinRPM is installed, and if so, see if binutils is installed + @windows_only try + using WinRPM + if WinRPM.installed("binutils") + ENV["PATH"] = "$(ENV["PATH"]):$(joinpath(WinRPM.installdir,"usr","$(Sys.ARCH)-w64-mingw32","sys-root","mingw","bin"))" + else + throw() + end + catch + warn("binutils package not installed! Install via WinRPM.install(\"binutils\") for faster sysimg load times" ) + end - for linker in poss_linkers - try - if success(`which $linker`) - return linker - end + + # See if `ld` exists + try + if success(`which ld`) + return "ld" end end - error( "No supported linker found (tried $(join(poss_linkers, ", "))), override with LD environment variable!" ) + warn( "No supported linker found; sysimg load times will be longer!" ) end if !isinteractive() From 5d8f73acef6eaf60b85ac5427a376a40b9ed3598 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Sat, 25 Oct 2014 12:02:02 -0700 Subject: [PATCH 30/40] Silently back out of linking on windows if LLVM < 3.5 --- contrib/build_sysimg.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/contrib/build_sysimg.jl b/contrib/build_sysimg.jl index 2ab68605fd1a0..9154888ea9d9f 100644 --- a/contrib/build_sysimg.jl +++ b/contrib/build_sysimg.jl @@ -81,6 +81,11 @@ function find_system_linker() # On Windows, check to see if WinRPM is installed, and if so, see if binutils is installed @windows_only try + # Silently fail on windows if LLVM < 3.5 + if convert(VersionNumber, Base.libllvm_version) < v"3.5.0" + return nothing + end + using WinRPM if WinRPM.installed("binutils") ENV["PATH"] = "$(ENV["PATH"]):$(joinpath(WinRPM.installdir,"usr","$(Sys.ARCH)-w64-mingw32","sys-root","mingw","bin"))" From 2b4a99188e3160bb54c16aa4b6a25e8980ccbdb5 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Sun, 26 Oct 2014 22:33:53 -0700 Subject: [PATCH 31/40] Fixes, cleanup, and resilliency. --- contrib/build_sysimg.jl | 106 +++++++++++++++++++++------------------- 1 file changed, 57 insertions(+), 49 deletions(-) diff --git a/contrib/build_sysimg.jl b/contrib/build_sysimg.jl index 9154888ea9d9f..ef4c091f8a3f7 100644 --- a/contrib/build_sysimg.jl +++ b/contrib/build_sysimg.jl @@ -7,63 +7,71 @@ function build_sysimg(sysimg_path; userimg_path=nothing, force=false, cpu_target sysimg = dlopen_e("sys") if !force && sysimg != C_NULL println("System image already loaded at $(Sys.dlpath(sysimg)), pass \"force=true\" to override") - return; + return end # Canonicalize userimg_path before we enter the base_dir - userimg_path = abspath(userimg_path) + if userimg_path != nothing + userimg_path = abspath(userimg_path) + end # Enter base/ and setup some useful paths base_dir = dirname(Base.find_source_file("sysimg.jl")) cd(base_dir) do - julia = joinpath(JULIA_HOME, "julia") - julia_libdir = dirname(Sys.dlpath("libjulia")) - ld = find_system_linker() - - # Ensure we have write-permissions to wherever we're trying to write to - if !success(`touch $sysimg_path.$(Sys.dlext)`) - error("$sysimg_path unwritable, ensure parent directory exists and is writable! (Do you need to run this with sudo?)") - end - - # Copy in userimg.jl if it exists... - if userimg_path != nothing - if !isreadable(userimg_path) - error("$userimg_path is not readable, ensure it is an absolute path!") + try + julia = joinpath(JULIA_HOME, "julia") + julia_libdir = dirname(Sys.dlpath("libjulia")) + ld = find_system_linker() + + # Ensure we have write-permissions to wherever we're trying to write to + try + touch("$sysimg_path.$(Sys.dlext)") + catch + error("$sysimg_path unwritable, ensure parent directory exists and is writable! (Do you need to run this with sudo?)") end - cp(userimg_path, "userimg.jl") - end - # Start by building sys0.{ji,o} - sys0_path = joinpath(dirname(sysimg_path), "sys0") - println("Building sys0.o...") - println("$julia -C $cpu_target --build $sys0_path sysimg.jl") - run(`$julia -C $cpu_target --build $sys0_path sysimg.jl`) - - # Bootstrap off of that to create sys.{ji,o} - println("Building sys.o...") - println("$julia -C $cpu_target --build $sysimg_path -J $sys0_path.ji -f sysimg.jl") - run(`$julia -C $cpu_target --build $sysimg_path -J $sys0_path.ji -f sysimg.jl`) - - # Link sys.o into sys.$(dlext) - FLAGS = ["-L$julia_libdir"] - if OS_NAME == :Darwin - push!(FLAGS, "-dylib") - push!(FLAGS, "-undefined") - push!(FLAGS, "dynamic_lookup") - else - push!(FLAGS, "--unresolved-symbols") - push!(FLAGS, "ignore-all") - end - @windows_only append!(FLAGS, ["-L$JULIA_HOME", "-ljulia", "-lssp"]) + # Copy in userimg.jl if it exists... + if userimg_path != nothing + if !isreadable(userimg_path) + error("$userimg_path is not readable, ensure it is an absolute path!") + end + cp(userimg_path, "userimg.jl") + end - if ld != nothing - println("Linking sys.$(Sys.dlext)") - run(`$ld $FLAGS -o $sysimg_path.$(Sys.dlext) $sysimg_path.o`) - end + # Start by building sys0.{ji,o} + sys0_path = joinpath(dirname(sysimg_path), "sys0") + println("Building sys0.o...") + println("$julia -C $cpu_target --build $sys0_path sysimg.jl") + run(`$julia -C $cpu_target --build $sys0_path sysimg.jl`) + + # Bootstrap off of that to create sys.{ji,o} + println("Building sys.o...") + println("$julia -C $cpu_target --build $sysimg_path -J $sys0_path.ji -f sysimg.jl") + run(`$julia -C $cpu_target --build $sysimg_path -J $sys0_path.ji -f sysimg.jl`) + + # Link sys.o into sys.$(dlext) + FLAGS = ["-L$julia_libdir"] + if OS_NAME == :Darwin + push!(FLAGS, "-dylib") + push!(FLAGS, "-undefined") + push!(FLAGS, "dynamic_lookup") + push!(FLAGS, "-macosx_version_min") + push!(FLAGS, "10.7") + else + push!(FLAGS, "--unresolved-symbols") + push!(FLAGS, "ignore-all") + end + @windows_only append!(FLAGS, ["-L$JULIA_HOME", "-ljulia", "-lssp"]) - # Cleanup userimg.jl - try - rm("userimg.jl") + if ld != nothing + println("Linking sys.$(Sys.dlext)") + run(`$ld $FLAGS -o $sysimg_path.$(Sys.dlext) $sysimg_path.o`) + end + finally + # Cleanup userimg.jl + try + rm("userimg.jl") + end end end @@ -73,8 +81,8 @@ end # Search for a linker to link sys.o into sys.dl_ext. Honor LD environment variable, otherwise search for something we know works function find_system_linker() if haskey( ENV, "LD" ) - if !success(`which $(ENV["LD"])`) - warn("Using linker override $(ENV["LD"]), but unable to find `$(ENV["LD"])`") + if !success(`$(ENV["LD"]) -v`) + warn("Using linker override $(ENV["LD"]), but unable to run `$(ENV["LD"]) -v`") end return ENV["LD"] end @@ -99,7 +107,7 @@ function find_system_linker() # See if `ld` exists try - if success(`which ld`) + if success(`ld -v`) return "ld" end end From c46da146e8fc3bdcc060e1189d4d203143ec7ae1 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Mon, 27 Oct 2014 13:55:06 -0700 Subject: [PATCH 32/40] We need to add `-shared` on Linux when invoking `ld`. --- contrib/build_sysimg.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contrib/build_sysimg.jl b/contrib/build_sysimg.jl index ef4c091f8a3f7..f7920cd4ccce5 100644 --- a/contrib/build_sysimg.jl +++ b/contrib/build_sysimg.jl @@ -58,6 +58,9 @@ function build_sysimg(sysimg_path; userimg_path=nothing, force=false, cpu_target push!(FLAGS, "-macosx_version_min") push!(FLAGS, "10.7") else + if OS_NAME == :Linux + push!(FLAGS, "-shared") + end push!(FLAGS, "--unresolved-symbols") push!(FLAGS, "ignore-all") end From b5eea233bb39183aae1de6b07cfab1ee24430d30 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Tue, 4 Nov 2014 01:07:51 -0800 Subject: [PATCH 33/40] Clean up script, add usage instructions, make noninteractive usage easy --- contrib/build_sysimg.jl | 51 ++++++++++++++++++++++++++++++----------- 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/contrib/build_sysimg.jl b/contrib/build_sysimg.jl index f7920cd4ccce5..b9245789a28c8 100644 --- a/contrib/build_sysimg.jl +++ b/contrib/build_sysimg.jl @@ -1,13 +1,17 @@ -# By default, put the system image next to libjulia -build_sysimg(; userimg_path=nothing, force=false, cpu_target="native") = build_sysimg(joinpath(dirname(Sys.dlpath("libjulia")),"sys"), userimg_path=userimg_path, force=force, cpu_target=cpu_target) - -# Build a system image binary at sysimg_path.dlext. If a system image is already loaded, error out, or continue if force = true -function build_sysimg(sysimg_path; userimg_path=nothing, force=false, cpu_target="native") - # Unless force == true, quit out if a sysimg is already loadable +#!/usr/bin/env julia + +# Build a system image binary at sysimg_path.dlext. By default, put the system image next to libjulia +# Allow insertion of a userimg via userimg_path. If sysimg_path.dlext is currently loaded into memory, +# don't continue unless force is set to true. Allow targeting of a CPU architecture via cpu_target +const default_sysimg_path = joinpath(dirname(Sys.dlpath("libjulia")),"sys") +function build_sysimg(sysimg_path=default_sysimg_path, cpu_target="native", userimg_path=nothing; force=false) + # Quit out if a sysimg is already loaded and is in the same spot as sysimg_path, unless forcing sysimg = dlopen_e("sys") - if !force && sysimg != C_NULL - println("System image already loaded at $(Sys.dlpath(sysimg)), pass \"force=true\" to override") - return + if sysimg != C_NULL + if !force && Sys.dlpath(sysimg) == "$(sysimg_path).$(Sys.dlext)" + println("System image already loaded at $(Sys.dlpath(sysimg)), set force to override") + return + end end # Canonicalize userimg_path before we enter the base_dir @@ -27,7 +31,9 @@ function build_sysimg(sysimg_path; userimg_path=nothing, force=false, cpu_target try touch("$sysimg_path.$(Sys.dlext)") catch - error("$sysimg_path unwritable, ensure parent directory exists and is writable! (Do you need to run this with sudo?)") + err_msg = "Unable to modify $sysimg_path.$(Sys.dlext), ensure parent directory exists " + err_msg *= "and is writable. Absolute paths work best. Do you need to run this with sudo?)" + error( err_msg ) end # Copy in userimg.jl if it exists... @@ -72,7 +78,7 @@ function build_sysimg(sysimg_path; userimg_path=nothing, force=false, cpu_target end finally # Cleanup userimg.jl - try + if isfile("userimg.jl") rm("userimg.jl") end end @@ -81,7 +87,7 @@ function build_sysimg(sysimg_path; userimg_path=nothing, force=false, cpu_target println("System image built; run julia -J $sysimg_path.ji") end -# Search for a linker to link sys.o into sys.dl_ext. Honor LD environment variable, otherwise search for something we know works +# Search for a linker to link sys.o into sys.dl_ext. Honor LD environment variable. function find_system_linker() if haskey( ENV, "LD" ) if !success(`$(ENV["LD"]) -v`) @@ -118,6 +124,25 @@ function find_system_linker() warn( "No supported linker found; sysimg load times will be longer!" ) end +# When running this file as a script, try to do so with default values. If arguments are passed +# in, use them as the arguments to build_sysimg above if !isinteractive() - build_sysimg() + if length(ARGS) > 4 + println("Usage: build_sysimg.jl --force") + println(" is an absolute, extensionless path to store the system image at") + println(" is an LLVM cpu target to build the system image against") + println(" is the path to a user image to be baked into the system image") + println(" --force Set if you wish to overwrite the default system image") + println() + println(" Example:") + println(" build_sysimg.jl /usr/local/lib/julia/sys core2 ~/my_usrimg.jl true") + println() + println(" Running this script with no arguments is equivalent to calling it via") + println(" build_sysimg.jl $(default_sysimg_path) native") + return 0 + end + + force_flag = "--force" in ARGS + filter!(x -> x != "--force", ARGS) + build_sysimg(ARGS..., force=force_flag) end From eff5fd5168710329d68712dd77be94cc25c7dd33 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Sun, 30 Nov 2014 12:08:45 -0800 Subject: [PATCH 34/40] Allow people to build sys.dll on LLVM < 3.5.0, but warn them profusely * Do a little bit of cleanup * add "--help" * Make closing remarks more helpful and intelligent * Rebase on top of master. Again. --- contrib/build_sysimg.jl | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/contrib/build_sysimg.jl b/contrib/build_sysimg.jl index b9245789a28c8..7aa432a398fbf 100644 --- a/contrib/build_sysimg.jl +++ b/contrib/build_sysimg.jl @@ -76,6 +76,21 @@ function build_sysimg(sysimg_path=default_sysimg_path, cpu_target="native", user println("Linking sys.$(Sys.dlext)") run(`$ld $FLAGS -o $sysimg_path.$(Sys.dlext) $sysimg_path.o`) end + + println("System image successfully built at $sysimg_path.$(Sys.dlext)") + @windows_only begin + if convert(VersionNumber, Base.libllvm_version) < v"3.5.0" + LLVM_msg = "Building sys.dll on Windows against LLVM < 3.5.0 can cause incorrect backtraces!" + LLVM_msg *= " Delete generated sys.dll to avoid these problems" + warn( LLVM_msg ) + end + end + + if default_sysimg_path != sysimg_path + println("To run Julia with this image loaded, run: julia -J $sysimg_path.ji") + else + println("Julia will automatically load this system image at next startup") + end finally # Cleanup userimg.jl if isfile("userimg.jl") @@ -83,8 +98,6 @@ function build_sysimg(sysimg_path=default_sysimg_path, cpu_target="native", user end end end - - println("System image built; run julia -J $sysimg_path.ji") end # Search for a linker to link sys.o into sys.dl_ext. Honor LD environment variable. @@ -98,9 +111,11 @@ function find_system_linker() # On Windows, check to see if WinRPM is installed, and if so, see if binutils is installed @windows_only try - # Silently fail on windows if LLVM < 3.5 + # Warn about LLVM < 3.5.0 if convert(VersionNumber, Base.libllvm_version) < v"3.5.0" - return nothing + LLVM_msg = "Building sys.dll on Windows against LLVM < 3.5.0 can cause incorrect backtraces!" + LLVM_msg *= " Delete generated sys.dll to avoid these problems" + warn( LLVM_msg ) end using WinRPM @@ -110,7 +125,9 @@ function find_system_linker() throw() end catch - warn("binutils package not installed! Install via WinRPM.install(\"binutils\") for faster sysimg load times" ) + if convert(VersionNumber, Base.libllvm_version) >= v"3.5.0" + warn("Install Binutils via WinRPM.install(\"binutils\") to generate sys.dll for faster startup times" ) + end end @@ -121,18 +138,19 @@ function find_system_linker() end end - warn( "No supported linker found; sysimg load times will be longer!" ) + warn( "No supported linker found; startup times will be longer" ) end # When running this file as a script, try to do so with default values. If arguments are passed # in, use them as the arguments to build_sysimg above if !isinteractive() - if length(ARGS) > 4 - println("Usage: build_sysimg.jl --force") + if length(ARGS) > 4 || ("--help" in ARGS || "-h" in ARGS) + println("Usage: build_sysimg.jl [--force] [--help]") println(" is an absolute, extensionless path to store the system image at") println(" is an LLVM cpu target to build the system image against") println(" is the path to a user image to be baked into the system image") println(" --force Set if you wish to overwrite the default system image") + println(" --help Print out this help text and exit") println() println(" Example:") println(" build_sysimg.jl /usr/local/lib/julia/sys core2 ~/my_usrimg.jl true") From 044bbb7d2c97005b0fd69c39e70fad3a96a16344 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Sat, 13 Dec 2014 02:48:11 -0800 Subject: [PATCH 35/40] Move warnings around a bit, and install as PREFIX/bin/build_sysimg.jl --- Makefile | 2 ++ contrib/build_sysimg.jl | 18 ++++++++---------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 44d8aa2a75af0..f104712be0aee 100644 --- a/Makefile +++ b/Makefile @@ -271,6 +271,8 @@ endif # Copy system image $(INSTALL_F) $(build_private_libdir)/sys.ji $(DESTDIR)$(private_libdir) $(INSTALL_M) $(build_private_libdir)/sys.$(SHLIB_EXT) $(DESTDIR)$(private_libdir) + # Copy in system image build script + $(INSTALL_M) contrib/build_sysimg.jl $(DESTDIR)$(bindir) # Copy in all .jl sources as well cp -R -L $(build_datarootdir)/julia $(DESTDIR)$(datarootdir)/ # Copy documentation diff --git a/contrib/build_sysimg.jl b/contrib/build_sysimg.jl index 7aa432a398fbf..09e92b77f8233 100644 --- a/contrib/build_sysimg.jl +++ b/contrib/build_sysimg.jl @@ -111,23 +111,21 @@ function find_system_linker() # On Windows, check to see if WinRPM is installed, and if so, see if binutils is installed @windows_only try - # Warn about LLVM < 3.5.0 - if convert(VersionNumber, Base.libllvm_version) < v"3.5.0" - LLVM_msg = "Building sys.dll on Windows against LLVM < 3.5.0 can cause incorrect backtraces!" - LLVM_msg *= " Delete generated sys.dll to avoid these problems" - warn( LLVM_msg ) - end - using WinRPM if WinRPM.installed("binutils") + # Warn about LLVM < 3.5.0 + if convert(VersionNumber, Base.libllvm_version) < v"3.5.0" + LLVM_msg = "Building sys.dll on Windows with LLVM < 3.5.0 can cause incorrect backtraces!" + LLVM_msg *= " Delete generated sys.dll to avoid these problems" + warn( LLVM_msg ) + end + ENV["PATH"] = "$(ENV["PATH"]):$(joinpath(WinRPM.installdir,"usr","$(Sys.ARCH)-w64-mingw32","sys-root","mingw","bin"))" else throw() end catch - if convert(VersionNumber, Base.libllvm_version) >= v"3.5.0" - warn("Install Binutils via WinRPM.install(\"binutils\") to generate sys.dll for faster startup times" ) - end + warn("Install Binutils via WinRPM.install(\"binutils\") to generate sys.dll for faster startup times" ) end From e15bd1d8972c70354b78859320e6c15cd76c7812 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Sat, 13 Dec 2014 11:54:54 -0800 Subject: [PATCH 36/40] Deduplicate warning --- contrib/build_sysimg.jl | 7 ------- 1 file changed, 7 deletions(-) diff --git a/contrib/build_sysimg.jl b/contrib/build_sysimg.jl index 09e92b77f8233..a73445a06564a 100644 --- a/contrib/build_sysimg.jl +++ b/contrib/build_sysimg.jl @@ -113,13 +113,6 @@ function find_system_linker() @windows_only try using WinRPM if WinRPM.installed("binutils") - # Warn about LLVM < 3.5.0 - if convert(VersionNumber, Base.libllvm_version) < v"3.5.0" - LLVM_msg = "Building sys.dll on Windows with LLVM < 3.5.0 can cause incorrect backtraces!" - LLVM_msg *= " Delete generated sys.dll to avoid these problems" - warn( LLVM_msg ) - end - ENV["PATH"] = "$(ENV["PATH"]):$(joinpath(WinRPM.installdir,"usr","$(Sys.ARCH)-w64-mingw32","sys-root","mingw","bin"))" else throw() From 364bb0d11232ba6ae3ac4b24b29883fbf640fc10 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Sun, 14 Dec 2014 03:08:06 -0800 Subject: [PATCH 37/40] Install to usr/share/julia after all --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f104712be0aee..83a972d8036c9 100644 --- a/Makefile +++ b/Makefile @@ -272,7 +272,7 @@ endif $(INSTALL_F) $(build_private_libdir)/sys.ji $(DESTDIR)$(private_libdir) $(INSTALL_M) $(build_private_libdir)/sys.$(SHLIB_EXT) $(DESTDIR)$(private_libdir) # Copy in system image build script - $(INSTALL_M) contrib/build_sysimg.jl $(DESTDIR)$(bindir) + $(INSTALL_M) contrib/build_sysimg.jl $(DESTDIR)$(datarootdir)/julia/ # Copy in all .jl sources as well cp -R -L $(build_datarootdir)/julia $(DESTDIR)$(datarootdir)/ # Copy documentation From e59653fe5fcec609ce5df3d82ed5904d2cc56652 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Sun, 14 Dec 2014 03:42:22 -0800 Subject: [PATCH 38/40] Add some basic documentation for build_sysimg.jl --- doc/devdocs/julia.rst | 1 + doc/devdocs/sysimg.rst | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 doc/devdocs/sysimg.rst diff --git a/doc/devdocs/julia.rst b/doc/devdocs/julia.rst index df37b96e0775f..e7b7f13fe6b5b 100644 --- a/doc/devdocs/julia.rst +++ b/doc/devdocs/julia.rst @@ -12,3 +12,4 @@ cartesian meta subarrays + sysimg diff --git a/doc/devdocs/sysimg.rst b/doc/devdocs/sysimg.rst new file mode 100644 index 0000000000000..2c432e0252d83 --- /dev/null +++ b/doc/devdocs/sysimg.rst @@ -0,0 +1,35 @@ +********************* +System Image Building +********************* + +Building the Julia system image +------------------------------- + +Julia ships with a preparsed system image containing the contents of the ``Base`` module, named ``sys.ji``. This file is also precompiled into a shared library called ``sys.{so,dll,dylib}`` on as many platforms as possible, so as to give vastly improved startup times. On systems that do not ship with a precompiled system image file, one can be generated from the source files shipped in Julia's ``DATAROOTDIR/julia/base`` folder. + +This operation is useful for multiple reasons. A user may: + +* Build a precompiled shared library system image on a platform that did not ship with one, thereby improving startup times. + +* Modify ``Base``, rebuild the system image and use the new ``Base`` next time Julia is started. + +* Include a ``usrimg.jl`` file that includes packages into the system image, thereby creating a system image that has packages embedded into the startup environment. + +Julia now ships with a script that automates the tasks of building the system image, wittingly named ``build_sysimg.jl`` that lives in ``DATAROOTDIR/julia/``. That is, to include it into a current Julia session, type: +:: + + include(joinpath(JULIA_HOME, Base.DATAROOTDIR, "julia", "build_sysimg.jl")) + +This will include a ``build_sysimg()`` function: + +.. function:: build_sysimg(sysimg_path=default_sysimg_path, cpu_target="native", userimg_path=nothing; force=false) + Rebuild the system image. + Store it in ``sysimg_path``, which defaults a file named ``sys.ji`` that sits in the same folder as ``libjulia.{so,dll,dylib}``. + Use the cpu instruction set given by ``cpu_target``. Valid CPU targets are the same as for the ``-C`` option to ``julia``, or the ``-march`` option to ``gcc``. Defaults to ``native``, which means to use all CPU instructions available on the current processor. + Include the user image file given by ``userimg_path``, which should contain directives such as ``using MyPackage`` to include that package in the new system image. + New system image will not replace an older image unless ``force`` is set to true. + +Note that this file can also be run as a script itself, with command line arguments taking the place of arguments passed to the ``build_sysimg`` function. For example, to build a system image in ``/tmp/sys.{so,dll,dylib}``, with the ``core2`` CPU instruction set, a user image of ``~/usrimg.jl`` and ``force`` set to ``true``, one would execute: +:: + + julia build_sysimg.jl /tmp/sys core2 ~/usrimg.jl --force \ No newline at end of file From b4ca61b4c1b3d0847f36de8a244ae80326642dd4 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Sun, 14 Dec 2014 03:49:43 -0800 Subject: [PATCH 39/40] I don't know where I got `usrimg` from. But it should be `userimg`. --- doc/devdocs/sysimg.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/devdocs/sysimg.rst b/doc/devdocs/sysimg.rst index 2c432e0252d83..162f2bddf6334 100644 --- a/doc/devdocs/sysimg.rst +++ b/doc/devdocs/sysimg.rst @@ -13,7 +13,7 @@ This operation is useful for multiple reasons. A user may: * Modify ``Base``, rebuild the system image and use the new ``Base`` next time Julia is started. -* Include a ``usrimg.jl`` file that includes packages into the system image, thereby creating a system image that has packages embedded into the startup environment. +* Include a ``userimg.jl`` file that includes packages into the system image, thereby creating a system image that has packages embedded into the startup environment. Julia now ships with a script that automates the tasks of building the system image, wittingly named ``build_sysimg.jl`` that lives in ``DATAROOTDIR/julia/``. That is, to include it into a current Julia session, type: :: @@ -29,7 +29,7 @@ This will include a ``build_sysimg()`` function: Include the user image file given by ``userimg_path``, which should contain directives such as ``using MyPackage`` to include that package in the new system image. New system image will not replace an older image unless ``force`` is set to true. -Note that this file can also be run as a script itself, with command line arguments taking the place of arguments passed to the ``build_sysimg`` function. For example, to build a system image in ``/tmp/sys.{so,dll,dylib}``, with the ``core2`` CPU instruction set, a user image of ``~/usrimg.jl`` and ``force`` set to ``true``, one would execute: +Note that this file can also be run as a script itself, with command line arguments taking the place of arguments passed to the ``build_sysimg`` function. For example, to build a system image in ``/tmp/sys.{so,dll,dylib}``, with the ``core2`` CPU instruction set, a user image of ``~/userimg.jl`` and ``force`` set to ``true``, one would execute: :: - julia build_sysimg.jl /tmp/sys core2 ~/usrimg.jl --force \ No newline at end of file + julia build_sysimg.jl /tmp/sys core2 ~/userimg.jl --force \ No newline at end of file From 48210168791f3f17dec13e96ae8c183d0be15d0e Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Sun, 14 Dec 2014 03:50:19 -0800 Subject: [PATCH 40/40] Let's see if I can fix this typo before Tony finds my next one --- doc/devdocs/sysimg.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/devdocs/sysimg.rst b/doc/devdocs/sysimg.rst index 162f2bddf6334..32bc40e58a24c 100644 --- a/doc/devdocs/sysimg.rst +++ b/doc/devdocs/sysimg.rst @@ -24,7 +24,7 @@ This will include a ``build_sysimg()`` function: .. function:: build_sysimg(sysimg_path=default_sysimg_path, cpu_target="native", userimg_path=nothing; force=false) Rebuild the system image. - Store it in ``sysimg_path``, which defaults a file named ``sys.ji`` that sits in the same folder as ``libjulia.{so,dll,dylib}``. + Store it in ``sysimg_path``, which defaults to a file named ``sys.ji`` that sits in the same folder as ``libjulia.{so,dll,dylib}``. Use the cpu instruction set given by ``cpu_target``. Valid CPU targets are the same as for the ``-C`` option to ``julia``, or the ``-march`` option to ``gcc``. Defaults to ``native``, which means to use all CPU instructions available on the current processor. Include the user image file given by ``userimg_path``, which should contain directives such as ``using MyPackage`` to include that package in the new system image. New system image will not replace an older image unless ``force`` is set to true.