dlopen should follow platform default behavior

For OSX the default behavior is to dlopen with RTLD_GLOBAL while
on most other POSIX platforms the default is RTLD_LOCAL.  This reverts
the behavior introduced in #11352 which made RTLD_LOCAL the default for
all platforms which caused issues when loading shared libraries on OSX.

This PR also cleans up some `Libdl` behavior and adds tests for the
`Libdl` module.  `Libdl.dlopen` now prints a detailed error message in
the thrown exception rather than printing to STDERR.  `Libdl.dlsym` now
throws an exception instead of printing an error message to STDERR if
the symbol is not found in the shared library.  `RTLD_*` enum values now
start at 1 instead of 0 to reflect the fact that RTLD_LOCAL is not
universal default behavior on all POSIX platforms and needs to be detected on
OSX.

cc: @vtjnash, @poulson

closes #11692
fixes #11689

base/libdl.jl

@@ -11,38 +11,45 @@ const DL_LOAD_PATH = ByteString[]
 @osx_only push!(DL_LOAD_PATH, "@executable_path/../lib")
 
 # constants to match JL_RTLD_* in src/julia.h
-const RTLD_LOCAL = 0x00000000
-const RTLD_GLOBAL = 0x00000001
-const RTLD_LAZY = 0x00000002
-const RTLD_NOW = 0x00000004
-const RTLD_NODELETE = 0x00000008
-const RTLD_NOLOAD = 0x00000010
-const RTLD_DEEPBIND = 0x00000020
-const RTLD_FIRST = 0x00000040
+const RTLD_LOCAL = 0x00000001
+const RTLD_GLOBAL = 0x00000002
+const RTLD_LAZY = 0x00000004
+const RTLD_NOW = 0x00000008
+const RTLD_NODELETE = 0x00000010
+const RTLD_NOLOAD = 0x00000020
+const RTLD_DEEPBIND = 0x00000040
+const RTLD_FIRST = 0x00000080
 
 function dlsym(hnd::Ptr, s::Union{Symbol,AbstractString})
- hnd == C_NULL && error("NULL library handle")
+ hnd == C_NULL && throw(ArgumentError("NULL library handle"))
  ccall(:jl_dlsym, Ptr{Void}, (Ptr{Void}, Cstring), hnd, s)
 end
 
 function dlsym_e(hnd::Ptr, s::Union{Symbol,AbstractString})
- hnd == C_NULL && error("NULL library handle")
+ hnd == C_NULL && throw(ArgumentError("NULL library handle"))
  ccall(:jl_dlsym_e, Ptr{Void}, (Ptr{Void}, Cstring), hnd, s)
 end
 
-dlopen(s::Symbol, flags::Integer = RTLD_LAZY | RTLD_DEEPBIND) = dlopen(string(s), flags)
+dlopen(s::Symbol, flags::Integer = RTLD_LAZY | RTLD_DEEPBIND) =
+ dlopen(string(s), flags)
 
 dlopen(s::AbstractString, flags::Integer = RTLD_LAZY | RTLD_DEEPBIND) =
  ccall(:jl_load_dynamic_library, Ptr{Void}, (Cstring,UInt32), s, flags)
 
+dlopen_e(s::Symbol, flags::Integer = RTLD_LAZY | RTLD_DEEPBIND) =
+ dlopen_e(string(s), flags)
+
 dlopen_e(s::AbstractString, flags::Integer = RTLD_LAZY | RTLD_DEEPBIND) =
  ccall(:jl_load_dynamic_library_e, Ptr{Void}, (Cstring,UInt32), s, flags)
 
-dlopen_e(s::Symbol, flags::Integer = RTLD_LAZY | RTLD_DEEPBIND) = dlopen_e(string(s), flags)
-
-dlclose(p::Ptr) = if p!=C_NULL; ccall(:uv_dlclose,Void,(Ptr{Void},),p); Libc.free(p); end
+function dlclose(p::Ptr)
+ if p != C_NULL
+ ccall(:uv_dlclose,Void,(Ptr{Void},),p)
+ Libc.free(p)
+ end
+end
 
-function find_library{T<:ByteString, S<:ByteString}(libnames::Array{T,1}, extrapaths::Array{S,1}=ASCIIString[])
+function find_library(libnames::Vector, extrapaths::Vector=ASCIIString[])
  for lib in libnames
  for path in extrapaths
  l = joinpath(path, lib)
@@ -61,8 +68,8 @@ function find_library{T<:ByteString, S<:ByteString}(libnames::Array{T,1}, extrap
  return ""
 end
 
-function dlpath( handle::Ptr{Void} )
- p = ccall( :jl_pathname_for_handle, Ptr{UInt8}, (Ptr{Void},), handle )
+function dlpath(handle::Ptr{Void})
+ p = ccall(:jl_pathname_for_handle, Ptr{UInt8}, (Ptr{Void},), handle)
  s = bytestring(p)
  @windows_only Libc.free(p)
  return s

doc/stdlib/libdl.rst

@@ -14,10 +14,13 @@
  the corresponding flags of the POSIX (and/or GNU libc and/or MacOS)
  dlopen command, if possible, or are ignored if the specified
  functionality is not available on the current platform. The
- default is ``RTLD_LAZY|RTLD_DEEPBIND|RTLD_LOCAL``. An important usage
- of these flags, on POSIX platforms, is to specify
- ``RTLD_LAZY|RTLD_DEEPBIND|RTLD_GLOBAL`` in order for the library's
- symbols to be available for usage in other shared libraries, in
+ default flags are platform specific. On MacOS the default ``dlopen`` flags are
+ ``RTLD_LAZY|RTLD_DEEPBIND|RTLD_GLOBAL`` while on other platforms the
+ defaults are ``RTLD_LAZY|RTLD_DEEPBIND|RTLD_LOCAL``. An important usage
+ of these flags is to specify non default behavior for when the dynamic library loader
+ binds library references to exported symbols and if the bound references are put into
+ process local or global scope. For instance ``RTLD_LAZY|RTLD_DEEPBIND|RTLD_GLOBAL``
+ allows the library's symbols to be available for usage in other shared libraries, addressing
  situations where there are dependencies between shared libraries.
 
 .. function:: dlopen_e(libfile::AbstractString [, flags::Integer])

src/dlload.c

@@ -50,7 +50,8 @@ DLLEXPORT int jl_uv_dlopen(const char *filename, jl_uv_libhandle lib_, unsigned
  dlerror(); /* Reset error status. */
  lib->handle = dlopen(filename,
  (flags & JL_RTLD_NOW ? RTLD_NOW : RTLD_LAZY)
- | (flags & JL_RTLD_GLOBAL ? RTLD_GLOBAL : RTLD_LOCAL)
+ | JL_RTLD(flags, LOCAL)
+ | JL_RTLD(flags, GLOBAL)
 #ifdef RTLD_NODELETE
  | JL_RTLD(flags, NODELETE)
 #endif
@@ -96,7 +97,7 @@ static uv_lib_t *jl_load_dynamic_library_(const char *modname, unsigned flags, i
  (LPCWSTR)(&jl_load_dynamic_library),
  &handle->handle)) {
  free(handle);
- jl_errorf("could not load base module", modname);
+ jl_error("could not load base module");
  }
 #else
  handle->handle = dlopen(NULL,RTLD_NOW);
@@ -178,14 +179,13 @@ static uv_lib_t *jl_load_dynamic_library_(const char *modname, unsigned flags, i
 #endif
 
 notfound:
- if (throw_err)
-  jl_printf(JL_STDERR, "dlerror loading '%s': %s\n", modname, uv_dlerror(handle));
-
+ // copy the error message into the path buffer so we can free the lib handle
+ path[0] = '\0';
+ snprintf(path, PATHBUF, "%s", uv_dlerror(handle));
  uv_dlclose(handle);
  free(handle);
-
  if (throw_err)
- jl_errorf("could not load library %s", modname);
+ jl_errorf("could not load library '%s': %s", modname, path);
  return NULL;
 done:
  return handle;
@@ -204,17 +204,17 @@ jl_uv_libhandle jl_load_dynamic_library(const char *modname, unsigned flags)
 void *jl_dlsym_e(jl_uv_libhandle handle, const char *symbol)
 {
  void *ptr;
- int  error=uv_dlsym((uv_lib_t *) handle, symbol, &ptr);
+ int error = uv_dlsym((uv_lib_t *) handle, symbol, &ptr);
  if (error) ptr=NULL;
  return ptr;
 }
 
 void *jl_dlsym(jl_uv_libhandle handle, const char *symbol)
 {
  void *ptr;
- int  error = uv_dlsym((uv_lib_t *) handle, symbol, &ptr);
- if (error != 0) {
- jl_printf(JL_STDERR, "symbol could not be found %s (%d): %s\n", symbol, error, uv_dlerror((uv_lib_t *) handle));
+ int error = uv_dlsym((uv_lib_t *) handle, symbol, &ptr);
+ if (error) {
+ jl_errorf("could not load symbol '%s': %s", symbol, uv_dlerror((uv_lib_t *) handle));
  }
  return ptr;
 }

src/julia.h

@@ -1158,12 +1158,17 @@ DLLEXPORT void *jl_eval_string(const char *str);
 
 // external libraries
 enum JL_RTLD_CONSTANT {
- JL_RTLD_LOCAL=0U, JL_RTLD_GLOBAL=1U, /* LOCAL=0 since it is the default */
- JL_RTLD_LAZY=2U, JL_RTLD_NOW=4U,
+ JL_RTLD_LOCAL=1U,
+ JL_RTLD_GLOBAL=2U,
+ JL_RTLD_LAZY=4U,
+ JL_RTLD_NOW=8U,
  /* Linux/glibc and MacOS X: */
- JL_RTLD_NODELETE=8U, JL_RTLD_NOLOAD=16U,
- /* Linux/glibc: */ JL_RTLD_DEEPBIND=32U,
- /* MacOS X 10.5+: */ JL_RTLD_FIRST=64U
+ JL_RTLD_NODELETE=16U,
+ JL_RTLD_NOLOAD=32U,
+ /* Linux/glibc: */
+ JL_RTLD_DEEPBIND=64U,
+ /* MacOS X 10.5+: */
+ JL_RTLD_FIRST=128U
 };
 #define JL_RTLD_DEFAULT (JL_RTLD_LAZY | JL_RTLD_DEEPBIND)
 

test/backtrace.jl

@@ -12,19 +12,3 @@ for l in bt
 end
 
 @test have_backtrace
-
-# these could fail on an embedded installation
-# but for now, we don't handle that case
-dlls = Libdl.dllist()
-@test !isempty(dlls)
-@test length(dlls) > 3 # at a bare minimum, probably have some version of libstdc, libgcc, libjulia, ...
-if @unix? true : (Base.windows_version() >= Base.WINDOWS_VISTA_VER)
- for dl in dlls
- if isfile(dl) && (Libdl.dlopen_e(dl) != C_NULL)
- @test Base.samefile(Libdl.dlpath(dl), dl)
- end
- end
-end
-@test length(filter(dlls) do dl
- return ismatch(Regex("^libjulia(?:.*)\.$(Libdl.dlext)(?:\..+)?\$"), basename(dl))
- end) == 1 # look for something libjulia-like (but only one)

test/choosetests.jl

@@ -30,7 +30,7 @@ function choosetests(choices = [])
  "replutil", "sets", "test", "goto", "llvmcall", "grisu",
  "nullable", "meta", "profile", "libgit2", "docs", "markdown",
  "base64", "parser", "serialize", "functors", "char", "misc",
- "enums", "cmdlineargs", "i18n", "workspace"
+ "enums", "cmdlineargs", "i18n", "workspace", "libdl"
  ]
 
  if Base.USE_GPL_LIBS

test/libdl.jl

@@ -0,0 +1,78 @@
+# these could fail on an embedded installation
+# but for now, we don't handle that case
+dlls = Libdl.dllist()
+@test !isempty(dlls)
+@test length(dlls) > 3 # at a bare minimum, probably have some version of libstdc, libgcc, libjulia, ...
+if @unix? true : (Base.windows_version() >= Base.WINDOWS_VISTA_VER)
+ for dl in dlls
+ if isfile(dl) && (Libdl.dlopen_e(dl) != C_NULL)
+ @test Base.samefile(Libdl.dlpath(dl), dl)
+ end
+ end
+end
+@test length(filter(dlls) do dl
+ return ismatch(Regex("^libjulia(?:.*)\.$(Libdl.dlext)(?:\..+)?\$"), basename(dl))
+ end) == 1 # look for something libjulia-like (but only one)
+
+# library handle pointer must not be NULL
+@test_throws ArgumentError Libdl.dlsym(C_NULL, :foo)
+@test_throws ArgumentError Libdl.dlsym_e(C_NULL, :foo)
+
+@test !isempty(Libdl.find_library(["libccalltest"], [dirname(@__FILE__)]))
+
+# dlopen should be able to handle absolute and relative paths, with and without dlext
+let dl = Libdl.dlopen_e(abspath("./libccalltest"))
+ @test dl != C_NULL
+ Libdl.dlclose(dl)
+end
+
+let dl = Libdl.dlopen_e(join((abspath("./libccalltest"), string(".", Libdl.dlext))))
+ @test dl != C_NULL
+ Libdl.dlclose(dl)
+end
+
+let dl = Libdl.dlopen_e("./libccalltest")
+ @test dl != C_NULL
+ Libdl.dlclose(dl)
+end
+
+let dl = Libdl.dlopen_e(join(("./libccalltest", string(".", Libdl.dlext))))
+ @test dl != C_NULL
+ Libdl.dlclose(dl)
+end
+
+let dl = Libdl.dlopen_e("./foo")
+ @test dl == C_NULL
+end
+
+# test dlpath
+let dl = Libdl.dlopen("./libccalltest")
+ try
+ @test dl != C_NULL
+ @test Base.samefile(abspath(Libdl.dlpath(dl)),
+ abspath(Libdl.dlpath("./libccalltest")))
+ @test Base.samefile(abspath(Libdl.dlpath(dl)),
+ abspath(join(("./libccalltest",Libdl.dlext), '.')))
+ finally
+ Libdl.dlclose(dl)
+ end
+end
+
+# opening a library that does not exist throws an ErrorException
+@test_throws ErrorException Libdl.dlopen("./foo")
+
+# test dlsym
+let dl = Libdl.dlopen("./libccalltest")
+ try
+ fptr = Libdl.dlsym(dl, :set_verbose)
+ @test fptr != C_NULL
+ @test_throws ErrorException Libdl.dlsym(dl, :foo)
+
+ fptr = Libdl.dlsym_e(dl, :set_verbose)
+ @test fptr != C_NULL
+ fptr = Libdl.dlsym_e(dl, :foo)
+ @test fptr == C_NULL
+ finally
+ Libdl.dlclose(dl)
+ end
+end