check for out-of-range bases in ndigits(::BigInt, b)

GMP doesn't sanitize itself the input (there is an ASSERT
in the low-level MPN_SIZEINBASE function, which is probably
disabled in the build). A wrong base may crash the system.

Also, use the documented types for mpz_sizeinbase in ccall.

base/gmp.jl

@@ -520,12 +520,12 @@ end
 
 function ndigits0z(x::BigInt, b::Integer=10)
  b < 2 && throw(DomainError())
- if ispow2(b)
- Int(ccall((:__gmpz_sizeinbase,:libgmp), Culong, (Ptr{BigInt}, Int32), &x, b))
+ if ispow2(b) && 2 <= b <= 62 # GMP assumes b is in this range
+ Int(ccall((:__gmpz_sizeinbase,:libgmp), Csize_t, (Ptr{BigInt}, Cint), &x, b))
  else
  # non-base 2 mpz_sizeinbase might return an answer 1 too big
  # use property that log(b, x) < ndigits(x, b) <= log(b, x) + 1
- n = Int(ccall((:__gmpz_sizeinbase,:libgmp), Culong, (Ptr{BigInt}, Int32), &x, 2))
+ n = Int(ccall((:__gmpz_sizeinbase,:libgmp), Csize_t, (Ptr{BigInt}, Cint), &x, 2))
  lb = log2(b) # assumed accurate to <1ulp (true for openlibm)
  q,r = divrem(n,lb)
  iq = Int(q)

test/bigint.jl

@@ -280,6 +280,12 @@ ndigits_mismatch(n) = ndigits(n) != ndigits(BigInt(n))
 @test !any(ndigits_mismatch, 512:999)
 @test !any(ndigits_mismatch, 8192:9999)
 
+# The following should not crash (#16579)
+ndigits(rand(big(-999:999)), rand(63:typemax(Int)))
+ndigits(rand(big(-999:999)), big(2)^rand(2:999))
+
+@test_throws DomainError ndigits(rand(big(-999:999)), rand(typemin(Int):1))
+
 # conversion from float
 @test BigInt(2.0) == BigInt(2.0f0) == BigInt(big(2.0)) == 2
 @test_throws InexactError convert(BigInt, 2.1)