Merge pull request JuliaLang#9157 from amartgon/base64_decode_#5656

adds base64 decoding (fixes JuliaLang#5656)

base/base64.jl

@@ -1,25 +1,25 @@
 module Base64
-import Base: read, write, close
-export Base64Pipe, base64
+import Base: read, write, close, eof, empty!
+export Base64EncodePipe, Base64DecodePipe, base64encode, base64decode
 
-# Base64Pipe is a pipe-like IO object, which converts writes (and
-# someday reads?) into base64 encoded (decoded) data send to a stream.
-# (You must close the pipe to complete the encode, separate from
-# closing the target stream). We also have a function base64(f,
+# Base64EncodePipe is a pipe-like IO object, which converts into base64 data sent
+# to a stream. (You must close the pipe to complete the encode, separate from
+# closing the target stream). We also have a function base64encode(f,
 # args...) which works like sprint except that it produces
-# base64-encoded data, along with base64(args...) which is equivalent
-# to base64(write, args...), to return base64 strings.
-
+# base64-encoded data, along with base64encode(args...) which is equivalent
+# to base64encode(write, args...), to return base64 strings.
+# A Base64DecodePipe object can be used to decode base64-encoded data read from a stream
+# , while function base64decode is useful for decoding strings
 #############################################################################
 
-type Base64Pipe <: IO
+type Base64EncodePipe <: IO
  io::IO
  # writing works in groups of 3, so we need to cache last two bytes written
  b0::UInt8
  b1::UInt8
  nb::UInt8 # number of bytes in cache: 0, 1, or 2
 
- function Base64Pipe(io::IO)
+ function Base64EncodePipe(io::IO)
  b = new(io,0,0,0)
  finalizer(b, close)
  return b
@@ -32,6 +32,8 @@ end
 
 const b64chars = ['A':'Z','a':'z','0':'9','+','/']
 
+const base64_pad = uint8('=')
+
 function b64(x::UInt8, y::UInt8, z::UInt8)
  n = int(x)<<16 | int(y)<<8 | int(z)
  b64chars[(n >> 18) + 1],
@@ -42,17 +44,46 @@ end
 
 function b64(x::UInt8, y::UInt8)
  a, b, c = b64(x, y, 0x0)
- a, b, c, '='
+ a, b, c, base64_pad
 end
 
 function b64(x::UInt8)
  a, b = b64(x, 0x0, 0x0)
- a, b, '=', '='
+ a, b, base64_pad, base64_pad
+end
+
+const sentinel = typemax(UInt8)
+const revb64chars = fill(sentinel, 256)
+# Fill revb64chars
+for (val, ch) in enumerate(b64chars)
+ revb64chars[uint8(ch)] = uint8(val - 1)
 end
 
+#Decode a block of at least 2 and at most 4 bytes, received in encvec
+#Returns the first decoded byte and stores up to two more in cache
+function b64decode!(encvec::Vector{UInt8}, cache::Vector{UInt8})
+ if length(encvec) < 2
+ error("Incorrect base64 format")
+ end
+ @inbounds u = revb64chars[encvec[1]]
+ @inbounds v = revb64chars[encvec[2]]
+ empty!(cache)
+ res = (u << 2) | (v >> 4)
+ if length(encvec) > 2
+ @inbounds w = revb64chars[encvec[3]]
+ push!(cache, (v << 4) | (w >> 2))
+ end
+ if length(encvec) > 3
+ @inbounds z = revb64chars[encvec[4]]
+ push!(cache, (w << 6) | z)
+ end
+ res
+end
+
+
 #############################################################################
 
-function write(b::Base64Pipe, x::AbstractVector{UInt8})
+function write(b::Base64EncodePipe, x::AbstractVector{UInt8})
  n = length(x)
  s = 1 # starting index
  # finish any cached data to write:
@@ -93,7 +124,7 @@ function write(b::Base64Pipe, x::AbstractVector{UInt8})
  end
 end
 
-function write(b::Base64Pipe, x::UInt8)
+function write(b::Base64EncodePipe, x::UInt8)
  if b.nb == 0
  b.b0 = x
  b.nb = 1
@@ -106,7 +137,7 @@ function write(b::Base64Pipe, x::UInt8)
  end
 end
 
-function close(b::Base64Pipe)
+function close(b::Base64EncodePipe)
  if b.nb > 0
  # write leftover bytes + padding
  if b.nb == 1
@@ -119,19 +150,63 @@ function close(b::Base64Pipe)
 end
 
 # like sprint, but returns base64 string
-function base64(f::Function, args...)
+function base64encode(f::Function, args...)
  s = IOBuffer()
- b = Base64Pipe(s)
+ b = Base64EncodePipe(s)
  f(b, args...)
  close(b)
  takebuf_string(s)
 end
-base64(x...) = base64(write, x...)
+base64encode(x...) = base64encode(write, x...)
 
 #############################################################################
 
 # read(b::Base64Pipe, ::Type{UInt8}) = # TODO: decode base64
 
 #############################################################################
 
+type Base64DecodePipe <: IO
+ io::IO
+ # reading works in blocks of 4 characters that are decoded into 3 bytes and 2 of them cached
+ cache::Vector{UInt8}
+ encvec::Vector{UInt8}
+
+ function Base64DecodePipe(io::IO)
+ b = new(io,[],[])
+ finalizer(b, close)
+ return b
+ end
+end
+
+function read(b::Base64DecodePipe, t::Type{UInt8})
+ if length(b.cache) > 0
+ val = shift!(b.cache)
+ else
+ empty!(b.encvec)
+ while !eof(b.io) && length(b.encvec) < 4
+ c::UInt8 = read(b.io, t)
+ @inbounds if revb64chars[c] != sentinel
+ push!(b.encvec, c)
+ end
+ end
+ val = b64decode!(b.encvec,b.cache)
+ end
+ val
+end
+
+function eof(b::Base64DecodePipe)
+ return length(b.cache) == 0 && eof(b.io)
+end
+
+function close(b::Base64DecodePipe)
+end
+
+# Decodes a base64-encoded string
+function base64decode(s)
+ b = IOBuffer(s)
+ decoded = readall(Base64DecodePipe(b))
+ close(b)
+ decoded
+end
+
 end # module

base/deprecated.jl

@@ -246,6 +246,10 @@ const Uint128 = UInt128
 @deprecate iround(x) round(Integer,x)
 @deprecate iround{T}(::Type{T},x) round(T,x)
 
+export Base64Pipe, base64
+const Base64Pipe = Base64EncodePipe
+const base64 = base64encode
+
 @deprecate prevind(a::Any, i::Integer) i-1
 @deprecate nextind(a::Any, i::Integer) i+1
 
@@ -254,3 +258,4 @@ const Uint128 = UInt128
 @deprecate squeeze(X, dims) squeeze(X, tuple(dims...))
 
 @deprecate sizehint(A, n) sizehint!(A, n)
+

base/exports.jl

@@ -807,8 +807,10 @@ export
 # strings and text output
  ascii,
  base,
- base64,
- Base64Pipe,
+ base64encode,
+ base64decode,
+ Base64EncodePipe,
+ Base64DecodePipe,
  beginswith,
  bin,
  bits,

base/multimedia.jl

@@ -79,8 +79,8 @@ function reprmime(m::MIME, x)
  takebuf_array(s)
 end
 reprmime(m::MIME, x::Vector{UInt8}) = x
-stringmime(m::MIME, x) = base64(writemime, m, x)
-stringmime(m::MIME, x::Vector{UInt8}) = base64(write, x)
+stringmime(m::MIME, x) = base64encode(writemime, m, x)
+stringmime(m::MIME, x::Vector{UInt8}) = base64encode(write, x)
 
 # it is convenient to accept strings instead of ::MIME
 istext(m::AbstractString) = istext(MIME(m))

doc/manual/types.rst

@@ -253,15 +253,15 @@ Assuming no method more specific than the above is found, Julia next internally
 defines and compiles a method called ``myplus`` specifically for two :class:`Int`
 arguments based on the generic function given above, i.e., it implicitly
 defines and compiles::
- 
+
  function myplus(x::Int,y::Int)
  x+y
  end
- 
+
 and finally, it invokes this specific method.
 
 Thus, abstract types allow programmers to write generic functions that can
-later be used as the default method by many combinations of concrete types. 
+later be used as the default method by many combinations of concrete types.
 Thanks to multiple dispatch, the programmer has full control over whether the
 default or more specific method is used.
 
@@ -412,7 +412,7 @@ You may find a list of field names using the ``names`` function.
  :bar
  :baz
  :qux
- 
+
 You can access the field values of a composite object using the
 traditional ``foo.bar`` notation:
 
@@ -510,7 +510,7 @@ i.e., if the fields of objects passed around by copying could be modified,
 then it would become more difficult to reason about certain instances of generic code. For example,
 suppose ``x`` is a function argument of an abstract type, and suppose that the function
 changes a field: ``x.isprocessed = true``. Depending on whether ``x`` is passed by copying
-or by reference, this statement may or may not alter the actual argument in the 
+or by reference, this statement may or may not alter the actual argument in the
 calling routine. Julia
 sidesteps the possibility of creating functions with unknown effects in this
 scenario by forbidding modification of fields
@@ -1135,7 +1135,7 @@ This declaration of ``Vector`` creates a subtype relation
 ``typealias`` statement creates such a relation; for example, the statement::
 
  typealias AA{T} Array{Array{T,1},1}
- 
+
 does not create the relation ``AA{Int} <: AA``. The reason is that ``Array{Array{T,1},1}`` is not
 an abstract type at all; in fact, it is a concrete type describing a
 1-dimensional array in which each entry

doc/stdlib/base.rst

@@ -2335,15 +2335,20 @@ Text I/O
 
  Equivalent to ``writedlm`` with ``delim`` set to comma.
 
-.. function:: Base64Pipe(ostream)
+.. function:: Base64EncodePipe(ostream)
 
  Returns a new write-only I/O stream, which converts any bytes written
  to it into base64-encoded ASCII bytes written to ``ostream``. Calling
  ``close`` on the ``Base64Pipe`` stream is necessary to complete the
  encoding (but does not close ``ostream``).
 
-.. function:: base64(writefunc, args...)
- base64(args...)
+.. function:: Base64DecodePipe(istream)
+
+ Returns a new read-only I/O stream, which decodes base64-encoded data
+ read from ``istream``.
+
+.. function:: base64encode(writefunc, args...)
+ base64encode(args...)
 
  Given a ``write``-like function ``writefunc``, which takes an I/O
  stream as its first argument, ``base64(writefunc, args...)``
@@ -2353,6 +2358,10 @@ Text I/O
  using the standard ``write`` functions and returns the base64-encoded
  string.
 
+.. function:: base64decode(string)
+
+ Decodes the base64-encoded ``string`` and returns the obtained bytes.
+
 Multimedia I/O
 --------------
 

test/base64.jl

@@ -0,0 +1,41 @@
+
+const inputText = "Man is distinguished, not only by his reason, but by this singular passion from other animals, which is a lust of the mind, that by a perseverance of delight in the continued and indefatigable generation of knowledge, exceeds the short vehemence of any carnal pleasure."
+const encodedMaxLine76 =
+"""TWFuIGlzIGRpc3Rpbmd1aXNoZWQsIG5vdCBvbmx5IGJ5IGhpcyByZWFzb24sIGJ1dCBieSB0aGlz
+IHNpbmd1bGFyIHBhc3Npb24gZnJvbSBvdGhlciBhbmltYWxzLCB3aGljaCBpcyBhIGx1c3Qgb2Yg
+dGhlIG1pbmQsIHRoYXQgYnkgYSBwZXJzZXZlcmFuY2Ugb2YgZGVsaWdodCBpbiB0aGUgY29udGlu
+dWVkIGFuZCBpbmRlZmF0aWdhYmxlIGdlbmVyYXRpb24gb2Yga25vd2xlZGdlLCBleGNlZWRzIHRo
+ZSBzaG9ydCB2ZWhlbWVuY2Ugb2YgYW55IGNhcm5hbCBwbGVhc3VyZS4="""
+
+# Encode and decode
+fname = tempname()
+f = open(fname, "w")
+opipe = Base64EncodePipe(f)
+write(opipe,inputText)
+close(opipe)
+close(f)
+f = open(fname, "r")
+ipipe = Base64DecodePipe(f)
+@test readall(ipipe) == inputText
+close(ipipe)
+close(f)
+rm(fname)
+
+# Encode to string and decode
+@test base64decode(base64encode(inputText)) == inputText
+
+# Decode with max line chars = 76 and padding
+ipipe = Base64DecodePipe(IOBuffer(encodedMaxLine76))
+@test readall(ipipe) == inputText
+
+# Decode with max line chars = 76 and no padding
+ipipe = Base64DecodePipe(IOBuffer(encodedMaxLine76[1:end-1]))
+@test readall(ipipe) == inputText
+
+# Decode with two padding characters ("==")
+ipipe = Base64DecodePipe(IOBuffer(string(encodedMaxLine76[1:end-2],"==")))
+@test readall(ipipe) == inputText[1:end-1]
+
+# Test incorrect format
+ipipe = Base64DecodePipe(IOBuffer(encodedMaxLine76[1:end-3]))
+@test_throws ErrorException readall(ipipe)

test/runtests.jl

@@ -11,7 +11,7 @@ testnames = [
  "sysinfo", "rounding", "ranges", "mod2pi", "euler", "show",
  "lineedit", "replcompletions", "repl", "test", "goto",
  "llvmcall", "grisu", "nullable", "meta", "profile",
- "libgit2", "docs"
+ "libgit2", "docs", "base64"
 ]
 
 if isdir(joinpath(JULIA_HOME, Base.DOCDIR, "examples"))