readuntil() for any delim type

returns a string for Char delim, otherwise an array
readuntil(s, uint8('\n')) provides a way to read a line without an encoding check
streams are encouraged to provide an efficient readuntil(s, ::Uint8),
  then reading until ASCII delimiters is fast, leaving the encoding
  logic to higher-level layers in io.jl and string.jl.
for JuliaLang#1792

base/io.jl

@@ -122,7 +122,12 @@ function read(s::IO, ::Type{Char})
  char(c)
 end
 
-function readuntil(s::IO, delim)
+function readuntil(s::IO, delim::Char)
+ if delim < 0x80
+ data = readuntil(s, uint8(delim))
+ enc = byte_string_classify(data)
+ return (enc==1) ? ASCIIString(data) : UTF8String(data)
+ end
  out = memio()
  while !eof(s)
  c = read(s, Char)
@@ -134,6 +139,18 @@ function readuntil(s::IO, delim)
  takebuf_string(out)
 end
 
+function readuntil{T}(s::IO, delim::T)
+ out = T[]
+ while !eof(s)
+ c = read(s, T)
+ push(out, c)
+ if c == delim
+ break
+ end
+ end
+ out
+end
+
 readline(s::IO) = readuntil(s, '\n')
 
 function readall(s::IO)
@@ -395,9 +412,8 @@ end
 
 write(x) = write(OUTPUT_STREAM::IOStream, x)
 
-function readuntil(s::IOStream, delim)
- # TODO: faster versions that avoid the encoding check
- ccall(:jl_readuntil, ByteString, (Ptr{Void}, Uint8), s.ios, delim)
+function readuntil(s::IOStream, delim::Uint8)
+ ccall(:jl_readuntil, Array{Uint8,1}, (Ptr{Void}, Uint8), s.ios, delim)
 end
 
 function readall(s::IOStream)

base/iostring.jl

@@ -196,11 +196,10 @@ function memchr(buf::IOString, delim)
  q = ccall(:memchr,Ptr{Uint8},(Ptr{Uint8},Int32,Int32),p,delim,nb_available(buf))
  nb = (q == C_NULL ? 0 : q-p+1)
 end
-function readuntil(io::IOString, delim)
+function readuntil(io::IOString, delim::Uint8)
  nb = memchr(io, delim)
  if nb == 0
  nb = nb_available(io)
  end
- readbytes(io,nb)
+ read(io, Array(Uint8, nb))
 end
-

base/string.jl

@@ -682,8 +682,9 @@ unescape_string(s::String) = sprint(length(s), print_unescaped, s)
 
 ## checking UTF-8 & ACSII validity ##
 
-byte_string_classify(s::ByteString) =
- ccall(:u8_isvalid, Int32, (Ptr{Uint8}, Int), s.data, length(s))
+byte_string_classify(data::Array{Uint8,1}) =
+ ccall(:u8_isvalid, Int32, (Ptr{Uint8}, Int), data, length(data))
+byte_string_classify(s::ByteString) = byte_string_classify(s.data)
  # 0: neither valid ASCII nor UTF-8
  # 1: valid ASCII
  # 2: valid UTF-8

extras/gzip.jl

@@ -402,46 +402,23 @@ end
 readall(s::GZipStream) = readall(s, Z_BIG_BUFSIZE)
 
 # TODO: Create a c-wrapper based on gzreadline
-function readuntil(s::GZipStream, delim)
- if delim == '\n'
- return readline(s)
- else
- buf = memio(GZ_LINE_BUFSIZE, false)
- c = read(s, Char)
- print(buf, c)
- while c != delim && !eof(s)
- try
- c = read(s, Char)
- print(buf, c)
- catch e
- if !isa(e, EOFError)
- throw(e)
- end
- end
- end
- check_eof(s)
- takebuf_string(buf)
- end
-end
-
-
-function readline(s::GZipStream)
+function readuntil(s::GZipStream, c::Uint8)
  buf = Array(Uint8, GZ_LINE_BUFSIZE)
  pos = 1
 
  if gzgets(s, buf) == C_NULL # Throws an exception on error
- return ""
+ return buf[1:0]
  end
 
  while(true)
  # since gzgets didn't return C_NULL, there must be a \0 in the buffer
  eos = memchr(buf, '\0', pos)
- if eos == 1 || buf[eos-1] == '\n'
- return bytestring(buf[1:eos-1])
+ if eos == 1 || buf[eos-1] == c
+ return buf[1:eos-1]
  end
 
  # If we're at the end of the file, return the string
- if eof(s) return bytestring(buf[1:eos-1]) end
+ if eof(s) return buf[1:eos-1] end
 
  # Otherwise, append to the end of the previous buffer
 
@@ -454,7 +431,7 @@ function readline(s::GZipStream)
  if gzgets(s, pointer(buf)+pos-1, GZ_LINE_BUFSIZE) == C_NULL
  # eof(s); remove extra buffer space
  grow(buf, -GZ_LINE_BUFSIZE)
- return bytestring(buf)
+ return buf
  end
  end
 end

src/sys.c

@@ -310,14 +310,7 @@ jl_value_t *jl_readuntil(ios_t *s, uint8_t delim)
  ((char*)a->data)[n] = '\0';
  }
  }
- JL_GC_PUSH(&a);
- jl_struct_type_t* string_type = u8_isvalid(jl_array_data(a), jl_array_len(a)) == 1 ? // ASCII
- jl_ascii_string_type : jl_utf8_string_type;
- jl_value_t *str = alloc_2w();
- str->type = (jl_type_t*)string_type;
- jl_set_nth_field(str, 0, (jl_value_t*)a);
- JL_GC_POP();
- return str;
+ return (jl_value_t*)a;
 }
 
 void jl_free2(void *p, void *hint)

test/perf2/perf2.jl

@@ -18,10 +18,10 @@ srand(1776) # get more consistent times
 
 require("$JULIA_HOME/../../examples/list.jl")
 
-function listn1n2(n1::Int64,n2::Int64)
- l1 = Nil{Int64}()
+function listn1n2(n1::Int,n2::Int)
+ l1 = Nil{Int}()
  for i=n2:-1:n1
- l1 = Cons{Int64}(i,l1)
+ l1 = Cons{Int}(i,l1)
  end
  l1
 end