iostream: Fix \r\n handling in reading buffers

src/flisp/iostream.c

@@ -333,7 +333,7 @@ value_t fl_ioreaduntil(fl_context_t *fl_ctx, value_t *args, uint32_t nargs)
  ios_setbuf(&dest, data, 80, 0);
  char delim = get_delim_arg(fl_ctx, args[1], "io.readuntil");
  ios_t *src = toiostream(fl_ctx, args[0], "io.readuntil");
- size_t n = ios_copyuntil(&dest, src, delim, 0);
+ size_t n = ios_copyuntil(&dest, src, delim);
  cv->len = n;
  if (dest.buf != data) {
  // outgrew initial space
@@ -352,7 +352,7 @@ value_t fl_iocopyuntil(fl_context_t *fl_ctx, value_t *args, uint32_t nargs)
  ios_t *dest = toiostream(fl_ctx, args[0], "io.copyuntil");
  ios_t *src = toiostream(fl_ctx, args[1], "io.copyuntil");
  char delim = get_delim_arg(fl_ctx, args[2], "io.copyuntil");
- return size_wrap(fl_ctx, ios_copyuntil(dest, src, delim, 0));
+ return size_wrap(fl_ctx, ios_copyuntil(dest, src, delim));
 }
 
 value_t fl_iocopy(fl_context_t *fl_ctx, value_t *args, uint32_t nargs)

src/support/ios.c

@@ -803,7 +803,7 @@ size_t ios_copyall(ios_t *to, ios_t *from)
 
 #define LINE_CHUNK_SIZE 160
 
-size_t ios_copyuntil(ios_t *to, ios_t *from, char delim, uint8_t chomp)
+size_t ios_copyuntil(ios_t *to, ios_t *from, char delim)
 {
  size_t total = 0, avail = (size_t)(from->size - from->bpos);
  while (!ios_eof(from)) {
@@ -822,11 +822,7 @@ size_t ios_copyuntil(ios_t *to, ios_t *from, char delim, uint8_t chomp)
  }
  else {
  size_t ntowrite = pd - (from->buf+from->bpos) + 1;
- size_t nchomp = 0;
- if (chomp) {
- nchomp = ios_nchomp(from, ntowrite);
- }
- written = ios_write(to, from->buf+from->bpos, ntowrite - nchomp);
+ written = ios_write(to, from->buf+from->bpos, ntowrite);
  from->bpos += ntowrite;
  total += written;
  return total;
@@ -1167,7 +1163,7 @@ char *ios_readline(ios_t *s)
 {
  ios_t dest;
  ios_mem(&dest, 0);
- ios_copyuntil(&dest, s, '\n', 0);
+ ios_copyuntil(&dest, s, '\n');
  size_t n;
  return ios_take_buffer(&dest, &n);
 }

src/support/ios.h

@@ -94,7 +94,7 @@ JL_DLLEXPORT int ios_get_writable(ios_t *s);
 JL_DLLEXPORT void ios_set_readonly(ios_t *s);
 JL_DLLEXPORT size_t ios_copy(ios_t *to, ios_t *from, size_t nbytes);
 JL_DLLEXPORT size_t ios_copyall(ios_t *to, ios_t *from);
-JL_DLLEXPORT size_t ios_copyuntil(ios_t *to, ios_t *from, char delim, uint8_t chomp);
+JL_DLLEXPORT size_t ios_copyuntil(ios_t *to, ios_t *from, char delim);
 JL_DLLEXPORT size_t ios_nchomp(ios_t *from, size_t ntowrite);
 // ensure at least n bytes are buffered if possible. returns # available.
 JL_DLLEXPORT size_t ios_readprep(ios_t *from, size_t n);

src/sys.c

@@ -256,9 +256,9 @@ JL_DLLEXPORT jl_value_t *jl_readuntil(ios_t *s, uint8_t delim, uint8_t str, uint
 {
  jl_array_t *a;
  // manually inlined common case
- char *pd = (char*)memchr(s->buf+s->bpos, delim, (size_t)(s->size - s->bpos));
+ char *pd = (char*)memchr(s->buf + s->bpos, delim, (size_t)(s->size - s->bpos));
  if (pd) {
- size_t n = pd-(s->buf+s->bpos)+1;
+ size_t n = pd - (s->buf + s->bpos) + 1;
  if (str) {
  size_t nchomp = 0;
  if (chomp) {
@@ -277,7 +277,15 @@ JL_DLLEXPORT jl_value_t *jl_readuntil(ios_t *s, uint8_t delim, uint8_t str, uint
  ios_t dest;
  ios_mem(&dest, 0);
  ios_setbuf(&dest, (char*)a->data, 80, 0);
- size_t n = ios_copyuntil(&dest, s, delim, chomp);
+ size_t n = ios_copyuntil(&dest, s, delim);
+ if (chomp && n > 0 && dest.buf[n - 1] == '\n') {
+ n--;
+ if (n > 0 && dest.buf[n - 1] == '\r') {
+ n--;
+ }
+ int truncret = ios_trunc(&dest, n); // it should always be possible to truncate dest
+ assert(truncret == 0);
+ }
  if (dest.buf != a->data) {
  a = jl_take_buffer(&dest);
  }

test/iobuffer.jl

@@ -90,6 +90,25 @@ write(io,"\n\r\n\n\r \n") > 0
 @test readlines(IOBuffer(""), chomp=true) == []
 @test readlines(IOBuffer("first\nsecond"), chomp=false) == String["first\n", "second"]
 @test readlines(IOBuffer("first\nsecond"), chomp=true) == String["first", "second"]
+
+let fname = tempname()
+ for dochomp in [true, false],
+ endline in ["\n", "\r\n"],
+ i in -5:5
+
+ ref = ("1"^(2^17 - i)) * endline
+ open(fname, "w") do io
+ write(io, ref)
+ end
+ x = readlines(fname, chomp = dochomp)
+ if dochomp
+ ref = chomp(ref)
+ end
+ @test ref == x[1]
+ end
+ rm(fname)
+end
+
 Base.compact(io)
 @test position(io) == 0
 @test ioslength(io) == 0