add positional chomp argument to readline (squashed JuliaLang#19944)

NEWS.md

@@ -46,6 +46,9 @@ Breaking changes
 
 This section lists changes that do not have deprecation warnings.
 
+ * `readline`, `readlines` and `eachline` return lines without line endings by default.
+ You can use `readline(s, false)` to get the old behavior and include EOL character(s). ([#19944]).
+
  * `String`s no longer have a `.data` field (as part of a significant performance
  improvement). Use `Vector{UInt8}(str)` to access a string as a byte array.
  However, allocating the `Vector` object has overhead. You can also use

base/LineEdit.jl

@@ -211,7 +211,7 @@ function refresh_multi_line(termbuf::TerminalBuffer, terminal::UnixTerminal, buf
  seek(buf, 0)
  moreinput = true # add a blank line if there is a trailing newline on the last line
  while moreinput
- l = readline(buf)
+ l = readline(buf, false)
  moreinput = endswith(l, "\n")
  # We need to deal with on-screen characters, so use strwidth to compute occupied columns
  llength = strwidth(l)
@@ -549,7 +549,7 @@ end
 function edit_kill_line(s::MIState)
  buf = buffer(s)
  pos = position(buf)
- killbuf = readline(buf)
+ killbuf = readline(buf, false)
  if length(killbuf) > 1 && killbuf[end] == '\n'
  killbuf = killbuf[1:end-1]
  char_move_left(buf)

base/REPL.jl

@@ -205,7 +205,7 @@ function run_frontend(repl::BasicREPL, backend::REPLBackendRef)
  interrupted = false
  while true
  try
- line *= readline(repl.terminal)
+ line *= readline(repl.terminal, false)
  catch e
  if isa(e,InterruptException)
  try # raise the debugger if present
@@ -337,7 +337,7 @@ An editor may have converted tabs to spaces at line """
 
 function hist_getline(file)
  while !eof(file)
- line = readline(file)
+ line = readline(file, false)
  isempty(line) && return line
  line[1] in "\r\n" || return line
  end
@@ -995,7 +995,7 @@ function run_frontend(repl::StreamREPL, backend::REPLBackendRef)
  if have_color
  print(repl.stream, input_color(repl))
  end
- line = readline(repl.stream)
+ line = readline(repl.stream, false)
  if !isempty(line)
  ast = Base.parse_input_line(line)
  if have_color

base/client.jl

@@ -216,7 +216,7 @@ parse_input_line(s::AbstractString) = parse_input_line(String(s))
 function parse_input_line(io::IO)
  s = ""
  while !eof(io)
- s = s*readline(io)
+ s = s*readline(io, false)
  e = parse_input_line(s)
  if !(isa(e,Expr) && e.head === :incomplete)
  return e

base/io.jl

@@ -170,24 +170,53 @@ The text is assumed to be encoded in UTF-8.
 readuntil(filename::AbstractString, args...) = open(io->readuntil(io, args...), filename)
 
 """
- readline(stream::IO=STDIN)
- readline(filename::AbstractString)
+ readline()
+ readline(stream, chomp::Bool=true)
+ readline(filename::AbstractString, chomp::Bool=true)
 
-Read a single line of text, including a trailing newline character (if one is reached before
-the end of the input), from the given I/O stream or file (defaults to `STDIN`).
-When reading from a file, the text is assumed to be encoded in UTF-8.
+Read a single line of text from the given I/O stream or file (defaults to `STDIN`).
+Lines in the input can end in `'\\n'` or `"\\r\\n"`. When reading from a file, the text is
+assumed to be encoded in UTF-8.
+
+If `chomp=false` trailing newline character(s) will be included in the output
+(if reached before the end of the input); otherwise newline characters(s)
+are stripped from result.
 """
-readline(filename::AbstractString) = open(readline, filename)
+function readline(filename::AbstractString, chomp::Bool=true)
+ open(filename) do f
+ readline(f, chomp)
+ end
+end
+readline() = readline(STDIN, false)
+
+function readline(s::IO, chomp::Bool=true)
+ line = readuntil(s, 0x0a)
+ i = length(line)
+ if !chomp || i == 0 || line[i] != 0x0a
+ return String(line)
+ elseif i < 2 || line[i-1] != 0x0d
+ return String(resize!(line,i-1))
+ else
+ return String(resize!(line,i-2))
+ end
+end
 
 """
- readlines(stream::IO)
- readlines(filename::AbstractString)
+ readlines(stream::IO, chomp::Bool=true)
+ readlines(filename::AbstractString, chomp::Bool=true)
 
 Read all lines of an I/O stream or a file as a vector of strings.
+Lines in the input can end in `'\\n'` or `"\\r\\n"`.
 The text is assumed to be encoded in UTF-8.
-"""
-readlines(filename::AbstractString) = open(readlines, filename)
 
+If `chomp=false` trailing newline character(s) will be included in the output;
+otherwise newline characters(s) are stripped from result.
+"""
+function readlines(filename::AbstractString, chomp::Bool=true)
+ open(filename) do f
+ readlines(f, chomp)
+ end
+end
 
 ## byte-order mark, ntoh & hton ##
 
@@ -454,9 +483,6 @@ function readuntil(s::IO, t::AbstractString)
  return String(take!(out))
 end
 
-readline() = readline(STDIN)
-readline(s::IO) = readuntil(s, '\n')
-
 """
  readchomp(x)
 
@@ -519,22 +545,28 @@ readstring(filename::AbstractString) = open(readstring, filename)
 
 type EachLine
  stream::IO
+ chomp::Bool
  ondone::Function
- EachLine(stream) = EachLine(stream, ()->nothing)
- EachLine(stream, ondone) = new(stream, ondone)
+ EachLine(stream, chomp) = EachLine(stream, chomp, ()->nothing)
+ EachLine(stream, chomp, ondone) = new(stream, chomp, ondone)
 end
 
 """
- eachline(stream::IO)
- eachline(filename::AbstractString)
+ eachline(stream::IO, chomp::Bool=true)
+ eachline(filename::AbstractString, chomp::Bool=true)
 
 Create an iterable object that will yield each line from an I/O stream or a file.
+Lines in the input can end in `'\\n'` or `"\\r\\n"`.
 The text is assumed to be encoded in UTF-8.
+
+If `chomp=false` trailing newline character(s) will be included in the output;
+otherwise newline characters(s) are stripped from result.
 """
-eachline(stream::IO) = EachLine(stream)
-function eachline(filename::AbstractString)
+eachline(stream::IO, chomp::Bool=true) = EachLine(stream, chomp)
+
+function eachline(filename::AbstractString, chomp::Bool=true)
  s = open(filename)
- EachLine(s, ()->close(s))
+ EachLine(s, chomp, ()->close(s))
 end
 
 start(itr::EachLine) = nothing
@@ -545,10 +577,11 @@ function done(itr::EachLine, nada)
  itr.ondone()
  true
 end
-next(itr::EachLine, nada) = (readline(itr.stream), nothing)
+
+next(itr::EachLine, nada) = (readline(itr.stream, itr.chomp), nothing)
 eltype(::Type{EachLine}) = String
 
-readlines(s=STDIN) = collect(eachline(s))
+readlines(s::IO, chomp::Bool=true) = collect(eachline(s, chomp))
 
 iteratorsize(::Type{EachLine}) = SizeUnknown()
 

base/iostream.jl

@@ -222,16 +222,16 @@ take!(s::IOStream) =
  ccall(:jl_take_buffer, Vector{UInt8}, (Ptr{Void},), s.ios)
 
 function readuntil(s::IOStream, delim::UInt8)
- ccall(:jl_readuntil, Array{UInt8,1}, (Ptr{Void}, UInt8, UInt8), s.ios, delim, 0)
+ ccall(:jl_readuntil, Array{UInt8,1}, (Ptr{Void}, UInt8, UInt8, UInt8), s.ios, delim, 0, 0)
 end
 
 # like readuntil, above, but returns a String without requiring a copy
 function readuntil_string(s::IOStream, delim::UInt8)
- ccall(:jl_readuntil, Ref{String}, (Ptr{Void}, UInt8, UInt8), s.ios, delim, 1)
+ ccall(:jl_readuntil, Ref{String}, (Ptr{Void}, UInt8, UInt8, UInt8), s.ios, delim, 1, false)
 end
 
-function readline(s::IOStream)
- ccall(:jl_readuntil, Ref{String}, (Ptr{Void}, UInt8, UInt8), s.ios, '\n', 1)
+function readline(s::IOStream, chomp::Bool=true)
+ ccall(:jl_readuntil, Ref{String}, (Ptr{Void}, UInt8, UInt8, UInt8), s.ios, '\n', 1, chomp)
 end
 
 function readbytes_all!(s::IOStream, b::Array{UInt8}, nb)
@@ -325,7 +325,7 @@ function skipchars(s::IOStream, pred; linecomment::Char=Char(0xffffffff))
  ch = peekchar(s); status = Int(ch)
  while status >= 0 && (pred(ch) || ch == linecomment)
  if ch == linecomment
- readline(s)
+ readline(s, false)
  else
  read(s, Char) # advance one character
  end

base/libgit2/callbacks.jl

@@ -99,7 +99,7 @@ function authenticate_ssh(creds::SSHCredentials, libgit2credptr::Ptr{Ptr{Void}},
  else
  # In encrypted private keys, the second line is "Proc-Type: 4,ENCRYPTED"
  open(privatekey) do f
- passphrase_required = (readline(f); chomp(readline(f)) == "Proc-Type: 4,ENCRYPTED")
+ passphrase_required = (readline(f, false); readline(f) == "Proc-Type: 4,ENCRYPTED")
  end
  end
 

base/libgit2/utils.jl

@@ -31,7 +31,7 @@ function prompt(msg::AbstractString; default::AbstractString="", password::Bool=
  Base.getpass(msg)
  else
  print(msg)
- chomp(readline(STDIN))
+ readline(STDIN)
  end
  isempty(uinput) ? default : uinput
 end

base/markdown/Common/block.jl

@@ -61,7 +61,7 @@ function hashheader(stream::IO, md::MD)
  return false
 
  if c != '\n' # Empty header
- h = readline(stream) |> strip
+ h = readline(stream, false) |> strip
  h = match(r"(.*?)( +#+)?$", h).captures[1]
  buffer = IOBuffer()
  print(buffer, h)
@@ -76,11 +76,11 @@ end
 function setextheader(stream::IO, md::MD)
  withstream(stream) do
  eatindent(stream) || return false
- header = readline(stream) |> strip
+ header = readline(stream, false) |> strip
  header == "" && return false
 
  eatindent(stream) || return false
- underline = readline(stream) |> strip
+ underline = readline(stream, false) |> strip
  length(underline) < 3 && return false
  u = underline[1]
  u in "-=" || return false
@@ -108,7 +108,7 @@ function indentcode(stream::IO, block::MD)
  buffer = IOBuffer()
  while !eof(stream)
  if startswith(stream, " ") || startswith(stream, "\t")
- write(buffer, readline(stream))
+ write(buffer, readline(stream, false))
  elseif blankline(stream)
  write(buffer, '\n')
  else
@@ -139,10 +139,10 @@ function footnote(stream::IO, block::MD)
  else
  ref = match(regex, str).captures[1]
  buffer = IOBuffer()
- write(buffer, readline(stream))
+ write(buffer, readline(stream, false))
  while !eof(stream)
  if startswith(stream, " ")
- write(buffer, readline(stream))
+ write(buffer, readline(stream, false))
  elseif blankline(stream)
  write(buffer, '\n')
  else
@@ -174,7 +174,7 @@ function blockquote(stream::IO, block::MD)
  empty = true
  while eatindent(stream) && startswith(stream, '>')
  startswith(stream, " ")
- write(buffer, readline(stream))
+ write(buffer, readline(stream, false))
  empty = false
  end
  empty && return false
@@ -210,7 +210,7 @@ function admonition(stream::IO, block::MD)
  category, title =
  let untitled = r"^([a-z]+)$", # !!! <CATEGORY_NAME>
  titled = r"^([a-z]+) \"(.*)\"$", # !!! <CATEGORY_NAME> "<TITLE>"
- line = strip(readline(stream))
+ line = strip(readline(stream, false))
  if ismatch(untitled, line)
  m = match(untitled, line)
  # When no title is provided we use CATEGORY_NAME, capitalising it.
@@ -229,7 +229,7 @@ function admonition(stream::IO, block::MD)
  buffer = IOBuffer()
  while !eof(stream)
  if startswith(stream, " ")
- write(buffer, readline(stream))
+ write(buffer, readline(stream, false))
  elseif blankline(stream)
  write(buffer, '\n')
  else
@@ -305,7 +305,7 @@ function list(stream::IO, block::MD)
  newline = false
  if startswith(stream, " "^indent)
  # Indented text that is part of the current list item.
- print(buffer, readline(stream))
+ print(buffer, readline(stream, false))
  else
  matched = startswith(stream, regex)
  if isempty(matched)
@@ -316,7 +316,7 @@ function list(stream::IO, block::MD)
  # Start of a new list item.
  count += 1
  count > 1 && pushitem!(list, buffer)
- print(buffer, readline(stream))
+ print(buffer, readline(stream, false))
  end
  end
  end

base/markdown/GitHub/GitHub.jl

@@ -8,7 +8,7 @@ function fencedcode(stream::IO, block::MD)
  startswith(stream, "~~~", padding = true) || startswith(stream, "```", padding = true) || return false
  skip(stream, -1)
  ch = read(stream, Char)
- trailing = strip(readline(stream))
+ trailing = strip(readline(stream, false))
  flavor = lstrip(trailing, ch)
  n = 3 + length(trailing) - length(flavor)
 
@@ -30,7 +30,7 @@ function fencedcode(stream::IO, block::MD)
  seek(stream, line_start)
  end
  end
- write(buffer, readline(stream))
+ write(buffer, readline(stream, false))
  end
  return false
  end

base/markdown/GitHub/table.jl

@@ -7,7 +7,7 @@ end
 
 function parserow(stream::IO)
  withstream(stream) do
- line = readline(stream) |> chomp
+ line = readline(stream)
  row = split(line, r"(?<!\\)\|")
  length(row) == 1 && return
  row[1] == "" && shift!(row)

base/markdown/parse/util.jl

@@ -46,7 +46,7 @@ function linecontains(io::IO, chars; allow_whitespace = true,
  eat = true,
  allowempty = false)
  start = position(io)
- l = readline(io) |> chomp
+ l = readline(io)
  length(l) == 0 && return allowempty
 
  result = allowempty
@@ -99,7 +99,7 @@ function startswith(stream::IO, r::Regex; eat = true, padding = false)
  @assert Base.startswith(r.pattern, "^")
  start = position(stream)
  padding && skipwhitespace(stream)
- line = chomp(readline(stream))
+ line = readline(stream)
  seek(stream, start)
  m = match(r, line)
  m === nothing && return ""

base/multi.jl

@@ -1624,7 +1624,7 @@ end
 
 function redirect_worker_output(ident, stream)
  @schedule while !eof(stream)
- line = readline(stream)
+ line = readline(stream, false)
  if startswith(line, "\tFrom worker ")
  # STDOUT's of "additional" workers started from an initial worker on a host are not available
  # on the master directly - they are routed via the initial worker's STDOUT.
@@ -1642,7 +1642,7 @@ end
 # setup a all-to-all network.
 function read_worker_host_port(io::IO)
  while true
- conninfo = readline(io)
+ conninfo = readline(io, false)
  bind_addr, port = parse_connection_info(conninfo)
  if bind_addr != ""
  return bind_addr, port

base/pkg/dir.jl

@@ -67,7 +67,7 @@ end
 
 function getmetabranch()
  try
- chomp(readline(joinpath(path(),"META_BRANCH")))
+ readline(joinpath(path(),"META_BRANCH"))
  catch err
  META_BRANCH
  end