fix some issues with color configuration (JuliaLang#36689)

- `--color=no` did not remove all color in the REPL
- color was not fully disabled by default when stdout was a pipe
- `--color=yes` did not enable color when stdout was a pipe (fixes JuliaLang#30703)

base/client.jl

@@ -371,13 +371,12 @@ function run_main_repl(interactive::Bool, quiet::Bool, banner::Bool, history_fil
  invokelatest(REPL_MODULE_REF[]) do REPL
  term_env = get(ENV, "TERM", @static Sys.iswindows() ? "" : "dumb")
  term = REPL.Terminals.TTYTerminal(term_env, stdin, stdout, stderr)
- color_set || (global have_color = REPL.Terminals.hascolor(term))
  banner && Base.banner(term)
  if term.term_type == "dumb"
  active_repl = REPL.BasicREPL(term)
  quiet || @warn "Terminal not fully functional"
  else
- active_repl = REPL.LineEditREPL(term, have_color, true)
+ active_repl = REPL.LineEditREPL(term, get(stdout, :color, false), true)
  active_repl.history_file = history_file
  end
  # Make sure any displays pushed in .julia/config/startup.jl ends up above the
@@ -487,7 +486,7 @@ function _start()
  invokelatest(display_error, catch_stack())
  exit(1)
  end
- if is_interactive && have_color === true
+ if is_interactive && get(stdout, :color, false)
  print(color_normal)
  end
 end

base/libuv.jl

@@ -115,6 +115,16 @@ function reinit_stdio()
  global stdin = init_stdio(ccall(:jl_stdin_stream, Ptr{Cvoid}, ()))
  global stdout = init_stdio(ccall(:jl_stdout_stream, Ptr{Cvoid}, ()))
  global stderr = init_stdio(ccall(:jl_stderr_stream, Ptr{Cvoid}, ()))
+ opts = JLOptions()
+ if opts.color != 0
+ have_color = (opts.color == 1)
+ if !isa(stdout, TTY)
+ global stdout = IOContext(stdout, :color => have_color)
+ end
+ if !isa(stderr, TTY)
+ global stderr = IOContext(stderr, :color => have_color)
+ end
+ end
  nothing
 end
 

stdlib/REPL/src/LineEdit.jl

@@ -395,7 +395,7 @@ function refresh_multi_line(termbuf::TerminalBuffer, terminal::UnixTerminal, buf
  regstart, regstop = region(buf)
  written = 0
  # Write out the prompt string
- lindent = write_prompt(termbuf, prompt)
+ lindent = write_prompt(termbuf, prompt, hascolor(terminal))
  # Count the '\n' at the end of the line if the terminal emulator does (specific to DOS cmd prompt)
  miscountnl = @static Sys.iswindows() ? (isa(Terminals.pipe_reader(terminal), Base.TTY) && !Base.ispty(Terminals.pipe_reader(terminal))) : false
 
@@ -1259,15 +1259,15 @@ refresh_line(s, termbuf) = refresh_multi_line(termbuf, s)
 default_completion_cb(::IOBuffer) = []
 default_enter_cb(_) = true
 
-write_prompt(terminal, s::PromptState) = write_prompt(terminal, s.p)
+write_prompt(terminal, s::PromptState, color::Bool) = write_prompt(terminal, s.p, color)
 
-function write_prompt(terminal, p::Prompt)
+function write_prompt(terminal, p::Prompt, color::Bool)
  prefix = prompt_string(p.prompt_prefix)
  suffix = prompt_string(p.prompt_suffix)
  write(terminal, prefix)
- write(terminal, Base.text_colors[:bold])
- width = write_prompt(terminal, p.prompt)
- write(terminal, Base.text_colors[:normal])
+ color && write(terminal, Base.text_colors[:bold])
+ width = write_prompt(terminal, p.prompt, color)
+ color && write(terminal, Base.text_colors[:normal])
  write(terminal, suffix)
  return width
 end
@@ -1303,7 +1303,7 @@ end
 end
 
 # returns the width of the written prompt
-function write_prompt(terminal, s::Union{AbstractString,Function})
+function write_prompt(terminal, s::Union{AbstractString,Function}, color::Bool)
  @static Sys.iswindows() && _reset_console_mode()
  promptstr = prompt_string(s)
  write(terminal, promptstr)
@@ -1740,7 +1740,7 @@ end
 
 input_string(s::PrefixSearchState) = String(take!(copy(s.response_buffer)))
 
-write_prompt(terminal, s::PrefixSearchState) = write_prompt(terminal, s.histprompt.parent_prompt)
+write_prompt(terminal, s::PrefixSearchState, color::Bool) = write_prompt(terminal, s.histprompt.parent_prompt, color)
 prompt_string(s::PrefixSearchState) = prompt_string(s.histprompt.parent_prompt.prompt)
 
 terminal(s::PrefixSearchState) = s.terminal

stdlib/REPL/src/REPL.jl

@@ -436,9 +436,15 @@ mutable struct LineEditREPL <: AbstractREPL
  last_shown_line_infos::Vector{Tuple{String,Int}}
  interface::ModalInterface
  backendref::REPLBackendRef
- LineEditREPL(t,hascolor,prompt_color,input_color,answer_color,shell_color,help_color,history_file,in_shell,in_help,envcolors) =
+ function LineEditREPL(t,hascolor,prompt_color,input_color,answer_color,shell_color,help_color,history_file,in_shell,in_help,envcolors)
+ opts = Options()
+ opts.hascolor = hascolor
+ if !hascolor
+ opts.beep_colors = [""]
+ end
  new(t,hascolor,prompt_color,input_color,answer_color,shell_color,help_color,history_file,in_shell,
- in_help,envcolors,false,nothing, Options(), nothing, Tuple{String,Int}[])
+ in_help,envcolors,false,nothing, opts, nothing, Tuple{String,Int}[])
+ end
 end
 outstream(r::LineEditREPL) = r.t
 specialdisplay(r::LineEditREPL) = r.specialdisplay
@@ -849,7 +855,8 @@ end
 
 function reset(repl::LineEditREPL)
  raw!(repl.t, false)
- print(repl.t, Base.text_colors[:normal])
+ hascolor(repl) && print(repl.t, Base.text_colors[:normal])
+ nothing
 end
 
 function prepare_next(repl::LineEditREPL)

stdlib/REPL/src/Terminals.jl

@@ -152,7 +152,7 @@ beep(t::UnixTerminal) = write(t.err_stream,"\x7")
 
 Base.displaysize(t::UnixTerminal) = displaysize(t.out_stream)
 
-hascolor(t::TTYTerminal) = Base.ttyhascolor(t.term_type)
+hascolor(t::TTYTerminal) = get(t.out_stream, :color, false)::Bool
 
 # use cached value of have_color
 Base.in(key_value::Pair, t::TTYTerminal) = in(key_value, pipe_writer(t))

stdlib/REPL/test/repl.jl

@@ -90,7 +90,7 @@ end
 # in the mix. If verification needs to be done, keep it to the bare minimum. Basically
 # this should make sure nothing crashes without depending on how exactly the control
 # characters are being used.
-fake_repl(options = REPL.Options(confirm_exit=false,hascolor=false)) do stdin_write, stdout_read, repl
+fake_repl(options = REPL.Options(confirm_exit=false,hascolor=true)) do stdin_write, stdout_read, repl
  repl.specialdisplay = REPL.REPLDisplay(repl)
  repl.history_file = false
 
@@ -1279,7 +1279,7 @@ end
  repl.backendref = REPL.REPLBackendRef(Channel(1), Channel(1))
  put!(repl.backendref.response_channel, (bt, true))
 
- REPL.print_response(repl, (err, true), true, false)
+ REPL.print_response(repl, (bt, true), true, false)
  seekstart(out_stream)
  @test count(
  contains(