reorganize Base with default packages, installed in site/

as a demo, moves DataFmt out of Base. adds a line to default juliarc
so the functions are still available by default.

Makefile

@@ -19,7 +19,7 @@ default: $(JULIA_BUILD_MODE) # contains either "debug" or "release"
 all: debug release
 
 # sort is used to remove potential duplicates
-DIRS := $(sort $(build_bindir) $(build_depsbindir) $(build_libdir) $(build_private_libdir) $(build_libexecdir) $(build_includedir) $(build_includedir)/julia $(build_sysconfdir)/julia $(build_datarootdir)/julia $(build_man1dir))
+DIRS := $(sort $(build_bindir) $(build_depsbindir) $(build_libdir) $(build_private_libdir) $(build_libexecdir) $(build_includedir) $(build_includedir)/julia $(build_sysconfdir)/julia $(build_datarootdir)/julia $(build_datarootdir)/julia/site $(build_man1dir))
 ifneq ($(BUILDROOT),$(JULIAHOME))
 BUILDDIRS := $(BUILDROOT) $(addprefix $(BUILDROOT)/,base src ui doc deps test test/perf examples examples/embedding)
 BUILDDIRMAKE := $(addsuffix /Makefile,$(BUILDDIRS))
@@ -50,6 +50,11 @@ endif
 
 $(foreach dir,$(DIRS),$(eval $(call dir_target,$(dir))))
 $(foreach link,base test,$(eval $(call symlink_target,$(link),$(build_datarootdir)/julia)))
+$(eval $(call symlink_target,stdlib,$(build_datarootdir)/julia/site))
+
+build_defaultpkgdir = $(build_datarootdir)/julia/site/$(shell echo $(VERSDIR))
+$(build_defaultpkgdir): $(build_datarootdir)/julia/site/stdlib
+ @mv $(build_datarootdir)/julia/site/stdlib $@
 
 julia_flisp.boot.inc.phony: julia-deps
  @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/src julia_flisp.boot.inc.phony
@@ -81,7 +86,7 @@ ifndef JULIA_VAGRANT_BUILD
 endif
 endif
 
-julia-deps: | $(DIRS) $(build_datarootdir)/julia/base $(build_datarootdir)/julia/test
+julia-deps: | $(DIRS) $(build_datarootdir)/julia/base $(build_datarootdir)/julia/test $(build_defaultpkgdir)
  @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/deps
 
 julia-base: julia-deps $(build_sysconfdir)/julia/juliarc.jl $(build_man1dir)/julia.1 $(build_datarootdir)/julia/julia-config.jl

README.md

@@ -372,11 +372,12 @@ It is highly recommended to start with a fresh clone of the Julia repository.
 
 The Julia source code is organized as follows:
 
- base/ source code for Julia's standard library
+ base/ source code for the Base module (part of Julia's standard library)
+ stdlib/ source code for other standard library packages
  contrib/ editor support for Julia source, miscellaneous scripts
  deps/ external dependencies
- doc/manual  source for the user manual
- doc/stdlib  source for standard library function help text
+ doc/src/manual source for the user manual
+ doc/src/stdlib source for standard library function reference
  examples/ example Julia programs
  src/ source for Julia language core
  test/ test suites

base/deprecated.jl

@@ -1724,19 +1724,12 @@ import .LinAlg: diagm
 @deprecate_binding φ MathConstants.φ
 @deprecate_binding golden MathConstants.golden
 
-# deprecate writecsv
-@deprecate writecsv(io, a; opts...) writedlm(io, a, ','; opts...)
-
 # PR #23271
 function IOContext(io::IO; kws...)
  depwarn("IOContext(io, k=v, ...) is deprecated, use IOContext(io, :k => v, ...) instead.", :IOContext)
  IOContext(io, (k=>v for (k, v) in kws)...)
 end
 
-# deprecate readcsv
-@deprecate readcsv(io; opts...) readdlm(io, ','; opts...)
-@deprecate readcsv(io, T::Type; opts...) readdlm(io, ',', T; opts...)
-
 @deprecate IOContext(io::IO, key, value) IOContext(io, key=>value)
 
 # PR #23485

base/exports.jl

@@ -1048,7 +1048,6 @@ export
  readbytes!,
  readchomp,
  readdir,
- readdlm,
  readline,
  readlines,
  readuntil,
@@ -1070,7 +1069,6 @@ export
  unmark,
  watch_file,
  write,
- writedlm,
  TCPSocket,
  UDPSocket,
 

base/initdefs.jl

@@ -69,7 +69,7 @@ functionality is experimental and may break or change in Julia 1.0.
 const LOAD_PATH = Any[]
 const LOAD_CACHE_PATH = String[]
 
-function init_load_path()
+function init_load_path(JULIA_HOME = JULIA_HOME)
  vers = "v$(VERSION.major).$(VERSION.minor)"
  if haskey(ENV, "JULIA_LOAD_PATH")
  prepend!(LOAD_PATH, split(ENV["JULIA_LOAD_PATH"], @static Sys.iswindows() ? ';' : ':'))

base/io.jl

@@ -779,3 +779,26 @@ function skipchars(io::IO, pred; linecomment=nothing)
  end
  return io
 end
+
+"""
+ countlines(io::IO, eol::Char='\\n')
+
+Read `io` until the end of the stream/file and count the number of lines. To specify a file
+pass the filename as the first argument. EOL markers other than `'\\n'` are supported by
+passing them as the second argument.
+"""
+function countlines(io::IO, eol::Char='\n')
+ isascii(eol) || throw(ArgumentError("only ASCII line terminators are supported"))
+ aeol = UInt8(eol)
+ a = Vector{UInt8}(8192)
+ nl = 0
+ while !eof(io)
+ nb = readbytes!(io, a)
+ @simd for i=1:nb
+ @inbounds nl += a[i] == aeol
+ end
+ end
+ nl
+end
+
+countlines(f::AbstractString, eol::Char='\n') = open(io->countlines(io,eol), f)::Int

base/sysimg.jl

@@ -360,8 +360,6 @@ include("mmap.jl")
 import .Mmap
 
 # utilities - timing, help, edit
-include("datafmt.jl")
-using .DataFmt
 include("deepcopy.jl")
 include("interactiveutil.jl")
 include("summarysize.jl")
@@ -453,4 +451,12 @@ end # baremodule Base
 
 using Base
 
+# set up load path to be able to find stdlib packages
+Base.init_load_path(ccall(:jl_get_julia_home, Any, ()))
+
+# load some stdlib packages but don't put their names in Main
+Base.require(:DelimitedFiles)
+
+empty!(LOAD_PATH)
+
 Base.isfile("userimg.jl") && Base.include(Main, "userimg.jl")

doc/make.jl

@@ -16,6 +16,13 @@ end
 
 # Documenter Setup.
 
+# make links for stdlib package docs
+if Sys.iswindows()
+ cp("../stdlib/DelimitedFiles/docs/src/index.md", "src/stdlib/delimitedfiles.md")
+else
+ symlink("../../../stdlib/DelimitedFiles/docs/src/index.md", "src/stdlib/delimitedfiles.md")
+end
+
 const PAGES = [
  "Home" => "index.md",
  "Manual" => [
@@ -69,6 +76,7 @@ const PAGES = [
  "stdlib/linalg.md",
  "stdlib/constants.md",
  "stdlib/file.md",
+ "stdlib/delimitedfiles.md",
  "stdlib/io-network.md",
  "stdlib/punctuation.md",
  "stdlib/sort.md",
@@ -116,9 +124,11 @@ const PAGES = [
  ],
 ]
 
+using DelimitedFiles
+
 makedocs(
  build = joinpath(pwd(), "_build/html/en"),
- modules = [Base, Core, BuildSysImg],
+ modules = [Base, Core, BuildSysImg, DelimitedFiles],
  clean = false,
  doctest = "doctest" in ARGS,
  linkcheck = "linkcheck" in ARGS,

doc/src/stdlib/.gitignore

@@ -0,0 +1 @@
+delimitedfiles.md

doc/src/stdlib/index.md

@@ -10,6 +10,7 @@
  * [Linear Algebra](@ref)
  * [Constants](@ref lib-constants)
  * [Filesystem](@ref)
+ * [Delimited Files](@ref)
  * [I/O and Network](@ref)
  * [Punctuation](@ref)
  * [Sorting and Related Functions](@ref)

doc/src/stdlib/io-network.md

@@ -47,7 +47,7 @@ Base.redirect_stdin(::Function, ::Any)
 Base.readchomp
 Base.truncate
 Base.skipchars
-Base.DataFmt.countlines
+Base.countlines
 Base.PipeBuffer
 Base.readavailable
 Base.IOContext
@@ -77,13 +77,6 @@ Base.readline
 Base.readuntil
 Base.readlines
 Base.eachline
-Base.DataFmt.readdlm(::Any, ::Char, ::Type, ::Char)
-Base.DataFmt.readdlm(::Any, ::Char, ::Char)
-Base.DataFmt.readdlm(::Any, ::Char, ::Type)
-Base.DataFmt.readdlm(::Any, ::Char)
-Base.DataFmt.readdlm(::Any, ::Type)
-Base.DataFmt.readdlm(::Any)
-Base.DataFmt.writedlm
 Base.Base64.Base64EncodePipe
 Base.Base64.Base64DecodePipe
 Base.Base64.base64encode

etc/juliarc.jl

@@ -1,3 +1,4 @@
 # This file should contain site-specific commands to be executed on Julia startup
 # Users may store their own personal commands in the user home directory `homedir()`, in a file named `.juliarc.jl`
 
+using DelimitedFiles

stdlib/DelimitedFiles/docs/src/index.md

@@ -0,0 +1,11 @@
+# Delimited Files
+
+```@docs
+DelimitedFiles.readdlm(::Any, ::Char, ::Type, ::Char)
+DelimitedFiles.readdlm(::Any, ::Char, ::Char)
+DelimitedFiles.readdlm(::Any, ::Char, ::Type)
+DelimitedFiles.readdlm(::Any, ::Char)
+DelimitedFiles.readdlm(::Any, ::Type)
+DelimitedFiles.readdlm(::Any)
+DelimitedFiles.writedlm
+```

base/datafmt.jl → stdlib/DelimitedFiles/src/DelimitedFiles.jl

@@ -1,12 +1,16 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-## file formats ##
+__precompile__(true)
 
-module DataFmt
+module DelimitedFiles
 
 import Base: _default_delims, tryparse_internal, show
 
-export countlines, readdlm, writedlm
+export readdlm, writedlm
+
+Base.@deprecate readcsv(io; opts...) readdlm(io, ','; opts...)
+Base.@deprecate readcsv(io, T::Type; opts...) readdlm(io, ',', T; opts...)
+Base.@deprecate writecsv(io, a; opts...) writedlm(io, a, ','; opts...)
 
 invalid_dlm(::Type{Char}) = reinterpret(Char, 0xfffffffe)
 invalid_dlm(::Type{UInt8}) = 0xfe
@@ -15,29 +19,6 @@ invalid_dlm(::Type{UInt32}) = 0xfffffffe
 
 const offs_chunk_size = 5000
 
-countlines(f::AbstractString, eol::Char='\n') = open(io->countlines(io,eol), f)::Int
-
-"""
- countlines(io::IO, eol::Char='\\n')
-
-Read `io` until the end of the stream/file and count the number of lines. To specify a file
-pass the filename as the first argument. EOL markers other than `'\\n'` are supported by
-passing them as the second argument.
-"""
-function countlines(io::IO, eol::Char='\n')
- isascii(eol) || throw(ArgumentError("only ASCII line terminators are supported"))
- aeol = UInt8(eol)
- a = Vector{UInt8}(8192)
- nl = 0
- while !eof(io)
- nb = readbytes!(io, a)
- @simd for i=1:nb
- @inbounds nl += a[i] == aeol
- end
- end
- nl
-end
-
 """
  readdlm(source, T::Type; options...)
 
@@ -698,4 +679,4 @@ writedlm(io, a; opts...) = writedlm(io, a, '\t'; opts...)
 show(io::IO, ::MIME"text/csv", a) = writedlm(io, a, ',')
 show(io::IO, ::MIME"text/tab-separated-values", a) = writedlm(io, a, '\t')
 
-end # module DataFmt
+end # module DelimitedFiles

test/datafmt.jl → stdlib/DelimitedFiles/test/runtests.jl

@@ -1,18 +1,7 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-@testset "countlines" begin
- @test countlines(IOBuffer("\n")) == 1
- @test countlines(IOBuffer("\n"),'\r') == 0
- @test countlines(IOBuffer("\n\n\n\n\n\n\n\n\n\n")) == 10
- @test countlines(IOBuffer("\n \n \n \n \n \n \n \n \n \n")) == 10
- @test countlines(IOBuffer("\r\n \r\n \r\n \r\n \r\n")) == 5
- file = tempname()
- write(file,"Spiffy header\nspectacular first row\neven better 2nd row\nalmost done\n")
- @test countlines(file) == 4
- @test countlines(file,'\r') == 0
- @test countlines(file,'\n') == 4
- rm(file)
-end
+using Base.Test
+using DelimitedFiles
 
 isequaldlm(m1, m2, t) = isequal(m1, m2) && (eltype(m1) == eltype(m2) == t)
 
@@ -295,3 +284,9 @@ end
 
 # issue #11484: useful error message for invalid readdlm filepath arguments
 @test_throws ArgumentError readdlm(tempdir())
+
+# displaying as text/csv
+let d = TextDisplay(IOBuffer())
+ display(d, "text/csv", [3 1 4])
+ @test String(take!(d.io)) == "3,1,4\n"
+end

test/choosetests.jl

@@ -32,7 +32,7 @@ function choosetests(choices = [])
  "bigfloat", "sorting", "statistics", "spawn", "backtrace",
  "file", "read", "mmap", "version", "resolve",
  "pollfd", "mpfr", "broadcast", "complex", "socket",
- "floatapprox", "datafmt", "reflection", "regex", "float16",
+ "floatapprox", "stdlib", "reflection", "regex", "float16",
  "combinatorics", "sysinfo", "env", "rounding", "ranges", "mod2pi",
  "euler", "show", "lineedit", "replcompletions", "repl",
  "replutil", "sets", "test", "goto", "llvmcall", "llvmcall2", "grisu",

test/compile.jl

@@ -207,8 +207,11 @@ try
  @test map(x -> x[1], sort(deps)) == [Foo_file, joinpath(dir, "bar.jl"), joinpath(dir, "foo.jl")]
 
  modules, deps1 = Base.cache_dependencies(cachefile)
- @test modules == Dict(s => Base.module_uuid(getfield(Foo, s)) for s in
- [:Base, :Core, Foo2_module, FooBase_module, :Main])
+ @test modules == merge(Dict(s => Base.module_uuid(getfield(Foo, s)) for s in
+ [:Base, :Core, Foo2_module, FooBase_module, :Main]),
+ # plus modules included in the system image
+ Dict(s => Base.module_uuid(Base.root_module(s)) for s in
+ [:DelimitedFiles]))
  @test deps == deps1
 
  @test current_task()(0x01, 0x4000, 0x30031234) == 2

test/offsetarray.jl

@@ -2,6 +2,7 @@
 
 isdefined(Main, :TestHelpers) || @eval Main include(joinpath(dirname(@__FILE__), "TestHelpers.jl"))
 using Main.TestHelpers.OAs
+using DelimitedFiles
 
 const OAs_name = join(fullname(OAs), ".")
 

test/read.jl

@@ -1,5 +1,7 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
+using DelimitedFiles
+
 mktempdir() do dir
 
 tasks = []
@@ -521,3 +523,17 @@ close(f2)
 rm(f)
 
 end # mktempdir() do dir
+
+@testset "countlines" begin
+ @test countlines(IOBuffer("\n")) == 1
+ @test countlines(IOBuffer("\n"),'\r') == 0
+ @test countlines(IOBuffer("\n\n\n\n\n\n\n\n\n\n")) == 10
+ @test countlines(IOBuffer("\n \n \n \n \n \n \n \n \n \n")) == 10
+ @test countlines(IOBuffer("\r\n \r\n \r\n \r\n \r\n")) == 5
+ file = tempname()
+ write(file,"Spiffy header\nspectacular first row\neven better 2nd row\nalmost done\n")
+ @test countlines(file) == 4
+ @test countlines(file,'\r') == 0
+ @test countlines(file,'\n') == 4
+ rm(file)
+end

test/show.jl

@@ -786,10 +786,7 @@ end
 # don't use julia-specific `f` in Float32 printing (PR #18053)
 @test sprint(print, 1f-7) == "1.0e-7"
 
-# test that the REPL TextDisplay works for displaying arbitrary textual MIME types
 let d = TextDisplay(IOBuffer())
- display(d, "text/csv", [3 1 4])
- @test String(take!(d.io)) == "3,1,4\n"
  @test_throws MethodError display(d, "text/foobar", [3 1 4])
  try
  display(d, "text/foobar", [3 1 4])

test/sparse/cholmod.jl

@@ -3,6 +3,7 @@
 srand(123)
 
 using Base.SparseArrays.CHOLMOD
+using DelimitedFiles
 
 # based on deps/SuiteSparse-4.0.2/CHOLMOD/Demo/
 

test/stdlib.jl

@@ -0,0 +1,7 @@
+# This file is a part of Julia. License is MIT: https://julialang.org/license
+
+# test default packages
+
+cd(joinpath(JULIA_HOME,"..","share","julia","site","v$(VERSION.major).$(VERSION.minor)")) do
+ Pkg.Entry.test(convert(Vector{AbstractString}, readdir()))
+end