diff --git a/Makefile b/Makefile index d439ccec7c285..c100eb8072061 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ default: $(JULIA_BUILD_MODE) # contains either "debug" or "release" all: debug release # sort is used to remove potential duplicates -DIRS := $(sort $(build_bindir) $(build_depsbindir) $(build_libdir) $(build_private_libdir) $(build_libexecdir) $(build_includedir) $(build_includedir)/julia $(build_sysconfdir)/julia $(build_datarootdir)/julia $(build_man1dir)) +DIRS := $(sort $(build_bindir) $(build_depsbindir) $(build_libdir) $(build_private_libdir) $(build_libexecdir) $(build_includedir) $(build_includedir)/julia $(build_sysconfdir)/julia $(build_datarootdir)/julia $(build_datarootdir)/julia/site $(build_man1dir)) ifneq ($(BUILDROOT),$(JULIAHOME)) BUILDDIRS := $(BUILDROOT) $(addprefix $(BUILDROOT)/,base src ui doc deps test test/perf examples examples/embedding) BUILDDIRMAKE := $(addsuffix /Makefile,$(BUILDDIRS)) @@ -50,6 +50,11 @@ endif $(foreach dir,$(DIRS),$(eval $(call dir_target,$(dir)))) $(foreach link,base test,$(eval $(call symlink_target,$(link),$(build_datarootdir)/julia))) +$(eval $(call symlink_target,stdlib,$(build_datarootdir)/julia/site)) + +build_defaultpkgdir = $(build_datarootdir)/julia/site/$(shell echo $(VERSDIR)) +$(build_defaultpkgdir): $(build_datarootdir)/julia/site/stdlib + @mv $(build_datarootdir)/julia/site/stdlib $@ julia_flisp.boot.inc.phony: julia-deps @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/src julia_flisp.boot.inc.phony @@ -81,7 +86,7 @@ ifndef JULIA_VAGRANT_BUILD endif endif -julia-deps: | $(DIRS) $(build_datarootdir)/julia/base $(build_datarootdir)/julia/test +julia-deps: | $(DIRS) $(build_datarootdir)/julia/base $(build_datarootdir)/julia/test $(build_defaultpkgdir) @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/deps julia-base: julia-deps $(build_sysconfdir)/julia/juliarc.jl $(build_man1dir)/julia.1 $(build_datarootdir)/julia/julia-config.jl diff --git a/README.md b/README.md index 353067458def1..6d0901179bd0c 100644 --- a/README.md +++ b/README.md @@ -372,11 +372,12 @@ It is highly recommended to start with a fresh clone of the Julia repository. The Julia source code is organized as follows: - base/ source code for Julia's standard library + base/ source code for the Base module (part of Julia's standard library) + stdlib/ source code for other standard library packages contrib/ editor support for Julia source, miscellaneous scripts deps/ external dependencies - doc/manual source for the user manual - doc/stdlib source for standard library function help text + doc/src/manual source for the user manual + doc/src/stdlib source for standard library function reference examples/ example Julia programs src/ source for Julia language core test/ test suites diff --git a/base/deprecated.jl b/base/deprecated.jl index c8b8a1084c770..36b21d454b420 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -1724,19 +1724,12 @@ import .LinAlg: diagm @deprecate_binding φ MathConstants.φ @deprecate_binding golden MathConstants.golden -# deprecate writecsv -@deprecate writecsv(io, a; opts...) writedlm(io, a, ','; opts...) - # PR #23271 function IOContext(io::IO; kws...) depwarn("IOContext(io, k=v, ...) is deprecated, use IOContext(io, :k => v, ...) instead.", :IOContext) IOContext(io, (k=>v for (k, v) in kws)...) end -# deprecate readcsv -@deprecate readcsv(io; opts...) readdlm(io, ','; opts...) -@deprecate readcsv(io, T::Type; opts...) readdlm(io, ',', T; opts...) - @deprecate IOContext(io::IO, key, value) IOContext(io, key=>value) # PR #23485 diff --git a/base/exports.jl b/base/exports.jl index b3e3b288cd65b..195c536e30d0d 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -1048,7 +1048,6 @@ export readbytes!, readchomp, readdir, - readdlm, readline, readlines, readuntil, @@ -1070,7 +1069,6 @@ export unmark, watch_file, write, - writedlm, TCPSocket, UDPSocket, diff --git a/base/initdefs.jl b/base/initdefs.jl index ff96f21855e3a..ef223bd70a7c9 100644 --- a/base/initdefs.jl +++ b/base/initdefs.jl @@ -69,7 +69,7 @@ functionality is experimental and may break or change in Julia 1.0. const LOAD_PATH = Any[] const LOAD_CACHE_PATH = String[] -function init_load_path() +function init_load_path(JULIA_HOME = JULIA_HOME) vers = "v$(VERSION.major).$(VERSION.minor)" if haskey(ENV, "JULIA_LOAD_PATH") prepend!(LOAD_PATH, split(ENV["JULIA_LOAD_PATH"], @static Sys.iswindows() ? ';' : ':')) diff --git a/base/io.jl b/base/io.jl index 80a97b8be3517..4b3a74e30e264 100644 --- a/base/io.jl +++ b/base/io.jl @@ -779,3 +779,26 @@ function skipchars(io::IO, pred; linecomment=nothing) end return io end + +""" + countlines(io::IO, eol::Char='\\n') + +Read `io` until the end of the stream/file and count the number of lines. To specify a file +pass the filename as the first argument. EOL markers other than `'\\n'` are supported by +passing them as the second argument. +""" +function countlines(io::IO, eol::Char='\n') + isascii(eol) || throw(ArgumentError("only ASCII line terminators are supported")) + aeol = UInt8(eol) + a = Vector{UInt8}(8192) + nl = 0 + while !eof(io) + nb = readbytes!(io, a) + @simd for i=1:nb + @inbounds nl += a[i] == aeol + end + end + nl +end + +countlines(f::AbstractString, eol::Char='\n') = open(io->countlines(io,eol), f)::Int diff --git a/base/sysimg.jl b/base/sysimg.jl index 55e64328aac3a..013ef3da83e6d 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -360,8 +360,6 @@ include("mmap.jl") import .Mmap # utilities - timing, help, edit -include("datafmt.jl") -using .DataFmt include("deepcopy.jl") include("interactiveutil.jl") include("summarysize.jl") @@ -453,4 +451,12 @@ end # baremodule Base using Base +# set up load path to be able to find stdlib packages +Base.init_load_path(ccall(:jl_get_julia_home, Any, ())) + +# load some stdlib packages but don't put their names in Main +Base.require(:DelimitedFiles) + +empty!(LOAD_PATH) + Base.isfile("userimg.jl") && Base.include(Main, "userimg.jl") diff --git a/doc/make.jl b/doc/make.jl index ad966f96d51cb..5de0371d68068 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -16,6 +16,13 @@ end # Documenter Setup. +# make links for stdlib package docs +if Sys.iswindows() + cp("../stdlib/DelimitedFiles/docs/src/index.md", "src/stdlib/delimitedfiles.md") +else + symlink("../../../stdlib/DelimitedFiles/docs/src/index.md", "src/stdlib/delimitedfiles.md") +end + const PAGES = [ "Home" => "index.md", "Manual" => [ @@ -69,6 +76,7 @@ const PAGES = [ "stdlib/linalg.md", "stdlib/constants.md", "stdlib/file.md", + "stdlib/delimitedfiles.md", "stdlib/io-network.md", "stdlib/punctuation.md", "stdlib/sort.md", @@ -116,9 +124,11 @@ const PAGES = [ ], ] +using DelimitedFiles + makedocs( build = joinpath(pwd(), "_build/html/en"), - modules = [Base, Core, BuildSysImg], + modules = [Base, Core, BuildSysImg, DelimitedFiles], clean = false, doctest = "doctest" in ARGS, linkcheck = "linkcheck" in ARGS, diff --git a/doc/src/stdlib/.gitignore b/doc/src/stdlib/.gitignore new file mode 100644 index 0000000000000..014a6f161ae01 --- /dev/null +++ b/doc/src/stdlib/.gitignore @@ -0,0 +1 @@ +delimitedfiles.md diff --git a/doc/src/stdlib/index.md b/doc/src/stdlib/index.md index c4b9d525c3b8e..05346024aff82 100644 --- a/doc/src/stdlib/index.md +++ b/doc/src/stdlib/index.md @@ -10,6 +10,7 @@ * [Linear Algebra](@ref) * [Constants](@ref lib-constants) * [Filesystem](@ref) + * [Delimited Files](@ref) * [I/O and Network](@ref) * [Punctuation](@ref) * [Sorting and Related Functions](@ref) diff --git a/doc/src/stdlib/io-network.md b/doc/src/stdlib/io-network.md index f56b014d2a439..91ad6151eb247 100644 --- a/doc/src/stdlib/io-network.md +++ b/doc/src/stdlib/io-network.md @@ -47,7 +47,7 @@ Base.redirect_stdin(::Function, ::Any) Base.readchomp Base.truncate Base.skipchars -Base.DataFmt.countlines +Base.countlines Base.PipeBuffer Base.readavailable Base.IOContext @@ -77,13 +77,6 @@ Base.readline Base.readuntil Base.readlines Base.eachline -Base.DataFmt.readdlm(::Any, ::Char, ::Type, ::Char) -Base.DataFmt.readdlm(::Any, ::Char, ::Char) -Base.DataFmt.readdlm(::Any, ::Char, ::Type) -Base.DataFmt.readdlm(::Any, ::Char) -Base.DataFmt.readdlm(::Any, ::Type) -Base.DataFmt.readdlm(::Any) -Base.DataFmt.writedlm Base.Base64.Base64EncodePipe Base.Base64.Base64DecodePipe Base.Base64.base64encode diff --git a/etc/juliarc.jl b/etc/juliarc.jl index 3993e62867710..7e7eabcbb8526 100644 --- a/etc/juliarc.jl +++ b/etc/juliarc.jl @@ -1,3 +1,4 @@ # This file should contain site-specific commands to be executed on Julia startup # Users may store their own personal commands in the user home directory `homedir()`, in a file named `.juliarc.jl` +using DelimitedFiles diff --git a/stdlib/DelimitedFiles/docs/src/index.md b/stdlib/DelimitedFiles/docs/src/index.md new file mode 100644 index 0000000000000..7cd38cb01b25d --- /dev/null +++ b/stdlib/DelimitedFiles/docs/src/index.md @@ -0,0 +1,11 @@ +# Delimited Files + +```@docs +DelimitedFiles.readdlm(::Any, ::Char, ::Type, ::Char) +DelimitedFiles.readdlm(::Any, ::Char, ::Char) +DelimitedFiles.readdlm(::Any, ::Char, ::Type) +DelimitedFiles.readdlm(::Any, ::Char) +DelimitedFiles.readdlm(::Any, ::Type) +DelimitedFiles.readdlm(::Any) +DelimitedFiles.writedlm +``` diff --git a/base/datafmt.jl b/stdlib/DelimitedFiles/src/DelimitedFiles.jl similarity index 97% rename from base/datafmt.jl rename to stdlib/DelimitedFiles/src/DelimitedFiles.jl index 462eece25c695..4377288166e67 100644 --- a/base/datafmt.jl +++ b/stdlib/DelimitedFiles/src/DelimitedFiles.jl @@ -1,12 +1,16 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -## file formats ## +__precompile__(true) -module DataFmt +module DelimitedFiles import Base: _default_delims, tryparse_internal, show -export countlines, readdlm, writedlm +export readdlm, writedlm + +Base.@deprecate readcsv(io; opts...) readdlm(io, ','; opts...) +Base.@deprecate readcsv(io, T::Type; opts...) readdlm(io, ',', T; opts...) +Base.@deprecate writecsv(io, a; opts...) writedlm(io, a, ','; opts...) invalid_dlm(::Type{Char}) = reinterpret(Char, 0xfffffffe) invalid_dlm(::Type{UInt8}) = 0xfe @@ -15,29 +19,6 @@ invalid_dlm(::Type{UInt32}) = 0xfffffffe const offs_chunk_size = 5000 -countlines(f::AbstractString, eol::Char='\n') = open(io->countlines(io,eol), f)::Int - -""" - countlines(io::IO, eol::Char='\\n') - -Read `io` until the end of the stream/file and count the number of lines. To specify a file -pass the filename as the first argument. EOL markers other than `'\\n'` are supported by -passing them as the second argument. -""" -function countlines(io::IO, eol::Char='\n') - isascii(eol) || throw(ArgumentError("only ASCII line terminators are supported")) - aeol = UInt8(eol) - a = Vector{UInt8}(8192) - nl = 0 - while !eof(io) - nb = readbytes!(io, a) - @simd for i=1:nb - @inbounds nl += a[i] == aeol - end - end - nl -end - """ readdlm(source, T::Type; options...) @@ -698,4 +679,4 @@ writedlm(io, a; opts...) = writedlm(io, a, '\t'; opts...) show(io::IO, ::MIME"text/csv", a) = writedlm(io, a, ',') show(io::IO, ::MIME"text/tab-separated-values", a) = writedlm(io, a, '\t') -end # module DataFmt +end # module DelimitedFiles diff --git a/test/datafmt.jl b/stdlib/DelimitedFiles/test/runtests.jl similarity index 96% rename from test/datafmt.jl rename to stdlib/DelimitedFiles/test/runtests.jl index eee6f4e1bcf7e..aa39df87b6371 100644 --- a/test/datafmt.jl +++ b/stdlib/DelimitedFiles/test/runtests.jl @@ -1,18 +1,7 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -@testset "countlines" begin - @test countlines(IOBuffer("\n")) == 1 - @test countlines(IOBuffer("\n"),'\r') == 0 - @test countlines(IOBuffer("\n\n\n\n\n\n\n\n\n\n")) == 10 - @test countlines(IOBuffer("\n \n \n \n \n \n \n \n \n \n")) == 10 - @test countlines(IOBuffer("\r\n \r\n \r\n \r\n \r\n")) == 5 - file = tempname() - write(file,"Spiffy header\nspectacular first row\neven better 2nd row\nalmost done\n") - @test countlines(file) == 4 - @test countlines(file,'\r') == 0 - @test countlines(file,'\n') == 4 - rm(file) -end +using Base.Test +using DelimitedFiles isequaldlm(m1, m2, t) = isequal(m1, m2) && (eltype(m1) == eltype(m2) == t) @@ -295,3 +284,9 @@ end # issue #11484: useful error message for invalid readdlm filepath arguments @test_throws ArgumentError readdlm(tempdir()) + +# displaying as text/csv +let d = TextDisplay(IOBuffer()) + display(d, "text/csv", [3 1 4]) + @test String(take!(d.io)) == "3,1,4\n" +end diff --git a/test/choosetests.jl b/test/choosetests.jl index b527dc8d5b216..af9d5df891367 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -32,7 +32,7 @@ function choosetests(choices = []) "bigfloat", "sorting", "statistics", "spawn", "backtrace", "file", "read", "mmap", "version", "resolve", "pollfd", "mpfr", "broadcast", "complex", "socket", - "floatapprox", "datafmt", "reflection", "regex", "float16", + "floatapprox", "stdlib", "reflection", "regex", "float16", "combinatorics", "sysinfo", "env", "rounding", "ranges", "mod2pi", "euler", "show", "lineedit", "replcompletions", "repl", "replutil", "sets", "test", "goto", "llvmcall", "llvmcall2", "grisu", diff --git a/test/compile.jl b/test/compile.jl index 075a33b82aed6..8fa1829c24426 100644 --- a/test/compile.jl +++ b/test/compile.jl @@ -207,8 +207,11 @@ try @test map(x -> x[1], sort(deps)) == [Foo_file, joinpath(dir, "bar.jl"), joinpath(dir, "foo.jl")] modules, deps1 = Base.cache_dependencies(cachefile) - @test modules == Dict(s => Base.module_uuid(getfield(Foo, s)) for s in - [:Base, :Core, Foo2_module, FooBase_module, :Main]) + @test modules == merge(Dict(s => Base.module_uuid(getfield(Foo, s)) for s in + [:Base, :Core, Foo2_module, FooBase_module, :Main]), + # plus modules included in the system image + Dict(s => Base.module_uuid(Base.root_module(s)) for s in + [:DelimitedFiles])) @test deps == deps1 @test current_task()(0x01, 0x4000, 0x30031234) == 2 diff --git a/test/offsetarray.jl b/test/offsetarray.jl index a15727bfa2c97..62010e73f3d63 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -2,6 +2,7 @@ isdefined(Main, :TestHelpers) || @eval Main include(joinpath(dirname(@__FILE__), "TestHelpers.jl")) using Main.TestHelpers.OAs +using DelimitedFiles const OAs_name = join(fullname(OAs), ".") diff --git a/test/read.jl b/test/read.jl index dfe38094e4598..8014affdc5f35 100644 --- a/test/read.jl +++ b/test/read.jl @@ -1,5 +1,7 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license +using DelimitedFiles + mktempdir() do dir tasks = [] @@ -521,3 +523,17 @@ close(f2) rm(f) end # mktempdir() do dir + +@testset "countlines" begin + @test countlines(IOBuffer("\n")) == 1 + @test countlines(IOBuffer("\n"),'\r') == 0 + @test countlines(IOBuffer("\n\n\n\n\n\n\n\n\n\n")) == 10 + @test countlines(IOBuffer("\n \n \n \n \n \n \n \n \n \n")) == 10 + @test countlines(IOBuffer("\r\n \r\n \r\n \r\n \r\n")) == 5 + file = tempname() + write(file,"Spiffy header\nspectacular first row\neven better 2nd row\nalmost done\n") + @test countlines(file) == 4 + @test countlines(file,'\r') == 0 + @test countlines(file,'\n') == 4 + rm(file) +end diff --git a/test/show.jl b/test/show.jl index e0cfb88d9479f..f466136719b53 100644 --- a/test/show.jl +++ b/test/show.jl @@ -786,10 +786,7 @@ end # don't use julia-specific `f` in Float32 printing (PR #18053) @test sprint(print, 1f-7) == "1.0e-7" -# test that the REPL TextDisplay works for displaying arbitrary textual MIME types let d = TextDisplay(IOBuffer()) - display(d, "text/csv", [3 1 4]) - @test String(take!(d.io)) == "3,1,4\n" @test_throws MethodError display(d, "text/foobar", [3 1 4]) try display(d, "text/foobar", [3 1 4]) diff --git a/test/sparse/cholmod.jl b/test/sparse/cholmod.jl index b8b321ec6b86d..cb49c6ca627c7 100644 --- a/test/sparse/cholmod.jl +++ b/test/sparse/cholmod.jl @@ -3,6 +3,7 @@ srand(123) using Base.SparseArrays.CHOLMOD +using DelimitedFiles # based on deps/SuiteSparse-4.0.2/CHOLMOD/Demo/ diff --git a/test/stdlib.jl b/test/stdlib.jl new file mode 100644 index 0000000000000..7f5cec2ada287 --- /dev/null +++ b/test/stdlib.jl @@ -0,0 +1,7 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +# test default packages + +cd(joinpath(JULIA_HOME,"..","share","julia","site","v$(VERSION.major).$(VERSION.minor)")) do + Pkg.Entry.test(convert(Vector{AbstractString}, readdir())) +end