update doc string syntax (#25938)

- disallow extra lines between a docstring and the documented object
- parse an extra expression after a newline in calls to `@doc`
- deprecate special parsing of `doc" "`
- deprecate `->` for doc strings

base/int.jl

@@ -452,7 +452,7 @@ end
 # doesn't work either at this point.
 if nameof(@__MODULE__) === :Base
  for fname in (:mod, :rem)
- @eval @doc ("""
+ @eval @doc """
  rem(x::Integer, T::Type{<:Integer}) -> T
  mod(x::Integer, T::Type{<:Integer}) -> T
  %(x::Integer, T::Type{<:Integer}) -> T
@@ -466,7 +466,7 @@ if nameof(@__MODULE__) === :Base
  julia> 129 % Int8
  -127
  ```
- """ -> $fname(x::Integer, T::Type{<:Integer}))
+ """ $fname(x::Integer, T::Type{<:Integer})
  end
 end
 

base/libc.jl

@@ -3,7 +3,7 @@
 module Libc
 @doc """
 Interface to libc, the C standard library.
-""" -> Libc
+""" Libc
 
 import Base: transcode
 

base/statistics.jl

@@ -546,8 +546,8 @@ cor(x::AbstractVecOrMat, y::AbstractVecOrMat, vardim::Int=1) =
 
 Compute the middle of a scalar value, which is equivalent to `x` itself, but of the type of `middle(x, x)` for consistency.
 """
-# Specialized functions for real types allow for improved performance
 middle(x::Union{Bool,Int8,Int16,Int32,Int64,Int128,UInt8,UInt16,UInt32,UInt64,UInt128}) = Float64(x)
+# Specialized functions for real types allow for improved performance
 middle(x::AbstractFloat) = x
 middle(x::Real) = (x + zero(x)) / 1
 

base/sysinfo.jl

@@ -3,7 +3,7 @@
 module Sys
 @doc """
 Provide methods for retrieving information about hardware and the operating system.
-""" -> Sys
+""" Sys
 
 export BINDIR,
  CPU_CORES,

doc/src/manual/documentation.md

@@ -3,9 +3,10 @@
 Julia enables package developers and users to document functions, types and other objects easily
 via a built-in documentation system since Julia 0.4.
 
-The basic syntax is very simple: any string appearing at the top-level right before an object
+The basic syntax is simple: any string appearing at the top-level right before an object
 (function, macro, type or instance) will be interpreted as documenting it (these are called *docstrings*).
-Here is a very simple example:
+Note that no blank lines or comments may intervene between a docstring and the documented object.
+Here is a basic example:
 
 ```julia
 "Tell whether there are too foo items in the array."
@@ -248,6 +249,20 @@ The `@doc` macro associates its first argument with its second in a per-module d
 macro simply creates an object representing the Markdown content. In the future it is likely to
 do more advanced things such as allowing for relative image or link paths.
 
+To make it easier to write documentation, the parser treats the macro name `@doc` specially:
+if a call to `@doc` has one argument, but another expression appears after a single line
+break, then that additional expression is added as an argument to the macro.
+Therefore the following syntax is parsed as a 2-argument call to `@doc`:
+
+```julia
+@doc raw"""
+...
+"""
+f(x) = x
+```
+
+This makes it easy to use an arbitrary object (here a `raw` string) as a docstring.
+
 When used for retrieving documentation, the `@doc` macro (or equally, the `doc` function) will
 search all `META` dictionaries for metadata relevant to the given object and return it. The returned
 object (some Markdown content, for example) will by default display itself intelligently. This
@@ -308,24 +323,9 @@ y = MyType("y")
 
 A comprehensive overview of all documentable Julia syntax.
 
-In the following examples `"..."` is used to illustrate an arbitrary docstring which may be one
-of the follow four variants and contain arbitrary text:
-
-```julia
-"..."
-
-doc"..."
-
-"""
-...
-"""
-
-doc"""
-...
-"""
-```
+In the following examples `"..."` is used to illustrate an arbitrary docstring.
 
-`@doc_str` should only be used when the docstring contains `$` or `\` characters that should not
+`doc""` should only be used when the docstring contains `$` or `\` characters that should not
 be parsed by Julia such as LaTeX syntax or Julia source code examples containing interpolation.
 
 ### Functions and Methods

src/julia-parser.scm

@@ -2317,17 +2317,19 @@
  (parse-unary-prefix s))))
  (peek-token s)
  (if (ts:space? s)
- `(macrocall ,(macroify-name head)
- ,startloc
- ,@(parse-space-separated-exprs s))
+ (maybe-docstring
+ s `(macrocall ,(macroify-name head)
+ ,startloc
+ ,@(parse-space-separated-exprs s)))
  (let ((call (parse-call-chain s head #t)))
  (if (and (pair? call) (eq? (car call) 'call))
  `(macrocall ,(macroify-name (cadr call))
  ,startloc
  ,@(cddr call))
- `(macrocall ,(macroify-name call)
- ,startloc
- ,@(parse-space-separated-exprs s))))))))
+ (maybe-docstring
+ s `(macrocall ,(macroify-name call)
+ ,startloc
+ ,@(parse-space-separated-exprs s)))))))))
  ;; command syntax
  ((eqv? t #\`)
  (take-token s)
@@ -2352,30 +2354,60 @@
  (quote ,(apply macroify-name (cadr (caddr e)) suffixes))))
  (else (error (string "invalid macro usage \"@(" (deparse e) ")\"" )))))
 
-(define (simple-string-literal? e) (string? e))
+(define (called-macro-name e)
+ (if (and (length= e 3) (eq? (car e) '|.|)
+ (pair? (caddr e)) (eq? (car (caddr e)) 'quote))
+ (called-macro-name (cadr (caddr e)))
+ e))
 
-(define (string-macro-name? x)
- (let ((x (string x)))
- (and (> (string-length x) 4)
- (eqv? (string.char x 0) #\@)
- (eqv? (string.sub x (string.dec x (sizeof x) 4)) "_str"))))
+(define (maybe-docstring s e)
+ (if (and (length= e 4)
+ (eq? (called-macro-name (cadr e)) '@doc))
+ (let ((arg (cadddr e)))
+ (if (and (pair? arg) (eq? (car arg) '->))
+ (begin (parser-depwarn s "@doc call with ->" "a line break")
+ e)
+ (let loop ((t (peek-token s))
+ (nl 0))
+ (cond ((closing-token? t) e)
+ ((newline? t)
+ (if (> nl 0)
+ e
+ (begin (take-token s)
+ (loop (peek-token s) 1))))
+ (else
+ `(,@e ,(parse-eq s)))))))
+ e))
+
+(define (simple-string-literal? e) (string? e))
 
-(define (doc-string-literal? e)
+(define (doc-string-literal? s e)
  (or (simple-string-literal? e)
  (and (pair? e)
  (or (eq? (car e) 'string) ; string interpolation
  (and (eq? (car e) 'macrocall)
- (string-macro-name? (cadr e))
  (or (and (length= e 3) (simple-string-literal? (caddr e)))
- (and (length= e 4) (simple-string-literal? (cadddr e)))))))))
+ (and (length= e 4) (simple-string-literal? (cadddr e))))
+ (eq? (cadr e) '@doc_str)
+ (begin (parser-depwarn s "doc\" \"" "@doc doc\" \"")
+ #t))))))
 
 (define (parse-docstring s production)
  (let ((startloc (line-number-node s)) ; be sure to use the line number from the head of the docstring
- (ex (production s)))
- (if (and (doc-string-literal? ex)
- (let loop ((t (peek-token s)))
+ (ex (production s)))
+ (if (and (doc-string-literal? s ex)
+ (let loop ((t (peek-token s))
+ (nl 0))
  (cond ((closing-token? t) #f)
- ((newline? t) (take-token s) (loop (peek-token s)))
+ ((newline? t)
+ (if (= nl 1)
+ ;;#f ;; extra line => not a doc string. enable when deprecation is removed.
+ (begin (parser-depwarn s "multiple line breaks between doc string and object"
+ "at most one line break")
+ (take-token s)
+ (loop (peek-token s) 2))
+ (begin (take-token s)
+ (loop (peek-token s) 1))))
  (else #t))))
  `(macrocall (core @doc) ,startloc ,ex ,(production s))
  ex)))

stdlib/LibGit2/src/consts.jl

@@ -362,7 +362,7 @@ if version() >= v"0.24.0"
  * `CONFIG_LEVEL_LOCAL` - Repository specific configuration file; `\$WORK_DIR/.git/config` on non-bare repos
  * `CONFIG_LEVEL_APP` - Application specific configuration file; freely defined by applications
  * `CONFIG_HIGHEST_LEVEL` - Represents the highest level available config file (i.e. the most specific config file available that actually is loaded)
- """ ->
+ """
  @enum(GIT_CONFIG, CONFIG_LEVEL_DEFAULT = 0,
  CONFIG_LEVEL_PROGRAMDATA = 1,
  CONFIG_LEVEL_SYSTEM = 2,
@@ -384,7 +384,7 @@ else
  * `CONFIG_LEVEL_LOCAL` - Repository specific configuration file; `\$WORK_DIR/.git/config` on non-bare repos
  * `CONFIG_LEVEL_APP` - Application specific configuration file; freely defined by applications
  * `CONFIG_HIGHEST_LEVEL` - Represents the highest level available config file (i.e. the most specific config file available that actually is loaded)
- """ ->
+ """
  @enum(GIT_CONFIG, CONFIG_LEVEL_DEFAULT = 0,
  CONFIG_LEVEL_SYSTEM = 1,
  CONFIG_LEVEL_XDG = 2,

stdlib/Libdl/src/Libdl.jl

@@ -5,7 +5,7 @@ __precompile__(true)
 module Libdl
 @doc """
 Interface to libdl. Provides dynamic linking support.
-""" -> Libdl
+""" Libdl
 
 import Base.DL_LOAD_PATH
 
@@ -32,7 +32,7 @@ const RTLD_NOLOAD = 0x00000020
 const RTLD_DEEPBIND = 0x00000040
 const RTLD_FIRST = 0x00000080
 
-@doc """
+"""
  RTLD_DEEPBIND
  RTLD_FIRST
  RTLD_GLOBAL
@@ -44,7 +44,7 @@ const RTLD_FIRST = 0x00000080
 
 Enum constant for [`dlopen`](@ref). See your platform man page for details, if
 applicable.
-""" ->
+"""
 (RTLD_DEEPBIND, RTLD_FIRST, RTLD_GLOBAL, RTLD_LAZY, RTLD_LOCAL, RTLD_NODELETE, RTLD_NOLOAD, RTLD_NOW)
 
 """

stdlib/LinearAlgebra/src/blas.jl

@@ -3,7 +3,7 @@
 module BLAS
 @doc """
 Interface to BLAS subroutines.
-""" -> BLAS
+""" BLAS
 
 import ..axpy!, ..axpby!
 import Base: copyto!

stdlib/LinearAlgebra/src/lapack.jl

@@ -3,7 +3,7 @@
 module LAPACK
 @doc """
 Interfaces to LAPACK subroutines.
-""" -> LAPACK
+""" LAPACK
 
 const liblapack = Base.liblapack_name
 

test/choosetests.jl

@@ -5,7 +5,7 @@ using Random
 const STDLIB_DIR = joinpath(Sys.BINDIR, "..", "share", "julia", "site", "v$(VERSION.major).$(VERSION.minor)")
 const STDLIBS = readdir(STDLIB_DIR)
 
-@doc """
+"""
 
 `tests, net_on, exit_on_error, seed = choosetests(choices)` selects a set of tests to be
 run. `choices` should be a vector of test names; if empty or set to
@@ -30,7 +30,7 @@ in the `choices` argument:
  - "--seed=SEED", which sets the value of `seed` to `SEED`
  (parsed as an `UInt128`); `seed` is otherwise initialized randomly.
  This option can be used to reproduce failed tests.
-""" ->
+"""
 function choosetests(choices = [])
  testnames = [
  "subarray", "core", "compiler", "worlds",

test/docs.jl

@@ -30,19 +30,19 @@ function docstring_startswith(d1, d2)
 end
 docstring_startswith(d1::DocStr, d2) = docstring_startswith(parsedoc(d1), d2)
 
-@doc "Doc abstract type" ->
+@doc "Doc abstract type"
 abstract type C74685{T,N} <: AbstractArray{T,N} end
 @test stringmime("text/plain", Docs.doc(C74685))==" Doc abstract type\n"
 @test string(Docs.doc(C74685))=="Doc abstract type\n"
 
 macro macro_doctest() end
-@doc "Helps test if macros can be documented with `@doc \"...\" -> @...`." ->
+@doc "Helps test if macros can be documented with `@doc \"...\" @...`."
 :@macro_doctest
 
 @test (@doc @macro_doctest) !== nothing
 
 # test that random stuff interpolated into docstrings doesn't break search or other methods here
-doc"""
+@doc doc"""
 break me:
 
  code
@@ -602,7 +602,7 @@ end
 
 f1_11993()
 
-@doc "Document inline function with old syntax" ->
+@doc "Document inline function with old syntax"
 @inline f2_11993() = nothing
 
 @test (@doc f2_11993) !== nothing
@@ -839,7 +839,7 @@ $(curmod_prefix)Undocumented.D <: $(curmod_prefix)Undocumented.B <: $(curmod_pre
 """)
 @test docstrings_equal(@doc(Undocumented.D), doc"$doc_str")
 
-let d = @doc Undocumented.f
+let d = @doc(Undocumented.f)
  io = IOBuffer()
  show(io, MIME"text/markdown"(), d)
  @test startswith(String(take!(io)),"""
@@ -849,7 +849,7 @@ let d = @doc Undocumented.f
  """)
 end
 
-let d = @doc Undocumented.undocumented
+let d = @doc(Undocumented.undocumented)
  io = IOBuffer()
  show(io, MIME"text/markdown"(), d)
  @test startswith(String(take!(io)), """

test/syntax.jl

@@ -1269,7 +1269,34 @@ end
 @test_throws ParseError Meta.parse("using Colors()")
 
 let ex = Meta.parse("md\"x\"
+ f(x) = x", 1)[1] # custom string literal is not a docstring
+ @test Meta.isexpr(ex, :macrocall)
+ @test ex.args[1] === Symbol("@md_str")
+ @test length(ex.args) == 3
+end
+
+let ex = Meta.parse("@doc raw\"
+ \"
  f(x) = x")
  @test Meta.isexpr(ex, :macrocall)
- @test ex.args[1] == Core.GlobalRef(Core, Symbol("@doc"))
+ @test ex.args[1] === Symbol("@doc")
+ @test length(ex.args) == 4
+ @test Meta.isexpr(ex.args[4], :(=))
+end
+
+let ex = Meta.parse("@doc raw\"
+ \"
+
+ f(x) = x", 1)[1]
+ @test Meta.isexpr(ex, :macrocall)
+ @test ex.args[1] === Symbol("@doc")
+ @test length(ex.args) == 3
 end
+
+# TODO: enable when 0.7 deprecations are removed
+#@test Meta.parse("\"x\"
+# # extra line, not a doc string
+# f(x) = x", 1)[1] === "x"
+#@test Meta.parse("\"x\"
+#
+# f(x) = x", 1)[1] === "x"