fix JuliaLang#37664: treat comments as whitespace

belamenso · Sep 25, 2020 · 639c536 · 639c536
1 parent 0ef55f5
commit 639c536
Show file tree

Hide file tree

Showing 2 changed files with 76 additions and 35 deletions.
diff --git a/src/julia-parser.scm b/src/julia-parser.scm
@@ -529,42 +529,44 @@
 (define (scolno port) (string " near column " (input-port-column port)))
 
 (define (next-token port s)
- (aset! s 2 (eq? (skip-ws port whitespace-newline) #t))
- (let ((c (peek-char port)))
- (cond ((or (eof-object? c) (eqv? c #\newline)) (read-char port))
-
- ((identifier-start-char? c) (accum-julia-symbol c port))
-
- ((string.find "()[]{},;\"`@" c) (read-char port))
-
- ((string.find "0123456789" c) (read-number port #f #f))
-
- ((eqv? c #\#) (skip-comment port) (next-token port s))
+ (let loop ((comment-induced-whitespace #f))
+ (aset! s 2 (or (eq? (skip-ws port whitespace-newline) #t)
+ comment-induced-whitespace))
+ (let ((c (peek-char port)))
+ (cond ((or (eof-object? c) (eqv? c #\newline)) (read-char port))
+
+ ((identifier-start-char? c) (accum-julia-symbol c port))
+
+ ((string.find "()[]{},;\"`@" c) (read-char port))
+
+ ((string.find "0123456789" c) (read-number port #f #f))
+
+ ((eqv? c #\#) (skip-comment port) (loop #t))
+
+ ;; . is difficult to handle; it could start a number or operator
+ ((and (eqv? c #\.)
+ (let ((c (read-char port))
+ (nextc (peek-char port)))
+ (cond ((eof-object? nextc)
+ '|.|)
+ ((char-numeric? nextc)
+ (read-number port #t #f))
+ ((opchar? nextc)
+ (let* ((op (read-operator port c))
+ (nx (peek-char port)))
+ (if (and (eq? op '..) (opchar? nx) (not (memv nx '(#\' #\:))))
+ (error (string "invalid operator \"" op nx "\"" (scolno port))))
+ op))
+ (else '|.|)))))
+
+ ((opchar? c) (read-operator port (read-char port)))
 
- ;; . is difficult to handle; it could start a number or operator
- ((and (eqv? c #\.)
- (let ((c (read-char port))
- (nextc (peek-char port)))
- (cond ((eof-object? nextc)
- '|.|)
- ((char-numeric? nextc)
- (read-number port #t #f))
- ((opchar? nextc)
- (let* ((op (read-operator port c))
- (nx (peek-char port)))
- (if (and (eq? op '..) (opchar? nx) (not (memv nx '(#\' #\:))))
- (error (string "invalid operator \"" op nx "\"" (scolno port))))
- op))
- (else '|.|)))))
-
- ((opchar? c) (read-operator port (read-char port)))
-
- (else
- (let ((cn (input-port-column port)))
- (read-char port)
- (if (default-ignorable-char? c)
- (error (string "invisible character \\u" (number->string (fixnum c) 16) " near column " (+ 1 cn)))
- (error (string "invalid character \"" c "\" near column " (+ 1 cn)))))))))
+ (else
+ (let ((cn (input-port-column port)))
+ (read-char port)
+ (if (default-ignorable-char? c)
+ (error (string "invisible character \\u" (number->string (fixnum c) 16) " near column " (+ 1 cn)))
+ (error (string "invalid character \"" c "\" near column " (+ 1 cn))))))))))
 
 ;; --- token stream ---
 

diff --git a/test/syntax.jl b/test/syntax.jl
@@ -2345,3 +2345,42 @@ end
 
 # issue #37656
 @test :(if true 'a' else 1 end) == Expr(:if, true, quote 'a' end, quote 1 end)
+
+# issue #37664
+@test_throws ParseError("extra token \"b\" after end of expression") Meta.parse("a b")
+@test_throws ParseError("extra token \"b\" after end of expression") Meta.parse("a#==#b")
+@test_throws ParseError("extra token \"b\" after end of expression") Meta.parse("a #==#b")
+@test_throws ParseError("extra token \"b\" after end of expression") Meta.parse("a#==# b")
+
+@test_throws ParseError("extra token \"2\" after end of expression") Meta.parse("1 2")
+@test_throws ParseError("extra token \"2\" after end of expression") Meta.parse("1#==#2")
+@test_throws ParseError("extra token \"2\" after end of expression") Meta.parse("1 #==#2")
+@test_throws ParseError("extra token \"2\" after end of expression") Meta.parse("1#==# 2")
+
+@test size([1#==#2#==#3]) == size([1 2 3])
+@test size([1#==#2#==#3]) == size([1 2 3]) # tabs
+@test size([1#==#2#==#3]) == size([1 2 3]) # tabs ans spaces
+@test size([1#==#2#==#3]) == size([1 2 3]) # tabs ans spaces
+@test [zeros(Int,2,2)#==#[1;2]
+ [3#==#4]#==#5] == [zeros(Int,2,2) [1; 2]
+ [3 4] 5 ] == [0 0 1
+ 0 0 2
+ 3 4 5]
+
+@test Meta.parse("for x in 1:10 g(x) end") ==
+ Meta.parse("for#==#x#==#in#==#1:10#==#g(x)#==#end")
+@test Meta.parse("(f->f(1))() do x x+1 end") ==
+ Meta.parse("(f->f(1))()#==#do#==#x#==#x+1#==#end")
+@test Meta.parse("while i < 10 i += 1 end") ==
+ Meta.parse("while#==#i#==#<#==#10#==#i#==#+=#==#1#==#end")
+@test Meta.parse("begin x=1 end") == Meta.parse("begin#==#x=1#==#end")
+@test Meta.parse("if x<y x+1 elseif y>0 y+1 else z end") ==
+ Meta.parse("if#==#x<y#==#x+1#==#elseif#==#y>0#==#y+1#==#else#==#z#==#end")
+@test Meta.parse("function(x) x end") == Meta.parse("function(x)#==#x#==#end")
+@test Meta.parse("a ? b : c") == Meta.parse("a#==#?#==#b#==#:#==#c")
+@test_throws ParseError("space before \"(\" not allowed in \"f (\" at none:1") begin
+ Meta.parse("f#==#(x)=x")
+end
+@test Meta.parse("try f() catch e g() finally h() end") ==
+ Meta.parse("try#==#f()#==#catch#==#e#==#g()#==#finally#==#h()#==#end")
+@test Meta.parse("@m a b") == Meta.parse("@m#==#a#==#b")