Merge pull request hashicorp#240 from octo/scanner-next

scanner: Update prevPos even when returning utf8.RuneError.

hcl/scanner/scanner.go

@@ -74,21 +74,18 @@ func (s *Scanner) next() rune {
  return eof
  }
 
- if ch == utf8.RuneError && size == 1 {
- s.srcPos.Column++
- s.srcPos.Offset += size
- s.lastCharLen = size
- s.err("illegal UTF-8 encoding")
- return ch
- }
-
  // remember last position
  s.prevPos = s.srcPos
 
  s.srcPos.Column++
  s.lastCharLen = size
  s.srcPos.Offset += size
 
+ if ch == utf8.RuneError && size == 1 {
+ s.err("illegal UTF-8 encoding")
+ return ch
+ }
+
  if ch == '\n' {
  s.srcPos.Line++
  s.lastLineLen = s.srcPos.Column

hcl/scanner/scanner_test.go

@@ -592,3 +592,22 @@ func countNewlines(s string) int {
  }
  return n
 }
+
+func TestScanHeredocRegexpCompile(t *testing.T) {
+ cases := []string{
+ "0\xe1\n<<ȸ\nhello\nworld\nȸ",
+ }
+
+ for _, c := range cases {
+ s := New([]byte(c))
+ fmt.Printf("START %q\n", c)
+
+ for {
+ tok := s.Scan()
+ if tok.Type == token.EOF {
+ break
+ }
+ t.Logf("s.Scan() = %s", tok)
+ }
+ }
+}