feat[lexer]: Support Booleans.

watcol · Nov 22, 2021 · da1e3b0 · da1e3b0
1 parent ef4ba9b
commit da1e3b0
Showing 1 changed file with 69 additions and 38 deletions.
diff --git a/src/lex.rs b/src/lex.rs
@@ -35,20 +35,25 @@ peg::parser! { grammar lexer() for str {
  / "_" { Symbol::UnderLine }
  / "@" { Symbol::At }
 
+ rule boolean() -> bool = "true" { true } / "false" { false }
+
  rule comment() = "#" [^ '\n'|'\r']*
- rule _ = ([' '|'\t'] / ("\\" [' '|'\t']* __))*
- rule __ = comment()? ['\n'|'\r'] ([' '|'\t'|'\n'|'\r'] / comment())*
+ rule continuous() = "\\" [' '|'\t']* __
+ rule _ = ([' '|'\t'] / continuous())*
+ rule __ = _ comment()? ['\n'|'\r'] ([' '|'\t'|'\n'|'\r'] / comment() / continuous())*
 
  rule token(file_id: usize) -> PosToken
  = s:position!()
- t:(sym:symbol() { Token::Symbol(sym) })
+ t:(s:symbol() { Token::Symbol(s) }
+ / b:boolean() { Token::Bool(b) })
  e:position!() { PosToken{ file_id, pos: s..e, token: t } }
 
  rule statement(file_id: usize) -> Vec<PosToken>
- = _ ts:(token(file_id) ++ _) _ { ts }
+ = ts:(token(file_id) ++ _) { ts }
 
  pub rule tokens(file_id: usize) -> Vec<Vec<PosToken>>
  = __? s:(statement(file_id) ** __) __? { s }
+ / __? { Vec::new() }
 }}
 
 #[derive(Clone, Debug, PartialEq)]
@@ -107,164 +112,190 @@ mod tests {
  use super::*;
 
  #[test]
- fn symbols() {
+ fn whitespace() {
  let code = indoc::indoc! {"
- # Comment Line
- \t \r = + - * / % ** == != < > <= >= \\ # After Comment
+ # Comment
+ \r \t \\ # Comment
 
- ! & | ^ << >> ( ) { } [ ] , . : _ @
+ # Comment
  "};
+ assert_eq!(lex(code, 0), Ok(Vec::new()));
+ }
+
+ #[test]
+ fn bools() {
+ let code = "true false";
  assert_eq!(
  lex(code, 0),
  Ok(vec![vec![
  PosToken {
  file_id: 0,
- pos: 21..22,
+ pos: 0..4,
+ token: Token::Bool(true),
+ },
+ PosToken {
+ file_id: 0,
+ pos: 5..10,
+ token: Token::Bool(false),
+ },
+ ]])
+ )
+ }
+
+ #[test]
+ fn symbols() {
+ let code = "= + - * / % ** == != < > <= >= ! & | ^ << >> ( ) { } [ ] , . : _ @";
+ assert_eq!(
+ lex(code, 0),
+ Ok(vec![vec![
+ PosToken {
+ file_id: 0,
+ pos: 0..1,
  token: Token::Symbol(Symbol::Assign)
  },
  PosToken {
  file_id: 0,
- pos: 23..24,
+ pos: 2..3,
  token: Token::Symbol(Symbol::Plus)
  },
  PosToken {
  file_id: 0,
- pos: 25..26,
+ pos: 4..5,
  token: Token::Symbol(Symbol::Minus)
  },
  PosToken {
  file_id: 0,
- pos: 27..28,
+ pos: 6..7,
  token: Token::Symbol(Symbol::Multiply)
  },
  PosToken {
  file_id: 0,
- pos: 29..30,
+ pos: 8..9,
  token: Token::Symbol(Symbol::Divide),
  },
  PosToken {
  file_id: 0,
- pos: 31..32,
+ pos: 10..11,
  token: Token::Symbol(Symbol::Remains),
  },
  PosToken {
  file_id: 0,
- pos: 33..35,
+ pos: 12..14,
  token: Token::Symbol(Symbol::Exponent),
  },
  PosToken {
  file_id: 0,
- pos: 36..38,
+ pos: 15..17,
  token: Token::Symbol(Symbol::Equals),
  },
  PosToken {
  file_id: 0,
- pos: 39..41,
+ pos: 18..20,
  token: Token::Symbol(Symbol::NotEquals),
  },
  PosToken {
  file_id: 0,
- pos: 42..43,
+ pos: 21..22,
  token: Token::Symbol(Symbol::LessThan),
  },
  PosToken {
  file_id: 0,
- pos: 44..45,
+ pos: 23..24,
  token: Token::Symbol(Symbol::GreaterThan),
  },
  PosToken {
  file_id: 0,
- pos: 46..48,
+ pos: 25..27,
  token: Token::Symbol(Symbol::LessThanEquals),
  },
  PosToken {
  file_id: 0,
- pos: 49..51,
+ pos: 28..30,
  token: Token::Symbol(Symbol::GreaterThanEquals),
  },
  PosToken {
  file_id: 0,
- pos: 71..72,
+ pos: 31..32,
  token: Token::Symbol(Symbol::Not),
  },
  PosToken {
  file_id: 0,
- pos: 73..74,
+ pos: 33..34,
  token: Token::Symbol(Symbol::And),
  },
  PosToken {
  file_id: 0,
- pos: 75..76,
+ pos: 35..36,
  token: Token::Symbol(Symbol::Or),
  },
  PosToken {
  file_id: 0,
- pos: 77..78,
+ pos: 37..38,
  token: Token::Symbol(Symbol::Xor),
  },
  PosToken {
  file_id: 0,
- pos: 79..81,
+ pos: 39..41,
  token: Token::Symbol(Symbol::LeftShift),
  },
  PosToken {
  file_id: 0,
- pos: 82..84,
+ pos: 42..44,
  token: Token::Symbol(Symbol::RightShift),
  },
  PosToken {
  file_id: 0,
- pos: 85..86,
+ pos: 45..46,
  token: Token::Symbol(Symbol::LeftParenthesis),
  },
  PosToken {
  file_id: 0,
- pos: 87..88,
+ pos: 47..48,
  token: Token::Symbol(Symbol::RightParenthesis),
  },
  PosToken {
  file_id: 0,
- pos: 89..90,
+ pos: 49..50,
  token: Token::Symbol(Symbol::LeftBrace),
  },
  PosToken {
  file_id: 0,
- pos: 91..92,
+ pos: 51..52,
  token: Token::Symbol(Symbol::RightBrace),
  },
  PosToken {
  file_id: 0,
- pos: 93..94,
+ pos: 53..54,
  token: Token::Symbol(Symbol::LeftBracket),
  },
  PosToken {
  file_id: 0,
- pos: 95..96,
+ pos: 55..56,
  token: Token::Symbol(Symbol::RightBracket),
  },
  PosToken {
  file_id: 0,
- pos: 97..98,
+ pos: 57..58,
  token: Token::Symbol(Symbol::Comma),
  },
  PosToken {
  file_id: 0,
- pos: 99..100,
+ pos: 59..60,
  token: Token::Symbol(Symbol::Dot),
  },
  PosToken {
  file_id: 0,
- pos: 101..102,
+ pos: 61..62,
  token: Token::Symbol(Symbol::Colon),
  },
  PosToken {
  file_id: 0,
- pos: 103..104,
+ pos: 63..64,
  token: Token::Symbol(Symbol::UnderLine),
  },
  PosToken {
  file_id: 0,
- pos: 105..106,
+ pos: 65..66,
  token: Token::Symbol(Symbol::At),
  },
  ]])