Fail early (ie. don't backtrack) after a keyword or an indentation er…

…ror.

src/bytes.rs

@@ -96,16 +96,16 @@ named!(pub bytes<StrSpan, Vec<u8>>,
  is_raw: call!(|i, s:StrSpan| Ok((i, s.fragment.0.contains('r') || s.fragment.0.contains('R'))), prefix) >>
  content: switch!(call!(|i| Ok((i, is_raw))),
  false => alt!(
- delimited!(tag!("'''"), call!(longbytes, '\''), tag!("'''"))
- | delimited!(tag!("\"\"\""), call!(longbytes, '"'), tag!("\"\"\""))
- | delimited!(char!('\''), call!(shortbytes, '\''), char!('\''))
- | delimited!(char!('"'), call!(shortbytes, '"'), char!('"'))
+ delimited!(tag!("'''"), return_error!(call!(longbytes, '\'')), tag!("'''"))
+ | delimited!(tag!("\"\"\""), return_error!(call!(longbytes, '"')), tag!("\"\"\""))
+ | delimited!(char!('\''), return_error!(call!(shortbytes, '\'')), char!('\''))
+ | delimited!(char!('"'), return_error!(call!(shortbytes, '"')), char!('"'))
  )
  | true => alt!(
- delimited!(tag!("'''"), call!(longrawbytes, '\''), tag!("'''"))
- | delimited!(tag!("\"\"\""), call!(longrawbytes, '"'), tag!("\"\"\""))
- | delimited!(char!('\''), call!(shortrawbytes, '\''), char!('\''))
- | delimited!(char!('"'), call!(shortrawbytes, '"'), char!('"'))
+ delimited!(tag!("'''"), return_error!(call!(longrawbytes, '\'')), tag!("'''"))
+ | delimited!(tag!("\"\"\""), return_error!(call!(longrawbytes, '"')), tag!("\"\"\""))
+ | delimited!(char!('\''), return_error!(call!(shortrawbytes, '\'')), char!('\''))
+ | delimited!(char!('"'), return_error!(call!(shortrawbytes, '"')), char!('"'))
  )
  ) >> (content)
  )

src/errors.rs

@@ -0,0 +1,36 @@
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+#[repr(u32)]
+pub enum PyParseError {
+ UnexpectedIndent,
+ ExpectedIndent,
+}
+impl From<PyParseError> for u32 {
+ fn from(e: PyParseError) -> u32 {
+ e as u32
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use nom;
+ use nom::{Context, ErrorKind};
+ use nom::types::CompleteStr;
+ use nom_locate::LocatedSpan;
+
+ use helpers::*;
+ use statements::statement;
+
+
+ #[test]
+ fn if_no_condition() {
+ assert_eq!(statement(make_strspan("if:\n foo"), 0), Err(
+ nom::Err::Failure(
+ Context::Code(
+ LocatedSpan { offset: 2, line: 1, fragment: CompleteStr(":\n foo") },
+ ErrorKind::Alt
+ )
+ )
+ ));
+ }
+}

src/expressions.rs

@@ -32,9 +32,9 @@ named!(pub test<StrSpan, Box<Expression>>,
  left: call!(Self::or_test) >>
  right: opt!(do_parse!(
  ws_auto!(keyword!("if")) >>
- cond: call!(Self::or_test) >>
+ cond: return_error!(call!(Self::or_test)) >>
  ws_auto!(keyword!("else")) >>
- right: call!(Self::test) >> (
+ right: return_error!(call!(Self::test)) >> (
  (cond, right)
  )
  )) >> (
@@ -57,28 +57,26 @@ named!(test_nocond<StrSpan, Box<Expression>>,
 
 // lambdef: 'lambda' [varargslist] ':' test
 named!(lambdef<StrSpan, Box<Expression>>,
- ws_auto!(do_parse!(
- keyword!("lambda") >>
+ ws_auto!(preceded!(keyword!("lambda"), return_error!(do_parse!(
  args: opt!(varargslist) >>
  spaces!() >>
  char!(':') >>
  spaces!() >>
  code: call!(Self::test) >> (
  Box::new(Expression::Lambdef(args.unwrap_or_default(), code))
  )
- ))
+ ))))
 );
 
 // lambdef_nocond: 'lambda' [varargslist] ':' test_nocond
 named!(lambdef_nocond<StrSpan, Box<Expression>>,
- do_parse!(
- keyword!("lambda") >>
+ ws_auto!(preceded!(keyword!("lambda"), return_error!(do_parse!(
  args: opt!(varargslist) >>
  char!(':') >>
  code: call!(Self::test_nocond) >> (
  Box::new(Expression::Lambdef(args.unwrap_or_default(), code))
  )
- )
+ ))))
 );
 
 } // End ExpressionParser
@@ -426,7 +424,7 @@ named_args!(dictmaker(item1: DictItem) <StrSpan, Box<Expression>>,
  v.insert(0, item1.clone()); // FIXME: do not clone
  Box::new(Expression::DictLiteral(v))
  }}
- | preceded!(peek!(keyword!("for")), call!(Self::comp_for)) => { |comp| {
+ | preceded!(peek!(keyword!("for")), return_error!(call!(Self::comp_for))) => { |comp| {
  Box::new(Expression::DictComp(Box::new(item1.clone()), comp)) // FIXME: do not clone
  }}
  )),
@@ -529,7 +527,7 @@ named_args!(comp_iter(acc: Vec<ComprehensionChunk>) <StrSpan, Vec<ComprehensionC
 );
 
 named_args!(opt_comp_iter(acc: Vec<ComprehensionChunk>) <StrSpan, Vec<ComprehensionChunk>>,
- map!(opt!(call!(Self::comp_iter, acc.clone())), |r| r.unwrap_or(acc)) // FIXME: do not clone
+ return_error!(map!(opt!(call!(Self::comp_iter, acc.clone())), |r| r.unwrap_or(acc))) // FIXME: do not clone
 );
 
 // comp_for: [ASYNC] 'for' exprlist 'in' or_test [comp_iter]
@@ -541,11 +539,11 @@ named_args!(comp_for2(acc: Vec<ComprehensionChunk>) <StrSpan, Vec<ComprehensionC
  async: map!(opt!(terminated!(tag!("async"), space_sep!())), |o| o.is_some()) >>
  keyword!("for") >>
  spaces!() >>
- item: call!(Self::exprlist) >>
+ item: return_error!(call!(Self::exprlist)) >>
  spaces!() >>
  keyword!("in") >>
  spaces!() >>
- iterator: map!(call!(Self::or_test), |e| *e) >>
+ iterator: return_error!(map!(call!(Self::or_test), |e| *e)) >>
  spaces!() >>
  r: call!(Self::opt_comp_iter, { let mut acc = acc; acc.push(ComprehensionChunk::For { async, item, iterator }); acc }) >> (
  r
@@ -558,7 +556,7 @@ named_args!(comp_if(acc: Vec<ComprehensionChunk>) <StrSpan, Vec<ComprehensionChu
  do_parse!(
  keyword!("if") >>
  spaces!() >>
- cond: map!(call!(Self::test_nocond), |e| *e) >>
+ cond: return_error!(map!(call!(Self::test_nocond), |e| *e)) >>
  spaces!() >>
  r: call!(Self::opt_comp_iter, { let mut acc = acc; acc.push(ComprehensionChunk::If { cond }); acc }) >> (
  r

src/functions.rs

@@ -14,7 +14,7 @@ use ast::*;
 // decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
 named_args!(decorator(indent: usize) <StrSpan, Decorator>,
  do_parse!(
- count!(char!(' '), indent) >>
+ indent!(indent) >>
  char!('@') >>
  name: ws_nonl!(call!(ImportParser::<NewlinesAreNotSpaces>::dotted_name)) >>
  args: opt!(ws_nonl!(delimited!(char!('('), ws_comm!(call!(ExpressionParser::<NewlinesAreSpaces>::arglist)), char!(')')))) >>
@@ -49,7 +49,7 @@ named_args!(pub decorated(indent: usize) <StrSpan, CompoundStatement>,
 // funcdef: 'def' NAME parameters ['->' test] ':' suite
 named_args!(funcdef(indent: usize, decorators: Vec<Decorator>) <StrSpan, CompoundStatement>,
  do_parse!(
- count!(char!(' '), indent) >>
+ indent!(indent) >>
  async: opt!(tuple!(tag!("async"), space_sep_nonl)) >>
  tag!("def") >>
  space_sep_nonl >>
@@ -68,7 +68,7 @@ named_args!(funcdef(indent: usize, decorators: Vec<Decorator>) <StrSpan, Compoun
 // classdef: 'class' NAME ['(' [arglist] ')'] ':' suite
 named_args!(classdef(indent: usize, decorators: Vec<Decorator>) <StrSpan, CompoundStatement>,
  do_parse!(
- count!(char!(' '), indent) >>
+ indent!(indent) >>
  tag!("class") >>
  space_sep_nonl >>
  name: name >>

src/helpers.rs

@@ -205,3 +205,82 @@ pub(crate) fn first_word(i: StrSpan) -> Result<(StrSpan, &str), ::nom::Err<StrSp
  Err(e) => Err(e),
  }
 }
+
+// https://github.com/Geal/nom/pull/800
+macro_rules! fold_many1_fixed(
+ ($i:expr, $submac:ident!( $($args:tt)* ), $init:expr, $f:expr) => (
+ {
+ use nom;
+ use nom::lib::std::result::Result::*;
+ use nom::{Err,Needed,InputLength,Context,AtEof};
+
+ match $submac!($i, $($args)*) {
+ Err(Err::Error(_)) => Err(Err::Error(
+ error_position!($i, nom::ErrorKind::Many1)
+ )),
+ Err(Err::Failure(_)) => Err(Err::Failure(
+ error_position!($i, nom::ErrorKind::Many1)
+ )),
+ Err(Err::Incomplete(i)) => Err(Err::Incomplete(i)),
+ Ok((i1,o1)) => {
+ let f = $f;
+ let mut acc = f($init, o1);
+ let mut input = i1;
+ let mut incomplete: nom::lib::std::option::Option<Needed> =
+ nom::lib::std::option::Option::None;
+ let mut failure: nom::lib::std::option::Option<Context<_,_>> =
+ nom::lib::std::option::Option::None;
+ loop {
+ match $submac!(input, $($args)*) {
+ Err(Err::Error(_)) => {
+ break;
+ },
+ Err(Err::Incomplete(i)) => {
+ incomplete = nom::lib::std::option::Option::Some(i);
+ break;
+ },
+ Err(Err::Failure(e)) => {
+ failure = nom::lib::std::option::Option::Some(e);
+ break;
+ },
+ Ok((i, o)) => {
+ if i.input_len() == input.input_len() {
+ if !i.at_eof() {
+ failure = nom::lib::std::option::Option::Some(error_position!(i, nom::ErrorKind::Many1));
+ }
+ break;
+ }
+ acc = f(acc, o);
+ input = i;
+ }
+ }
+ }
+
+ match failure {
+ nom::lib::std::option::Option::Some(e) => Err(Err::Failure(e)),
+ nom::lib::std::option::Option::None => match incomplete {
+ nom::lib::std::option::Option::Some(i) => nom::need_more($i, i),
+ nom::lib::std::option::Option::None => Ok((input, acc))
+ }
+ }
+ }
+ }
+ }
+ );
+ ($i:expr, $f:expr, $init:expr, $fold_f:expr) => (
+ fold_many_fixed1!($i, call!($f), $init, $fold_f);
+ );
+);
+
+macro_rules! indent {
+ ($i:expr, $nb_spaces:expr) => {{
+ use nom::ErrorKind;
+ use $crate::errors::PyParseError;
+ count!($i, char!(' '), $nb_spaces).and_then(|(i2,_)|
+ return_error!(i2,
+ ErrorKind::Custom(PyParseError::UnexpectedIndent.into()),
+ not!(peek!(char!(' ')))
+ )
+ )
+ }}
+}

src/lib.rs

@@ -91,6 +91,7 @@ mod bytes;
 mod numbers;
 pub mod ast;
 pub mod visitors;
+pub mod errors;
 
 use helpers::*;
 use statements::*;
@@ -113,8 +114,9 @@ named_attr!(#[doc = "Parses a module or sequence of commands."],
 pub file_input <StrSpan, Vec<Statement>>,
  fold_many0!(
  alt!(
- call!(statement, 0) => { |s| Some(s) }
- | newline => { |_| None }
+ newline => { |_| None }
+ | eof!() => { |_| None }
+ | call!(statement, 0) => { |s| Some(s) }
  ),
  Vec::new(),
  |acc: Vec<_>, item| { let mut acc = acc; if let Some(s) = item { acc.extend(s); } acc }