Merge pull request #393 from dtolnay/whitespace

Require \r whitespace to be followed by \n

Author: dtolnay

src/parse.rs

@@ -104,10 +104,14 @@ fn skip_whitespace(input: Cursor) -> Cursor {
             }
         }
         match byte {
-            b' ' | 0x09..=0x0d => {
+            b' ' | 0x09..=0x0c => {
                 s = s.advance(1);
                 continue;
             }
+            b'\r' if s.as_bytes().get(1) == Some(&b'\n') => {
+                s = s.advance(2);
+                continue;
+            }
             b if b <= 0x7f => {}
             _ => {
                 let ch = s.chars().next().unwrap();

tests/test.rs

@@ -694,6 +694,18 @@ fn check_spans_internal(ts: TokenStream, lines: &mut &[(usize, usize, usize, usi
     }
 }
 
+#[test]
+fn whitespace() {
+    // space, horizontal tab, vertical tab, form feed, carriage return, line
+    // feed, non-breaking space, left-to-right mark, right-to-left mark
+    let various_spaces = " \t\u{b}\u{c}\r\n\u{a0}\u{200e}\u{200f}";
+    let tokens = various_spaces.parse::<TokenStream>().unwrap();
+    assert_eq!(tokens.into_iter().count(), 0);
+
+    let lone_carriage_return = " \r ";
+    lone_carriage_return.parse::<TokenStream>().unwrap_err();
+}
+
 #[test]
 fn byte_order_mark() {
     let string = "\u{feff}foo";