widen CharReader buffer (#1859)

diff --git a/src/parser/char_reader.rs b/src/parser/char_reader.rs
index a2a7c2e277d606231d2aba0c3ee26c33faa92a3a..8a74db377c80e45eb03e2e6fbd7bf98afeec781e 100644 (file)
--- a/src/parser/char_reader.rs
+++ b/src/parser/char_reader.rs
@@ -20,7 +20,7 @@ use std::str;
 
 pub struct CharReader<R> {
     inner: R,
-    buf: SmallVec<[u8;4]>,
+    buf: SmallVec<[u8;32]>,
     pos: usize,
 }
 
@@ -121,7 +121,7 @@ impl<R: Read> CharReader<R> {
 
             self.buf.clear();
 
-            let mut word = [0u8;4];
+            let mut word = [0u8; std::mem::size_of::<char>()];
             let nread = self.inner.read(&mut word)?;
 
             self.buf.extend_from_slice(&word[..nread]);
@@ -234,10 +234,10 @@ impl<R: Read> CharRead for CharReader<R> {
     #[inline(always)]
     fn put_back_char(&mut self, c: char) {
         let src_len = self.buf.len() - self.pos;
-        debug_assert!(src_len <= 4);
+        debug_assert!(src_len <= self.buf.capacity());
 
         let c_len = c.len_utf8();
-        let mut shifted_slice = [0u8; 4];
+        let mut shifted_slice = [0u8; 32];
 
         shifted_slice[0..src_len].copy_from_slice(&self.buf[self.pos .. self.buf.len()]);
 

diff --git a/src/parser/lexer.rs b/src/parser/lexer.rs
index 7f5d73ed656dbb0d60dc3174b6662016c5da9048..3fbddc1dd106743bbbc8b394584e848b77d45ab1 100644 (file)
--- a/src/parser/lexer.rs
+++ b/src/parser/lexer.rs
@@ -988,6 +988,17 @@ impl<'a, R: CharRead> Lexer<'a, R> {
 
                             return Ok(Token::End);
                         }
+                        Ok(c) if c == '\\' => {
+                            self.skip_char(c);
+
+                            if self.lookahead_char().ok() == Some('n') {
+                                self.skip_char('n');
+                                return Ok(Token::End);
+                            } else {
+                                self.return_char(c);
+                                self.return_char('.');
+                            }
+                        }
                         Err(ParserError::UnexpectedEOF) => {
                             return Ok(Token::End);
                         }