Merge pull request #6 from Akanoa/carriage_return

feat: Add support to carriage return and tab recognizer

Author: Akanoa

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "noa-parser"
-version = "0.3.0"
+version = "0.4.0"
 edition = "2024"
 homepage = "https://github.com/Akanoa/noa-parser"
 repository = "https://github.com/Akanoa/noa-parser"

Cargo.toml

@@ -1,5 +1,9 @@
 # Changelog
 
+**0.4.0**
+
+- Add support to carriage return and tab recognizer
+
 **0.3.0**
 
 - Add support to delimited group

Changelog.md

@@ -4,7 +4,9 @@ use noa_parser::peek::peek;
 fn main() {
     let data = b"(2 * 3)";
     let mut scanner = noa_parser::scanner::Scanner::new(data);
-    let result = peek(GroupKind::Parenthesis, &mut scanner).expect("failed to parse").expect("failed to peek");
+    let result = peek(GroupKind::Parenthesis, &mut scanner)
+        .expect("failed to parse")
+        .expect("failed to peek");
     println!(
         "{}",
         String::from_utf8_lossy(result.peeked_slice()) // 2 * 3

examples/delimited_group.rs

@@ -233,9 +233,9 @@ impl<'a> Peekable<'a, u8, Token, Token> for GroupKind {
 
 #[cfg(test)]
 mod tests {
-    use crate::bytes::components::groups::{match_for_delimited_group, match_group, GroupKind};
+    use crate::bytes::components::groups::{GroupKind, match_for_delimited_group, match_group};
     use crate::bytes::token::Token;
-    use crate::peek::{peek, PeekResult, Peeking};
+    use crate::peek::{PeekResult, Peeking, peek};
     use crate::scanner::Scanner;
 
     #[test]

src/bytes/components/groups.rs

@@ -1,6 +1,6 @@
 //! Classic tokens
 
-use crate::bytes::matchers::match_char;
+use crate::bytes::matchers::{match_char, match_pattern};
 use crate::matcher::{Match, MatchSize};
 
 #[derive(Copy, Clone)]
@@ -65,6 +65,14 @@ pub enum Token {
     Underscore,
     /// The `#` character
     Sharp,
+    /// The `\n` character
+    Ln,
+    /// The `\r` character
+    Cr,
+    /// The `\t` character
+    Tab,
+    /// The `\r\n` character
+    CrLn,
 }
 
 impl Match<u8> for Token {
@@ -99,6 +107,10 @@ impl Match<u8> for Token {
             Token::Backslash => match_char('\\', data),
             Token::Underscore => match_char('_', data),
             Token::Sharp => match_char('#', data),
+            Token::Ln => match_char('\n', data),
+            Token::Cr => match_char('\r', data),
+            Token::Tab => match_char('\t', data),
+            Token::CrLn => match_pattern(b"\r\n", data),
         }
     }
 }
@@ -135,6 +147,10 @@ impl MatchSize for Token {
             Token::Backslash => 1,
             Token::Underscore => 1,
             Token::Sharp => 1,
+            Token::Ln => 1,
+            Token::Cr => 1,
+            Token::Tab => 1,
+            Token::CrLn => 2,
         }
     }
 }

src/bytes/token.rs

@@ -5,7 +5,7 @@
 
 use crate::errors::ParseResult;
 use crate::matcher::MatchSize;
-use crate::recognizer::{recognize, Recognizable};
+use crate::recognizer::{Recognizable, recognize};
 use crate::scanner::Scanner;
 use std::marker::PhantomData;