add regex

Author: bea4dev

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "bnf_rules"
-version = "0.1.4"
+version = "0.1.5"
 edition = "2021"
 repository = "https://github.com/bea4dev/bnf_rules"
 description = "LR(1) parser generator"
@@ -13,6 +13,6 @@ members = [
 ]
 
 [dependencies]
-bnf_rules_parser = { path = "./bnf_rules_parser", version = "0.1.4" }
-bnf_rules_macro = { path = "./bnf_rules_macro", version = "0.1.4" }
+bnf_rules_parser = { path = "./bnf_rules_parser", version = "0.1.5" }
+bnf_rules_macro = { path = "./bnf_rules_macro", version = "0.1.5" }
 regex = "^1.7.1"

README.md

@@ -14,26 +14,9 @@ bnf_rules!(
     expr     ::= factor { "+" factor }
     factor   ::= "-" primary | primary
     primary  ::= "(" expr ")" | number
-    number   ::= fn (number_tokenizer) // custom tokenizer with function
+    number   ::= r"\d+" // regex
 );
 
-/// Custom tokenizer for numeric literal
-fn number_tokenizer(source: &Vec<char>, mut current_position: usize) -> usize {
-    let mut iteration_count = 0;
-    loop {
-        let current_char = match source.get(current_position) {
-            Some(ch) => ch.clone(),
-            _ => break
-        };
-        if !current_char.is_numeric() {
-            break;
-        }
-        iteration_count += 1;
-        current_position += 1;
-    }
-    return iteration_count; // 0 means 'rejected', other means 'accepted' and 'length of token'.
-}
-
 pub fn parse() {
 
     // A function named "parse_source" is automatically generated.
@@ -45,7 +28,7 @@ pub fn parse() {
 
 ### Usage
 ```toml
-bnf_rules = "0.1.4"
+bnf_rules = "0.1.5"
 ```
 
 ### Extended BNF
@@ -54,6 +37,7 @@ bnf_rules = "0.1.4"
 |          source          |          An entire input source.           |
 |          ident           |    A non-terminal symbol named "ident".    |
 |       "something"        |        A terminal symbol for text.         |
+|          r"\d+"          |       A terminal symbol for regex.         |
 |    fn (function_name)    | A custom tokenizer with user function.[^1] |
 |       { pattern }        |   Zero or more repetitions of "pattern".   |
 |      \[ pattern \]       |             "pattern" or null.             |
@@ -64,4 +48,4 @@ bnf_rules = "0.1.4"
 
 > Example 1: https://github.com/bea4dev/bnf_rules/blob/master/src/lib.rs
 
-> Example 2: https://github.com/bea4dev/catla/blob/master/catla_parser/src/grammar.rs
+> Example 2: https://github.com/bea4dev/catla/blob/master/catla_parser/src/grammar.rs

bnf_rules_macro/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "bnf_rules_macro"
-version = "0.1.4"
+version = "0.1.5"
 edition = "2021"
 repository = "https://github.com/bea4dev/bnf_rules"
 description = "LR(1) parser generator"

bnf_rules_macro/src/lib.rs

@@ -20,26 +20,9 @@ use syn::parse_macro_input;
 ///     expr     ::= factor { "+" factor }
 ///     factor   ::= "-" primary | primary
 ///     primary  ::= "(" expr ")" | number
-///     number   ::= fn (number_tokenizer) // custom tokenizer with function
+///     number   ::= r"\d+" // regex
 /// );
 ///
-/// /// Custom tokenizer for numeric literal
-/// fn number_tokenizer(source: &Vec<char>, mut current_position: usize) -> usize {
-///     let mut iteration_count = 0;
-///     loop {
-///         let current_char = match source.get(current_position) {
-///             Some(ch) => ch.clone(),
-///             _ => break
-///         };
-///         if !current_char.is_numeric() {
-///             break;
-///         }
-///         iteration_count += 1;
-///         current_position += 1;
-///     }
-///     return iteration_count; // 0 means 'rejected', other means 'accepted' and 'length of token'.
-/// }
-///
 /// pub fn parse() {
 ///
 ///     // A function named "parse_source" is automatically generated.
@@ -58,4 +41,3 @@ pub fn bnf_rules(input: TokenStream) -> TokenStream {
     let mut generator = ParserGenerator::new(map);
     return generator.generate(generate_code).unwrap().parse().unwrap();
 }
-

bnf_rules_parser/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "bnf_rules_parser"
-version = "0.1.4"
+version = "0.1.5"
 edition = "2021"
 repository = "https://github.com/bea4dev/bnf_rules"
 description = "LR(1) parser generator"