fix for cyrillic chars (#10)

Author: angelk

src/Potaka/BbCode/Tokenizer/Tokenizer.php

@@ -15,12 +15,14 @@ public function tokenize($text)
     {
         $this->rootTag = new Tag(null);
         $curentElement = 0;
-        $textLenght = mb_strlen($text);
+        $textAsArray = preg_split('//u', $text);
+        $textLenght = count($textAsArray);
+        
         $bufferText = '';
         $currentTag = $this->rootTag;
         while ($curentElement < $textLenght) {
-            $currentChar = $text[$curentElement];
-            if ($currentChar === '[' && ($curentElement+1 < $textLenght) && $text[$curentElement+1] !== ']') {
+            $currentChar = $textAsArray[$curentElement];
+            if ($currentChar === '[' && ($curentElement+1 < $textLenght) && $textAsArray[$curentElement+1] !== ']') {
                 // get the close bracket
                 $closeTagFound = false;
                 // [tag=argumen]
@@ -30,16 +32,16 @@ public function tokenize($text)
                 $tagText = '';
                 $tmpPosion++;
                 while ($tmpPosion < $textLenght) {
-                    if ($text[$tmpPosion] === ']') {
+                    if ($textAsArray[$tmpPosion] === ']') {
                         $closeTagFound = true;
                         break;
-                    } elseif ($text[$tmpPosion] === '=') {
+                    } elseif ($textAsArray[$tmpPosion] === '=') {
                         $argumentFound = true;
                     } else {
                         if ($argumentFound) {
-                            $argumentValue .= $text[$tmpPosion];
+                            $argumentValue .= $textAsArray[$tmpPosion];
                         } else {
-                            $tagText .= $text[$tmpPosion];
+                            $tagText .= $textAsArray[$tmpPosion];
                         }
                     }
                     $tmpPosion++;

tests/phpUnit/Test/Potaka/BbCode/Tokenizer/TokenizerTest.php

@@ -218,4 +218,17 @@ public function testTokenizerReuse()
 
         $this->assertSameTokenized($expectedReuse, $resultReuse);
     }
+
+    public function testCyrillicChars()
+    {
+        $tokenizer = new Tokenizer();
+        $text = 'удебелен текст';
+        $result = $tokenizer->tokenize($text);
+        $expected = new Tag(null);
+        $expected->addTag(
+            (new Tag(null))->setText('удебелен текст')
+        );
+
+        $this->assertSameTokenized($expected, $result);
+    }
 }