feat: add FixedLengthPreTokenizer for fixed-length tokenization

Author: CodeWithKyrian

src/Factories/PreTokenizerFactory.php

@@ -8,6 +8,7 @@
 use Codewithkyrian\Tokenizers\PreTokenizers\BertPreTokenizer;
 use Codewithkyrian\Tokenizers\PreTokenizers\ByteLevelPreTokenizer;
 use Codewithkyrian\Tokenizers\PreTokenizers\DigitsPreTokenizer;
+use Codewithkyrian\Tokenizers\PreTokenizers\FixedLengthPreTokenizer;
 use Codewithkyrian\Tokenizers\PreTokenizers\MetaspacePreTokenizer;
 use Codewithkyrian\Tokenizers\PreTokenizers\PreTokenizerSequence;
 use Codewithkyrian\Tokenizers\PreTokenizers\PunctuationPreTokenizer;
@@ -37,6 +38,9 @@ public static function create(array $config): PreTokenizerInterface
             'Digits' => new DigitsPreTokenizer(
                 individualDigits: $config['individual_digits'] ?? false
             ),
+            'FixedLength' => new FixedLengthPreTokenizer(
+                length: $config['length']
+            ),
             'Metaspace' => new MetaspacePreTokenizer(
                 replacement: $config['replacement'] ?? ' ',
                 addPrefixSpace: $config['add_prefix_space'] ?? true,

src/PreTokenizers/FixedLengthPreTokenizer.php

@@ -0,0 +1,38 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Codewithkyrian\Tokenizers\PreTokenizers;
+
+use Codewithkyrian\Tokenizers\Contracts\PreTokenizerInterface;
+
+/**
+ * Splits text into fixed-length tokens.
+ */
+class FixedLengthPreTokenizer implements PreTokenizerInterface
+{
+    public function __construct(
+        protected int $length
+    ) {}
+
+    public function preTokenize(array|string $text, array $options = []): array
+    {
+        if (\is_array($text)) {
+            $result = [];
+            foreach ($text as $t) {
+                $result = array_merge($result, $this->preTokenize($t, $options));
+            }
+
+            return $result;
+        }
+
+        $tokens = [];
+        $len = mb_strlen($text);
+
+        for ($i = 0; $i < $len; $i += $this->length) {
+            $tokens[] = mb_substr($text, $i, $this->length);
+        }
+
+        return $tokens;
+    }
+}