feat: implement getConfig for tokenizer and all components

Author: CodeWithKyrian

src/Contracts/DecoderInterface.php

@@ -12,4 +12,14 @@ interface DecoderInterface
      * @param string[] $tokens
      */
     public function decode(array $tokens): string;
+
+    /**
+     * Get configuration value(s).
+     *
+     * @param null|string $key     The configuration key. If null, returns all config.
+     * @param mixed       $default The default value if the key doesn't exist
+     *
+     * @return mixed the configuration value, or full config array if $key is null
+     */
+    public function getConfig(?string $key = null, mixed $default = null): mixed;
 }

src/Contracts/ModelInterface.php

@@ -54,10 +54,12 @@ public function getVocabSize(): int;
     public function addToken(string $token, int $id): void;
 
     /**
-     * Get the end of word suffix, if any.
-     * Only some models (like BPE) have this property.
+     * Get configuration value(s).
      *
-     * @return null|string the end of word suffix
+     * @param null|string $key     The configuration key (e.g., 'dropout'). If null, returns all config.
+     * @param mixed       $default The default value if the key doesn't exist (ignored when $key is null)
+     *
+     * @return mixed the configuration value, or full config array if $key is null
      */
-    public function getEndOfWordSuffix(): ?string;
+    public function getConfig(?string $key = null, mixed $default = null): mixed;
 }

src/Contracts/NormalizerInterface.php

@@ -7,4 +7,14 @@
 interface NormalizerInterface
 {
     public function normalize(string $text): string;
+
+    /**
+     * Get configuration value(s).
+     *
+     * @param null|string $key     The configuration key. If null, returns all config.
+     * @param mixed       $default The default value if the key doesn't exist
+     *
+     * @return mixed the configuration value, or full config array if $key is null
+     */
+    public function getConfig(?string $key = null, mixed $default = null): mixed;
 }

src/Contracts/PostProcessorInterface.php

@@ -15,4 +15,14 @@ interface PostProcessorInterface
      * @return array{0: string[], 1: int[]} the processed tokens and type IDs
      */
     public function process(array $tokens, ?array $pair = null, bool $addSpecialTokens = true): array;
+
+    /**
+     * Get configuration value(s).
+     *
+     * @param null|string $key     The configuration key. If null, returns all config.
+     * @param mixed       $default The default value if the key doesn't exist
+     *
+     * @return mixed the configuration value, or full config array if $key is null
+     */
+    public function getConfig(?string $key = null, mixed $default = null): mixed;
 }

src/Contracts/PreTokenizerInterface.php

@@ -15,4 +15,14 @@ interface PreTokenizerInterface
      * @return string[]
      */
     public function preTokenize(array|string $text, array $options = []): array;
+
+    /**
+     * Get configuration value(s).
+     *
+     * @param null|string $key     The configuration key. If null, returns all config.
+     * @param mixed       $default The default value if the key doesn't exist
+     *
+     * @return mixed the configuration value, or full config array if $key is null
+     */
+    public function getConfig(?string $key = null, mixed $default = null): mixed;
 }

src/DataStructures/AddedToken.php

@@ -10,7 +10,7 @@
  *   - Whether they should only match single words
  *   - Whether to include any whitespace on its left or right.
  */
-readonly class AddedToken
+class AddedToken implements \JsonSerializable
 {
     public function __construct(
         /**
@@ -58,4 +58,20 @@ public static function fromArray(array $data): self
             $data['special'] ?? false,
         );
     }
+
+    /**
+     * @return array<string, mixed>
+     */
+    public function jsonSerialize(): array
+    {
+        return [
+            'id' => $this->id,
+            'content' => $this->content,
+            'single_word' => $this->singleWord,
+            'lstrip' => $this->lStrip,
+            'rstrip' => $this->rStrip,
+            'normalized' => $this->normalized,
+            'special' => $this->special,
+        ];
+    }
 }

src/Decoders/BPEDecoder.php

@@ -8,6 +8,22 @@ class BPEDecoder extends BaseDecoder
 {
     public function __construct(protected string $suffix = '') {}
 
+    public function getConfig(?string $key = null, mixed $default = null): mixed
+    {
+        if (null !== $key) {
+            return match ($key) {
+                'type' => 'BPEDecoder',
+                'suffix' => $this->suffix,
+                default => $default,
+            };
+        }
+
+        return [
+            'type' => 'BPEDecoder',
+            'suffix' => $this->suffix,
+        ];
+    }
+
     protected function processTokens(array $tokens): array
     {
         $decoded = [];

src/Decoders/ByteFallbackDecoder.php

@@ -6,6 +6,15 @@
 
 class ByteFallbackDecoder extends BaseDecoder
 {
+    public function getConfig(?string $key = null, mixed $default = null): mixed
+    {
+        if (null !== $key) {
+            return 'type' === $key ? 'ByteFallback' : $default;
+        }
+
+        return ['type' => 'ByteFallback'];
+    }
+
     protected function processTokens(array $tokens): array
     {
         $newTokens = [];

src/Decoders/ByteLevelDecoder.php

@@ -275,11 +275,16 @@ public function __construct(protected array $addedTokens = [], protected ?string
 
     /**
      * Convert an array of tokens to a string by decoding each byte.
-     *
-     * @param string[] $tokens array of tokens to be decoded
-     *
-     * @return string the decoded string
      */
+    public function getConfig(?string $key = null, mixed $default = null): mixed
+    {
+        if (null !== $key) {
+            return 'type' === $key ? 'ByteLevel' : $default;
+        }
+
+        return ['type' => 'ByteLevel'];
+    }
+
     public function decode(array $tokens): string
     {
         $decoded = parent::decode($tokens);

src/Decoders/CTCDecoder.php

@@ -14,6 +14,26 @@ public function __construct(
         protected bool $cleanup = true
     ) {}
 
+    public function getConfig(?string $key = null, mixed $default = null): mixed
+    {
+        if (null !== $key) {
+            return match ($key) {
+                'type' => 'CTC',
+                'pad_token' => $this->padToken,
+                'word_delimiter_token' => $this->wordDelimiterToken,
+                'cleanup' => $this->cleanup,
+                default => $default,
+            };
+        }
+
+        return [
+            'type' => 'CTC',
+            'pad_token' => $this->padToken,
+            'word_delimiter_token' => $this->wordDelimiterToken,
+            'cleanup' => $this->cleanup,
+        ];
+    }
+
     protected function processTokens(array $tokens): array
     {
         if (empty($tokens)) {