tokenizers.py

import os
from utils import save_json, load_txt, load_json
import torch      

def create_word_token_map(file_path:str,  punctuations=set('.,:;!?()[]"-§¶_\'')):
    """
    Function for tokenize the text using characterlevel tokenzier. 
    - Saves 2 JSON files (string to integers, integers to string)
    """
    # Reading the selected data
    text = load_txt(filepath=file_path)

    tokens = word_tokenize(text=text, punctuations=punctuations)

    unique_tokens = sorted(set(tokens))
    # Tokenizer (using chracter level tokenizer)
    string_to_int = dict()
    int_to_string = dict()
    for i, token in enumerate(unique_tokens):
        string_to_int[token] = i
        int_to_string[i] = token

    unk_idx = len(string_to_int)
    string_to_int["<unk>"] = unk_idx
    int_to_string[unk_idx] = "<unk>" 

    # Creates a folder if it not already exist
    folder = "artifacts"
    os.makedirs(folder, exist_ok=True)
    # Saves the dicts as Json 
    save_json(string_to_int, os.path.join(folder,"string_to_int_word.json" ))
    save_json(int_to_string, os.path.join(folder,"int_to_string_word.json" ))

def word_tokenize(text,  punctuations=set('.,:;!?()[]"-§¶_\'')):
    """
    Function for tokenixe a text, seperated the punctuations.
    """
    # Create tokens by looping through the text and set punctuations to be single tokens aswell
    tokens = []
    current_word = ""
    for ch in text:
        if ch.isalnum():
            current_word += ch
        else:
            if current_word:
                tokens.append(current_word)
                current_word = ""
            if ch in punctuations:
                tokens.append(ch)

    if current_word:
        tokens.append(current_word)

    return tokens

def word_encoder(text, tokenizer_path:str):
    """
    Function for encode a text snipet
    """
    tokens = word_tokenize(text)
    string_to_int = load_json(tokenizer_path)
    output = list()
    for word in tokens:
        try: 
            token = string_to_int[word]
        except:
            token = string_to_int["<unk>"]
        output.append(token)
    return output


def word_decoder(text, tokenizer_path:str, sep="", punctuations=set('.,:;!?()[]"-§¶_\'')): 
    """
    Function for decode a text snipet
    """
    int_to_string = load_json(tokenizer_path)
    output = list()
    n =len(text)
    for idx, token in enumerate(text):
        output.append(int_to_string[str(token)])
        if idx < n-1:
            nxt = int_to_string[str(text[idx + 1])]
            if nxt in punctuations:
                continue
            output.append(sep)

    return "".join(output)


""" Character-level tokenization """

def create_char_token_map(file_path:str):
    """
    Function for tokenize the text using characterlevel tokenzier. 
    - Saves 2 JSON files (string to integers, integers to string)
    """
    # Reading the selected data
    text = load_txt(filepath=file_path)

    chars = sorted(set(text))

    # Tokenizer (using chracter level tokenizer)
    string_to_int = dict()
    int_to_string = dict()
    for i, char in enumerate(chars):
        string_to_int[char] = i
        int_to_string[i] = char

    unk_idx = len(string_to_int)
    string_to_int["<unk>"] = unk_idx
    int_to_string[unk_idx] = "<unk>" 

    # Creates a folder if it not already exist
    folder = "artifacts"
    os.makedirs(folder, exist_ok=True)
    # Saves the dicts as Json 
    save_json(string_to_int, os.path.join(folder,"string_to_int_char.json" ))
    save_json(int_to_string, os.path.join(folder,"int_to_string_char.json" ))


def char_encoder(text, tokenizer_path:str):
    """
    Function for encode a text snipet
    """
    string_to_int = load_json(tokenizer_path)
    output = list()
    for ch in text:
        try: 
            token = string_to_int[ch]
        except:
            token = string_to_int["<unk>"]
        output.append(token)
    return output


def char_decoder(text, tokenizer_path:str): 
    """
    Function for decode a text snipet
    """

    int_to_string = load_json(tokenizer_path)
    output = str()
    for ch in text:
        output += int_to_string[str(ch)]
    return output


if __name__ == "__main__": 

    file_path = "data/wikitext2_train.txt"
    #create_word_token_map(file_path=file_path,  punctuations=set('.,:;!?()[]"-§¶_\''))
    create_char_token_map(file_path=file_path)