text_cleaning

import pandas as pd
import numpy as np
import re
import nltk
from nltk.corpus import stopwords
from nltk.corpus import wordnet
nltk.download('stopwords')
nltk.download('wordnet')
nltk.download('omw-1.4')
from tqdm.auto import tqdm

# Load Data
df = pd.read_csv('path/data.csv')

# Assign Cleaning Criteria
REPLACE_BY_SPACE_RE = re.compile('[/(){}\[\]\|@,;.&!--#$%^*_+"”:<>"'"'"'?`~½¾]')
STOPWORDS = set(stopwords.words('english'))
MONTH = {'january','jan','february','feb','march','mar','april','apr','may','june','jun','july','jul','august','aug','september','sept','sep','october','oct','november','nov','december','dec'}

# Define Cleaning Sequence
def clean_text(text):
    text = re.sub('\d+', ' ', text) # remove digits
    text = text.lower() # lowercase text
    text = REPLACE_BY_SPACE_RE.sub(' ', text) # replace REPLACE_BY_SPACE_RE symbols by space in text
    text = re.sub(r"\b[a-zA-Z]\b", ' ', text) # single character removal
    text = re.sub(r'\s+', ' ', text) # remove multiple spaces
    text = ' '.join(word for word in text.split() if word not in STOPWORDS) # delete stopwords from text
    text = ' '.join(word for word in text.split() if word not in MONTH) # delete month names from text
    return text

# Define Function to Retain only English Words
def check_for_word(s):
    return ' '.join(w for w in str(s).split(' ') if len(wordnet.synsets(w)) > 0) # retain only english words

# Clean Text
df['description'] = df['description'].apply(clean_text)

# Keep only English Words
tqdm.pandas(desc="Filtering only English Words")
df['description'] = df['description'].progress_apply(check_for_word)

# Remove Blank Rows
df['description'].replace('', np.nan, inplace=True)
df.dropna(subset=['description'], inplace=True)