bert_metrics_voting.py

# -*- coding: utf-8 -*-
"""
Created on Wed May 25 14:02:52 2022

@author: iliaskaloup
"""

import tensorflow
print("tensorflow version: ",tensorflow.version.VERSION)
print("\nNum GPUs Available: ", len(tensorflow.config.experimental.list_physical_devices('GPU')))

gpus = tensorflow.config.experimental.list_physical_devices('GPU')
for gpu in gpus:
    tensorflow.config.experimental.set_memory_growth(gpu, True)
    logical_gpus = tensorflow.config.experimental.list_logical_devices('GPU')

print("Gpus: ",gpus)

import tensorflow as tf
import numpy as np
import csv
from transformers import AutoTokenizer, TFAutoModelForSequenceClassification #, BertModel, BertTokenizer, TFBertForSequenceClassification
import matplotlib.pyplot as plt
import random
from tensorflow.keras.callbacks import CSVLogger
from tensorflow.keras.callbacks import EarlyStopping
import tensorflow.keras.backend as K
from collections import OrderedDict
import time
from sklearn.metrics import accuracy_score, recall_score, f1_score, precision_score, \
roc_auc_score, confusion_matrix, classification_report
from sklearn.model_selection import StratifiedKFold

import pandas as pd
import io
import json

from sklearn.ensemble import RandomForestClassifier


# user parameters
n_folds = 5
n_epochs = 4
batch_size = 6
lr = 1e-5
seed = 123
model_variation = "microsoft/graphcodebert-base"
tokenizer = AutoTokenizer.from_pretrained(model_variation) #Tokenizer
#bert-base-uncased # roberta-base # distilbert-base-uncased # microsoft/codebert-base-mlm # microsoft/graphcodebert-base
# TFAutoModelForSequenceClassification # BertModel

def seeder():
    return 0.1

def dropEmpty(tokens0):
    tokens = []
    for i in range(0, len(tokens0)):
        temp = tokens0[i]
        if temp != []:
            tokens.append(temp)
    return tokens

def getLengths(data):
    lens = []
    for i in range(len(data)):
        lens.append(len(data[i]))
    lens = pd.DataFrame(lens)
    lensFreq = lens[0].value_counts()
    lensFreq=pd.DataFrame(lensFreq)
    return lens, lensFreq

def getLabels(data):
    values = []
    for i in range(len(data)):
        values.append(int(data[i][0]))
    serVal = pd.DataFrame(values)
    return serVal

def prepareData(data):
        
    # lowercase
    lines = []
    labels = []
    headlines = []
    for i in range(0, len(data)):
        labels.append(int(data[i][0]))
        headlines.append(data[i][3])
        line = data[i][44:]
        lows = [w.lower() for w in line]
        lines.append(lows)
    
    texts = []
    for i in range(0, len(lines)):
        texts.append(listToString(lines[i]))
    
    return texts, labels, headlines

def makeTfData(texts, indices):
    inputs = tokenizer(texts, padding=True, truncation=True, return_tensors='tf') #Tokenized text
    dataset = tf.data.Dataset.from_tensor_slices((dict(inputs), indices)) #Create a tensorflow dataset
    return dataset

def listToString(s): 
    
    # initialize an empty string
    str1 = "" 
    
    # traverse in the string 
    count = 0
    for ele in s: 
        if count==0:
            str1 = str1 + ele
        else:
            str1 = str1 + ' ' + ele
        count = count + 1
        #str1 += ele  
    
    # return string  
    return str1

def indicize_labels(labels):
    """Transforms string labels into indices"""
    indices=[]
    for j in range(len(labels)):
        for i in range(n_categories):
            if labels[j]==categories[i]:
                indices.append(i)
    return indices

def recall_metric(y_true, y_pred):
        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
        recall = (true_positives + K.epsilon()) / (possible_positives + K.epsilon())
        return recall

def precision_metric(y_true, y_pred):
        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
        precision = (true_positives + K.epsilon()) / (predicted_positives + K.epsilon())
        return precision

def f1_metric(y_true, y_pred):

    prec = precision_metric(y_true, y_pred)
    rec = recall_metric(y_true, y_pred)
    f1 = 2*((prec*rec)/(prec+rec+K.epsilon()))
    return f1

def f2_metric(y_true, y_pred):

    prec = precision_metric(y_true, y_pred)
    rec = recall_metric(y_true, y_pred)
    f2 = 5*((prec*rec)/(4*prec+rec+K.epsilon()))
    return f2
    
    
#main
with open('datajs.csv', newline='', encoding='utf-8') as f:
    reader = csv.reader(f)
    data = list(reader)
data = dropEmpty(data)
#data = data[0:100]

random.shuffle(data, seeder)

#sequences
texts, labels, headlines = prepareData(data)

#metrics
features = []
for i in range(0, len(data)):
    line = data[i][9:44]
    new_line = [float(j) for j in line]
    features.append(new_line)
X2 = pd.DataFrame(features)

#labels
sentiment = getLabels(data).iloc[:,0].values

# explore data
n_elements=len(headlines)
print('Elements in dataset:', n_elements)
categories=sorted(list(set(labels))) #set will return the unique different entries
n_categories=len(categories)
print("{} categories found:".format(n_categories))
for category in categories:
    print(category)
    
indices=indicize_labels(labels) #Integer label indices


############## cross validation
texts = np.array(texts)
indices = np.array(indices)
scores=['accuracy', 'precision', 'recall', 'f1', 'roc_auc', 'f2', 'fpr']
values = [np.array([]) for i in range(0, len(scores))]
score_dict = OrderedDict(zip(scores, values))
k=5
f=0
kfold = StratifiedKFold(n_splits=k,shuffle=True,random_state=seed)
print("Training...")
milli_sec1 = int(round(time.time() * 1000))

for train_index, test_index in kfold.split(texts, indices): # for train_index, test_index in kfold.split(X1, y):
    f = f + 1
    print('fold number= ',f)
    
    texts_train, texts_test = texts[train_index], texts[test_index]
    indices_train, indices_test = indices[train_index], indices[test_index]
    
    texts_train = texts_train.tolist()
    indices_train = indices_train.tolist()
    texts_test = texts_test.tolist()
    indices_test = indices_test.tolist()
    
    # tokenize the input text
    trainset = makeTfData(texts_train, indices_train)
    testset = makeTfData(texts_test, indices_test)
    train_data_size = len(texts_train)
    train_ds = trainset.take(train_data_size).batch(batch_size, drop_remainder=True)
    val_data_size = len(texts_test)
    val_ds = testset.take(val_data_size).batch(batch_size, drop_remainder=True)
    
    # train model
    model = TFAutoModelForSequenceClassification.from_pretrained(model_variation, num_labels=n_categories)  
    
    model.compile(
            optimizer=tf.keras.optimizers.Adam(learning_rate=lr, epsilon=1e-08, clipnorm=1.),
            loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), #f2_loss,
            metrics=[f2_metric] #tf.metrics.SparseCategoricalAccuracy()
            )
    
    #csv_logger = CSVLogger('log.csv', append=True, separator=',')
    #es = EarlyStopping(monitor='val_f2_metric', mode='max', verbose=1, patience=2)
    #history = model.fit(train_ds, validation_data=val_ds, epochs=n_epochs, verbose=1, callbacks=[csv_logger,es])
    history = model.fit(train_ds, validation_data=val_ds, epochs=n_epochs, verbose=1)
    
    # save model's weights
    model.save_weights('./saved_weights.h5')
    
    # load trained model's weights
    trained_model = TFAutoModelForSequenceClassification.from_pretrained(model_variation, num_labels=n_categories)
    trained_model.load_weights('./saved_weights.h5')
    
    # evaluation
    y_test=[]
    x_test=[]
    for i in range(val_data_size):
        y_test.append(indices_test[i])
        x_test.append(texts_test[i])
    #y_test=indicize_labels(y_test)
    tokens=tokenizer(x_test, padding=True,truncation=True, return_tensors='tf') 
    logits=trained_model.predict(dict(tokens), verbose=1).logits
    prob=tf.nn.softmax(logits, axis=1).numpy()   
    predictions=np.argmax(prob, axis=1)  
    predictions = predictions.astype(np.int32)
           
    
    # 2nd classifier
    X_train_metrics, X_test_metrics = X2.iloc[train_index], X2.iloc[test_index]
    model2 = RandomForestClassifier(n_estimators=100, bootstrap = True, max_features = 'sqrt')
    model2.fit(X_train_metrics, indices_train)
    X_test_metrics = np.array(X_test_metrics)
    print("Evaluation on Validation set")
    predictions2 = model2.predict(X_test_metrics)
    predScores2 = model2.predict_proba(X_test_metrics)
    #print("predScores2:", predScores2[0][0])
    #print("predScores:", predScores2)
    #print("predictions:", predictions2)
    
    # Voting process
    preds = []
    for i in range(0, len(predictions)):
        pred = predictions[i]
        if pred == predictions2[i]:
            preds.append(predictions2[i])
        else:
            if predictions2[i] == 0:
                if predScores2[i][0] >= prob[i][1]:
                    preds.append(0)
                else:
                    preds.append(1)
            else:
                if predScores2[i][1] >= prob[i][0]:
                    preds.append(1)
                else:
                    preds.append(0)
    
    accuracy=accuracy_score(indices_test, preds)
    precision=precision_score(indices_test, preds)
    recall=recall_score(indices_test, preds)
    f1=f1_score(indices_test, preds)
    roc_auc=roc_auc_score(indices_test, preds)
    f2=5*precision*recall / (4*precision+recall)
    print(confusion_matrix(indices_test, preds, labels=[0, 1]))
    tn, fp, fn, tp = confusion_matrix(indices_test, preds).ravel()
    fpr = fp / (fp+tn)
    acc = ((tp+tn)/(tp+tn+fp+fn))
    print("Accuracy:%.2f%%"%(acc*100))
    print("Precision:%.2f%%"%(precision*100))
    print("Recall:%.2f%%"%(recall*100))
    print("F1 score:%.2f%%"%(f1*100))
    print("Roc_Auc score:%.2f%%"%(roc_auc*100))
    print("F2 score:%.2f%%"%(f2*100))
    print("FPR score:%.2f%%"%(fpr*100))
    print(classification_report(indices_test, preds))
    del trained_model
    del model2
    score_dict['accuracy'] = np.append(score_dict['accuracy'], accuracy)
    score_dict['precision'] = np.append(score_dict['precision'], precision)
    score_dict['recall'] = np.append(score_dict['recall'], recall)
    score_dict['f1'] = np.append(score_dict['f1'], f1)
    score_dict['roc_auc'] = np.append(score_dict['roc_auc'], roc_auc)
    score_dict['f2'] = np.append(score_dict['f2'], f2)
    score_dict['fpr'] = np.append(score_dict['fpr'], fpr)
    
milli_sec2 = int(round(time.time() * 1000))
print("Training is completed after", milli_sec2-milli_sec1)

print("accuracy: %.2f%% (%.2f%%)" % (score_dict['accuracy'].mean()*100, score_dict['accuracy'].std()*100))
print("precision: %.2f%% (%.2f%%)" % (score_dict['precision'].mean()*100, score_dict['precision'].std()*100))
print("recall: %.2f%% (%.2f%%)" % (score_dict['recall'].mean()*100, score_dict['recall'].std()*100))
print("f1: %.2f%% (%.2f%%)" % (score_dict['f1'].mean()*100, score_dict['f1'].std()*100))
print("roc_auc: %.2f%% (%.2f%%)" % (score_dict['roc_auc'].mean()*100, score_dict['roc_auc'].std()*100))
print("f2: %.2f%% (%.2f%%)" % (score_dict['f2'].mean()*100, score_dict['f2'].std()*100))
print("fpr: %.2f%% (%.2f%%)" % (score_dict['fpr'].mean()*100, score_dict['fpr'].std()*100))