get_feedback_sentiment.py

# %%
from transformers import AutoTokenizer, AutoModelForSequenceClassification
import torch
import pandas as pd
from tqdm import tqdm
if torch.backends.mps.is_available():
    mps_device = torch.device("mps")
elif torch.cuda.is_available():
    mps_device = torch.device("cuda")
else:
    mps_device = torch.device("cpu")

# %%
df = pd.read_excel('cleaned_essay_data.xlsx', index_col=0)

# %%
model_name = "tabularisai/multilingual-sentiment-analysis"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForSequenceClassification.from_pretrained(model_name).to(mps_device)
model.eval()

def predict_sentiment(texts, batch_size=32):
    sentiments = []
    sent_probs = []
    sentiment_map = {0: "Very Negative", 1: "Negative", 2: "Neutral", 3: "Positive", 4: "Very Positive"}

    for i in tqdm(range(0, len(texts), batch_size)):
        batch_texts = texts[i:i+batch_size]
        try:
            inputs = tokenizer(batch_texts, return_tensors="pt", truncation=True, padding=True, max_length=512).to(mps_device)

            with torch.no_grad():
                outputs = model(**inputs)

            probabilities = torch.nn.functional.softmax(outputs.logits, dim=-1)
            preds = torch.argmax(probabilities, dim=-1).tolist()
            confidences = torch.max(probabilities, dim=-1).values.tolist()

            sentiments.extend([sentiment_map[p] for p in preds])
            sent_probs.extend(confidences)
        except:
            print(f"Error processing batch {i} to {i+batch_size}")
            print(batch_texts)
            exit()

    return sentiments, sent_probs

# %
df['sentiment_fb1'], df['sentiment_prob_fb1'] = predict_sentiment(df['feedback1'].fillna("").tolist(), batch_size=128)
df['sentiment_fb2'], df['sentiment_prob_fb2'] = predict_sentiment(df['feedback2'].fillna("").tolist(), batch_size=128)
df['sentiment_fb3'], df['sentiment_prob_fb3'] = predict_sentiment(df['feedback3'].fillna("").tolist(), batch_size=128)

df.to_excel('essay_data_with_sentiment.xlsx', index=True)

# %%