1_plan_label_tfidf.py

# -*- coding: utf-8 -*-
"""1_plan_label_tfidf.ipynb

Automatically generated by Colab.

Original file is located at
    https://colab.research.google.com/drive/13mBWymAGiduoYMxirjbFkj5Ocx3h030B
"""

!apt-get install python3.8.10
!ln -sf /usr/bin/python3.8 /usr/bin/python3

!python3 --version

import re
#str = input("Type word: ")
str = "hello world in research @paula"
for i in ["hello", "world", "@[w]"]:
  str = re.sub(i, "", str, flags=re.IGNORECASE)

str = re.sub(r'(\s)?@\w+', r'\1', str)
str

import nltk
nltk.download('wordnet')

# import these modules
from nltk.stem import WordNetLemmatizer

lemmatizer = WordNetLemmatizer()

print("rocks :", lemmatizer.lemmatize("rocks"))
print("corpora :", lemmatizer.lemmatize("corpora"))

# a denotes adjective in "pos"
print("better :", lemmatizer.lemmatize("better", pos ="a"))

from nltk.stem import WordNetLemmatizer

lemmatizer = WordNetLemmatizer()
english_stopwords = ["is","are"]
sentence = "mangoes are my world"

tokens = sentence.split()

tokens = [token for token in tokens if token.lower() not in english_stopwords]

for i in range(len(tokens)):
  tokens[i] = lemmatizer.lemmatize(tokens[i])

tokens.remove("my")

cleaned_sentence = " ".join(tokens)
cleaned_sentence

import gensim
from gensim.models import word2vec
from gensim.models.word2vec import Word2Vec
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import spacy
import string
import re

from sklearn.feature_extraction.text import TfidfVectorizer

#data = pd.read_csv("/content/drive/MyDrive/MSc_study/Summer_2024/man_try1_sec.csv",error_bad_lines=False, engine="python")
data = pd.read_csv("/content/drive/MyDrive/MSc_study/Fall_2023/final_plan_dataset.csv",error_bad_lines=False, engine="python")
#data = pd.read_csv("/content/drive/MyDrive/MSc_study/Fall_2023/2final_plan_dataset.csv",error_bad_lines=False, engine="python")
data.head()

import nltk
nltk.download('stopwords')

st_nir = []
dt = pd.read_csv("/content/drive/MyDrive/MSc_study/Fall_2023/smart_stop_words.csv")

for i in range(dt['Column1'].size):
  st_nir.append(dt['Column1'][i])
print(st_nir)

for i in range(dt['Column2'].size):
  st_nir.append(dt['Column2'][i])

for i in range(dt['Column3'].size):
  st_nir.append(dt['Column3'][i])

for i in range(dt['Column4'].size):
  st_nir.append(dt['Column4'][i])

for i in range(dt['Column5'].size):
  st_nir.append(dt['Column5'][i])

for i in range(dt['Column6'].size):
  st_nir.append(dt['Column6'][i])

#SE STOP WORDS
#st_nir = []
dt = pd.read_csv("/content/drive/MyDrive/MSc_study/Fall_2023/se_stop_words.csv")

for i in range(dt['stopwords'].size):
  st_nir.append(dt['stopwords'][i])
st_nir

from nltk.stem import WordNetLemmatizer

corpus = data['Body'].to_list()
lemmatizer = WordNetLemmatizer()
sentence = ""
#len(corpus)
print(corpus[3])

'''
#TRY
sentence = corpus[3]
tokens = sentence.split()
print("STEP 1: ",tokens)
  #tokens = [re.sub([0-9], '', i) for i in tokens]
tokens = [token for token in tokens if token.lower() not in (st_nir or [0-9])]
cleaned_sentence = ""
for j in range(len(tokens)):
  tokens[j] = lemmatizer.lemmatize(tokens[j])
  print("STEP 2: ",tokens[j])
  cleaned_sentence = cleaned_sentence + " " + tokens[j]
  print("STEP 3: ", cleaned_sentence)
  corpus[3] = cleaned_sentence
'''
for i in range(len(corpus)):
  sentence = corpus[i]
  tokens = sentence.split()
  #tokens = [re.sub([0-9], '', i) for i in tokens]
  tokens = [token for token in tokens if token.lower() not in (st_nir or [0-9])]
  cleaned_sentence = ""
  for j in range(len(tokens)):
    tokens[j] = lemmatizer.lemmatize(tokens[j])
    result = re.sub(r'[()\[\]]', '', tokens[j])
    cleaned_sentence = cleaned_sentence + " " + result
    corpus[i] = cleaned_sentence
print(corpus[3])



#let's create the vectorizer and fit the corpus and transform them accordingly
v = TfidfVectorizer()
v.fit(corpus)
transform_output = v.transform(corpus)

transform_output

print(v.vocabulary_)

dir(v)

#let's print the idf of each word:

all_feature_names = v.get_feature_names_out()

for word in all_feature_names:

    #let's get the index in the vocabulary
    indx = v.vocabulary_.get(word)

    #get the score
    idf_score = v.idf_[indx]

idf_scores = v.idf_

# Create a DataFrame with words and their TF-IDF scores
word_importance_df = pd.DataFrame(list(zip(all_feature_names, idf_scores)), columns=['Word', 'TF-IDF'])

# Sort the DataFrame by TF-IDF score in descending order
word_importance_df = word_importance_df.sort_values(by='TF-IDF', ascending=False)

# Specify the file path where you want to save the CSV file
csv_file_path = "plan_word_importance.csv"

# Save the DataFrame to a CSV file
word_importance_df.to_csv(csv_file_path, index=False)

#let's print the transformed output from tf-idf
print(transform_output.toarray()[:2])

#let's print the transformed output from tf-idf
print(transform_output.toarray())

#START FROM HERE - FOR THESIS
import gensim
from gensim.models import word2vec
from gensim.models.word2vec import Word2Vec
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import spacy
import string

from sklearn.model_selection import train_test_split

#df  = pd.read_csv("/content/small_plan_dataset.csv",error_bad_lines=False, engine="python")
#df  = pd.read_csv("/content/bugreport_labeled.csv",error_bad_lines=False, engine="python")
#df1 = pd.read_csv("/content/drive/MyDrive/MSc_study/Summer_2024/man_try1_sec.csv",error_bad_lines=False, engine="python")
df1 = pd.read_csv("/content/drive/MyDrive/MSc_study/Fall_2023/4final_plan_dataset.csv",error_bad_lines=False, engine="python")
df2 = pd.read_csv("/content/drive/MyDrive/MSc_study/Summer_2024/rastkar_bugs_400.csv",error_bad_lines=False, engine="python")
#print(df['COMMENTS'])
'''
X_train = df1['Body']
X_test = df2['Body']
y_train = df1['raf_PLAN']
y_test = df2['PLAN']
'''
X_train = df1['Body']
X_test = df2['Body']
y_train = df1['OT']
y_test = df2['sh_OT']
'''
X_train, X_test, y_train, y_test = train_test_split(
    df.Body,
    df.PLAN,
    test_size=0.2, # 20% samples will go to test dataset
    random_state=2022
)
'''

df1 = pd.read_csv("/content/predict_ot.csv",error_bad_lines=False, engine="python")

#print(df1)
#for i in range(2360):
for number in df1:
    print(number)

print("Shape of X_train: ", X_train.shape)
print("Shape of X_test: ", X_test.shape)

y_train.value_counts()

y_test.value_counts()

from sklearn.neighbors import KNeighborsClassifier
from sklearn.pipeline import Pipeline
from sklearn.metrics import classification_report

#1. create a pipeline object
clf = Pipeline([
     ('vectorizer_tfidf',TfidfVectorizer()),
     ('KNN', KNeighborsClassifier())
])

#2. fit with X_train and y_train
clf.fit(X_train, y_train)


#3. get the predictions for X_test and store it in y_pred
y_pred = clf.predict(X_test)


#4. print the classfication report
print(classification_report(y_test, y_pred))

X_test[:5]

y_test[:5]

y_pred[:5]

from sklearn.naive_bayes import MultinomialNB


#1. create a pipeline object
clf = Pipeline([
     ('vectorizer_tfidf',TfidfVectorizer()),
     ('Multi NB', MultinomialNB())
])

#2. fit with X_train and y_train
clf.fit(X_train, y_train)


#3. get the predictions for X_test and store it in y_pred
y_pred = clf.predict(X_test)


#4. print the classfication report
print(classification_report(y_test, y_pred))

from sklearn import svm


#1. create a pipeline object
clf = Pipeline([
     ('vectorizer_tfidf',TfidfVectorizer()),
     ('Multi NB', svm.SVC())
])

#2. fit with X_train and y_train
clf.fit(X_train, y_train)


#3. get the predictions for X_test and store it in y_pred
y_pred = clf.predict(X_test)


#4. print the classfication report
print(classification_report(y_test, y_pred))

from sklearn.ensemble import RandomForestClassifier
import pandas as pd

#1. create a pipeline object
clf = Pipeline([
     ('vectorizer_tfidf',TfidfVectorizer()),        #using the ngram_range parameter
     ('Random Forest', RandomForestClassifier())
])

#2. fit with X_train and y_train
clf.fit(X_train, y_train)

'''
#3. get the predictions for X_test and store it in y_pred
y_pred = clf.predict(X_test)
print(y_pred)
#my = pd.DataFrame.from_records(y_pred)
my = y_pred.tolist()
for i in range(2360):
  #file1.writelines(my[i])
  print(my[i])
  with open("myfile.txt", "w") as file1:
    file1.writelines(my[0])
'''
'''
with open("myfile.txt", "w") as file1:
    # Writing data to a file
    for i in range(2360):
      file1.writelines(my[i])

with open('predict_ot.csv', 'w', newline='') as file:
    writer = csv.writer(file)
    field = ["SH_LABEL"]
    for i in range(0, 2360):
      me = my[i]
      writer.writerow(me)
'''

# Specify the file path where you want to save the CSV file
csv_file_path2 = "plan_predict.csv"
df = pd.DataFrame(y_pred)
# Save the DataFrame to a CSV file
df.to_csv(csv_file_path2, index=False)

'''
# Import Joblib
import joblib

# Save the model to a file
joblib.dump(clf, 'sh_model_3_8.joblib')

# Load the model using Joblib
loaded_model_joblib = joblib.load('sh_model_3_8.joblib')
'''

import pickle

# Save the model to a file
with open('my_model.pkl', 'wb') as file:
    pickle.dump(clf, file)

with open('my_model.pkl', 'rb') as file:
    loaded_model_pickle = pickle.load(file)

#3. get the predictions for X_test and store it in y_pred
#y_pred = clf.predict(X_test)



y_pred = loaded_model_pickle.predict(X_test)

#4. print the classfication report
print(classification_report(y_test, y_pred))



#Pema grg
from sklearn.feature_extraction.text import TfidfVectorizer
import pandas as pd
from sklearn import svm
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
import pickle
from sklearn import linear_model
#Project_path = "<path to the project folder>"
from sklearn.ensemble import RandomForestClassifier



#2. fit with X_train and y_train
clf.fit(X_train, y_train)

data = pd.read_csv('/content/drive/MyDrive/MSc_study/Fall_2023/4final_plan_dataset.csv',error_bad_lines=False, engine="python")
#print (data.category.unique())
vectorizer = TfidfVectorizer(sublinear_tf=True, encoding='utf-8',
                             decode_error='ignore')


def train_bpsd(df, vectorizer):
    tfidf = vectorizer.fit(df["Body"].values.astype('U'))

    X = vectorizer.fit_transform(df["Body"].values.astype('U'))
    y = df['OT']
    #train_test(X, y)
    model = svm.LinearSVC()

    print("==fitting the model===")
    model.fit(X.A, y)

    print("==fit done\=")
    return model, tfidf


Project_path = "/content/drive/MyDrive/MSc_study"

model_path = Project_path + "/sh_models/sh.pickle"
vectorizer_path = Project_path + "/sh_models/sh_vectorizer.pickle"
model, vectorizer_model = train_bpsd(data, vectorizer)
pickle.dump(model, open(model_path, 'wb'))
pickle.dump(vectorizer_model, open(vectorizer_path, "wb"))





from sklearn import tree

#1. create a pipeline object
clf = Pipeline([
     ('vectorizer_tfidf',TfidfVectorizer()),
     ('Multi NB', tree.DecisionTreeClassifier())
])

#2. fit with X_train and y_train
clf.fit(X_train, y_train)


#3. get the predictions for X_test and store it in y_pred
y_pred = clf.predict(X_test)


#4. print the classfication report
print(classification_report(y_test, y_pred))

import matplotlib.pyplot as plt

from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split

X, y = make_classification(
    n_samples=1000,
    n_features=10,
    n_informative=3,
    n_redundant=0,
    n_repeated=0,
    n_classes=2,
    random_state=0,
    shuffle=False,
)
X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=42)

from sklearn.ensemble import RandomForestClassifier

feature_names = [f"{X[i]}" for i in range(X.shape[1])]
forest = RandomForestClassifier(random_state=0)
forest.fit(X_train, y_train)

import time

import numpy as np

start_time = time.time()
importances = forest.feature_importances_
std = np.std([tree.feature_importances_ for tree in forest.estimators_], axis=0)
elapsed_time = time.time() - start_time

print(f"Elapsed time to compute the importances: {elapsed_time:.3f} seconds")

import pandas as pd

forest_importances = pd.Series(importances, index=feature_names)

feature_scores = pd.Series(forest.feature_importances_, index=feature_names).sort_values(ascending=False)

feature_scores

from google.colab import drive
drive.mount('/content/drive')





from sklearn import svm
import csv

#1. create a pipeline object
clf = Pipeline([
     ('vectorizer_tfidf',TfidfVectorizer()),        #using the ngram_range parameter
     ('svm', svm.SVC())
])

#2. fit with X_train and y_train
clf.fit(X_train, y_train)


#3. get the predictions for X_test and store it in y_pred
y_pred = clf.predict(X_test)
#df = pd.DataFrame.from_records(y_pred)
'''
with open('predict_ot.csv', 'w', newline='') as file:
    writer = csv.writer(file)
    field = ["SH_LABEL"]
    writer.writerow(field)
    for i in range(2360):
      writer.writerow(df[i])
'''
#4. print the classfication report
print(classification_report(y_test, y_pred))

x_train = df1['Body']
x_test = df2['Body']
y_train = df1['OT']
y_test = df2['sh_OT']