twitterSentimentAnalysis/PreProcess.py at master · ethankeystone/twitterSentimentAnalysis · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
import string
import pandas as pd
from spellchecker import SpellChecker
from langdetect import detect
def proccessTweets(tweets):
    retweet = []

    for tweet in tweets:
        if(str(tweet)[0:4] == 'b\"RT' or str(tweet)[0:4] == 'b\'RT'):
            retweet.append(True)
        else:
            retweet.append(False)

    retweet_person = []

    for i in range(len(retweet)):
        if(retweet[i]):
            bottom = 5
            top = 0
            for j in range(5,len(tweets[i])):
                if(str(tweets[i])[j] == ' '):
                    top = j - 1
                    break
            retweet_person.append(str(tweets[i])[bottom:top + 1])
        else:
            retweet_person.append('None')


    cleaned_tweets = []

    for text in tweets:
        cleaned_tweets.append(text)

    for i in range(len(tweets)):

        cleaned_tweets[i] = (processTweet(cleaned_tweets[i]))


    df = pd.DataFrame({'Text' : cleaned_tweets, 'Retweet' : retweet,'Retweeted Person' : retweet_person})

    return(df)
def processTweet(tweet):
    if(tweet[0] == 'b'):
        if(tweet[0:4] == 'b\'RT' or tweet[0:4] == 'b\"RT'):
            tweet = tweet[4:len(tweet)]
        elif(tweet[0:2] == 'b\''):
            tweet = tweet[2:len(tweet)]

    tweet = removeLink(tweet)

    tweet = removeRandomBitCode(tweet)

    tweet = removePunctuation(tweet)

    tweet = removeStart(tweet)

    #   tweet = fixSpelling(tweet)
    return(tweet)
def removeStart(tweet):
    if(tweet[0] == 'b'):
        if(tweet[0:3] == 'bRT'):
            tweet = tweet[4:len(tweet)]
        else:
            tweet = tweet[1:len(tweet)]

    return tweet
def removeLink(tweet):
    returnString = str(tweet)
    while((returnString).find('https') != -1):
        bottom = (returnString).find('https')

        top = 0

        for i in range(bottom,len(returnString)):
            if(returnString[i] == ' '):
                top = i
                break
        if(top == 0):
            top = len(returnString)

        returnString = returnString[0:bottom] + returnString[top:len(returnString)]

    return(returnString)

def removeRandomBitCode(retweet):
    tweet = str(retweet)
    for i in range(len(tweet)):
        if(i >= len(tweet)):
            break
        if(tweet[i] == '\\'):
            bottom = i
            top = 0

            if(tweet[i:i+2] != '\\n'):
                for j in range(i,len(tweet)):
                    if(tweet[j] == ' '):
                        top = j
                        break
                if(top == 0):
                    top = len(tweet)

                tweet = tweet[0:bottom] + tweet[top:len(tweet)]
    return(tweet)

def removePunctuation(tweet):

    for char in string.punctuation:
        tweet = str(tweet).replace(char,'')
    return(tweet)


def fixSpelling(tweet):
    spell = SpellChecker()
    returnTweet = ''
    for word in tweet.split():
        returnTweet = returnTweet + ' ' + (spell.correction(word))
    return(returnTweet)

def isEnglish(tweet):
    if(detect(str(tweet)) == 'en'):
        return True
    return False