1
2
3

%load_ext rpy2.ipython
import warnings
warnings.filterwarnings('ignore')

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

import itertools

import numpy
import pandas as pd

from wordcloud import WordCloud, STOPWORDS

from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, classification_report
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.naive_bayes import MultinomialNB

import nltk
from nltk.tree import *

import matplotlib.pyplot as plt
import seaborn as sns

import re
import string

from collections import Counter

1
2
3
4
5

%%R
library(dplyr)
library(reticulate)
library(ggplot2)
library(rtweet)

1
2
3
4
5
6
7

nltk.download("stopwords") # Load StopWords
from nltk.corpus import stopwords
set(stopwords.words('english'))

tweets_dataset = pd.read_csv("data/US_Airline_Tweets.csv") # Load Data

# %Rpush tweets_dataset

1

tweets_dataset.info()

1

tweets_dataset.describe()

1

tweets_dataset.head()

1
2
3

%%R

tweets_df <- read.csv("data/US_Airline_Tweets.csv", encoding = "UTF-8", header=TRUE, stringsAsFactors=FALSE)

1
2

%%R
tweets_df %>% colnames() # Columns

1
2

%%R
tweets_df %>% dim() # Shape / Dimension

1
2

%%R
tweets_df %>% summary() # Data Summary

1
2
3
4
5
6
7
8

positive_tweets = tweets_dataset['airline_sentiment'].str.contains("positive")
negative_tweets = tweets_dataset['airline_sentiment'].str.contains("negative")
neutral_tweets = tweets_dataset['airline_sentiment'].str.contains("neutral")


print(f'positive tweets shape: {positive_tweets.shape}')
print(f'negative tweets shape: {negative_tweets.shape}')
print(f'neutral tweets shape: {neutral_tweets.shape}')

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

def word_cloud(content):
    wordcloud = WordCloud(
    width = 3000,
    height = 2000,
    background_color = 'black',
    stopwords = STOPWORDS).generate(str(content))
    fig = plt.figure(
        figsize = (40, 30),
        facecolor = 'k',
        edgecolor = 'k')
    plt.imshow(wordcloud, interpolation = 'bilinear')
    plt.axis('off')
    plt.tight_layout(pad=0)
    plt.show()

1

word_cloud(tweets_dataset.loc[tweets_dataset['airline_sentiment'] == 'positive'].text.values)

1

word_cloud(tweets_dataset.loc[tweets_dataset['airline_sentiment'] == 'neutral'].text.values)

1

word_cloud(tweets_dataset.loc[tweets_dataset['airline_sentiment'] == 'negative'].text.values)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

American = tweets_dataset['airline'].str.contains("American")
Delta = tweets_dataset['airline'].str.contains("Delta")
Southwest = tweets_dataset['airline'].str.contains("Southwest")
United = tweets_dataset['airline'].str.contains("United")
VAmerica = tweets_dataset['airline'].str.contains("Virgin America")
USair = tweets_dataset['airline'].str.contains("US Airways")

print(f'American Tweets Shape: {American.shape}')
print(f'Delta Tweets Shape: {Delta.shape}')
print(f'Southwest Tweets Shape: {Southwest.shape}')
print(f'United Tweets Shape: {United.shape}')
print(f'VAmerica Tweets Shape: {VAmerica.shape}')
print(f'USair Tweets Shape: {USair.shape}')

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

%%R
tweets_sentiment_groups_df <- tweets_df %>%
    group_by(airline_sentiment) %>%
    rowwise() %>%
    mutate(SENTIMENT_COUNT = n()) %>%
    select(airline, airline_sentiment, SENTIMENT_COUNT)

# tweets_sentiment_groups_df %>% head()
    
ggplot(tweets_sentiment_groups_df, aes(x = airline, y = SENTIMENT_COUNT, fill=airline_sentiment)) +
  geom_bar(stat="identity") +
  coord_flip() +
  scale_y_continuous(name="Sentiment Tallies", labels = scales::comma) +
  ggtitle("Airline Sentiment Type Counts")

1
2
3
4
5
6

tweets_dataset_train, tweets_dataset_test = train_test_split(
    tweets_dataset, test_size=0.2
)

print(f'tweets_dataset_train tweets shape: {tweets_dataset_train.shape}')
print(f'tweets_dataset_test tweets shape: {tweets_dataset_test.shape}')

1

dataset = tweets_dataset.copy()[['airline_sentiment', 'text']]

1
2
3

positive = dataset['airline_sentiment'].str.contains("positive")
negative = dataset['airline_sentiment'].str.contains("negative")
neutral = dataset['airline_sentiment'].str.contains("neutral")

1
2
3
4
5
6
7
8

dataset_positive = dataset.copy()[dataset.airline_sentiment == 'positive'][:2363]
dataset_negative = dataset.copy()[dataset.airline_sentiment == 'negative'][:2363]
dataset_neutral = dataset.copy()[dataset.airline_sentiment == 'neutral'][:2363]

dataset = pd.concat(
    [dataset_positive, dataset_negative, dataset_neutral],
    ignore_index=True
).reset_index(drop=True)

1
2
3
4
5
6

def tweet_clean(tweet):
    tweet = ''.join(c for c in tweet if c not in string. punctuation) #Removing special characters
    tweet = re.sub(r'\d+', '', tweet)  #Removing digits
    tweet = re.sub('((www\S+)|(http\S+))', '', tweet) #Removing urls
    tweet = tweet.lower().strip()  #Convering to lower case
    return tweet

1
2
3

stop_words = nltk.corpus.stopwords.words("english")
stop_words = [''.join(c for c in s if c not in string.punctuation) for s in stop_words]
stop_words = [t.encode('utf-8') for t in stop_words]

1
2
3
4

def remove_sw(tweet, stop_words):
    text = tweet.split()
    text = ' '.join(word for word in text if word not in stop_words)
    return text

1
2
3
4
5
6
7

processed_tweets = []
for tweet in dataset['text']:
    cleaned = tweet_clean(tweet)
    stopwords_removed = remove_sw(cleaned, stop_words)
    processed_tweets.append(stopwords_removed)

dataset['text'] = processed_tweets

1
2
3
4
5
6
7
8

X_train, X_test, Y_train, Y_test = train_test_split(
    dataset['text'], dataset['airline_sentiment'], test_size=0.2
)

print(f'X_train tweets shape: {X_train.shape}')
print(f'X_test tweets shape: {X_test.shape}')
print(f'Y_train tweets shape: {Y_train.shape}')
print(f'Y_test tweets shape: {Y_test.shape}')

1
2
3
4
5
6
7
8
9

data_train = pd.DataFrame()
data_train['text'] = X_train
data_train['airline_sentiment'] = Y_train
data_train = data_train.reset_index(drop=True)

data_test = pd.DataFrame()
data_test['text'] = X_test
data_test['airline_sentiment'] = Y_test
data_test = data_test.reset_index(drop=True)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72

class NaiveBayesClassifier(object):
    def __init__(self, data_train):
        self.data_train = data_train
        self.dataset_positive = data_train.copy()[data_train.airline_sentiment == 'positive']
        self.dataset_negative = data_train.copy()[data_train.airline_sentiment == 'negative']
        self.dataset_neutral = data_train.copy()[data_train.airline_sentiment == 'neutral']
    def fit(self):
        Pr_pos = dataset_positive.shape[0]/self.data_train.shape[0]
        Pr_neg = dataset_negative.shape[0]/self.data_train.shape[0]
        Pr_neu = dataset_neutral.shape[0]/self.data_train.shape[0]
        self.Prior  = (Pr_pos, Pr_neg, Pr_neu)
        self.pos_words = ' '.join(self.dataset_positive['text'].tolist()).split()
        self.neg_words = ' '.join(self.dataset_negative['text'].tolist()).split()
        self.neu_words = ' '.join(self.dataset_neutral['text'].tolist()).split()
        words_list = ' '.join(self.data_train['text'].tolist()).split()
        self.vocab = len(Counter(words_list))
        wc_pos = len(' '.join(self.dataset_positive['text'].tolist()).split())
        wc_neg = len(' '.join(self.dataset_negative['text'].tolist()).split())
        wc_neu = len(' '.join(self.dataset_neutral['text'].tolist()).split())
        self.word_count = (wc_pos, wc_neg, wc_neu)
        return self
    def predict(self, data_test):
        category = ['positive', 'negative', 'neutral']
        classification = []
        for tweet in data_test['text']:
            text = tweet.split()
            val_pos = numpy.array([])
            val_neg = numpy.array([])
            val_neu = numpy.array([])
            for word in text:
                tmp_pos = numpy.log((self.pos_words.count(word)+1)/(self.word_count[0]+self.vocab))
                tmp_neg = numpy.log((self.neg_words.count(word)+1)/(self.word_count[1]+self.vocab))
                tmp_neu = numpy.log((self.neu_words.count(word)+1)/(self.word_count[2]+self.vocab))
                val_pos = numpy.append(val_pos, tmp_pos)
                val_neg = numpy.append(val_neg, tmp_neg)
                val_neu = numpy.append(val_neu, tmp_neu)
            val_pos = numpy.log(self.Prior[0]) + numpy.sum(val_pos)
            val_neg = numpy.log(self.Prior[1]) + numpy.sum(val_neg)
            val_neu = numpy.log(self.Prior[2]) + numpy.sum(val_neu)
            probability = (val_pos, val_neg, val_neu)
            classification.append(category[numpy.argmax(probability)])
        return classification
    def score(self, feature, target):
        compare = []
        for i in range(0,len(feature)):
            if feature[i] == target[i]:
                tmp ='correct'
                compare.append(tmp)
            else:
                tmp ='incorrect'
                compare.append(tmp)
        r = Counter(compare)
        accuracy = r['correct']/(r['correct']+r['incorrect'])
        return accuracy

def plot_confusion_matrix(cm, classes, title='Confusion matrix', cmap=plt.cm.Blues):
    cm = cm.astype('float') / cm.sum(axis=1)[:, numpy.newaxis]
    print(cm)
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = numpy.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)
    fmt = '.2f'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt), horizontalalignment="center", color="white" if cm[i, j] > thresh else "black")
    plt.ylabel('True')
    plt.xlabel('Predicted')
    plt.tight_layout()
    plt.show()

1
2
3
4
5

run = NaiveBayesClassifier(data_train)
run = run.fit()
predict = run.predict(data_test)
score = run.score(predict,data_test.airline_sentiment.tolist())
print(f'Naive Bayes Classifier Score: {score}')

1
2
3

class_labels = ['Positive', 'Negative', 'Neutral']
cm = confusion_matrix(data_test['airline_sentiment'], predict).T
plot_confusion_matrix(cm, classes = class_labels)

1
2
3
4
5
6
7

dataset_positive2 = dataset.copy()[dataset.airline_sentiment == 'positive'][:2363]
dataset_negative2 = dataset.copy()[dataset.airline_sentiment == 'negative'][:2363]

dataset2 = pd.concat(
    [dataset_positive2, dataset_negative2],
    ignore_index=True
).reset_index(drop=True)

1
2
3
4
5
6
7

processed_tweets2 = []
for tweet in dataset2['text']:
    cleaned = tweet_clean(tweet)
    stopwords_removed = remove_sw(cleaned, stop_words)
    processed_tweets2.append(stopwords_removed)

dataset2['text'] = processed_tweets2

1
2
3
4
5
6
7
8

X_train2, X_test2, Y_train2, Y_test2 = train_test_split(
    dataset2['text'], dataset2['airline_sentiment'], test_size=0.2
)

print(f'X_train2 tweets shape: {X_train2.shape}')
print(f'X_test2 tweets shape: {X_test2.shape}')
print(f'Y_train2 tweets shape: {Y_train2.shape}')
print(f'Y_test2 tweets shape: {Y_test2.shape}')

1
2
3
4
5
6
7
8
9

data_train2 = pd.DataFrame()
data_train2['text'] = X_train2
data_train2['airline_sentiment'] = Y_train2
data_train2 = data_train2.reset_index(drop=True)

data_test2 = pd.DataFrame()
data_test2['text'] = X_test2
data_test2['airline_sentiment'] = Y_test2
data_test2 = data_test2.reset_index(drop=True)

1
2
3
4
5

run2 = NaiveBayesClassifier(data_train2)
run2 = run2.fit()
predict2 = run2.predict(data_test2)
score2 = run2.score(predict2,data_test2.airline_sentiment.tolist())
print(f'Naive Bayes Classifier Score2: {score2}')

1
2
3

class_labels2 = ['Positive', 'Negative']
cm2 = confusion_matrix(data_test2['airline_sentiment'], predict2).T
plot_confusion_matrix(cm2, classes = class_labels2)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

def text_process(text):
    '''
    Takes in a string of text, then performs the following:
    1. Remove all punctuation
    2. Remove all stopwords
    3. Return the cleaned text as a list of words
    '''
    nopunc = [char for char in text if char not in string.punctuation]
    nopunc = ''.join(nopunc)
    
    return [word for word in nopunc.split() if word.lower() not in stopwords.words('english')]

1
2
3
4

def vectorize_text(text_array):
    transformer = CountVectorizer(analyzer=text_process).fit(text_array)
    vectorized_text_array = transformer.transform(text_array)
    return vectorized_text_array

1
2
3
4
5
6
7

Txt = vectorize_text(tweets_dataset['text'])

print('Shape of Sparse Matrix: ', Txt.shape)
print('Amount of Non-Zero occurrences: ', Txt.nnz)
# Percentage of non-zero values
density = (100.0 * Txt.nnz / (Txt.shape[0] * Txt.shape[1]))
print('Density: {}'.format((density)))

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

sent = tweets_dataset['airline_sentiment']

Txt_train, Txt_test, sent_train, sent_test = train_test_split(Txt, sent, test_size=0.2)

airline_sent_nb = MultinomialNB()

airline_sent_nb.fit(Txt_train, sent_train)

airline_sent_preds = airline_sent_nb.predict(Txt_test)

mat = confusion_matrix(sent_test, airline_sent_preds)
# Confusion Matrix
print(mat)
# Classification Report
print(classification_report(sent_test, airline_sent_preds))

1
2

# Confusion Matrix Plot
plot_confusion_matrix(mat.T, classes = ['negative', 'neutral', 'positive'])

1

sns.heatmap(mat.T, square=True, annot=True, fmt='d', cbar=False, xticklabels = True, yticklabels = True)