← Sugeerth Murugesan Twitter Airline Sentiment Portfolio 
%reload_ext autoreload
%autoreload 2
%matplotlib inline
from fastai.tabular import *
import pandas as pd
bs = 64

# Reading the tweets into the pandas dataframes

path = ('drive/My Drive/Colab Notebooks/twitter_airline_sentiment') 

data_panda = pd.read_csv(path+'/tweets.csv')

conversion = {
    "object": "TEXT",
    "float64": "NUMERIC",
    "int64": "INTEGER"
}

# Initializing variables that can be used 
dep_var = 'airline_sentiment'
cat_names = ['airline', 'negativereason', 'tweet_location', 'user_timezone']
cont_names = ['tweet_id', 'airline_sentiment_confidence', 'negativereason_confidence', 'retweet_count']
procs = [FillMissing, Categorify, Normalize]

# Making a test dataset that can be used for validation 

path
test = TabularList.from_df(data_panda.iloc[400:700].copy(), path=path, cat_names=cat_names, cont_names=cont_names)

# Creating the tabularlist that can be used to do training and testing 

data = (TabularList.from_df(data_panda, path=path, cat_names=cat_names, cont_names=cont_names, procs=procs)
                           .split_by_idx(list(range(800,1000)))
                           .label_from_df(cols=dep_var)
                           .add_test(test)
                           .databunch())

len(data_panda)

14640
data.show_batch(rows=10)
airline negativereason tweet_location user_timezone negativereason_confidence_na tweet_id airline_sentiment_confidence negativereason_confidence retweet_count target
United Lost Luggage Chorley, Lancashire Amsterdam False 0.8046 0.6134 1.2568 -0.1098 negative
American #na# Charlottesville, VA Hawaii False 0.9356 -1.1414 -2.3108 -0.1098 neutral
United Customer Service Issue Chicago Central Time (US & Canada) False 0.8839 -1.4831 0.0399 -0.1098 negative
American Cancelled Flight #na# Atlantic Time (Canada) False 0.8577 -3.2686 -0.9965 1.2236 negative
United Bad Flight New York/New Jersey Eastern Time (US & Canada) False 0.7322 0.6134 0.1437 -0.1098 negative
United Customer Service Issue #na# Pacific Time (US & Canada) False 0.3830 -1.1696 0.2219 -0.1098 negative
Southwest Customer Service Issue San Diego, CA #na# False -0.0045 0.6134 0.0396 -0.1098 negative
US Airways Late Flight Bedford, NH Eastern Time (US & Canada) False 1.0429 0.6134 1.2568 -0.1098 negative
American Customer Service Issue Brooklyn, NY Eastern Time (US & Canada) False 0.9839 -1.2219 0.1915 -0.1098 negative
Southwest Late Flight #na# Central Time (US & Canada) False -1.6949 0.6134 -1.0564 -0.1098 negative 
# The tabular learner making the predictions based on neural networks 

learn = tabular_learner(data, layers=[53,10], metrics=accuracy)
learn.lr_find()
learn.recorder.plot()
0.00% [0/1 00:00<00:00]
epoch train_loss valid_loss accuracy time

39.56% [89/225 00:01<00:02 0.7800] 
LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.

 learn.fit(4, 1e-1)
 
row = data_panda.iloc[51]
row
epoch train_loss valid_loss accuracy time
0 0.254427 0.206726 0.885000 00:03
1 0.246003 0.231012 0.880000 00:02
2 0.217732 0.236338 0.875000 00:03
3 0.208154 0.232266 0.860000 00:02 
tweet_id                                                       570006886012973056
airline_sentiment                                                        positive
airline_sentiment_confidence                                                0.657
negativereason                                                                NaN
negativereason_confidence                                                     NaN
airline                                                            Virgin America
airline_sentiment_gold                                                        NaN
name                                                                   joyabsalon
negativereason_gold                                                           NaN
retweet_count                                                                   0
text                            @VirginAmerica @ladygaga @carrieunderwood Juli...
tweet_coord                                                                   NaN
tweet_created                                           2015-02-23 15:46:46 -0800
tweet_location                                                  Northern Virginia
user_timezone                                          Eastern Time (US & Canada)
Name: 51, dtype: object
learn.predict(row)

(Category positive, tensor(2), tensor([1.0597e-05, 3.1680e-01, 6.8319e-01]))