# Importing all the relvant libraries and features that is responseible for the jupyter notebook 
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from scipy import stats
import pylab
import seaborn as sns

import sklearn
%matplotlib inline
import pandas
from sklearn.model_selection import train_test_split
import numpy

#Import the 2 csv files into dataframes
test_table = pd.read_csv('instacart/test_trips.csv',parse_dates=['shopping_started_at'])
train_table = pd.read_csv('instacart/train_trips.csv',parse_dates=['shopping_started_at','shopping_ended_at'])
order_items = pd.read_csv('instacart/order_items.csv')

# Take a look at the data for my test set
test_table.head(5)

# This is my training set where I have the data about when the shopper started shopping and when the shopper ended 
train_table.head(5)

order_items.head(5)

train_table['fulfillment_model']=train_table['fulfillment_model'].astype('str')

# Identifying the total time in seconds that a shopper spent within a given trip 
train_table['time_difference'] = (train_table.shopping_ended_at- train_table.shopping_started_at).astype('timedelta64[s]')

# Looking at the distribution of the store data, it seems like there are few stores that attract a lot of shoppers 
sns.distplot(train_table['store_id']);

# A statistical measure indicating the asymmetry of the probability distribution of a random-variable about its mean 
print("Skewness: %f" % train_table['store_id'].skew())

# While now we can look at other distributions of the test dataset, we could see a similar pattern 
hist = test_table.hist(bins=9)

# Understanding the distributions of what people actually buy in the store
hist = order_items.hist(bins=49)

# Plotting the distribution of the models  
sns.catplot(x="fulfillment_model", kind="count", palette="ch:.25", data=train_table);

# Looking at the distribution of the store data, it seems like there are few stores that attract a lot of shoppers 
sns.distplot(train_table['time_difference']);

train_table.count()

train_table.head(3)

train_table['day_of_week'] = train_table['shopping_started_at'].dt.day_name()

sns.catplot(x="day_of_week", kind="count", palette="ch:.25", data=train_table);

# Adding numerical value to the feature
train_table['day_of_week'] = train_table['shopping_started_at'].dt.dayofweek

train_table.dtypes

# Understanding the segment of the day 
train_table = train_table.assign(day_segment=pd.cut(train_table.shopping_started_at.dt.hour,[0,6,12,18,24],labels=[1,2,3,4]))

train_table['day_segment']=train_table['day_segment'].astype('int')

sns.catplot(x="day_segment", kind="count", palette="ch:.25", data=train_table);

train_table.dtypes

train_table.head(9)

train_table = train_table.merge(order_items,how='left',on='trip_id')
train_table.head()

# Convert the date time, categorize them into time of day

train_table.isnull().sum()

# Create a new training set that can group by trip_id, store_id','department_name','item_id','fulfillment_model','shopping_started_at' 
cleaned_train_data = train_table.groupby(['trip_id','store_id','department_name','item_id','fulfillment_model','shopping_started_at','day_of_week','day_segment']).agg({'quantity':np.sum,'time_difference':np.mean,'shopper_id':np.median}).reset_index()

cleaned_train_data.head(9)

cleaned_train_data.count()

sns.scatterplot(x="store_id", y="time_difference", hue="day_of_week", data=train_table)

shopper_average_data_cleaned = cleaned_train_data.groupby(['shopper_id']).agg({'time_difference':np.mean}).reset_index()

store_average_data_cleaned = cleaned_train_data.groupby(['store_id']).agg({'time_difference':np.mean}).reset_index()

sns.scatterplot(x="time_difference", y="shopper_id", data=shopper_average_data_cleaned)

unique_trip_id_data = cleaned_train_data.groupby(['trip_id','store_id','shopper_id','shopping_started_at']).agg({'day_segment':np.mean, 'day_of_week':np.mean,'time_difference':np.mean,'quantity': np.sum,'department_name': 'nunique'}).reset_index()
unique_trip_id_data.head(10)

hist = unique_trip_id_data.hist(bins=9)

finalTrainData = newTrainData.groupby(['trip_id','store_id','department_name']).agg({'quantity':np.sum,'shopping_trip_time':np.mean,'shopper_id':np.mean,'item_id':'nunique'}).reset_index()
finalTrainData = finalTrainData.rename(columns = {'item_id':'num_item_department'})
departmentDf = finalTrainData[['trip_id','department_name']]
uniqueDepartments = finalTrainData['department_name'].unique()
uniqueTripIds = finalTrainData['trip_id'].unique()
pivotTrainData_department = pd.pivot_table(finalTrainData,columns = ['department_name'],values=['quantity'],index='trip_id')
pivotTrainData_department = pivotTrainData_department.fillna(0)
pivotTrainData_numItems = pd.pivot_table(finalTrainData,columns = ['department_name'],values=['num_item_department'],index='trip_id')
pivotTrainData_numItems = pivotTrainData_numItems.fillna(0)
pivotTrainData = pd.concat([pivotTrainData_department,pivotTrainData_numItems],axis = 1)
flat_pivotData = pd.DataFrame(pivotTrainData.to_records())
flat_pivotData.columns = [hdr.replace("('num_item_department', '", "num_item_dept.").replace("')", "") for hdr in flat_pivotData.columns]
flat_pivotData.columns = [hdr.replace("('num_item_department',", "num_item_dept.").replace(")", "") for hdr in flat_pivotData.columns]
flat_pivotData.columns = [hdr.replace("('quantity', '", "quantity.").replace("')", "") for hdr in flat_pivotData.columns]

# Identify the number of unique departments 
unique_department = working_training_data.department_name.unique()
unique_department

# The goal of this code is to tease a one-hot encoding of the store-id and department_name such that these features 
# uniquely identify patterns underlying a trip id

# Here the combination of the quantity and item_id with the department_name to create more descriptive features for the data

# The goal is to mention for a given trip how many quantities are present for department_name or what item_id is present for a department_name

working_training_data = cleaned_train_data.groupby(['trip_id','store_id','department_name','day_of_week','day_segment']).agg({'quantity':np.sum,'time_difference':np.mean,'shopper_id':np.mean,'item_id':'nunique'}).reset_index()

# Combining features department_name, quantity
training_data_department_name_quantity = pd.pivot_table(finalTrainData,columns = ['department_name'],values=['quantity'],index='trip_id')
training_data_department_name_quantity = training_data_department_name_quantity.fillna(0)

# Combining features department_name, item_id
training_data_department_name_item_id = pd.pivot_table(finalTrainData,columns = ['department_name'],values=['item_id'],index='trip_id')
training_data_department_name_item_id = training_data_department_name_item_id.fillna(0)

train_data = pd.concat([training_data_department_name_quantity,training_data_department_name_item_id],axis = 1)

# Instantiating the value so that the data convert to records
flattened_data = pd.DataFrame(train_data.to_records())
flattened_data

# Rename the columns such that the columns are more readable
flattened_data.columns = [hdr.replace("('item_id',", "item_id_").replace(")", "") for hdr in flattened_data.columns]
flattened_data.columns = [hdr.replace("('quantity', '", "quantity_").replace("')", "") for hdr in flattened_data.columns]

# Removing any duplicate columns or data that one may have
flattened_data.columns.duplicated()

# While day_segment is avaiable, the hour of the time is discriminative enough to identify as a feature
unique_trip_id_data['hour'] = unique_trip_id_data['shopping_started_at'].dt.hour

combined_train_data = pd.concat([unique_trip_id_data,flattened_data],axis = 1)

# AllTrainData = AllTrainData.iloc[:,~AllTrainData.columns.duplicated()]
# AllTrainData['day_of_week'] = AllTrainData['shopping_started_at'].dt.dayofweek
# AllTrainData['date'] = AllTrainData['shopping_started_at'].dt.date
# AllTrainData['hour'] = AllTrainData['shopping_started_at'].dt.hour
# collapsedTrainData = AllTrainData[['shopper_id','store_id','date','hour','day_of_week','quantity','time_difference','num_dept_visited']]

y = combined_train_data['time_difference'].values
X = combined_train_data.drop(['time_difference','trip_id','shopping_started_at'],axis=1)
X

# y = train_table['time_difference'].values
# #X = collapsedTrainData[['shopper_id','store_id','hour','day_of_week','quantity','num_dept_visited']]
# X = train_table.drop(['time_difference','fulfillment_model','shopping_ended_at','shopping_started_at','department_name'],axis=1)
# X

from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Lasso
from sklearn.model_selection import train_test_split

X_train,X_val,y_train,y_val = train_test_split(X,y,test_size = 0.20)

reg_9 = LinearRegression().fit(X_train,y_train)

print('The R2 score for this model is', reg_9.score(X_val,y_val))

# A regularized version of the Linear Regression
lasso_params = {'alpha':[0.02, 0.024, 0.025, 0.026, 0.03]}

reg = GridSearchCV(sklearn.linear_model.Lasso(), param_grid=lasso_params, verbose = 2, cv = 3, n_jobs = -1,).fit(X_train, y_train).best_estimator_
 
print('The R2 score for this model is', reg.score(X_val,y_val))

from sklearn.ensemble import GradientBoostingRegressor
GBRegressor = GradientBoostingRegressor(learning_rate = 0.1,n_estimators = 100,max_depth=3,min_samples_split = 2,loss='ls')
GBRegressor.fit(X_train,y_train)
print('The GB R2 score is: ', GBRegressor.score(X_val,y_val))

from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import GradientBoostingRegressor


param_grid = {
    'learning_rate': [0.3],
    'max_depth': [4],
    'n_estimators': [2000],
    'min_samples_split':[5],
    'loss':['ls']
}

GBRegressor = GradientBoostingRegressor()

grid_search = GridSearchCV(estimator = GBRegressor, param_grid = param_grid, 
                          cv = 3, n_jobs = -1, verbose = 2)
grid_search.fit(X_train,y_train)
print('The GB R2 score is: ', grid_search.score(X_val,y_val))

grid_search.score(X_val,y_val)

from sklearn.model_selection import GridSearchCV
import xgboost as xgb
param_grid = {
    'learning_rate': [0.05],
    'n_estimators': [1000]
}

XGBR = xgb.XGBRegressor()
XGBR.fit(X_train,y_train, verbose=False)

grid_search_xg = GridSearchCV(estimator = XGBR, param_grid = param_grid, 
                          cv = 3, n_jobs = -1, verbose = 2)
grid_search_xg.fit(X, y)
print('The XG R2 score is: ', grid_search_xg.score(X_val,y_val))

dictonary = {'Linear_Regression': 0.32184440260690717, 'Linear_Regression_CV':0.322 , 'GBR_CV': 0.4500336622192095, 'GBR': 0.36, 'XG_CV': 0.43571771162071565}
new_dictonary = {str(k1):v1 for k1,v1 in dictonary.items()}
plt.figure(figsize=(15,5))
plt.bar(new_dictonary.keys(), new_dictonary.values(), width=.5, color='g')

# A possible future work is learning more about the model, how is behaves performs and 
# why does it predict the way it does? This goes into the whole
# theory of XAI where we better want to understand the best way to explore the model itself
# A simple approach would be to visulize gradients and the residuals that is being learnt by the model

# Todo
# take care of nans 
# merge item id into a list and quantity sum 
# segment of the day
# remove the percent missing values
# sum the quantity value 
# One hot encoding of department and adding to a list 
# one row per trip id 
# drop trip_id, shopper_id
# add segment of the data 
# day of the weekend vs weekday or day of the week 
# Without null values 
# Plot the residuals, longer durations and need more features 
# modelling skills and caliberations 
# EDA, visualizations 
# histogram of quantity, lesser quantity more or more quantitity less 
# K fold with different random search and different parameters 

test_table.head(9)

test_table['fulfillment_model']=test_table['fulfillment_model'].astype('str')