3.1 Mean values of the features

#Overview of the mean values of the features grouped by final_subscribed
df = bank.groupby('final_subscribed').mean().T

#Compilation of the difference between the mean values
col_names = df.columns
df = pd.DataFrame(df, columns = col_names)
df['Score_diff'] = df['no'] - df['yes'] 
df

#Filter for differences below -2 and above 2
threshold = ['2', '-2']
df[(df["Score_diff"] >= float(threshold[0])) | (df["Score_diff"] <= float(threshold[1]))]

3.2 Description of the target variable

sns.countplot(x='final_subscribed', data=bank, palette='hls')
print(plt.show())
print(bank['final_subscribed'].value_counts())

#get percentage of male and female within our dataset

bank['final_subscribed'].value_counts(normalize=True) * 100

3.3 Description of the predictor variables

%matplotlib inline
pd.crosstab(bank.job, bank.final_subscribed).plot(kind='bar')
plt.title('Subscription Frequency for Job Title')
plt.xlabel('Job')
plt.ylabel('Frequency of subscriptions')

The job title can be a good predictor for final subscription.

%matplotlib inline
pd.crosstab(bank.day_of_week, bank.final_subscribed).plot(kind='bar')
plt.title('Subscription Frequency for Day of Week')
plt.xlabel('Day of Week')
plt.ylabel('Frequency of subscriptions')

Day of week may not be a good predictor of for final subscription.

%matplotlib inline
pd.crosstab(bank.month, bank.final_subscribed).plot(kind='bar')
plt.title('Subscription Frequency for Month')
plt.xlabel('Month')
plt.ylabel('Frequency of subscriptions')

The month can be a good predictor for final subscription.

%matplotlib inline
pd.crosstab(bank.education, bank.final_subscribed).plot(kind='bar')
plt.title('Subscription Frequency for Education')
plt.xlabel('Education')
plt.ylabel('Frequency of subscriptions')

The Education can be a good predictor for final subscription.

bank.age.hist()
plt.title('Histogram of Age')
plt.xlabel('Age')
plt.ylabel('Frequency')

On the more detailed examination of the remaining variables is omitted in this post.

4.1 Conversion of the target variable

vals_to_replace = {'no':'0', 'yes':'1'}
bank['final_subscribed'] = bank['final_subscribed'].map(vals_to_replace)
bank['final_subscribed'] = bank.final_subscribed.astype('int64')
bank.head()

Now we have 0 and 1 as int64 for our target variable

4.2 Creation of dummy variables

There are some categorical variables in the data set. A logistic regression model only works with numeric variables, so we have to convert the non-numeric ones to dummy variables. If you want to learn more about one-hot-encoding / dummy variables, read this post from me: “The use of dummy variables”

#Just select the categorical variables
cat_col = ['object']
cat_columns = list(bank.select_dtypes(include=cat_col).columns)
cat_data = bank[cat_columns]
cat_vars = cat_data.columns

#Create dummy variables for each cat. variable
for var in cat_vars:
    cat_list = pd.get_dummies(bank[var], prefix=var)
    bank=bank.join(cat_list)

    
data_vars=bank.columns.values.tolist()
to_keep=[i for i in data_vars if i not in cat_vars]

#Create final dataframe
bank_final=bank[to_keep]
bank_final.columns.values

4.3 Feature Selection

Now we have created some new variables. However, not all are relevant for the planned classification. We use the Recursive Feature Elimination (RFE) algorithm to eliminate the redundant features.

#Here we select 20 best features

x = bank_final.drop('final_subscribed', axis=1)
y = bank_final['final_subscribed']

trainX, testX, trainY, testY = train_test_split(x, y, test_size = 0.2)

model = LogisticRegression()
rfe = RFE(model, 20)
fit = rfe.fit(trainX, trainY)

How the most important variables can be identified and assigned to an object is explained “here (chapter 4.2.3)” step by step.

Columns = x.columns
RFE_support = rfe.support_
RFE_ranking = rfe.ranking_

dataset = pd.DataFrame({'Columns': Columns, 'RFE_support': RFE_support, 'RFE_ranking': RFE_ranking}, columns=['Columns', 'RFE_support', 'RFE_ranking'])
df = dataset[(dataset["RFE_support"] == True) & (dataset["RFE_ranking"] == 1)]
filtered_features = df['Columns']
filtered_features

Eventually, the features identified as good will be assigned to a new x.

new_train_x = trainX[filtered_features]
new_test_x = testX[filtered_features]

6.1 Over-sampling using SMOTE

SMOTE is an over-sampling method and stands for Synthetic Minority Over-sampling Technique . What it does is, it creates synthetic (not duplicate) samples of the minority class. Hence making the minority class equal to the majority class. SMOTE does this by selecting similar records and altering that record one column at a time by a random amount within the difference to the neighbouring records.

There are two important points to note here:

• SMOTE is only applied to the training data.Because by oversampling only on the training data, none of the information in the test data is being used to create synthetic observations, therefore, no information will bleed from test data into the model training.

• Do not use SMOTE before using feature selection methods. Performing variable selection after using SMOTE should be done with some care because most variable selection methods assume that the samples are independent.

columns_x = new_train_x2.columns


sm = SMOTE(random_state=0)
trainX_smote ,trainY_smote = sm.fit_resample(new_train_x2, trainY)

trainX_smote = pd.DataFrame(data=trainX_smote,columns=columns_x)
trainY_smote = pd.DataFrame(data=trainY_smote,columns=['final_subscribed'])

The previously imbalanced dataset is now balanced.

print("Before OverSampling, counts of label '1': {}".format(sum(trainY==1)))
print("Before OverSampling, counts of label '0': {} \n".format(sum(trainY==0)))

print("After OverSampling, counts of label '1':", trainY_smote[(trainY_smote["final_subscribed"] == 1)].shape[0])
print("After OverSampling, counts of label '0':", trainY_smote[(trainY_smote["final_subscribed"] == 0)].shape[0])

sns.countplot(x='final_subscribed', data=trainY_smote, palette='hls')
print(plt.show())
print("Proportion of no subscription data in oversampled data is ",len(trainY_smote[trainY_smote['final_subscribed']==0])/len(trainX_smote))
print("Proportion of subscription data in oversampled data is ",len(trainY_smote[trainY_smote['final_subscribed']==1])/len(trainX_smote))

The complete dataset has become bigger:

print('Shape bevore OverSampling:')
print("Number of observations trainX dataset:", new_train_x2.shape[0])
print("Number of observations trainY dataset:", trainY.shape[0])


print('\nShape after OverSampling:')
print("Number of observations trainX_os dataset:", trainX_smote.shape[0])
print("Number of observations trainY_os dataset:", trainY_smote.shape[0])

6.2 Model Fitting

After all the selections of the features and creation of the synthetic samples, here again an overview of the train and test predictors as well as the train and test criterion. For a better overview, we assign the objects more descriptive / known names.

trainX_final = trainX_smote
testX_final = new_test_x2
trainY_final = trainY_smote
testY_final = testY

logreg = LogisticRegression()
logreg.fit(trainX_final, trainY_final)

y_pred = logreg.predict(testX_final)

Here is a nice overview of the features, their associated coefficients and odds ratios:

coef = pd.DataFrame({'features': trainX_final.columns, 
                     'coef': logreg.coef_[0], 
                     'odds_ratio': np.exp(logreg.coef_[0])})

coef[['features', 'coef', 'odds_ratio']]

6.3 Model evaluation

confusion_matrix = confusion_matrix(testY_final, y_pred)
print(confusion_matrix)

fig, ax = plt.subplots(figsize=(6, 6))
ax.matshow(confusion_matrix, cmap=plt.cm.Blues, alpha=0.3)
for i in range(confusion_matrix.shape[0]):
    for j in range(confusion_matrix.shape[1]):
        ax.text(x=j, y=i,s=confusion_matrix[i, j], va='center', ha='center', size='xx-large')
 
plt.xlabel('Predictions', fontsize=18)
plt.ylabel('Actuals', fontsize=18)
plt.title('Confusion Matrix', fontsize=18)
plt.show()

print('Accuracy: {:.2f}'.format(accuracy_score(testY_final, y_pred)))
print('Error rate: {:.2f}'.format(1 - accuracy_score(testY_final, y_pred)))
print('Precision: {:.2f}'.format(precision_score(testY_final, y_pred)))
print('Recall: {:.2f}'.format(recall_score(testY_final, y_pred)))
print('f1_score: {:.2f}'.format(f1_score(testY_final, y_pred)))

logit_roc_auc = roc_auc_score(testY_final, logreg.predict(testX_final))
fpr, tpr, thresholds = roc_curve(testY_final, logreg.predict_proba(testX_final)[:,1])
plt.figure()
plt.plot(fpr, tpr, label='Logistic Regression (area = %0.2f)' % logit_roc_auc)
plt.plot([0, 1], [0, 1],'r--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver operating characteristic ROC')
plt.legend(loc="lower right")

plt.show()

y_pred_proba = logreg.predict_proba(testX_final)[::,1]
roc_auc_score(testY_final, y_pred_proba)

lr = LogisticRegression()
dt = DecisionTreeClassifier()
rf = RandomForestClassifier()

lr.fit(trainX_final, trainY_final)
dt.fit(trainX_final, trainY_final)
rf.fit(trainX_final, trainY_final)

disp = plot_roc_curve(lr, testX_final, testY_final)

disp = plot_roc_curve(lr, testX_final, testY_final);
plot_roc_curve(dt, testX_final, testY_final, ax=disp.ax_);
plot_roc_curve(rf, testX_final, testY_final, ax=disp.ax_)

clf = LogisticRegression()
scores = cross_val_score(clf, trainX_final, trainY_final, cv=5)
scores

print("Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std()))