
pip install xgboost


# Importing packages and settings: 
import warnings 
warnings.filterwarnings(action= 'ignore')
import pandas as pd
import numpy as np
from sklearn.impute import SimpleImputer
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.compose import ColumnTransformer
from sklearn.model_selection import train_test_split
import xgboost as xgb
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import cross_val_score
from sklearn import metrics
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import roc_curve
import joblib


# Loading the data:
df_prep = pd.read_csv('https://bhciaaablob.blob.core.windows.net/featurenegineeringfiles/df_prepared.csv')
df = df_prep.drop(columns=['Unnamed: 0','city', 'experience', 'enrollee_id'])  
df.head()


# Define the target vector y
y = df['target']  

# Creating a dataset without the DV:
X = df.drop('target', axis = 1)

# Split X and y into training and testing sets
X_train, X_test, y_train, y_test = train_test_split(
  X, y, test_size=0.25, stratify=y, random_state=1121218 
)

# Creating an object with the column labels of only the categorical features and one with only the numeric features:
categorical_features = X.select_dtypes(exclude="number").columns.tolist()
numeric_features = X.select_dtypes(include="number").columns.tolist()

# Create the categorical pipeline, for the categorical variables Aki imputes the missing values with a constant value and we encode them with One-Hot encoding:
categorical_pipeline = Pipeline(
    steps=[
        ("impute", SimpleImputer(strategy= 'constant', fill_value= 'unknown')), 
        ("one-hot", OneHotEncoder(handle_unknown="ignore", sparse=False)) 
    ]
)
# Create the numeric pipeline, for the numeric variables Aki imputes the missings with the mean of the column and standardize them, so that the features have a mean of 0 and a variance of 1: 
numeric_pipeline = Pipeline(
    steps=[("impute", SimpleImputer(strategy="mean")), 
           ("scale", StandardScaler())]
)

# Combining the two pipelines with a column transformer:
full_processor = ColumnTransformer(transformers=[
        ("numeric", numeric_pipeline, numeric_features),
        ("categorical", categorical_pipeline, categorical_features),
    ]
)


# Instantiate the XGBClassifier:
xgb_cl = xgb.XGBClassifier(eval_metric='logloss', seed=7) 

# Create XGBoost pipeline:
xgb_pipeline = Pipeline(steps=[
    ('preprocess', full_processor),
    ('model', xgb_cl)
])

# Evaluate the model with the use of cv:
cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=7)  #, shuffle=True with or without shuffle??
scores = cross_val_score(xgb_pipeline, X_train, y_train, cv=cv, scoring = 'roc_auc')
print("roc_auc = %f (%f)" % (scores.mean(), scores.std()))


# The default hyperparameters of the XGBoost:
xgb_cl


def print_results_gridsearch(gridsearch, list_param1, list_param2, name_param1, name_param2):
  
  # Checking the results from each run in the gridsearch: 
  means = gridsearch.cv_results_['mean_test_score']
  stds = gridsearch.cv_results_['std_test_score']
  params = gridsearch.cv_results_['params']
  print("The results from each run in the gridsearch:")
  for mean, stdev, param in zip(means, stds, params):
    print("roc_auc = %f (%f) with: %r" % (mean, stdev, param))
  
  #Visualizing the results from each run in the gridsearch: 
  scores = np.array(means).reshape(len(list_param1), len(list_param2))
  for i, value in enumerate(list_param1):
    plt.plot(list_param2, scores[i], label= str(name_param1) + ': ' + str(value))
  plt.legend()
  plt.xlabel(str(name_param2))
  plt.ylabel('ROC AUC')  
  plt.show()
    
  # Checking the best performing model:
  print("\n")
  print("Best model: roc_auc = %f using %s" % (gridsearch.best_score_, gridsearch.best_params_))


# Step 1: Searching for the optimum parameters for the learning rate and the number of estimators:

# Defining the parameter grid to be used in GridSearch:
param_grid = {"model__subsample": [0.8], "model__colsample_bytree": [0.8]
              , "model__learning_rate": [0.0001, 0.001, 0.01, 0.1, 0.2, 0.3]
              , "model__n_estimators": range(50,500,50)
              }

#instantiate the Grid Search:
cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=7) 
grid_cv1 = GridSearchCV(xgb_pipeline
                        , param_grid
                        , n_jobs= -1 
                        , cv = cv
                        , scoring="roc_auc") 

# Fit
_ = grid_cv1.fit(X_train, y_train) 

# Checking the results from each run in the gridsearch:  
print_results_gridsearch(gridsearch=grid_cv1, list_param1 = param_grid["model__learning_rate"], list_param2 = param_grid["model__n_estimators"]
                         , name_param1 = 'learning_rate' , name_param2 = 'n_estimators')


# Step 2: Searching for the optimum parameters for max_depth and min_child_weight:

# Defining the parameter grid to be used in GridSearch:
param_grid = {"model__subsample": [0.8], "model__colsample_bytree": [0.8], "model__learning_rate": [0.01], "model__n_estimators": [250]
              , 'model__max_depth': range(3,10,2)
              , 'model__min_child_weight': range(1,6,2)
             }

#instantiate the Grid Search:
cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=7)
grid_cv2 = GridSearchCV(xgb_pipeline
                        , param_grid
                        , n_jobs=-1
                        , cv=cv
                        , scoring="roc_auc") 

# Fit
_ = grid_cv2.fit(X_train, y_train) 

# Checking the results from each run in the gridsearch:  
print_results_gridsearch(gridsearch=grid_cv2, list_param1 = param_grid["model__max_depth"], list_param2 = param_grid["model__min_child_weight"]
                         , name_param1 = 'max_depth' , name_param2 = 'min_child_weight')


# Step 3: Searching for the optimum parameters for subsample and colsample_bytree:

# Defining the parameter grid to be used in GridSearch:
param_grid = {"model__learning_rate": [0.01], "model__n_estimators": [250], 'model__max_depth': [5], 'model__min_child_weight': [1]
              , 'model__subsample':[i/10.0 for i in range(4,10)]
              , 'model__colsample_bytree':[i/10.0 for i in range(4,10)] 
             }

#instantiate the Grid Search:
cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=7)
grid_cv3 = GridSearchCV(xgb_pipeline
                        , param_grid
                        , n_jobs=-1
                        , cv=cv
                        , scoring="roc_auc") 

# Fit
_ = grid_cv3.fit(X_train, y_train) 

# Checking the results from each run in the gridsearch:  
print_results_gridsearch(gridsearch=grid_cv3, list_param1 = param_grid["model__subsample"], list_param2 = param_grid["model__colsample_bytree"]
                         , name_param1 = 'subsample' , name_param2 = 'colsample_bytree')


# Step 4: Searching for the optimum parameters for gamma and lambda: 

# Defining the parameter grid to be used in GridSearch:
param_grid = {"model__learning_rate": [0.01], "model__n_estimators": [250], 'model__max_depth': [5], 'model__min_child_weight': [1], 'model__subsample':[0.5], 'model__colsample_bytree':[0.9] 
              , "model__gamma": [i/10.0 for i in range(0,6)]
              , "model__reg_lambda": [0, 0.5, 1, 1.5, 2, 3, 4.5]
             }
              
#instantiate the Grid Search:
cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=7)
grid_cv4 = GridSearchCV(xgb_pipeline
                        , param_grid
                        , n_jobs=-1
                        , cv=cv
                        , scoring="roc_auc") 


# Fit
_ = grid_cv4.fit(X_train, y_train) 

# Checking the results from each run in the gridsearch:  
print_results_gridsearch(gridsearch=grid_cv4, list_param1 = param_grid["model__gamma"], list_param2 = param_grid["model__reg_lambda"]
                         , name_param1 = 'Gamma' , name_param2 = 'Lambda')


grid_cv4.best_params_


# Predict with Aki's final XGBoost with the best parameters resulting from the GridSearch: 
y_pred_aki = grid_cv4.predict(X_test)
y_pred_prob_aki = grid_cv4.predict_proba(X_test)[::,1]

# Evaluate:
print("roc_auc_score:",metrics.roc_auc_score(y_test, y_pred_aki))


# Fit Meta's default XGBoost pipeline:
xgb_pipeline.fit(X_train, y_train)

# Predict:
y_pred_meta = xgb_pipeline.predict(X_test)
y_pred_prob_meta = xgb_pipeline.predict_proba(X_test)[::,1]

# Evaluate:
print("roc_auc_score:",metrics.roc_auc_score(y_test, y_pred_meta))


# Compute False postive rate, and True positive rate
fpr1 , tpr1, thresholds1 = roc_curve(y_test, y_pred_prob_aki)
fpr2 , tpr2, thresholds2 = roc_curve(y_test, y_pred_prob_meta)

# Calculate Area under the curve to display on the plot
roc_auc_aki = metrics.roc_auc_score(y_test, y_pred_aki)
roc_auc_meta = metrics.roc_auc_score(y_test, y_pred_meta)

# Now, plot the computed values
plt.plot([0,1],[0,1], 'k--')
plt.plot(fpr1, tpr1, label= '%s ROC (area = %0.2f)' % ("XGBoost Aki", roc_auc_aki)) #"XGBoost Aki")
plt.plot(fpr2, tpr2, label= '%s ROC (area = %0.2f)' % ("XGBoost Meta", roc_auc_meta)) #label= "XGBoost Meta")
plt.legend()
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver Operating Characteristic')
plt.show()


#Saving Aki's final XGBoost pipeline:
best_pipe_aki = grid_cv4.best_estimator_
joblib.dump(best_pipe_aki, 'best_pipe_aki.joblib')


#Testing if Aki's model is correctly saved:

# Load the models:  
upload_pipe_aki = joblib.load('best_pipe_aki.joblib')

# Use it to make the same predictions:
print(upload_pipe_aki.predict(X_test))

	city_development_index	gender	enrolled_university	education_level	major_discipline	company_size	company_type	last_new_job	training_hours	target	ind_relevent_experience	experience_num	city name
0	0.920	Male	no_enrollment	Graduate	STEM	NaN	NaN	1	36	1	1	22.0	Denver-Aurora-Lakewood CO (Metro)
1	0.776	Male	no_enrollment	Graduate	STEM	50-99	Pvt Ltd	>4	47	0	0	15.0	Odessa TX (Metro)
2	0.624	NaN	Full time course	Graduate	STEM	NaN	NaN	never	83	0	0	5.0	Auburn-Opelika AL (Metro)
3	0.789	NaN	NaN	Graduate	Business Degree	NaN	Pvt Ltd	never	52	1	0	0.0	Corvallis OR (Metro)
4	0.767	Male	no_enrollment	Masters	STEM	50-99	Funded Startup	4	8	0	1	22.0	Tulsa OK (Metro)

Aki Razzi: 'Accuracy is what truly matters'¶

Using XGBoost to predict which Data Scientists are likely to change jobs¶