1. import pandas as pd
  2. import numpy as np
  3. from sklearn.datasets import load_boston
  5. # Suppress warnings for now
  6. import warnings
  7. warnings.simplefilter(action='ignore', category=UserWarning)
  8. warnings.simplefilter(action='ignore', category=FutureWarning)
  1. # Loading data
  2. train = pd.read_csv('../data/train_final.csv', engine='python')
  3. test = pd.read_csv('../data/test_final.csv', engine='python')
  4. TRAIN_IDX=train.shape[0]
  5. TEST_IDX = TRAIN_IDX + test.shape[0]
  6. data = pd.concat([train, test], axis=0)

Baseline Predictions

  1. from sklearn.model_selection import train_test_split
  2. train = data.iloc[:TRAIN_IDX, :]
  3. test = data.iloc[TRAIN_IDX:TEST_IDX, :]
  5. # Getting the output variable
  6. y = train['loan_status']
  8. # Getting the input variables
  9. X = train.drop(['loan_status'], axis=1).fillna(0)
  11. # Diving our input and output into training and testing sets
  12. X_train, X_test, y_train, y_test = train_test_split(
  13.                                     X, y, 
  14.                                     test_size=0.33, 
  15.                                     random_state=42
  16.                                    )
  1. from sklearn.model_selection import GridSearchCV
  2. from sklearn.metrics import accuracy_score
  5. def algorithm_pipeline(X_train_data,
  6.                        X_test_data,
  7.                        y_train_data,
  8.                        y_test_data,
  9.                        model,
  10.                        param_grid,
  11.                        cv=10,
  12.                        scoring_fit='accuracy',
  13.                        scoring_test=accuracy_score,
  14.                        do_probabilities=False):
  15.     gs = GridSearchCV(estimator=model,
  16.                       param_grid=param_grid,
  17.                       cv=cv,
  18.                       n_jobs=4,
  19.                       scoring=scoring_fit,
  20.                       verbose=2)
  21.     fitted_model = gs.fit(X_train_data, y_train_data)
  22.     best_model = fitted_model.best_estimator_
  24.     if do_probabilities:
  25.         pred = fitted_model.predict_proba(X_test_data)
  26.     else:
  27.         pred = fitted_model.predict(X_test_data)
  29.     score = scoring_test(y_test_data, pred)
  31.     return [best_model, pred, score]
  1. from sklearn.ensemble import RandomForestClassifier
  2. from lightgbm import LGBMClassifier
  3. from xgboost import XGBClassifier
  5. # Defining our estimator, the algorithm to optimize
  6. models_to_train = [XGBClassifier(), LGBMClassifier(), RandomForestClassifier()]
  8. # Defining the hyperparameters to optimize
  9. grid_parameters = [
  10.     { # XGBoost
  11.         'n_estimators': [400, 700, 1000],
  12.         'colsample_bytree': [0.7, 0.8],
  13.         'max_depth': [15,20,25],
  14.         'reg_alpha': [1.1, 1.2, 1.3],
  15.         'reg_lambda': [1.1, 1.2, 1.3],
  16.         'subsample': [0.7, 0.8, 0.9]
  17.     },
  18.     { # LightGBM
  19.         'n_estimators': [400, 700, 1000],
  20.         'learning_rate': [0.12],
  21.         'colsample_bytree': [0.7, 0.8],
  22.         'max_depth': [4],
  23.         'num_leaves': [10, 20],
  24.         'reg_alpha': [1.1, 1.2],
  25.         'reg_lambda': [1.1, 1.2],
  26.         'min_split_gain': [0.3, 0.4],
  27.         'subsample': [0.8, 0.9],
  28.         'subsample_freq': [10, 20]
  29.     }, 
  30.     { # Random Forest
  31.         'max_depth':[3, 5, 10, 13], 
  32.         'n_estimators':[100, 200, 400, 600, 900],
  33.         'max_features':[2, 4, 6, 8, 10]
  34.     }
  35. ]
  1. %%time
  2. models_preds_scores = []
  4. for i, model in enumerate(models_to_train):
  5.     params = grid_parameters[i]
  7.     result = algorithm_pipeline(X_train, X_test, y_train, y_test, 
  8.                                  model, params, cv=3)
  9.     models_preds_scores.append(result)
  1. for result in models_preds_scores:
  2.     print('Model: {0}, Score: {1}'.format(type(result[0]).__name__, result[2]))
  3. # Model: XGBClassifier, Score: 0.9094545454545454
  4. # Model: LGBMClassifier, Score: 0.9153333333333333
  5. # Model: RandomForestClassifier, Score: 0.9166666666666666

Improving baseline with stacking

  1. %%time
  2. from mlxtend.classifier import StackingCVClassifier
  3. from sklearn.linear_model import LogisticRegression
  5. RANDOM_SEED = 42
  7. xgb = XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,
  8.                     colsample_bynode=1, colsample_bytree=1, gamma=0, gpu_id=-1,
  9.                     importance_type='gain', interaction_constraints='',
  10.                     learning_rate=0.300000012, max_delta_step=0, max_depth=6,
  11.                     min_child_weight=1,  monotone_constraints='()',
  12.                     n_estimators=400, n_jobs=8, num_parallel_tree=1, random_state=0,
  13.                     reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=1,
  14.                     tree_method='exact', validate_parameters=1, verbosity=None)
  15. lgbm  = LGBMClassifier(boosting_type='gbdt', objective='binary', metric='binary_error',verbose=-1,
  16.             n_estimators=2000,learning_rate=0.001, num_leaves=11, max_depth=5,reg_alpha=1e-5,reg_lambda=1e-3)
  17. lgbm1 = LGBMClassifier(boosting_type='gbdt', objective='binary', metric='binary_error',verbose=-1,
  18.             n_estimators=2000,learning_rate=0.003, num_leaves=12, max_depth=5,reg_alpha=1e-5,reg_lambda=1e-3)
  19. lgbm2 = LGBMClassifier(boosting_type='gbdt', objective='binary', metric='binary_error',verbose=-1,
  20.             n_estimators=5000,learning_rate=0.0005, num_leaves=12, max_depth=5,reg_alpha=1e-5,reg_lambda=1e-3)
  21. rf = RandomForestClassifier(max_depth=13, max_features=10)
  22. lr = LogisticRegression()
  24. stack = StackingCVClassifier(classifiers=(rf, lgbm, lgbm1, lgbm2, xgb),
  25.                             meta_classifier=lr, cv=5,
  26.                             use_features_in_secondary=True,
  27.                             store_train_meta_features=True,
  28.                             shuffle=True,
  29.                             random_state=RANDOM_SEED)
  31. stack.fit(X_train.fillna(0), y_train)
  33. pred = stack.predict(X_test.fillna(0))
  34. score = accuracy_score(y_test, pred)
  35. print(score)
  36. print(accuracy_score(test['loan_status'],
  37.                      stack.predict(test.drop(['loan_status'], axis=1).fillna(0))))
  38. # Validation accuracy_score:0.9115151515151515
  39. # Test accuracy_score:0.91132