NOTE - Categorical Variables

image.png

Split the training and testing set

  1. import pandas as pd
  2. from sklearn.model_selection import train_test_split
  4. # Read the data
  5. X = pd.read_csv('../input/train.csv', index_col='Id') 
  6. X_test = pd.read_csv('../input/test.csv', index_col='Id')
  8. # Remove rows with missing target, separate target from predictors
  9. X.dropna(axis=0, subset=['SalePrice'], inplace=True)
  10. y = X.SalePrice
  11. X.drop(['SalePrice'], axis=1, inplace=True)
  13. # To keep things simple, we'll drop columns with missing values
  14. cols_with_missing = [col for col in X.columns if X[col].isnull().any()] 
  15. X.drop(cols_with_missing, axis=1, inplace=True)
  16. X_test.drop(cols_with_missing, axis=1, inplace=True)
  18. # Break off validation set from training data
  19. X_train, X_valid, y_train, y_valid = train_test_split(X, y,
  20.                                                       train_size=0.8, test_size=0.2,
  21.                                                       random_state=0)

Function to comparing different approaches and accuracy

  1. from sklearn.ensemble import RandomForestRegressor
  2. from sklearn.metrics import mean_absolute_error
  4. # function for comparing different approaches
  5. def score_dataset(X_train, X_valid, y_train, y_valid):
  6.     model = RandomForestRegressor(n_estimators=100, random_state=0)
  7.     model.fit(X_train, y_train)
  8.     preds = model.predict(X_valid)
  9.     return round(mean_absolute_error(y_valid, preds),6)

Investigating cardinality

  1. # Get number of unique entries in each column with categorical data
  2. object_nunique = list(map(lambda col: X_train[col].nunique(), object_cols))
  3. d = dict(zip(object_cols, object_nunique))
  5. # Print number of unique entries by column, in ascending order
  6. sorted(d.items(), key=lambda x: x[1])

Set up the One-Hot encoding

  1. # Columns that will be one-hot encoded
  2. low_cardinality_cols = [col for col in object_cols if X_train[col].nunique() < 10]
  4. # Columns that will be dropped from the dataset
  5. high_cardinality_cols = list(set(object_cols)-set(low_cardinality_cols))
  7. print('Categorical columns that will be one-hot encoded:', low_cardinality_cols)
  8. print('\nCategorical columns that will be dropped from the dataset:', high_cardinality_cols)
  1. from sklearn.preprocessing import OneHotEncoder
  3. # Use as many lines of code as you need!
  4. OH_encoder = OneHotEncoder(handle_unknown = 'ignore', sparse = False)
  6. OH_X_train = pd.DataFrame(
  7.     OH_encoder.fit_transform(X_train[low_cardinality_cols])
  8. )
  9. OH_X_valid = pd.DataFrame(
  10.     OH_encoder.transform(X_valid[low_cardinality_cols])
  11. )
  13. OH_X_train.index = X_train.index # Your code here
  14. OH_X_valid.index = X_valid.index # Your code here
  16. num_X_train = X_train.drop(object_cols, axis = 1)
  17. num_X_valid = X_valid.drop(object_cols, axis = 1)
  19. OH_X_train = pd.concat([num_X_train, OH_X_train], axis = 1)
  20. OH_X_valid = pd.concat([num_X_valid, OH_X_valid], axis = 1)