Count_or_frequency_encoding.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "# to split the datasets\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# to encode with feature-engine\n",
    "from feature_engine.categorical_encoders import CountFrequencyCategoricalEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Neighborhood</th>\n",
       "      <th>Exterior1st</th>\n",
       "      <th>Exterior2nd</th>\n",
       "      <th>SalePrice</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CollgCr</td>\n",
       "      <td>VinylSd</td>\n",
       "      <td>VinylSd</td>\n",
       "      <td>208500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Veenker</td>\n",
       "      <td>MetalSd</td>\n",
       "      <td>MetalSd</td>\n",
       "      <td>181500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>CollgCr</td>\n",
       "      <td>VinylSd</td>\n",
       "      <td>VinylSd</td>\n",
       "      <td>223500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Crawfor</td>\n",
       "      <td>Wd Sdng</td>\n",
       "      <td>Wd Shng</td>\n",
       "      <td>140000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NoRidge</td>\n",
       "      <td>VinylSd</td>\n",
       "      <td>VinylSd</td>\n",
       "      <td>250000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Neighborhood Exterior1st Exterior2nd  SalePrice\n",
       "0      CollgCr     VinylSd     VinylSd     208500\n",
       "1      Veenker     MetalSd     MetalSd     181500\n",
       "2      CollgCr     VinylSd     VinylSd     223500\n",
       "3      Crawfor     Wd Sdng     Wd Shng     140000\n",
       "4      NoRidge     VinylSd     VinylSd     250000"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load dataset\n",
    "\n",
    "data = pd.read_csv(\n",
    "    '../houseprice.csv',\n",
    "    usecols=['Neighborhood', 'Exterior1st', 'Exterior2nd', 'SalePrice'])\n",
    "\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Neighborhood :  25  labels\n",
      "Exterior1st :  15  labels\n",
      "Exterior2nd :  16  labels\n",
      "SalePrice :  663  labels\n"
     ]
    }
   ],
   "source": [
    "# let's have a look at how many labels each variable has\n",
    "\n",
    "for col in data.columns:\n",
    "    print(col, ': ', len(data[col].unique()), ' labels')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((1022, 3), (438, 3))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# let's separate into training and testing set\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    data[['Neighborhood', 'Exterior1st', 'Exterior2nd']], # predictors\n",
    "    data['SalePrice'],  # target\n",
    "    test_size=0.3,  # percentage of obs in test set\n",
    "    random_state=0)  # seed to ensure reproducibility\n",
    "\n",
    "X_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Count and Frequency encoding with pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'NAmes': 151,\n",
       " 'CollgCr': 105,\n",
       " 'OldTown': 73,\n",
       " 'Edwards': 71,\n",
       " 'Sawyer': 61,\n",
       " 'Somerst': 56,\n",
       " 'Gilbert': 55,\n",
       " 'NridgHt': 51,\n",
       " 'NWAmes': 51,\n",
       " 'SawyerW': 45,\n",
       " 'BrkSide': 41,\n",
       " 'Mitchel': 36,\n",
       " 'Crawfor': 35,\n",
       " 'Timber': 30,\n",
       " 'NoRidge': 30,\n",
       " 'ClearCr': 24,\n",
       " 'IDOTRR': 24,\n",
       " 'SWISU': 18,\n",
       " 'StoneBr': 16,\n",
       " 'Blmngtn': 12,\n",
       " 'MeadowV': 12,\n",
       " 'BrDale': 10,\n",
       " 'NPkVill': 7,\n",
       " 'Veenker': 6,\n",
       " 'Blueste': 2}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# let's obtain the counts for each one of the labels\n",
    "# in the variable Neigbourhood\n",
    "\n",
    "count_map = X_train['Neighborhood'].value_counts().to_dict()\n",
    "\n",
    "count_map"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The dictionary contains the number of observations per category in Neighbourhood."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# replace the labels with the counts\n",
    "\n",
    "X_train['Neighborhood'] = X_train['Neighborhood'].map(count_map)\n",
    "X_test['Neighborhood'] = X_test['Neighborhood'].map(count_map)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "64      105\n",
       "682      24\n",
       "960      41\n",
       "1384     71\n",
       "1100     18\n",
       "416      61\n",
       "1034     35\n",
       "853     151\n",
       "472      71\n",
       "1011     71\n",
       "Name: Neighborhood, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# let's explore the result\n",
    "\n",
    "X_train['Neighborhood'].head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'VinylSd': 0.3561643835616438,\n",
       " 'HdBoard': 0.149706457925636,\n",
       " 'Wd Sdng': 0.14481409001956946,\n",
       " 'MetalSd': 0.1350293542074364,\n",
       " 'Plywood': 0.08414872798434442,\n",
       " 'CemntBd': 0.03816046966731898,\n",
       " 'BrkFace': 0.03424657534246575,\n",
       " 'WdShing': 0.02054794520547945,\n",
       " 'Stucco': 0.016634050880626222,\n",
       " 'AsbShng': 0.014677103718199608,\n",
       " 'Stone': 0.0019569471624266144,\n",
       " 'BrkComm': 0.0009784735812133072,\n",
       " 'ImStucc': 0.0009784735812133072,\n",
       " 'CBlock': 0.0009784735812133072,\n",
       " 'AsphShn': 0.0009784735812133072}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# if instead of the count we would like the frequency\n",
    "# we need only divide the count by the total number of observations:\n",
    "\n",
    "frequency_map = (X_train['Exterior1st'].value_counts() / len(X_train) ).to_dict()\n",
    "frequency_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# replace the labels with the frequencies\n",
    "\n",
    "X_train['Exterior1st'] = X_train['Exterior1st'].map(frequency_map)\n",
    "X_test['Exterior1st'] = X_test['Exterior1st'].map(frequency_map)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((1022, 3), (438, 3))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# let's separate into training and testing set\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    data[['Neighborhood', 'Exterior1st', 'Exterior2nd']], # predictors\n",
    "    data['SalePrice'],  # target\n",
    "    test_size=0.3,  # percentage of obs in test set\n",
    "    random_state=0)  # seed to ensure reproducibility\n",
    "\n",
    "X_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CountFrequencyCategoricalEncoder(encoding_method='count',\n",
       "                                 variables=['Neighborhood', 'Exterior1st',\n",
       "                                            'Exterior2nd'])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_enc = CountFrequencyCategoricalEncoder(\n",
    "    encoding_method='count', # to do frequency ==> encoding_method='frequency'\n",
    "    variables=['Neighborhood', 'Exterior1st', 'Exterior2nd'])\n",
    "\n",
    "count_enc.fit(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Neighborhood': {'NAmes': 151,\n",
       "  'CollgCr': 105,\n",
       "  'OldTown': 73,\n",
       "  'Edwards': 71,\n",
       "  'Sawyer': 61,\n",
       "  'Somerst': 56,\n",
       "  'Gilbert': 55,\n",
       "  'NridgHt': 51,\n",
       "  'NWAmes': 51,\n",
       "  'SawyerW': 45,\n",
       "  'BrkSide': 41,\n",
       "  'Mitchel': 36,\n",
       "  'Crawfor': 35,\n",
       "  'Timber': 30,\n",
       "  'NoRidge': 30,\n",
       "  'ClearCr': 24,\n",
       "  'IDOTRR': 24,\n",
       "  'SWISU': 18,\n",
       "  'StoneBr': 16,\n",
       "  'Blmngtn': 12,\n",
       "  'MeadowV': 12,\n",
       "  'BrDale': 10,\n",
       "  'NPkVill': 7,\n",
       "  'Veenker': 6,\n",
       "  'Blueste': 2},\n",
       " 'Exterior1st': {'VinylSd': 364,\n",
       "  'HdBoard': 153,\n",
       "  'Wd Sdng': 148,\n",
       "  'MetalSd': 138,\n",
       "  'Plywood': 86,\n",
       "  'CemntBd': 39,\n",
       "  'BrkFace': 35,\n",
       "  'WdShing': 21,\n",
       "  'Stucco': 17,\n",
       "  'AsbShng': 15,\n",
       "  'Stone': 2,\n",
       "  'BrkComm': 1,\n",
       "  'ImStucc': 1,\n",
       "  'CBlock': 1,\n",
       "  'AsphShn': 1},\n",
       " 'Exterior2nd': {'VinylSd': 353,\n",
       "  'Wd Sdng': 142,\n",
       "  'HdBoard': 141,\n",
       "  'MetalSd': 136,\n",
       "  'Plywood': 112,\n",
       "  'CmentBd': 39,\n",
       "  'Wd Shng': 29,\n",
       "  'BrkFace': 18,\n",
       "  'AsbShng': 17,\n",
       "  'Stucco': 16,\n",
       "  'ImStucc': 8,\n",
       "  'Brk Cmn': 4,\n",
       "  'Stone': 4,\n",
       "  'Other': 1,\n",
       "  'CBlock': 1,\n",
       "  'AsphShn': 1}}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# in the encoder dict we can observe the number of \n",
    "# observations per category for each variable\n",
    "\n",
    "count_enc.encoder_dict_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Neighborhood</th>\n",
       "      <th>Exterior1st</th>\n",
       "      <th>Exterior2nd</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>105</td>\n",
       "      <td>364</td>\n",
       "      <td>353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>682</th>\n",
       "      <td>24</td>\n",
       "      <td>148</td>\n",
       "      <td>142</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>960</th>\n",
       "      <td>41</td>\n",
       "      <td>148</td>\n",
       "      <td>112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1384</th>\n",
       "      <td>71</td>\n",
       "      <td>21</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1100</th>\n",
       "      <td>18</td>\n",
       "      <td>148</td>\n",
       "      <td>142</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Neighborhood  Exterior1st  Exterior2nd\n",
       "64             105          364          353\n",
       "682             24          148          142\n",
       "960             41          148          112\n",
       "1384            71           21           29\n",
       "1100            18          148          142"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train = count_enc.transform(X_train)\n",
    "X_test = count_enc.transform(X_test)\n",
    "\n",
    "# let's explore the result\n",
    "X_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}