fuzzy_knn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Introduction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pprint\n",
    "import operator\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.model_selection import train_test_split, cross_val_score\n",
    "from sklearn.datasets import load_iris, load_breast_cancer\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.base import BaseEstimator, ClassifierMixin"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "iris = load_iris()\n",
    "breast = load_breast_cancer()\n",
    "dataset = iris\n",
    "\n",
    "X = dataset.data\n",
    "y = dataset.target\n",
    "\n",
    "xTrain, xTest, yTrain, yTest = train_test_split(X,y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Fuzzy K-Nearest Neighbors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class FuzzyKNN(BaseEstimator, ClassifierMixin):\n",
    "    def __init__(self, k=3, plot=False):\n",
    "        self.k = k\n",
    "        self.plot = plot\n",
    "        \n",
    "        \n",
    "    def fit(self, X, y=None):\n",
    "        self._check_params(X,y)\n",
    "        self.X = X\n",
    "        self.y = y\n",
    "        \n",
    "        self.xdim = len(self.X[0])\n",
    "        self.n = len(y)\n",
    "        \n",
    "        classes = list(set(y))\n",
    "        classes.sort()\n",
    "        self.classes = classes\n",
    "        \n",
    "        self.df = pd.DataFrame(self.X)\n",
    "        self.df['y'] = self.y\n",
    "        \n",
    "        self.memberships = self._compute_memberships()\n",
    "        \n",
    "        self.df['membership'] = self.memberships\n",
    "        \n",
    "        self.fitted_ = True\n",
    "        return self\n",
    "    \n",
    "    \n",
    "    def predict(self, X):\n",
    "        if self.fitted_ == None:\n",
    "            raise Exception('predict() called before fit()')\n",
    "        else:\n",
    "            m = 2\n",
    "            y_pred = []\n",
    "            \n",
    "            for x in X:\n",
    "                neighbors = self._find_k_nearest_neighbors(pd.DataFrame.copy(self.df), x)\n",
    "                \n",
    "                votes = {}\n",
    "                for c in self.classes:\n",
    "                    den = 0\n",
    "                    for n in range(self.k):\n",
    "                        dist = np.linalg.norm(x - neighbors.iloc[n,0:self.xdim])\n",
    "                        den += 1 / (dist ** (2 / (m-1)))\n",
    "                    \n",
    "                    neighbors_votes = []\n",
    "                    for n in range(self.k):\n",
    "                        dist = np.linalg.norm(x - neighbors.iloc[n,0:self.xdim])\n",
    "                        num = (neighbors.iloc[n].membership[c]) / (dist ** (2 / (m-1)))\n",
    "                        \n",
    "                        vote = num/den\n",
    "                        neighbors_votes.append(vote)\n",
    "                    votes[c] = np.sum(neighbors_votes)\n",
    "                    \n",
    "                pred = max(votes.items(), key=operator.itemgetter(1))[0]\n",
    "                y_pred.append((pred, votes))\n",
    "                \n",
    "            return y_pred\n",
    "        \n",
    "        \n",
    "    def score(self, X, y):\n",
    "        if self.fitted_ == None:\n",
    "            raise Exception('score() called before fit()')\n",
    "        else:\n",
    "            predictions = self.predict(X)\n",
    "            y_pred = [t[0] for t in predictions]\n",
    "            confidences = [t[1] for t in predictions]\n",
    "            \n",
    "            return accuracy_score(y_pred=y_pred, y_true=y)\n",
    "    \n",
    "        \n",
    "    def _find_k_nearest_neighbors(self, df, x):\n",
    "        X = df.iloc[:,0:self.xdim].values\n",
    "        \n",
    "        df['distances'] = [np.linalg.norm(X[i] - x) for i in range(self.n)]\n",
    "        \n",
    "        df.sort_values(by='distances', ascending=True, inplace=True)\n",
    "        neighbors = df.iloc[0:self.k]\n",
    "        \n",
    "        return neighbors\n",
    "\n",
    "                \n",
    "    def _get_counts(self, neighbors):\n",
    "        groups = neighbors.groupby('y')\n",
    "        counts = {group[1]['y'].iloc[0]:group[1].count()[0] for group in groups}\n",
    "        \n",
    "        return counts\n",
    "        \n",
    "        \n",
    "    def _compute_memberships(self):\n",
    "        memberships = []\n",
    "        for i in range(self.n):\n",
    "            x = self.X[i]\n",
    "            y = self.y[i]\n",
    "            \n",
    "            neighbors = self._find_k_nearest_neighbors(pd.DataFrame.copy(self.df), x)\n",
    "            counts = self._get_counts(neighbors)\n",
    "        \n",
    "            membership = dict()\n",
    "            for c in self.classes:\n",
    "                try:\n",
    "                    uci = 0.49 * (counts[c] / self.k)\n",
    "                    if c == y:\n",
    "                        uci += 0.51\n",
    "                    membership[c] = uci\n",
    "                except:\n",
    "                    membership[c] = 0\n",
    "                    \n",
    "            memberships.append(membership)\n",
    "        return memberships\n",
    "        \n",
    "        \n",
    "    def _check_params(self, X, y):\n",
    "        if type(self.k) != int:\n",
    "            raise Exception('\"k\" should have type int')\n",
    "        elif self.k >= len(y):\n",
    "            raise Exception('\"k\" should be less than no of feature sets')\n",
    "        elif self.k % 2 == 0:\n",
    "            raise Exception('\"k\" should be odd')\n",
    "            \n",
    "        if type(self.plot) != bool:\n",
    "            raise Exception('\"plot\" should have type bool')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# FuzzyKNN vs KNeighborsClassifier (IRIS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.93333333  0.89655172  0.96428571  0.92857143  0.89285714]\n",
      "[ 0.93333333  0.89655172  0.96428571  0.92857143  0.89285714]\n"
     ]
    }
   ],
   "source": [
    "skModel = KNeighborsClassifier()\n",
    "custModel = FuzzyKNN()\n",
    "\n",
    "skModel.fit(xTrain, yTrain)\n",
    "custModel.fit(xTrain, yTrain)\n",
    "\n",
    "print(cross_val_score(cv=5, estimator=skModel, X=xTest, y=yTest))\n",
    "print(cross_val_score(cv=5, estimator=custModel, X=xTest, y=yTest))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# FuzzyKNN vs KNeighborsClassifier (Breast Cancer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dataset = breast\n",
    "X = dataset.data\n",
    "y = dataset.target\n",
    "\n",
    "xTrain, xTest, yTrain, yTest = train_test_split(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.93103448  0.89655172  0.96551724  0.86206897  0.92592593]\n",
      "[ 0.93103448  0.93103448  1.          0.86206897  0.92592593]\n"
     ]
    }
   ],
   "source": [
    "skModel.fit(xTrain, yTrain)\n",
    "custModel.fit(xTrain, yTrain)\n",
    "\n",
    "print(cross_val_score(cv=5, estimator=skModel, X=xTest, y=yTest))\n",
    "print(cross_val_score(cv=5, estimator=custModel, X=xTest, y=yTest))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}