homework_ensemble.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Titanic: Machine Learning from Disaster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# https://www.kaggle.com/c/titanic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sklearn\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Загружаем данные из файлов\n",
    "train = pd.read_csv('./titanic/train.csv')\n",
    "test = pd.read_csv('./titanic/test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Braund, Mr. Owen Harris</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Heikkinen, Miss. Laina</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td>\n",
       "      <td>female</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113803</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>C123</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Allen, Mr. William Henry</td>\n",
       "      <td>male</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>373450</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Survived  Pclass  \\\n",
       "0            1         0       3   \n",
       "1            2         1       1   \n",
       "2            3         1       3   \n",
       "3            4         1       1   \n",
       "4            5         0       3   \n",
       "\n",
       "                                                Name     Sex   Age  SibSp  \\\n",
       "0                            Braund, Mr. Owen Harris    male  22.0      1   \n",
       "1  Cumings, Mrs. John Bradley (Florence Briggs Th...  female  38.0      1   \n",
       "2                             Heikkinen, Miss. Laina  female  26.0      0   \n",
       "3       Futrelle, Mrs. Jacques Heath (Lily May Peel)  female  35.0      1   \n",
       "4                           Allen, Mr. William Henry    male  35.0      0   \n",
       "\n",
       "   Parch            Ticket     Fare Cabin Embarked  \n",
       "0      0         A/5 21171   7.2500   NaN        S  \n",
       "1      0          PC 17599  71.2833   C85        C  \n",
       "2      0  STON/O2. 3101282   7.9250   NaN        S  \n",
       "3      0            113803  53.1000  C123        S  \n",
       "4      0            373450   8.0500   NaN        S  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Предобработка данных"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Заполняем пропуски в данных медианными \n",
    "# значениями факторов на обучающей выборке\n",
    "train_median = train.median()\n",
    "train_imp = train.fillna(train_median)\n",
    "test_imp = test.fillna(train_median)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Бинаризуем категориальные признаки\n",
    "CATEGORY_COL = ['Sex', 'Pclass', 'Embarked']\n",
    "train_dummies = pd.get_dummies(train_imp, columns=CATEGORY_COL, drop_first=True)\n",
    "test_dummies = pd.get_dummies(test_imp, columns=CATEGORY_COL, drop_first=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Name</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Sex_male</th>\n",
       "      <th>Pclass_2</th>\n",
       "      <th>Pclass_3</th>\n",
       "      <th>Embarked_Q</th>\n",
       "      <th>Embarked_S</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Braund, Mr. Owen Harris</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>Heikkinen, Miss. Laina</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113803</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>C123</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>Allen, Mr. William Henry</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>373450</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Survived                                               Name  \\\n",
       "0            1         0                            Braund, Mr. Owen Harris   \n",
       "1            2         1  Cumings, Mrs. John Bradley (Florence Briggs Th...   \n",
       "2            3         1                             Heikkinen, Miss. Laina   \n",
       "3            4         1       Futrelle, Mrs. Jacques Heath (Lily May Peel)   \n",
       "4            5         0                           Allen, Mr. William Henry   \n",
       "\n",
       "    Age  SibSp  Parch            Ticket     Fare Cabin  Sex_male  Pclass_2  \\\n",
       "0  22.0      1      0         A/5 21171   7.2500   NaN         1         0   \n",
       "1  38.0      1      0          PC 17599  71.2833   C85         0         0   \n",
       "2  26.0      0      0  STON/O2. 3101282   7.9250   NaN         0         0   \n",
       "3  35.0      1      0            113803  53.1000  C123         0         0   \n",
       "4  35.0      0      0            373450   8.0500   NaN         1         0   \n",
       "\n",
       "   Pclass_3  Embarked_Q  Embarked_S  \n",
       "0         1           0           1  \n",
       "1         0           0           0  \n",
       "2         1           0           1  \n",
       "3         0           0           1  \n",
       "4         1           0           1  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dummies.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Удаляем лишние столбцы\n",
    "DROP_COL = ['PassengerId', 'Name', 'Ticket', 'Cabin']\n",
    "TARGET_COL = 'Survived'\n",
    "X_train = train_dummies.drop(DROP_COL + [TARGET_COL], axis=1)\n",
    "y_train = train_dummies[TARGET_COL]\n",
    "X_test = test_dummies.drop(DROP_COL, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.metrics import r2_score, scorer, accuracy_score, make_scorer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Предсказание моделей для стеккинга"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "\n",
    "def cross_val_predict_proba(estimator, X_train, y_train):\n",
    "    print(estimator)\n",
    "    kfold = KFold(n_splits=4, shuffle=True, random_state=None)\n",
    "    return cross_val_predict(estimator, X_train, y_train, cv=kfold, method='predict_proba', n_jobs=-1)\n",
    "\n",
    "\n",
    "def grid_search(model,params, X_train, y_train):\n",
    "    kfold = KFold(n_splits=4, shuffle=True, random_state=None)\n",
    "    local_grid = GridSearchCV(model, params,\\\n",
    "                              scoring=make_scorer(accuracy_score),\\\n",
    "                              cv=kfold, n_jobs=-1)\n",
    "    local_model = local_grid.fit(X_train, y_train)\n",
    "    print(model)\n",
    "    print(local_grid.best_params_, local_grid.best_score_)\n",
    "    return local_model\n",
    "\n",
    "\n",
    "# TODO: подобрать гиперпараметры для ансамблей\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
      "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
      "            min_impurity_split=1e-07, min_samples_leaf=1,\n",
      "            min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
      "            n_estimators=10, n_jobs=1, oob_score=False, random_state=None,\n",
      "            verbose=0, warm_start=False)\n",
      "{'n_estimators': 40} 0.814814814815\n"
     ]
    }
   ],
   "source": [
    "model1 = grid_search(RandomForestClassifier(),{'n_estimators': range(30, 120, 10)}, X_train, y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
      "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
      "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
      "          verbose=0, warm_start=False)\n",
      "{'C': 1} 0.801346801347\n"
     ]
    }
   ],
   "source": [
    "\n",
    "model2 = grid_search(LogisticRegression(),{'C': [0.01,0.1,1,10,100]}, X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GradientBoostingClassifier(criterion='friedman_mse', init=None,\n",
      "              learning_rate=0.1, loss='deviance', max_depth=3,\n",
      "              max_features=None, max_leaf_nodes=None,\n",
      "              min_impurity_split=1e-07, min_samples_leaf=1,\n",
      "              min_samples_split=100, min_weight_fraction_leaf=0.0,\n",
      "              n_estimators=100, presort='auto', random_state=None,\n",
      "              subsample=1.0, verbose=0, warm_start=True)\n",
      "{'learning_rate': 0.1, 'max_depth': 11, 'min_samples_leaf': 20, 'min_samples_split': 200} 0.841750841751\n"
     ]
    }
   ],
   "source": [
    "model3 = grid_search((GradientBoostingClassifier(#learning_rate=0.1, \\\n",
    "                                                                  min_samples_split=100,\\\n",
    "                                                 #                 min_samples_leaf=2,\\\n",
    "                                                 #                 max_features='sqrt',\\\n",
    "                                                 #                 subsample=0.8,\\\n",
    "                                                 #                 random_state=10,\\\n",
    "                                                                  warm_start=True)),\\\n",
    "                     {\n",
    "                    'max_depth': range(1, 50,10),\\\n",
    "                    'learning_rate': [0.01,0.1],\\\n",
    "                    'min_samples_split': [2,10,100,200],\\\n",
    "                    'min_samples_leaf':[8,10,15,20]\n",
    "                     }, \n",
    "                     X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GridSearchCV(cv=KFold(n_splits=4, random_state=None, shuffle=True),\n",
      "       error_score='raise',\n",
      "       estimator=RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
      "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
      "            min_impurity_split=1e-07, min_samples_leaf=1,\n",
      "            min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
      "            n_estimators=10, n_jobs=1, oob_score=False, random_state=None,\n",
      "            verbose=0, warm_start=False),\n",
      "       fit_params={}, iid=True, n_jobs=-1,\n",
      "       param_grid={'n_estimators': range(30, 120, 10)},\n",
      "       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
      "       scoring=make_scorer(accuracy_score), verbose=0)\n",
      "GridSearchCV(cv=KFold(n_splits=4, random_state=None, shuffle=True),\n",
      "       error_score='raise',\n",
      "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
      "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
      "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
      "          verbose=0, warm_start=False),\n",
      "       fit_params={}, iid=True, n_jobs=-1,\n",
      "       param_grid={'C': [0.01, 0.1, 1, 10, 100]}, pre_dispatch='2*n_jobs',\n",
      "       refit=True, return_train_score=True,\n",
      "       scoring=make_scorer(accuracy_score), verbose=0)\n",
      "GridSearchCV(cv=KFold(n_splits=4, random_state=None, shuffle=True),\n",
      "       error_score='raise',\n",
      "       estimator=GradientBoostingClassifier(criterion='friedman_mse', init=None,\n",
      "              learning_rate=0.1, loss='deviance', max_depth=3,\n",
      "              max_features=None, max_leaf_nodes=None,\n",
      "              min_impurity_split=1e-07, min_samples_leaf=1,\n",
      "              min_samples_split=100, min_weight_fraction_leaf=0.0,\n",
      "              n_estimators=100, presort='auto', random_state=None,\n",
      "              subsample=1.0, verbose=0, warm_start=True),\n",
      "       fit_params={}, iid=True, n_jobs=-1,\n",
      "       param_grid={'max_depth': range(1, 50, 10), 'learning_rate': [0.01, 0.1], 'min_samples_split': [2, 10, 100, 200], 'min_samples_leaf': [8, 10, 15, 20]},\n",
      "       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
      "       scoring=make_scorer(accuracy_score), verbose=0)\n"
     ]
    }
   ],
   "source": [
    "model1_train_pred = cross_val_predict_proba(model1, X_train, y_train)\n",
    "model2_train_pred = cross_val_predict_proba(model2, X_train, y_train)\n",
    "model3_train_pred = cross_val_predict_proba(model3, X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model1_test_pred = model1.fit(X_train, y_train).predict_proba(X_test)\n",
    "model2_test_pred = model2.fit(X_train, y_train).predict_proba(X_test)\n",
    "model3_test_pred = model3.fit(X_train, y_train).predict_proba(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_array = [model1_train_pred[:,1], model2_train_pred[:,1], model3_train_pred[:,1]]\n",
    "test_array = [model1_test_pred[:,1], model2_test_pred[:,1], model3_test_pred[:,1]]\n",
    "\n",
    "#train_array = [model1_train_pred[:,1], model3_train_pred[:,1]]\n",
    "#test_array = [model1_test_pred[:,1], model3_test_pred[:,1]]\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_stack = np.stack(train_array, axis=1)\n",
    "X_test_stack = np.stack(test_array, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Объединяем предсказания ансамблей с помощью логистической регрессии"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: подобрать гиперпараметры LogisticRegression\n",
    "\n",
    "\n",
    "kfold = KFold(n_splits=10, shuffle=True, random_state=1243)\n",
    "\n",
    "pipeline = Pipeline(steps=[\n",
    "                           ('model', LogisticRegression(C=1, penalty='l2'))\n",
    "                          ])\n",
    "\n",
    "param_grid = {'model__C': [0.001,0.01,0.1,1,10,100,1000,10000]}\n",
    "\n",
    "grid = GridSearchCV(pipeline, param_grid, \n",
    "                    scoring=make_scorer(accuracy_score),\n",
    "                    cv=kfold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.80583614,  0.82154882,  0.82603816,  0.82267116,  0.82042649,\n",
       "        0.82042649,  0.82042649,  0.82042649])"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "logreg = grid.fit(X_train_stack, y_train)\n",
    "predicted = logreg.predict(X_test_stack)\n",
    "logreg.cv_results_['mean_test_score']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([mean: 0.80584, std: 0.03621, params: {'model__C': 0.001},\n",
       "  mean: 0.82155, std: 0.03410, params: {'model__C': 0.01},\n",
       "  mean: 0.82604, std: 0.02972, params: {'model__C': 0.1},\n",
       "  mean: 0.82267, std: 0.03121, params: {'model__C': 1},\n",
       "  mean: 0.82043, std: 0.02879, params: {'model__C': 10},\n",
       "  mean: 0.82043, std: 0.02879, params: {'model__C': 100},\n",
       "  mean: 0.82043, std: 0.02879, params: {'model__C': 1000},\n",
       "  mean: 0.82043, std: 0.02879, params: {'model__C': 10000}],\n",
       " {'model__C': 0.1},\n",
       " 0.82603815937149272)"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logreg.grid_scores_, logreg.best_params_, logreg.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEVpJREFUeJzt3X+s3Xddx/Hni9sOlQ0GayW4rutIhtogIF4HxkVR4+jA\nOE38sVEdFEhZYGSGGBmZYgzOBAkEkW1Ng+VHrMwYFpmkWFFRIgRZJ2Nsw0EZ7BfIuqEgI7ErffvH\nOWNnl27nrJ/Tns895/lIbnrO9/vtPZ99zt0zn37POfebqkKSNF8eN+sBSJKmz7hL0hwy7pI0h4y7\nJM0h4y5Jc8i4S9IcMu6SNIeMuyTNIeMuSXNozaweeN26dbVp06ZZPbwkrUrXX3/9vVW1ftxxM4v7\npk2b2Ldv36weXpJWpSS3T3Kcp2UkaQ4Zd0maQ8ZdkuaQcZekOWTcJWkOjY17kl1J7kly0yPsT5J3\nJNmf5MYkz53+MLUodu+GE0+ExK8E1q0bzIna7N49mMtZP5/H87mdZOX+HmDLo+w/Fzhz+LUduKp9\nWFpEu3fDhRfC/ffPeiT9uO8+ePnLDXyL3bth27bBXPbkWD+3Y9/nXlUfS7LpUQ45D3hfDa7X98kk\nJyd5WlV9dUpj1IK47DI4fHjWo+jDEoe4hD/jSXwDDsK9rwFunfWoVqd73wG//8CsR/Fw/8bZfIRz\nOHhw8HO/dev0H2MaH2I6Fbhz5P5dw23fE/ck2xms7tm4ceMUHlrz5I47Zj2CfjyHG3grvwvAYQLf\nAP54tmNarV7b4WWi38zr+QjnAMfu5/64fkK1qnYCOwGWl5c7nHLN0saNcPtEn72bf2s4BMAWPsxe\ntnD66fDlL892TKvV0zf1/XN1rNa503i3zN3AaSP3Nwy3SY/J5ZfD43z/FgDhobXPCScM5kZH5/LL\nYe3aWY/iyI7lczuN/5WuBS4cvmvm+cA3PN+uo7F1K7zvffCEJ8x6JLP3YNxPOins2nVszskuiq1b\n4d3vhlNOmfVIHu6UUzimz20Gr4M+ygHJ+4EXAOuArwF/CKwFqKodSQK8k8E7ar4NbKuqsb8RbHl5\nufzFYdIj+PjH4eyzYe9eOOecWY9GHUlyfVUtjztuknfLXDBmfwGveQxjkzTOg4uuZLbj0KrlGU6p\nR8ZdjYy71CPjrkbGXeqRcVcj4y71yLirkXGXemTc1ci4Sz0y7mpk3KUeGXc1Mu5Sz4y7jpJxl3rk\nyl2NjLvUI+OuRsZd6pFxVyPjLvXIuKuRcZd6ZNzVyLhLPTLuamTcpR6Nuc6CNI5xl3rkyl2NjLvU\nI+OuRsZd6pFxVyPjLvXIuKuRcZd6ZNzVyLhLPTLuamTcpR4ZdzUy7lKPjLsaGXepR8ZdjYy71CPj\nrkbGXeqZcddRMu5Sj1y5q5Fxl3pk3NXIuEs9Mu5qZNylHhl3NZoo7km2JLk1yf4klx5h/5OS/F2S\nzyS5Ocm26Q9VWiDGXY3Gxj3JEnAFcC6wGbggyeYVh70GuKWqng28AHhrkhOmPFZpcRh3NZpk5X4W\nsL+qbquqg8DVwHkrjingpCQBTgS+Dhya6kilReKVmNRokrifCtw5cv+u4bZR7wR+FPgK8Fngkqo6\nPJURSovIlbsaTesF1RcCNwA/BDwHeGeSJ648KMn2JPuS7Dtw4MCUHlqaQ8ZdjSaJ+93AaSP3Nwy3\njdoGXFMD+4EvAT+y8htV1c6qWq6q5fXr1x/tmKX5Z9zVaJK4XwecmeSM4Yuk5wPXrjjmDuAXAJI8\nFfhh4LZpDlRaKMZdjdaMO6CqDiW5GNgLLAG7qurmJBcN9+8A3gS8J8lngQCvr6p7j+G4pflm3NVo\nbNwBqmoPsGfFth0jt78CnDPdoUkLzLirkZ9QlXpk3NXIuEs9Mu5qZNylHhl3NTLuUo+MuxoZd6ln\nxl1HybhLPXLlrkbGXeqRcVcj4y71yLirkXGXemTc1ci4Sz0y7mpk3KUeGXc1Mu5Sj7wSkxoZd6lH\nrtzVyLhLPTLuamTcpR4ZdzUy7lKPjLsaGXepR8ZdjYy71CPjrkbGXeqRcVcj4y71yLirkXGXemTc\n1ci4Sz0y7mpk3KWeGXcdJeMu9ciVuxoZd6lHxl2NjLvUI+OuRsZd6pFxVyPjLvXIuKuRcZd6ZNzV\nyLhLPfJKTGpk3KUeuXJXo4ninmRLkluT7E9y6SMc84IkNyS5Ocm/TneY0oIx7mq0ZtwBSZaAK4Bf\nBO4CrktybVXdMnLMycCVwJaquiPJDx6rAUsLwbir0SQr97OA/VV1W1UdBK4GzltxzEuAa6rqDoCq\nume6w5QWjHFXo0nifipw58j9u4bbRj0DeHKSf0lyfZILj/SNkmxPsi/JvgMHDhzdiKVF4AuqajSt\nF1TXAD8BvBh4IfAHSZ6x8qCq2llVy1W1vH79+ik9tDSHqly1q8nYc+7A3cBpI/c3DLeNugu4r6ru\nB+5P8jHg2cDnpzJKadEYdzWaZOV+HXBmkjOSnACcD1y74pgPAmcnWZPkB4DnAZ+b7lClBWLc1Wjs\nyr2qDiW5GNgLLAG7qurmJBcN9++oqs8l+XvgRuAw8K6quulYDlyaa8ZdjSY5LUNV7QH2rNi2Y8X9\ntwBvmd7QpAVm3NXIT6hKvTLuamDcpR65clcj4y71yLirkXGXemTc1ci4Sz0y7mpk3KUeGXc1Mu5S\nj4y7Ghl3qUf+4jA1Mu5Sj1y5q5Fxl3pk3NXIuEs9Mu5qZNylHhl3NTLuUo+MuxoZd6lHxl2NjLvU\nI+OuRsZd6pFxVyPjLvXIuKuRcZd6ZNzVyLhLPTLuamTcpV4ZdzUw7lKPXLmrkXGXemTc1ci4Sz0y\n7mpk3KUeGXc1Mu5Sj4y7Ghl3qUdeiUmNjLvUI1fuamTcpR4ZdzUy7lKPjLsaGXepR8ZdjSaKe5It\nSW5Nsj/JpY9y3E8mOZTk16Y3RGkBGXc1Ghv3JEvAFcC5wGbggiSbH+G4NwP/MO1BSgvHuKvRJCv3\ns4D9VXVbVR0ErgbOO8JxrwU+ANwzxfFJi8m4q9EkcT8VuHPk/l3Dbd+V5FTgV4Grpjc0aYEZdzWa\n1guqbwdeX1WHH+2gJNuT7Euy78CBA1N6aGkOGXc1WjPBMXcDp43c3zDcNmoZuDqDH8Z1wIuSHKqq\nvx09qKp2AjsBlpeX/Qie9EiMuxpNEvfrgDOTnMEg6ucDLxk9oKrOePB2kvcAH1oZdkmPgXFXo7Fx\nr6pDSS4G9gJLwK6qujnJRcP9O47xGKXFZNzVYJKVO1W1B9izYtsRo15VL2sflrTgXLmrkZ9QlXpk\n3NXIuEs9Mu5qZNylHhl3NTLuUo+MuxoZd6lHXolJjYy71CNX7mpk3KUeGXc1Mu5Sj4y7Ghl3qUfG\nXY2Mu9Qj465Gxl3qkXFXI+Mu9ci4q5Fxl3pk3NXIuEs9Mu5qZNylHhl3NTLuUo+MuxoZd6lXxl0N\njLvUI1fuamTcpR4ZdzUy7lKPjLsaGXepR8ZdjYy71CPjrkbGXeqRV2JSI+Mu9ciVuxoZd6lHxl2N\njLvUI+OuRsZd6pFxVyPjLvXIuKuRcZd6ZNzVyLhLPTLuamTcpR4ZdzWaKO5JtiS5Ncn+JJceYf/W\nJDcm+WySTyR59vSHKi0Q465GY+OeZAm4AjgX2AxckGTzisO+BPxsVf0Y8CZg57QHKi0U465Gk6zc\nzwL2V9VtVXUQuBo4b/SAqvpEVf338O4ngQ3THaa0YIy7Gk0S91OBO0fu3zXc9kheAXz4SDuSbE+y\nL8m+AwcOTD5KaREZdzWY6guqSX6OQdxff6T9VbWzqparann9+vXTfGhpvrhyV6M1ExxzN3DayP0N\nw20Pk+RZwLuAc6vqvukMT1pQxl2NJlm5XwecmeSMJCcA5wPXjh6QZCNwDfDbVfX56Q9TWjDGXY3G\nrtyr6lCSi4G9wBKwq6puTnLRcP8O4I3AKcCVGfxAHqqq5WM3bGnOGXc1muS0DFW1B9izYtuOkduv\nBF453aFJC8y4q5GfUJV65JWY1Mi4Sz1y5a5Gxl3qkXFXI+Mu9ci4q5Fxl3pk3NXIuEs9Mu5qZNyl\nHhl3NTLuUo+MuxoZd6lHxl2NjLvUI+OuRsZd6pFxVyPjLvXIuKuRcZd6ZdzVwLhLPXLlrkbGXeqR\ncVcj4y71yLirkXGXemTc1ci4Sz0y7mpk3KUeeSUmNTLuUo9cuauRcZd6ZNzVyLhLPTLuamTcpR4Z\ndzUy7lKPjLsaGXepR8ZdjYy71CPjrkbGXeqRcVcj4y71yLirkXGXemTc1ci4Sz0y7mo0UdyTbEly\na5L9SS49wv4kecdw/41Jnjv9oUoLxrirwdi4J1kCrgDOBTYDFyTZvOKwc4Ezh1/bgaumPE4Adu+G\ndesGP/N+zffXSScNnu9F9e37i3f8eWb+PMzb19ISvPrVs352j49JVu5nAfur6raqOghcDZy34pjz\ngPfVwCeBk5M8bZoD3b0btm2D++6b5ndVr771LXjZyxYz8K9+NRw8WBwmsx7K3Dl8GK66ajECv2aC\nY04F7hy5fxfwvAmOORX4atPoRlx2GTzwAJzDXt7G66b1bdWzQ7B2G/Ansx7I8fWaW+CJfJMy7sfM\nzp1w5ZWzHsWxNUncpybJdganbdi4ceNj+rt33DH485s8kVtYeVZIc+sBeMaCPd233AI38Uz+mt+c\n9VDm1ne+M+sRHHuTxP1u4LSR+xuG2x7rMVTVTmAnwPLy8mO6GsHGjXD77fBJforf4G8ey1/VKnb6\n6fDrC/Z0X7BmMeIzS0tLsx7BsTfJOffrgDOTnJHkBOB84NoVx1wLXDh818zzgW9U1dROyQBcfjms\nXTvN76jerVkzeN4Xzfbtsx7B/FuEOR67cq+qQ0kuBvYCS8Cuqro5yUXD/TuAPcCLgP3At4Ft0x7o\n1q2DPy+5xBdVF8GJJ8KOHQ8974vkwXPBO3Z4tb1pe9zj4FWvmv/z7QCpGf30LC8v1759+2by2JK0\nWiW5vqqWxx3nJ1QlaQ4Zd0maQ8ZdkuaQcZekOWTcJWkOzezdMkkOALcf5V9fB9w7xeHMI+doPOdo\nPOdovOM9R6dX1fpxB80s7i2S7JvkrUCLzDkazzkazzkar9c58rSMJM0h4y5Jc2i1xn3nrAewCjhH\n4zlH4zlH43U5R6vynLsk6dGt1pW7JOlRrLq4j7tY96JIsivJPUluGtn2lCQfSfKF4Z9PHtn3huGc\n3ZrkhbMZ9fGV5LQkH01yS5Kbk1wy3O48DSX5viSfSvKZ4Rz90XC7czQiyVKSTyf50PB+//NTVavm\ni8GvHP4i8HTgBOAzwOZZj2tGc/EzwHOBm0a2/Slw6fD2pcCbh7c3D+fq8cAZwzlcmvV/w3GYo6cB\nzx3ePgn4/HAunKeH5ijAicPba4F/B57vHH3PPL0O+CvgQ8P73c/Palu5T3Kx7oVQVR8Dvr5i83nA\ne4e33wv8ysj2q6vq/6rqSwx+7/5Zx2WgM1RVX62q/xje/l/gcwyu7es8DdXAt4Z31w6/Cufou5Js\nAF4MvGtkc/fzs9ri/kgX4tbAU+uhK2D9F/DU4e2Fn7ckm4AfZ7AydZ5GDE853ADcA3ykqpyjh3s7\n8HvA4ZFt3c/Paou7JlSDfyP6ViggyYnAB4Dfqapvju5znqCqvlNVz2Fw7eOzkjxzxf6FnaMkvwTc\nU1XXP9Ixvc7Paov7RBfiXmBfS/I0gOGf9wy3L+y8JVnLIOy7q+qa4Wbn6Qiq6n+AjwJbcI4e9NPA\nLyf5MoPTwD+f5C9ZBfOz2uI+ycW6F9m1wEuHt18KfHBk+/lJHp/kDOBM4FMzGN9xlSTAXwCfq6q3\njexynoaSrE9y8vD29wO/CPwnzhEAVfWGqtpQVZsY9Oafq+q3WA3zM+tXoY/iVesXMXjXwxeBy2Y9\nnhnOw/uBrwIPMDiv9wrgFOCfgC8A/wg8ZeT4y4Zzditw7qzHf5zm6GwG/1y+Ebhh+PUi5+lhc/Qs\n4NPDOboJeONwu3P0vXP1Ah56t0z38+MnVCVpDq220zKSpAkYd0maQ8ZdkuaQcZekOWTcJWkOGXdJ\nmkPGXZLmkHGXpDn0/z9h7Ec/QjfWAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1194d9a90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pairs = np.array(sorted(zip(y_train, predicted)))\n",
    "plt.plot(range(0, len(pairs)), pairs[:,1], 'bo',\n",
    "         range(0, len(pairs)), pairs[:,0], 'r-',)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Формируем фалй для отправки"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('submission.txt', 'w') as out:\n",
    "    out.write('PassengerId,Survived\\n')\n",
    "    for passenger, y in zip(test['PassengerId'], predicted):\n",
    "        out.write('%s,%s\\n' % (passenger, y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}