1.3) Preprocessing - Dataset Splitting and Size Filtering.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018-03-23 Random Splitting by Tile and Size Filtering"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Candidate plaques are grouped by their WSI-source, as well as their tile. This ensures that all plaques within a uniform 256 x 256 image are labeled (or multi-labeled) for the subsequent classification task. \n",
    "\n",
    "We use a pixel threshold of 1500 pixels, then randomly sample the tiled WSIs and copy them to a new directory for the dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "import shutil\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import cv2\n",
    "\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "BLOB_CSV = 'image_details.csv'\n",
    "CSV_DIR = 'data/seg/'\n",
    "\n",
    "NORM_SRC =  'data/seg/blobs_bboxes/'\n",
    "NORM_DEST = 'data/seg/size_filtered/blobs_bboxes/'\n",
    "\n",
    "RAW_SRC   = 'data/seg/blobs/'\n",
    "RAW_DEST  = 'data/seg/size_filtered/blobs/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will use a pixel threshold of 1500 pixels. We have a conversion rate of 0.5 microns per pixel, so 40 pixels wide = 20 microns wide. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Define a pixel threshold\n",
    "THRESHOLD = 1500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Load all the image data\n",
    "image_df = pd.read_csv(CSV_DIR+BLOB_CSV)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>imagename</th>\n",
       "      <th>source</th>\n",
       "      <th>tile_column</th>\n",
       "      <th>tile_row</th>\n",
       "      <th>image coordinates (xywh)</th>\n",
       "      <th>blob coordinates (xywh)</th>\n",
       "      <th>blob size</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NA4619-02_AB_19_27_0.jpg</td>\n",
       "      <td>NA4619-02_AB</td>\n",
       "      <td>27</td>\n",
       "      <td>19</td>\n",
       "      <td>[653   0 256 256]</td>\n",
       "      <td>[750   0  62  20]</td>\n",
       "      <td>908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NA4619-02_AB_19_27_1.jpg</td>\n",
       "      <td>NA4619-02_AB</td>\n",
       "      <td>27</td>\n",
       "      <td>19</td>\n",
       "      <td>[794  39 256 256]</td>\n",
       "      <td>[896 148  52  38]</td>\n",
       "      <td>7088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NA4619-02_AB_19_27_2.jpg</td>\n",
       "      <td>NA4619-02_AB</td>\n",
       "      <td>27</td>\n",
       "      <td>19</td>\n",
       "      <td>[820 115 256 256]</td>\n",
       "      <td>[920 224  56  38]</td>\n",
       "      <td>1260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NA4619-02_AB_19_27_3.jpg</td>\n",
       "      <td>NA4619-02_AB</td>\n",
       "      <td>27</td>\n",
       "      <td>19</td>\n",
       "      <td>[994 163 256 256]</td>\n",
       "      <td>[1108  276   28   30]</td>\n",
       "      <td>1300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NA4619-02_AB_19_27_4.jpg</td>\n",
       "      <td>NA4619-02_AB</td>\n",
       "      <td>27</td>\n",
       "      <td>19</td>\n",
       "      <td>[1217  277  256  256]</td>\n",
       "      <td>[1314  384   62   42]</td>\n",
       "      <td>452</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  imagename        source  tile_column  tile_row  \\\n",
       "0  NA4619-02_AB_19_27_0.jpg  NA4619-02_AB           27        19   \n",
       "1  NA4619-02_AB_19_27_1.jpg  NA4619-02_AB           27        19   \n",
       "2  NA4619-02_AB_19_27_2.jpg  NA4619-02_AB           27        19   \n",
       "3  NA4619-02_AB_19_27_3.jpg  NA4619-02_AB           27        19   \n",
       "4  NA4619-02_AB_19_27_4.jpg  NA4619-02_AB           27        19   \n",
       "\n",
       "  image coordinates (xywh) blob coordinates (xywh)  blob size  \n",
       "0        [653   0 256 256]       [750   0  62  20]        908  \n",
       "1        [794  39 256 256]       [896 148  52  38]       7088  \n",
       "2        [820 115 256 256]       [920 224  56  38]       1260  \n",
       "3        [994 163 256 256]   [1108  276   28   30]       1300  \n",
       "4    [1217  277  256  256]   [1314  384   62   42]        452  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Group by tile\n",
    "grouped = image_df.groupby(['source', 'tile_column', 'tile_row'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# This is a list of the group keys\n",
    "tiles = list(grouped.groups)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# set a random seed\n",
    "np.random.seed(42)\n",
    "\n",
    "# shuffle the tiles\n",
    "np.random.shuffle(tiles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "SPLIT = int(0.5 * len(tiles))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "imageset = []\n",
    "sources = []\n",
    "for key in tiles[:SPLIT]:\n",
    "    value = grouped.groups[key]\n",
    "    images = image_df.loc[value]\n",
    "    filtered = images[images['blob size'] > THRESHOLD]\n",
    "    images = filtered['imagename']\n",
    "    source = filtered['source']\n",
    "    imageset.extend(images)\n",
    "    sources.extend(source)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "103409"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# down to 103409 images to label of greater than 1500 pixels\n",
    "len(imageset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "103409it [01:24, 1222.50it/s]\n"
     ]
    }
   ],
   "source": [
    "for image, source in tqdm(zip(imageset, sources)):\n",
    "    rawdir = RAW_DEST + source\n",
    "    normdir = NORM_DEST + source\n",
    "    if not os.path.exists(rawdir):\n",
    "        os.makedirs(rawdir)\n",
    "    if not os.path.exists(normdir):\n",
    "        os.makedirs(normdir)\n",
    "    filename = source + \"/\" + image\n",
    "    shutil.copy(RAW_SRC + filename, RAW_DEST + filename)\n",
    "    shutil.copy(NORM_SRC + filename, NORM_DEST + filename)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}