4.1) Saliency Mapping - Feature Occlusion.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.1 Saliency Mapping - Feature Occlusion\n",
    "\n",
    "Feature occlusion studies were performed to show the influence of occluding regions of input image to the confidence score predicted by the CNN model. \n",
    "\n",
    "The occlusion map was computed by replacing a region of the image with a pure white patch and generating a prediction on the occluded image. A sliding window approach was applied on the 256x256 pixels input images. A white patch was systematically slided across the image to replace a specific region of the input image, the CNN model forward propagated and generated the prediction confidence score on the occluded image at each time. As systematically sliding the white patch across the image (stride = 1 pixel), the prediction confidence score on the occluded image was recorded as each pixel of the occlusion map. The results were ploted as heatmaps, with red being the most confidence, and blue the least."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, time\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "from mpl_toolkits.axes_grid1 import make_axes_locatable\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import PIL.Image as Image\n",
    "\n",
    "import torch\n",
    "from torch.autograd import Variable\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torchvision import transforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "IMG_DIR = 'data/tiles/hold-out/'\n",
    "MODEL_DIR = 'models/CNN_model_parameters.pkl'\n",
    "SAVE_DIR = 'data/outputs/selected_test_blobs/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists(SAVE_DIR):\n",
    "        os.makedirs(SAVE_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "image_classes = ['cored', 'diffuse', 'CAA']\n",
    "use_gpu = torch.cuda.is_available()\n",
    "\n",
    "norm = np.load('utils/normalization.npy').item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = pd.read_csv('data/CSVs/selected_test_blobs.csv')\n",
    "image_list = list(file['tilename'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def imshow(inp, ax=plt, title=None, pause=True):\n",
    "    \"\"\"Imshow for Tensor.\"\"\"\n",
    "    inp = inp.numpy().transpose((1, 2, 0))\n",
    "    mean = np.array(norm['mean'])\n",
    "    std = np.array(norm['std'])\n",
    "    inp = std * inp + mean\n",
    "    inp = np.clip(inp, 0, 1)\n",
    "    \n",
    "    ax.imshow(inp)\n",
    "    if title is not None:\n",
    "        try:\n",
    "            ax.title(title)\n",
    "        except:\n",
    "            ax.set(title=title)\n",
    "    if pause:\n",
    "        plt.pause(0.001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "\n",
    "    def __init__(self, fc_nodes=1024, num_classes=3, dropout=0.5):\n",
    "        super(Net, self).__init__()\n",
    "\n",
    "    def forward(self, x):\n",
    " \n",
    "        x = self.features(x)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = self.classifier(x)\n",
    "\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# instatiate the model \n",
    "model = torch.load(MODEL_DIR, map_location=lambda storage, loc: storage)\n",
    "\n",
    "if use_gpu:\n",
    "    model = model.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def single_prediction(img_name, verbose=False):\n",
    "    \"\"\"\n",
    "        Perform forward propagation on a single image \n",
    "    \"\"\"\n",
    "    im = Image.open(IMG_DIR+img_name)\n",
    "    imtensor = transforms.ToTensor()(im)\n",
    "    imtensor = transforms.Normalize(norm['mean'], norm['std'])(imtensor)\n",
    "    imtensor = imtensor.view(1,imtensor.shape[0],imtensor.shape[1],imtensor.shape[2])\n",
    "    output = F.sigmoid(model.module(Variable(imtensor.cuda())))\n",
    "    if verbose:\n",
    "        print(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def occluded_predictions(img_name, block_size=32, verbose=False):\n",
    "    \"\"\"\n",
    "        Sliding window approach to perform feature occlusion study.\n",
    "        A white patch was systematically slided across the image to replace a region of the image,\n",
    "        performing forward propagation on the occluded image at each time.\n",
    "    \"\"\"\n",
    "    \n",
    "    im = Image.open(IMG_DIR+img_name)\n",
    "    imtensor = transforms.ToTensor()(im)\n",
    "    \n",
    "    stride = 1\n",
    "    final_output = torch.zeros(3, (imtensor.shape[1]-block_size)//stride+1, (imtensor.shape[2]-block_size)//stride+1)\n",
    "\n",
    "    start = time.time()\n",
    "    model.train(False)\n",
    "    for row in range(0, imtensor.shape[1]-block_size+1, stride):\n",
    "        for col in range(0, imtensor.shape[2]-block_size+1, stride):\n",
    "            imtensor = transforms.ToTensor()(im)\n",
    "            imtensor[:,row:row+block_size,col:col+block_size] = torch.ones(3,block_size,block_size)\n",
    "            imtensor = transforms.Normalize(norm['mean'], norm['std'])(imtensor)\n",
    "            imtensor = imtensor.view(1,imtensor.shape[0],imtensor.shape[1],imtensor.shape[2])\n",
    "            output = F.sigmoid(model.module(Variable(imtensor.cuda())))\n",
    "            final_output[:,row//stride,col//stride] = output.data.cpu()[0]\n",
    "    end = time.time()\n",
    "    \n",
    "    if verbose:\n",
    "        print('time: {}m {}s'.format((end-start)//60, (end-start)%60))\n",
    "    \n",
    "    return final_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def show_heatmaps(img_name, block_size, final_output, path=None, show=False, verbose=False):\n",
    "    \"\"\"\n",
    "        plot and save the heatmaps\n",
    "    \"\"\"\n",
    "    if verbose:\n",
    "        print('size:', block_size)\n",
    "    \n",
    "    im = Image.open(IMG_DIR+img_name)\n",
    "    imtensor = transforms.ToTensor()(im)\n",
    "    imtensor[:,-block_size:,-block_size:] = torch.ones(3,block_size,block_size)\n",
    "    imtensor = transforms.Normalize(norm['mean'], norm['std'])(imtensor)\n",
    "    \n",
    "    colormap = 'RdYlBu_r'\n",
    "\n",
    "    if show:\n",
    "        fig = plt.figure(figsize=(21,5))\n",
    "\n",
    "        ax = fig.add_subplot(141)\n",
    "        imshow(imtensor, title=img_name, pause=False)\n",
    "\n",
    "        ax = fig.add_subplot(142)\n",
    "        ax.set(title='cored')\n",
    "        im1 = ax.imshow(final_output[0], cmap=colormap, vmin=0., vmax=1.)\n",
    "        divider = make_axes_locatable(ax)\n",
    "        cax = divider.append_axes(\"right\", size=\"5%\", pad=0.1)\n",
    "        plt.colorbar(im1, cax=cax, ticks=[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9])\n",
    "\n",
    "        ax = fig.add_subplot(143)\n",
    "        ax.set(title='Diffuse')\n",
    "        im2 = ax.imshow(final_output[1], cmap=colormap, vmin=0., vmax=1.)\n",
    "        divider = make_axes_locatable(ax)\n",
    "        cax = divider.append_axes(\"right\", size=\"5%\", pad=0.1)\n",
    "        plt.colorbar(im2, cax=cax, ticks=[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9])\n",
    "\n",
    "        ax = fig.add_subplot(144)\n",
    "        ax.set(title='CAA')\n",
    "        im3 = ax.imshow(final_output[2], cmap=colormap, vmin=0., vmax=1.)\n",
    "        divider = make_axes_locatable(ax)\n",
    "        cax = divider.append_axes(\"right\", size=\"5%\", pad=0.1)\n",
    "        plt.colorbar(im3, cax=cax, ticks=[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9])\n",
    "        \n",
    "        plt.pause(0.01)\n",
    "    \n",
    "        if path:\n",
    "            fig.savefig(path+'ablations.jpg')\n",
    "    \n",
    "    if path:\n",
    "        plt.imsave(path+'ablation_size-{}_cored.png'.format(block_size), \n",
    "                   final_output[0], cmap=colormap, vmin=0., vmax=1.)\n",
    "        plt.imsave(path+'ablation_size-{}_diffuse.png'.format(block_size), \n",
    "                   final_output[1], cmap=colormap, vmin=0., vmax=1.)\n",
    "        plt.imsave(path+'ablation_size-{}_caa.png'.format(block_size), \n",
    "                   final_output[2], cmap=colormap, vmin=0., vmax=1.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done\n"
     ]
    }
   ],
   "source": [
    "for img in image_list:\n",
    "    if img is np.nan:\n",
    "        continue\n",
    "        \n",
    "    wsi_name = img.split('/')[0]\n",
    "    source_name = ''.join(img.split('/')[-1].split('.jpg'))\n",
    "    img_name = wsi_name+'/'+source_name+'.jpg'\n",
    "    save_path = SAVE+DIR+'{}/'.format(source_name)\n",
    "    if not os.path.isdir(save_path):\n",
    "        os.makedirs(save_path)\n",
    "    \n",
    "    single_prediction(img_name)\n",
    "\n",
    "    block_sizes = [2, 4, 8, 16, 32, 64]\n",
    "\n",
    "    for block_size in block_sizes:\n",
    "        final_output = occluded_predictions(img_name, block_size)\n",
    "        np.save(save_path+'ablation_size-{}'.format(block_size), final_output)\n",
    "        \n",
    "        show_heatmaps(img_name, block_size, final_output, path=save_path)\n",
    "        \n",
    "print('done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}