1.1) Preprocessing - Reinhard Normalization and WSI Tiling.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.1 Preprocessing - Reinhard Normalization and WSI Tiling\n",
    "\n",
    "As a first preprocessing step, all slides were color normalized with respect to a reference image selected by an expert neuropathologist. Color normalization was performed using the method described by [Reinhard et. al](https://ieeexplore.ieee.org/document/946629).\n",
    "\n",
    "The resulting color normalized whole slide images were tiled using PyVips to generate 1536 x 1536 images patches."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "import numpy as np\n",
    "import cv2\n",
    "import matplotlib.pyplot as plt\n",
    "import pyvips as Vips\n",
    "from tqdm import tqdm\n",
    "\n",
    "from utils import vips_utils, normalize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "TRAIN_WSI_DIR = 'data/Dataset 1a Development_train/'              # WSIs in the training set\n",
    "VAL_WSI_DIR = 'data/Dataset 1b Development_validation/'           # WSIs in the validation set\n",
    "\n",
    "SAVE_DIR = 'data/norm_tiles/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists(SAVE_DIR):\n",
    "        os.makedirs(SAVE_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "ref_imagename = 'NA5002_2AB.svs'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "wsi_train = os.listdir(TRAIN_WSI_DIR)\n",
    "wsi_val = os.listdir(VAL_WSI_DIR)\n",
    "\n",
    "imagenames = sorted(wsi_val + wsi_train)\n",
    "imagenames.remove('NA5005-02_AB.svs')             # this WSI was digitalized at 40x, need resize down to 20x\n",
    "imagenames.append('NA5005-02_AB.svs')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 57min 9s, sys: 3min 15s, total: 1h 24s\n",
      "Wall time: 4min 52s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Load reference image, fit Reinhard normalizer\n",
    "ref_image = Vips.Image.new_from_file(TRAIN_WSI_DIR + ref_imagename, level=0)\n",
    "\n",
    "normalizer = normalize.Reinhard()\n",
    "normalizer.fit(ref_image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 32/32 [2:57:12<00:00, 332.26s/it]  \n"
     ]
    }
   ],
   "source": [
    "stats_dict = {}\n",
    "for imagename in tqdm(imagenames[:-1]):\n",
    "    try:\n",
    "        vips_img = Vips.Image.new_from_file(TRAIN_WSI_DIR + imagename, level=0)\n",
    "    except:\n",
    "        vips_img = Vips.Image.new_from_file(VAL_WSI_DIR + imagename, level=0)\n",
    "    out = normalizer.transform(vips_img)\n",
    "    out.filename = vips_img.filename\n",
    "    vips_utils.save_and_tile(out, SAVE_DIR)\n",
    "    stats_dict[imagename] = normalizer.image_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [29:59<00:00, 1799.83s/it]\n"
     ]
    }
   ],
   "source": [
    "# Resize the single 40x image down to 20x\n",
    "for imagename in tqdm(imagenames[-1:]):\n",
    "    vips_img = Vips.Image.new_from_file(TRAIN_WSI_DIR + imagename, level=0)\n",
    "    vips_img = vips_img.resize(0.5)\n",
    "    out = normalizer.transform(vips_img)\n",
    "    out.filename = vips_img.filename\n",
    "    vips_utils.save_and_tile(out, SAVE_DIR)\n",
    "    stats_dict[imagename] = normalizer.image_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "stats = pd.DataFrame(stats_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "stats = stats.transpose()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "stats.columns = 'means', 'stds'"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}