1_Visualisation_SpaCy_Tutorial.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Tutorial 1: Visualising Text with SpaCy\n",
    "#author: Dr Mahmoud El-Haj (with help from the Internet) as part of the \"Visualise My Corpus Tutorial\" an event by Lanacaster University's UCREL and DSG Seminars\n",
    "#GitHub repository: https://github.com/drelhaj/NLP_ML_Visualization_Tutorial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#We'll use SpaCy, a python package with libraries needed to analyse and anotate text.\n",
    "#http://spacy.io"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#installing spaCy\n",
    "#https://spacy.io/usage\n",
    "#!pip install -U spacy\n",
    "#!pip install -U spacy-lookups-data\n",
    "#!python -m spacy download en_core_web_sm\n",
    "#!python -m spacy download de_core_news_sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# SpaCy Tokenizer Construction\n",
    "# If running the following returns with ModuleNotFoundError then you need to download the language needed (see cell above)\n",
    "from spacy.tokenizer import Tokenizer\n",
    "\n",
    "import spacy    \n",
    "\n",
    "nlp = spacy.load(\"en_core_web_sm\") #loading language model. Use de_core_news_sm for German for e.g..\n",
    "\n",
    "#otherwise you can use import of spacy.load()\n",
    "    #import en_core_web_sm\n",
    "    #nlp = en_core_web_sm.load()\n",
    "\n",
    "# Create a blank Tokenizer with just the English vocab\n",
    "tokenizer = Tokenizer(nlp.vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence = \"Today is   March 18th 2021 and   Mahmoud, is showing us   how to visualise text online at Lancaster University.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens = tokenizer(sentence) #' '.join(sentence.split()) needed to avoid excess whitespaces. \n",
    "# notice that a punctuation such as ، (a comma in Arabic), is considered a token!\n",
    "print('Number of words: ',len(tokens))\n",
    "print('\\n>>>>>>>Tokens<<<<<<<:')\n",
    "for t in tokens:\n",
    "    print(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "sentence = ' '.join(sentence.split())#remove extra white spaces\n",
    "sentence = re.sub(r'[^\\w\\s]','',sentence)#use regex to remove puncatuations\n",
    "\n",
    "tokens = tokenizer(sentence) #we call the tokenizer again over the cleaned sentence\n",
    "\n",
    "print('Numer of words: ',len(tokens))\n",
    "print('\\n>>>>>>>Tokens<<<<<<<:')\n",
    "for t in tokens:\n",
    "    print(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#what about stop-words?\n",
    "#SpaCy's English language stop words (for other languages see: https://spacy.io/usage/models)\n",
    "from spacy.lang.en.stop_words import STOP_WORDS\n",
    "\n",
    "print(STOP_WORDS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Let's get tokens ignoring stop-words and punctuations (remember we used regex to remove puncations).\n",
    "tokens_no_stopwords = [token.text for token in tokens if token.is_stop != True and token.is_punct != True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Notice that the puncation disappears in the 2nd output as well as the new stop-word 'text'\n",
    "print('With stop-words\\n',*tokens, '>>>>>>',len(tokens), 'words.')\n",
    "print('Without stop-words\\n',*tokens_no_stopwords, '>>>>>>', len(tokens_no_stopwords), 'words.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#what if we want to add/remove to/from the default stop-words list?\n",
    "#assume the word 'text' is very frequent in our corpus to an extent that it becomes a stop-word\n",
    "#to add 'text' to the stop words list:\n",
    "nlp.Defaults.stop_words.add(\"text\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print the list, notice 'text' is now an entry\n",
    "#to remove a word from the list use: nlp.Defaults.stop_words.remove(\"word_to_be_removed\")\n",
    "print(STOP_WORDS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "tokenizer = Tokenizer(nlp.vocab)#recreating the tokenizer as the previous one used the unupdated stopwords list\n",
    "tokens = tokenizer(sentence)\n",
    "#loop through the tokens and only consider non-stop-words and non-punctuations.\n",
    "tokens_no_stopwords = [token.text for token in tokens if token.is_stop != True and token.is_punct != True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Without updated-stop-words\\n',*tokens_no_stopwords, '>>>>>>', len(tokens_no_stopwords), 'words.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Linguistic annotations (Part of speech tags and dependencies using the Universal Dependecies https://universaldependencies.org) \n",
    "#This will return a Language object containing all components and data needed to process text\n",
    "doc = nlp(sentence) #A Doc is a sequence of Token https://spacy.io/api/doc\n",
    "for token in doc:\n",
    "    print(token.text, token.pos_, token.dep_)\n",
    "    \n",
    "    \n",
    "'''nsubj: nominal subject.\\t nummod: numeric modifier ...etc. For more visit: https://universaldependencies.org'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Let's visualise the annotated sentence above\n",
    "\n",
    "from spacy import displacy\n",
    "\n",
    "#nlp = spacy.load(\"en_core_web_sm\") #uncomment if not loaded previously\n",
    "#doc = nlp(sentence)# check previous cell. That is the original cleaned sentence (only extra spaces and puncations were removed)\n",
    "displacy.render(doc, style=\"dep\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#Can we make it look a bit cooler? (for more options https://spacy.io/api/top-level#displacy_options)\n",
    "options = {\"compact\": True, \"bg\": \"#ebc334\",\n",
    "           \"color\": \"black\", \"font\": \"Source Sans Pro\"}\n",
    "displacy.render(doc, style=\"dep\", options=options,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#to save in Scalable Vector Graphics (SVG) so you can view it in full screen:\n",
    "from pathlib import Path\n",
    "svg = displacy.render(doc, style=\"dep\", options=options,jupyter=False)\n",
    "\n",
    "output_path = Path(\"./plots/dependency_plot.svg\")\n",
    "output_path.open(\"w\", encoding=\"utf-8\").write(svg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#what about named entities (NER)?\n",
    "\n",
    "for ent in doc.ents:\n",
    "    print('[',ent.text,']', 'from',ent.start_char,'to', ent.end_char,'[', ent.label_,']')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#can we visualise named entities? Well, of course! :-)\n",
    "displacy.render(doc, style=\"ent\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#to save in Scalable Vector Graphics (SVG) so you can view it in full screen:\n",
    "from pathlib import Path\n",
    "html = displacy.render(doc, style=\"ent\",jupyter=False)#withouth jupyter = False you'll get a \n",
    "\n",
    "output_path = Path(\"./plots/ner_plot.html\")\n",
    "output_path.open(\"w\", encoding=\"utf-8\").write(html)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}