2_Visualisation_Topic_Modelling_Tutorial.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Tutorial 2: Visualising with Topic Modelling using Latent Dirichlet Allocation (LDA)\n",
    "#author: Dr Mahmoud El-Haj (with help from the Internet) as part of the \"Visualise My Corpus Tutorial\" an event by Lanacaster University's UCREL and DSG Seminars\n",
    "#GitHub repository: https://github.com/drelhaj/NLP_ML_Visualization_Tutorial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "#The line above is used to catch warnings and errors (careful this will hide all cell's output)\n",
    "\n",
    "#This part 2 of the 6 parts tutorial shows you how to topic modelling! \n",
    "#Our data-set is a list of talks and abstracts from the CCC conference https://gitlab.com/maxigas/cccongresstalks/\n",
    "#The data is available as .csv file format, we'll play with the data to show what topics are being focused on \n",
    "#    using Latent Dirichlet Allocation (LDA).\n",
    "# In natural language processing, the latent Dirichlet allocation (LDA) is a generative statistical model that allows sets of observations to be explained by unobserved groups that explain why some parts of the data are similar. \n",
    "\n",
    "#this is to avoid showing warning, comment out otherwise.\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import numpy as np # adding support for large, multi-dimensional arrays and matrices, along with a large collection of high-level mathematical functions to operate on these arrays.\n",
    "import pandas as pd # needed for data processing and reading CSV files I/O (e.g. pd.read_csv)\n",
    "\n",
    "import os\n",
    "import matplotlib.pyplot as plt #plotting library\n",
    "import seaborn as sns#data visualization library\n",
    "#import mglearn #Helper functions for the book 'Introduction to machine learning with Python'\n",
    "from sklearn.feature_extraction.text import CountVectorizer #CountVectorizer used to transform a given text into a vector on the basis of the frequency (count) of each word that occurs in the entire text."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Reading the 2016.csv CCC talks, which is stored as a CSV file in the csvs directory.\n",
    "#notice the delimiter is not a comma, check your files first.\n",
    "#Our data-set is a list of talks and abstracts from the CCC conference https://gitlab.com/maxigas/cccongresstalks/\n",
    "\n",
    "File2016_df = pd.read_csv(\"csvs/2016.csv\", delimiter='|', header=0)\n",
    "print('Number of titles: {:,}\\n'.format(File2016_df.shape[0]))\n",
    "#print a sample of 5 random rows.\n",
    "File2016_df.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#reading titles (title is a column in the csv file as shown in the sample above)\n",
    "\n",
    "#loop through the titles and store them in a dictionary.\n",
    "title_dict = {}\n",
    "\n",
    "# loop through titles\n",
    "for i in range(len(File2016_df)):\n",
    "    if File2016_df[\"title\"][i] in title_dict.keys():\n",
    "        title_dict[File2016_df[\"title\"][i]] += 1\n",
    "    else:\n",
    "        title_dict.setdefault(File2016_df[\"title\"][i], 1)\n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#just to show you the dictionary items, you don't need to print it\n",
    "for i in title_dict:\n",
    "    print(i, ',', title_dict[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#we do the same thing for the abstract column (I will only only use the title_dict in this tutorial)\n",
    "#Try this again later using the abstract_dict\n",
    "\n",
    "abstract_dict = {}\n",
    "\n",
    "# classify that the article has recommends\n",
    "for i in range(len(File2016_df)):\n",
    "    if File2016_df[\"abstract\"][i] in abstract_dict.keys():\n",
    "        abstract_dict[File2016_df[\"abstract\"][i]] += 1\n",
    "    else:\n",
    "        abstract_dict.setdefault(File2016_df[\"abstract\"][i], 1)\n",
    "\n",
    "abstract_dict = [x for x in abstract_dict if str(x) != 'nan']#some talks have no abstracts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#spacy tokenizer\n",
    "#a spacy method to extract clean tokens from text (noticed I added some extra delimiters to the tokenizer)\n",
    "import spacy\n",
    "spacy.load('en_core_web_sm')\n",
    "from spacy.lang.en import English\n",
    "parser = English()\n",
    "def tokenize(text):\n",
    "    all_tokens = []\n",
    "    tokens = parser(text)\n",
    "    for token in tokens:\n",
    "        if token.orth_.isspace():\n",
    "            continue\n",
    "        elif token.like_url:#checks if a token resembels a URL\n",
    "            all_tokens.append('URL')\n",
    "        elif token.orth_.startswith('@'):#needed when mentions are used to anonymise users\n",
    "            all_tokens.append('SCREEN_NAME')\n",
    "        else:\n",
    "            all_tokens.append(token.lower_)\n",
    "    return all_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "#NLTK’s Wordnet used to find the meanings of words, synonyms, antonyms, and more. \n",
    "#I import English and German stop words lists since the talks contain German text, especially in the earlier conferences.\n",
    "\n",
    "import nltk\n",
    "nltk.download('wordnet')#you may turn this one off if you've already downloaded the wordnet\n",
    "from nltk.corpus import wordnet as wn\n",
    "def get_lemma(word):\n",
    "    lemma = wn.morphy(word)\n",
    "    if lemma is None:\n",
    "        return word\n",
    "    else:\n",
    "        return lemma\n",
    "    \n",
    "nltk.download('stopwords')#download the stopword lists from NLTK. Can be turned off if already downloaded\n",
    "en_stop = set(nltk.corpus.stopwords.words('english'))\n",
    "de_stop = set(nltk.corpus.stopwords.words('german'))\n",
    "\n",
    "#preprocess text by removing stop words and keeping words with more than 4 letters (my choice you can change it)\n",
    "def preprocess_text(text):\n",
    "    tokens = tokenize(text)\n",
    "    tokens = [token for token in tokens if len(token) >= 3]\n",
    "    tokens = [token for token in tokens if token not in en_stop or token in de_stop]\n",
    "    tokens = [get_lemma(token) for token in tokens]\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#here is an example on how the preprocessing works. \n",
    "sentenceTest = 'He and She study, studied, they also work and worked, they went to swim and swam, gone, goes, ran, run, visualizing, visualize, organizing, organizing'\n",
    "\n",
    "print(preprocess_text(sentenceTest))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#I convert the dictionary into a list with clean pre-processed text\n",
    "title_text_data = []\n",
    "print(type(title_text_data))\n",
    "counter = 0\n",
    "for title in title_dict:\n",
    "    counter = counter + 1\n",
    "    title_tokens = preprocess_text(title.replace('attackz', 'attacks').replace('hackz', 'hack').replace('securityz', 'security'))#I do some replacing for German-style words!\n",
    "    if (counter < 11):#this is just to print the first 10 lines\n",
    "        print(title_tokens)\n",
    "    title_text_data.append(title_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#The same is done to the abstracts dictionary (again I'm only using the talks titles in this tutorial but here just to show you how to prepare the abstracts as well (check the csv file if confused))\n",
    "abstract_text_data = []\n",
    "counter2 = 0\n",
    "for abstract in abstract_dict:\n",
    "    counter2 = counter2 + 1\n",
    "    abstract_tokens = preprocess_text(abstract.replace('attackz', 'attacks').replace('hackz', 'hack').replace('securityz', 'security'))\n",
    "    if (counter2 < 3):\n",
    "        print(abstract_tokens)\n",
    "    abstract_text_data.append(abstract_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#gensim for titles\n",
    "#what is a Gensim? The next important object you need to familiarize yourself with in order to work in gensim is the Corpus (a Bag of Words). \n",
    "#That is, it is a corpus object that contains the word id and its frequency in each document. \n",
    "#You can think of it as gensim's equivalent of a Document-Term matrix\n",
    "#The main function is doc2bow , which converts a collection of words to its bag-of-words representation: a list of (word_id, word_frequency) 2-tuples\n",
    "from gensim import corpora\n",
    "title_dictionary = corpora.Dictionary(title_text_data)\n",
    "title_corpus = [title_dictionary.doc2bow(text) for text in title_text_data]\n",
    "\n",
    "import pickle\n",
    "#Pickle is needed to serialize gensim into a stream of bytes to be stored as a file. Serialization refers to the process of converting an object in memory to a byte stream that can be stored on disk or sent over a network.\n",
    "#gensim and pkl files are stored in the 'models' directory. \n",
    "#we save those models to load them later when we plot the LDA model\n",
    "pickle.dump(title_corpus, open('models/title_corpus.pkl', 'wb'))\n",
    "title_dictionary.save('models/title_dictionary.gensim')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#gensim for abstracts\n",
    "from gensim import corpora\n",
    "abstract_dictionary = corpora.Dictionary(abstract_text_data)\n",
    "abstract_corpus = [abstract_dictionary.doc2bow(text) for text in abstract_text_data]\n",
    "\n",
    "import pickle\n",
    "pickle.dump(abstract_corpus, open('models/abstract_corpus.pkl', 'wb'))\n",
    "abstract_dictionary.save('models/abstract_dictionary.gensim')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#title LDA topics (I went with 5 topics and 10 passes, you can update that to show more topics or have several passes --> Number of passes through the corpus during training.)\n",
    "# probability distribution (Jensen–Shannon divergence, mesures similarity between two probability distributions https://en.wikipedia.org/wiki/Jensen%E2%80%93Shannon_divergence)\n",
    "# Discover abstract “topics” that occur in a collection of documents (why some parts of the data are similar)\n",
    "# What are the topics they are talking about in the conference talks.\n",
    "\n",
    "#to run this on the abstracts you need to replace the title_corpus and the title_dictionary with the abstract ones from the preivous cell\n",
    "import gensim\n",
    "NUM_TOPICS = 5\n",
    "ldamodel = gensim.models.ldamodel.LdaModel(title_corpus, num_topics = NUM_TOPICS, id2word=title_dictionary, passes=10)\n",
    "ldamodel.save('models/title_model.gensim')\n",
    "topics = ldamodel.print_topics(num_words=5)\n",
    "for topic in topics:\n",
    "    print(topic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Try a new sentence on networking security, \n",
    "#LDA will show that topic 3 has the highest probability assigned, and topic 4 has the second highest probability assigned. \n",
    "#The printed probabilities should add up to 1.\n",
    "new_doc = 'Our new technique offers a full spectrum of data privacy and security services and technology'\n",
    "new_doc = preprocess_text(new_doc)\n",
    "new_doc_bow = title_dictionary.doc2bow(new_doc)\n",
    "print(ldamodel.get_document_topics(new_doc_bow))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#display and plot title topics\n",
    "#to understand the model: http://bl.ocks.org/AlessandraSozzi/raw/ce1ace56e4aed6f2d614ae2243aab5a5/\n",
    "#and https://www.youtube.com/watch?v=IksL96ls4o0&feature=emb_title&ab_channel=statgraphics\n",
    "#Saliency: how important regardless of the term frequency (i.e. some terms are frequent but not informative)\n",
    "#Paper and more details: http://vis.stanford.edu/files/2012-Termite-AVI.pdf\n",
    "#Relevance: how relevant is a term to a certain topic\n",
    "#Paper and more details: https://nlp.stanford.edu/events/illvi2014/papers/sievert-illvi2014.pdf\n",
    "#The size of the bubble measures the importance of the topics, relative to the data.\n",
    "#more weight on lambda shows terms that are relevant to the topic. Less weight shows terms that are most frequent in that topic (beware of stop-words)\n",
    "\n",
    "#may struggle with numpy >=1.20.x\n",
    "dictionary = gensim.corpora.Dictionary.load('models/title_dictionary.gensim')\n",
    "corpus = pickle.load(open('models/title_corpus.pkl', 'rb'))\n",
    "lda = gensim.models.ldamodel.LdaModel.load('models/title_model.gensim')\n",
    "import pyLDAvis.gensim\n",
    "pyLDAvis.enable_notebook()#this to make it run with Jupyter notebook, not needed if using python IDE\n",
    "lda_display = pyLDAvis.gensim.prepare(lda, corpus, dictionary)\n",
    "pyLDAvis.save_html(lda_display, './plots/titles_lda.html')#save the LDA visualisation to the models directory for you to use later.\n",
    "pyLDAvis.display(lda_display, template_type='notebook')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "import gensim\n",
    "import warnings\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "#lda = gensim.models.ldamodel.LdaModel.load('models/title_model.gensim')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fiz=plt.figure(figsize=(15,30))\n",
    "for i in range(5):\n",
    "    df=pd.DataFrame(lda.show_topic(i), columns=['term','prob']).set_index('term')\n",
    "    #df=df.sort_values('prob')# uncomment to sort in ascending order\n",
    "    \n",
    "    plt.subplot(5,2,i+1)\n",
    "    plt.title('topic '+str(i+1))\n",
    "    sns.barplot(x='prob', y=df.index, data=df, palette='Reds_d')#'Greens_d', Blues_d\n",
    "    plt.xlabel('probability')\n",
    "    \n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#***********Try it yourself*******\n",
    "\n",
    "#so now try to replicate the LDA topics and visualisation process on the abstracts. I've already created the pkl and gensim files for you.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}