4_Visualisation_Machine_Learning_Tutorial.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Tutorial 4: Visualising with classical machine learning algorithms! \n",
    "#author: Dr Mahmoud El-Haj (with help from the Internet) as part of the \"Visualise My Corpus Tutorial\" an event by Lanacaster University's UCREL and DSG Seminars\n",
    "#GitHub repository: https://github.com/drelhaj/NLP_ML_Visualization_Tutorial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "#The task is to classify talks and abstracts into their years (or year spans). \n",
    "# For example a talk titled \"Systems security\" belongs to year 1999 or to the 1990s span.\n",
    "#We go a step forward by showing you how to create noun-clouds and verb-clouds using SpaCy.\n",
    "#Our data-set is a list of talks and abstracts from the CCC conference https://gitlab.com/maxigas/cccongresstalks/\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import mglearn\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Input data files are available in the \"../csvs/\" directory.\n",
    "#Our data-set is a list of talks and abstracts from the CCC conference https://gitlab.com/maxigas/cccongresstalks/\n",
    "\n",
    "directory = \"csvs/\"\n",
    "combinedFile = \"csvs/combined_csv.csv\"\n",
    "\n",
    "combinedSpanFile = \"csvs/combined_span_csv.csv\"\n",
    "\n",
    "# delete csv combined-spans if it exists so it doesn't duplicate its contents (I later explain what this file is)\n",
    "if os.path.exists(combinedSpanFile):\n",
    "    os.remove(combinedSpanFile)\n",
    "    \n",
    "# delete csv combined if it exists so it doesn't duplicate its contents (I later explain what this file is)\n",
    "if os.path.exists(combinedFile):\n",
    "    os.remove(combinedFile)\n",
    "\n",
    "#The following loops through the CSV files and shows how many talks (rows) are there in each year.\n",
    "for filename in os.listdir(directory):\n",
    "    if filename.endswith(\".csv\"): \n",
    "        print(filename)\n",
    "        df1 = pd.read_csv((directory+'/'+filename), delimiter='|', header=0, error_bad_lines=False)\n",
    "        print('Number of titles: {:,}\\n'.format(df1.shape[0]))\n",
    "        continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#The following loops through all the CSVs (per year) in the CSV file and creates a combined file with data from all csvs into one.\n",
    "import glob\n",
    "import pandas as pd\n",
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "\n",
    "# delete csv combined if it exists so it doesn't duplicate its contents\n",
    "if os.path.exists(combinedFile):\n",
    "    os.remove(combinedFile)\n",
    "    \n",
    "extension = 'csv'\n",
    "#notice that the code changes the delimiter from \"|\" to \",\" which is the norm.\n",
    "all_filenames = [f for f in listdir(directory) if isfile(join(directory, f))]\n",
    "#combine all files in the list\n",
    "combined_csv = pd.concat([pd.read_csv((directory+'/'+f), delimiter='|', error_bad_lines=False).replace({',': ' '}, regex=False) for f in all_filenames ])\n",
    "#export to csv\n",
    "combined_csv.to_csv(combinedFile, index=False,encoding='utf-8-sig')\n",
    "\n",
    "#read the newly created combined file and show a sample of 10 rows (notice teh year column)\n",
    "df_years = pd.read_csv(combinedFile, delimiter=',', header=0, error_bad_lines=False)\n",
    "print('Number of titles: {:,}\\n'.format(df_years.shape[0]))\n",
    "df_years.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#what the following code does is it plays with the years to make them look like a span of years.\n",
    "#The reason I do that to show you later that classifying text (titles or abstracts) into years will yield low accuracy results \n",
    "#   as there are 35 years of talks\n",
    "# What I will do is group the years into a span of 1980s, 1990s, 2000s and 2010s(this includes years from 2010 and later)\n",
    "import re\n",
    "\n",
    "combinedSpanFile = \"csvs/combined_span_csv.csv\"\n",
    "\n",
    "# delete csv combined if it exists so it doesn't duplicate its contents\n",
    "if os.path.exists(combinedSpanFile):\n",
    "    os.remove(combinedSpanFile)\n",
    "\n",
    "# here is a list of replace statements to convert years into year spans (i'm sure there is a better way to do this but this suffice :) \n",
    "# we save the updated year combined file to a new file called \"combined_span_csv.csv\".\n",
    "with open(combinedFile, \"rt\", encoding=\"utf8\") as fin:\n",
    "    with open(combinedSpanFile, \"wt\", encoding=\"utf8\") as fout:\n",
    "        for line in fin:\n",
    "            newString = re.sub(\"201\\d{1}\", \"2010\", line)\n",
    "            newString = re.sub(\"200\\d{1}\", \"2000\", newString)\n",
    "            newString = re.sub(\"198\\d{1}\", \"1980\", newString)\n",
    "            newString = re.sub(\"199\\d{1}\", \"1990\", newString)\n",
    "            fout.write(newString)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#here I print a sample of 10 rows from the combined_span_csv.csv file.\n",
    "combinedSpanFile = \"csvs/combined_span_csv.csv\"\n",
    "\n",
    "df_span = pd.read_csv(combinedSpanFile, delimiter=',', header=0, error_bad_lines=False)\n",
    "print('Number of titles: {:,}\\n'.format(df_span.shape[0]))\n",
    "df_span.sample(10)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This line changes the way the machine learning works\n",
    "#keepnig df as df_span will classify text into 4 classes 1980s,1990s,2000s,2010s\n",
    "#changing it to df_years will classify text into 35 classes which will of course result in lower scores.\n",
    "df = df_span\n",
    "\n",
    "#print(os.getcwd())\n",
    "plots = \"plots\"\n",
    "\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "#thsi prints how many samples are tehre in the combined (or span) file.\n",
    "df.head()\n",
    "print('Number of titles: {:,}\\n'.format(df.shape[0]))\n",
    "\n",
    "#we are only intersted in classifying the abstracts as they have more text than the titles. To change that replace abstract with title all over.\n",
    "from io import StringIO\n",
    "col = ['year', 'abstract'] #those are the two columns we are interested in bascially (label: Year, Text: abstract)\n",
    "df = df[col]\n",
    "#make sure we have no null abstracts to avoid any errors\n",
    "df = df[pd.notnull(df['abstract'])]\n",
    "df.columns = ['year', 'abstract']\n",
    "\n",
    "#create teh categories (labels) from the year column\n",
    "df['category_id'] = df['year'].factorize()[0]\n",
    "category_id_df = df[['year', 'category_id']].drop_duplicates().sort_values('category_id')\n",
    "category_to_id = dict(category_id_df.values)\n",
    "print(type(category_to_id))\n",
    "id_to_category = dict(category_id_df[['category_id', 'year']].values)\n",
    "df.head()\n",
    "\n",
    "#I'm only showing you the Support Vector Machines (SVM) and Naive Bayes (NB) classifiers, \n",
    "#The implementation for Logistic Regression (LR) is in there you just need to add \"LR\" to the array below.\n",
    "modelsArray = [\"SVM\",\"NB\",\"LR\"]\n",
    "for w in range(len(modelsArray)):\n",
    "    model_type = modelsArray[w]\n",
    "    import numpy as np\n",
    "    from sklearn.model_selection import train_test_split\n",
    "    from sklearn.feature_extraction.text import CountVectorizer\n",
    "    from sklearn.feature_extraction.text import TfidfVectorizer \n",
    "    from sklearn.feature_extraction.text import TfidfTransformer\n",
    "    from sklearn.naive_bayes import MultinomialNB\n",
    "    from nltk import pos_tag\n",
    "    from nltk.corpus import stopwords\n",
    "    from nltk.stem import WordNetLemmatizer\n",
    "    from sklearn import svm\n",
    "    from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "    #prepare the training and testing dataset\n",
    "    #This randomly splits our data into training and testing, here we choose to go with 0.30 (30%) for testing and the rest for training\n",
    "    #X in our case is the text (abstract), Y is the labels (years)\n",
    "    X_train, X_test, y_train, y_test = train_test_split(df['abstract'], df['year'],random_state = 1, test_size=0.30)\n",
    "    \n",
    "    from nltk.corpus import stopwords\n",
    "        \n",
    "    #we use a bag of words approach, here we go with 1 ngram, if you want bigrams (2,2), unigrams and bigrams (1,2)...etc\n",
    "    count_vect = CountVectorizer(analyzer='word', ngram_range=(1, 1))#ngram size, default 1,1, default word ngrams\n",
    "    \n",
    "    #you can use a tf, idf vectorizer which should do better than word frequency, I also show you how to remove stop words.\n",
    "    #count_vect = TfidfVectorizer(analyzer='word', ngram_range=(1, 3), min_df=0, stop_words = stopwords.words(\"english\"), sublinear_tf=True)#ngram size, default 1,1, default word ngrams\n",
    "    \n",
    "    count_vect.fit(X_train)    \n",
    "    #transforming data to be ready for analysis and machine learning\n",
    "    #handling missing data, remove string formatting, convert categorical data to numerical ....etc\n",
    "    X_train_tfidf = count_vect.transform(X_train)\n",
    "    X_train_tfidf = count_vect.fit_transform(X_train)\n",
    "    X_test_tfidf = count_vect.transform(X_test)\n",
    "    \n",
    "    #algorithms setup, you can change the C value for SVM\n",
    "    if model_type==\"SVM\":\n",
    "        clf = svm.SVC(C=1.0, kernel='linear', degree=3, gamma='auto')\n",
    "    if model_type==\"NB\":\n",
    "        clf = MultinomialNB()\n",
    "    if model_type==\"LR\":\n",
    "        clf = LogisticRegression(random_state=0, solver='lbfgs',multi_class='multinomial', max_iter=4000)\n",
    "\n",
    "    #train the model\n",
    "    train_model=clf.fit(X_train_tfidf, y_train)\n",
    "    #predicting years for testing data\n",
    "    test_accuracy=train_model.predict(X_test_tfidf)\n",
    "    #print training and testnig accuracy\n",
    "    print(\"Training/Testing Accuracy\" , '\\t' , model_type , '\\t' , train_model.score(X_train_tfidf, y_train) , '\\t' , train_model.score(X_test_tfidf, y_test))\n",
    "\n",
    "    #plot confusion matrices\n",
    "    import seaborn as sns\n",
    "    from sklearn.metrics import confusion_matrix\n",
    "    \n",
    "    conf_mat = confusion_matrix(y_test, test_accuracy)\n",
    "    fig, ax = plt.subplots(figsize=(9,9))\n",
    "    sns.heatmap(conf_mat, annot=True, fmt='d',cmap=\"RdBu_r\",\n",
    "                xticklabels=category_id_df.year.values, yticklabels=category_id_df.year.values)\n",
    "    plt.ylabel('Actual')\n",
    "    plt.xlabel('Predicted')\n",
    "    #plt.show(block=False)\n",
    "    pltFileName = plots+'/'+'combined'+'_'+model_type+'.pdf';\n",
    "    plt.savefig(pltFileName)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}