Synset Corpus.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied (use --upgrade to upgrade): zss in /home/mseal/.pylocal/lib/python2.7/site-packages\n",
      "Requirement already satisfied (use --upgrade to upgrade): nltk in /home/mseal/.pylocal/lib/python2.7/site-packages\n",
      "Requirement already satisfied (use --upgrade to upgrade): six in /home/mseal/.pylocal/lib/python2.7/site-packages (from zss)\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "if [[ ! -d \"stanford-corenlp-full-2015-12-09\" ]]; then\n",
    "    wget http://nlp.stanford.edu/software/stanford-corenlp-full-2015-12-09.zip\n",
    "    unzip stanford-corenlp-full-2015-12-09.zip\n",
    "fi\n",
    "\n",
    "if [[ ! -d \"stanford_corenlp_pywrapper\" ]]; then\n",
    "    git clone https://github.com/brendano/stanford_corenlp_pywrapper\n",
    "    cd stanford_corenlp_pywrapper\n",
    "    pip install .\n",
    "    cd ..\n",
    "fi\n",
    "\n",
    "pip install zss nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "from nltk.corpus import stopwords, wordnet as wn\n",
    "from zss import simple_distance, Node\n",
    "# Java version >= \"1.8\"\n",
    "from stanford_corenlp_pywrapper import CoreNLP\n",
    "\n",
    "# Setup our jvm parser\n",
    "proc = CoreNLP('parse', corenlp_jars=['stanford-corenlp-full-2015-12-09/*'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{u'char_offsets': [[0, 3],\n",
       "   [4, 6],\n",
       "   [7, 12],\n",
       "   [13, 20],\n",
       "   [21, 25],\n",
       "   [26, 32],\n",
       "   [32, 33]],\n",
       "  u'deps_basic': [[u'root', -1, 4],\n",
       "   [u'det', 3, 2],\n",
       "   [u'advmod', 4, 0],\n",
       "   [u'aux', 4, 1],\n",
       "   [u'nsubj', 4, 3],\n",
       "   [u'advmod', 4, 5],\n",
       "   [u'punct', 4, 6]],\n",
       "  u'deps_cc': [[u'root', -1, 4],\n",
       "   [u'det', 3, 2],\n",
       "   [u'advmod', 4, 0],\n",
       "   [u'aux', 4, 1],\n",
       "   [u'nsubj', 4, 3],\n",
       "   [u'advmod', 4, 5],\n",
       "   [u'punct', 4, 6]],\n",
       "  u'lemmas': [u'how', u'do', u'these', u'parser', u'work', u'anyway', u'?'],\n",
       "  u'parse': u'(ROOT (SBARQ (WHADVP (WRB How)) (SQ (VBP do) (NP (DT these) (NNS parsers)) (VP (VBP work) (ADVP (RB anyway)))) (. ?)))',\n",
       "  u'pos': [u'WRB', u'VBP', u'DT', u'NNS', u'VBP', u'RB', u'.'],\n",
       "  u'tokens': [u'How', u'do', u'these', u'parsers', u'work', u'anyway', u'?']}]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "proc.parse_doc(\"How do these parsers work anyway?\")['sentences']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def convert_tag(tag):\n",
    "    '''\n",
    "    Simplify many tags to four basic types: noun, verb, adjective, adverb\n",
    "    '''\n",
    "    if tag in ['NN', 'NNP', 'NNS', 'NNPS']:\n",
    "        return 'NN'\n",
    "    if tag in ['VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ']:\n",
    "        return 'VB'\n",
    "    if tag in ['RB', 'RBR', 'RBS']:\n",
    "        return 'RB'\n",
    "    if tag in ['JJ', 'JJR', 'JJS']:\n",
    "        return 'JJ'\n",
    "    return tag\n",
    "\n",
    "def pos_dep_tree(parsed):\n",
    "    '''\n",
    "    Convert a parse from Stanford dependency parser into a directional pos/relation zss tree\n",
    "    '''\n",
    "    root = Node('root')\n",
    "    pos_nodes = { -1: root }\n",
    "    for i, pos in enumerate(parsed['pos']):\n",
    "        pos_nodes[i] = Node(convert_tag(parsed['pos'][i]))\n",
    "    \n",
    "    for dependency_edge in parsed['deps_cc']:\n",
    "        relation, src_index, dest_index = dependency_edge\n",
    "        relation_node = Node(relation)\n",
    "        # NNS -> advmod -> VBP\n",
    "        pos_nodes[src_index].addkid(relation_node)\n",
    "        relation_node.addkid(pos_nodes[dest_index])\n",
    "    return root\n",
    "\n",
    "def sent_dep_tree(sent):\n",
    "    return pos_dep_tree(proc.parse_doc(sent)['sentences'][0])\n",
    "\n",
    "def dep_tree_similarity(dep1, dep2, smoothing=4.0):\n",
    "    return smoothing / (smoothing + simple_distance(dep1, dep2))\n",
    "\n",
    "def sentence_similarity(sent1, sent2, smoothing=4.0):\n",
    "    return dep_tree_similarity(sent_dep_tree(sent1), sent_dep_tree(sent2), smoothing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 13, 11)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p1 = proc.parse_doc(\"How do these parsers work anyway?\")['sentences'][0]\n",
    "p2 = proc.parse_doc(\"How do these parsers work?\")['sentences'][0]\n",
    "p3 = proc.parse_doc(\"Some wild changed thing\")['sentences'][0]\n",
    "\n",
    "(simple_distance(pos_dep_tree(p1), pos_dep_tree(p2)),\n",
    " simple_distance(pos_dep_tree(p1), pos_dep_tree(p3)),\n",
    " simple_distance(pos_dep_tree(p2), pos_dep_tree(p3)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1.0, 0.6666666666666666, 0.23529411764705882)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(sentence_similarity(\"How do these parsers work anyway?\", \"How do these parsers work anyway?\"),\n",
    " sentence_similarity(\"How do these parsers work anyway?\", \"How do these parsers work?\"),\n",
    " sentence_similarity(\"How do these parsers work anyway?\", \"Some wild changed thing\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def sentence_match_scores(sent, potential_sents, smoothing=4.0):\n",
    "    tree = sent_dep_tree(sent)\n",
    "    sent_trees = zip(potential_sents, map(sent_dep_tree, potential_sents))\n",
    "    for check_sent, check_tree in sent_trees:\n",
    "        yield check_sent, dep_tree_similarity(tree, check_tree, smoothing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('How do these parsers work anyway?', 1.0),\n",
       " ('How do these parsers work?', 0.6666666666666666),\n",
       " ('Some wild changed thing', 0.23529411764705882)]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(sentence_match_scores(\"How do these parsers work anyway?\",\n",
    "    [\"How do these parsers work anyway?\", \"How do these parsers work?\", \"Some wild changed thing\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from operator import itemgetter\n",
    "\n",
    "STOP_WORDS = set(stopwords.words('english'))\n",
    "\n",
    "def sentence_synset_expansion(sent):\n",
    "    for word in sent.split():\n",
    "        if word not in STOP_WORDS:\n",
    "            # Take first example as a simplification\n",
    "            yield ((syn.name(), syn.examples()[0])\n",
    "                    for syn in wn.synsets(word) if syn.examples())\n",
    "                \n",
    "def sentence_synset_matches(sent):\n",
    "    for syn_named_examples in map(tuple, sentence_synset_expansion(sent)):\n",
    "        if not syn_named_examples:\n",
    "            continue\n",
    "        names, examples = zip(*syn_named_examples)\n",
    "        # Very slow, we're repeating a lot of work on each synset example sentence\n",
    "        scores = map(itemgetter(1), sentence_match_scores(sent, examples))\n",
    "        total = sum(scores)\n",
    "        for name, score in zip(names, scores):\n",
    "            yield name, score / total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'parser.n.01', 1.0),\n",
       " (u'work.n.01', 0.02404719589263042),\n",
       " (u'work.n.02', 0.01827586887839912),\n",
       " (u'employment.n.02', 0.035145901689229084),\n",
       " (u'study.n.02', 0.02404719589263042),\n",
       " (u'work.n.05', 0.028556045122498628),\n",
       " (u'workplace.n.01', 0.030459781463998536),\n",
       " (u'oeuvre.n.01', 0.028556045122498628),\n",
       " (u'work.v.01', 0.02404719589263042),\n",
       " (u'work.v.02', 0.057112090244997256),\n",
       " (u'work.v.03', 0.01986507486782513),\n",
       " (u'function.v.01', 0.01986507486782513),\n",
       " (u'work.v.05', 0.028556045122498628),\n",
       " (u'exercise.v.03', 0.03263548013999843),\n",
       " (u'make.v.36', 0.022844836097998904),\n",
       " (u'work.v.08', 0.030459781463998536),\n",
       " (u'work.v.09', 0.022844836097998904),\n",
       " (u'work.v.10', 0.028556045122498628),\n",
       " (u'bring.v.03', 0.03263548013999843),\n",
       " (u'work.v.12', 0.04153606563272528),\n",
       " (u'cultivate.v.02', 0.035145901689229084),\n",
       " (u'work.v.14', 0.035145901689229084),\n",
       " (u'influence.v.01', 0.02538315121999878),\n",
       " (u'work.v.16', 0.035145901689229084),\n",
       " (u'work.v.17', 0.03263548013999843),\n",
       " (u'work.v.18', 0.01986507486782513),\n",
       " (u'work.v.19', 0.030459781463998536),\n",
       " (u'shape.v.02', 0.035145901689229084),\n",
       " (u'work.v.21', 0.030459781463998536),\n",
       " (u'knead.v.01', 0.028556045122498628),\n",
       " (u'exploit.v.01', 0.028556045122498628),\n",
       " (u'solve.v.01', 0.04153606563272528),\n",
       " (u'ferment.v.03', 0.01692210081333252),\n",
       " (u'sour.v.01', 0.03807472682999817),\n",
       " (u'work.v.27', 0.01692210081333252)]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(sentence_synset_matches(\"How do these parsers work anyway?\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "\n",
    "def synset_counts(sentences):\n",
    "    counts = defaultdict(float)\n",
    "    for sent in sentences:\n",
    "        for name, score in sentence_synset_matches(sent):\n",
    "            counts[name] += score\n",
    "    return counts\n",
    "\n",
    "def top_k(counter, k):\n",
    "    return sorted(counter.items(), key=itemgetter(1), reverse=True)[:k]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'parser.n.01', 3.0),\n",
       " (u'some.a.01', 0.24174121293230966),\n",
       " (u'some.s.04', 0.2197647390293724),\n",
       " (u'approximately.r.01', 0.2014510107769247),\n",
       " (u'some.s.02', 0.18595477917869974),\n",
       " (u'changed.s.03', 0.15857623202002766),\n",
       " (u'some.s.03', 0.15108825808269352),\n",
       " (u'thing.n.03', 0.13052873953703442),\n",
       " (u'changed.a.01', 0.11532816874183831),\n",
       " (u'work.v.02', 0.11422418048999451),\n",
       " (u'thing.n.11', 0.10877394961419533),\n",
       " (u'matter.n.01', 0.10877394961419533),\n",
       " (u'thing.n.01', 0.1004067227207957),\n",
       " (u'wild.s.04', 0.09847637083584403),\n",
       " (u'thing.n.04', 0.09323481395502457),\n",
       " (u'switch.v.03', 0.0906149897257301),\n",
       " (u'thing.n.07', 0.08701915969135626),\n",
       " (u'thing.n.10', 0.08701915969135626),\n",
       " (u'work.v.12', 0.08307213126545056),\n",
       " (u'solve.v.01', 0.08307213126545056)]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_k(synset_counts([\n",
    "    \"How do these parsers work anyway?\",\n",
    "    \"How do these parsers work anyway?\",\n",
    "    \"How do these parsers work?\",\n",
    "    \"Some wild changed thing\"\n",
    "]), 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'two.s.01', 1.0),\n",
       " (u'affectionate.s.01', 1.0),\n",
       " (u'affection.n.01', 1.0),\n",
       " (u'excellent.s.01', 1.0),\n",
       " (u'daughter.n.01', 1.0),\n",
       " (u'indistinct.a.01', 1.0),\n",
       " (u'beget.v.01', 0.6465923944749823),\n",
       " (u'mother.n.05', 0.5135599429266327),\n",
       " (u'consequence.n.03', 0.5034965034965034),\n",
       " (u'mother.v.01', 0.5013323252379033),\n",
       " (u'caress.v.01', 0.5),\n",
       " (u'caress.n.01', 0.5),\n",
       " (u'ago.s.01', 0.5),\n",
       " (u'ago.r.01', 0.5),\n",
       " (u'consequence.n.01', 0.49650349650349646),\n",
       " (u'mother.n.01', 0.49543429788216325),\n",
       " (u'indulgent.s.02', 0.33901988482016737),\n",
       " (u'sister.n.01', 0.33617021276595743),\n",
       " (u'sister.n.03', 0.33617021276595743),\n",
       " (u'indulgent.s.03', 0.33467347604042164)]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from nltk import Text, corpus\n",
    "\n",
    "'''\n",
    "Each of these sentence parse calls are slow\n",
    "We can cache and optimize these queries to get solid performance improvements\n",
    "And we can split these sentences into separate parser instances and paralellize the work\n",
    "'''\n",
    "emma = map(lambda s: ' '.join(s), Text(corpus.gutenberg.sents('austen-emma.txt')))[4:6]\n",
    "top_k(synset_counts(emma), 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# If we load sc into our notebook we can now parallelize this without changing our code:\n",
    "\n",
    "def load_sentences(pdf_name):\n",
    "    # Assume this file has been downloaded -- we could pull from s3 here first or load from HDFS\n",
    "    txt_name = pdf_file.replace('.pdf', '.txt')\n",
    "    pdftotext = Popen(['pdftotext', pdf_name, txt_name], shell=False, stdout=PIPE)\n",
    "    pdftotext.wait()\n",
    "    \n",
    "    tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')\n",
    "    with open(txt_name) as txt_file:\n",
    "        return tokenizer.tokenize(txt_file.read())\n",
    "    \n",
    "# Collect some small identifier and pass into the spark distributed store\n",
    "pdf_names = [f for f in os.listdir('./budgets/') if '.pdf' in f.lower()]\n",
    "pdf_names_rdd = sc.parallelize(pdf_names)\n",
    "\n",
    "# Do the transformation from file name into rows of sentences\n",
    "pdf_sent_rdd = pdf_names_rdd.flatMap(load_sentences)\n",
    "pdf_all_sense_rdd = pdf_sent_rdd.flatMap(lambda s: synset_counts(s).items())\n",
    "\n",
    "# Aggregate each distributed count of synsets\n",
    "adder = lambda a, b: a + b\n",
    "pdf_sense_rdd = pdf_all_sense_rdd.aggregateByKey(0, adder, adder)\n",
    "pdf_sense_rdd.take(5)\n",
    "'''\n",
    "(sense_1, global_count),\n",
    "(sense_2, global_count),\n",
    "(sense_3, global_count),\n",
    "(sense_4, global_count),\n",
    "(sense_5, global_count)\n",
    "'''"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}