work/4-hacker-news-dashboard.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 9,
        "hidden": true,
        "row": 0,
        "width": 12
       },
       "report_default": {}
      }
     }
    }
   },
   "source": [
    "## Widgets And Dashboards\n",
    "**Goal:** This notebook will demonstrate how we can use code output from Apache Toree and integrate into some front end application.\n",
    "\n",
    "We are going to use the [Jupyter Declarative Widgets](https://github.com/jupyter-incubator/declarativewidgets) and [Jupyter Dashboards](https://github.com/jupyter-incubator/dashboards) extensions to build a dynamic dashboard to display our data.  This dashboard will display the top 10 hacker news stories. The user will then be able to \"drill down\" by clicking on the story. This will then show the comments, word counts, and links from the comments.\n",
    "\n",
    "This demonstration does not go into detail about deploying dashboards. However, if users are interested in how acheive such results please refer to the [Jupyter Dashboards Server](https://github.com/jupyter-incubator/dashboards_server) and the [Jupyter Dashboard Bundlers](https://github.com/jupyter-incubator/dashboards_bundlers) extensions.\n",
    "\n",
    "Most of the code stays the same. We will just need to add markup for the display and some wrapper functions to get the data in our widgets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 12,
        "hidden": true,
        "row": 0,
        "width": 4
       },
       "report_default": {}
      }
     }
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%adddeps org.jsoup jsoup 1.9.2 --transitive\n",
    "%adddeps com.github.seratch hackernews4s_2.10 0.6.0 --transitive\n",
    "%addjar http://localhost:8888/nbextensions/declarativewidgets/declarativewidgets.jar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {}
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "val sqlC = sqlContext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {}
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "import hackernews4s.v0._\n",
    "import sqlC.implicits._\n",
    "import org.apache.spark.sql.functions._\n",
    "import org.apache.spark.sql.Row\n",
    "import org.jsoup.Jsoup\n",
    "import org.jsoup.nodes.Document\n",
    "import org.jsoup.nodes.Element\n",
    "import scala.collection.JavaConversions._\n",
    "import org.apache.spark.ml.feature.{HashingTF, IDF, Tokenizer}\n",
    "import org.apache.spark.ml.feature.StopWordsRemover\n",
    "import org.apache.spark.ml.feature.{CountVectorizer, CountVectorizerModel}\n",
    "import org.apache.spark.sql.DataFrame\n",
    "import org.apache.spark.rdd.RDD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {}
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "case class Comment(story: Long, id: Long, text: String)\n",
    "\n",
    "val tokenizer = new Tokenizer().setInputCol(\"_1\").setOutputCol(\"words\")\n",
    "val remover = new StopWordsRemover().setInputCol(\"words\").setOutputCol(\"filteredWords\")\n",
    "\n",
    "// A function to transform an item into a tuple of that item and a list of comments on that item\n",
    "val getComments: (Item) => Seq[Comment] = (story: Item) => {\n",
    "    def _getComments:  (Item) => Seq[Comment] = (item: Item) => {\n",
    "        val commentIds = item.commentIds\n",
    "        if(commentIds.size == 0){\n",
    "            Seq(Comment(story.id.id, item.id.id, item.text))\n",
    "        } else {\n",
    "            val comments: Seq[Comment] = commentIds.flatMap((itemId: ItemId) => { \n",
    "                _getComments(HackerNews.getItem(itemId).get)\n",
    "            })\n",
    "            if(\"Story\".equals(item.itemType.toString)){\n",
    "                comments\n",
    "            } else {\n",
    "                Comment(story.id.id, item.id.id, item.text) +: comments\n",
    "            }\n",
    "            \n",
    "        }   \n",
    "    }\n",
    "    \n",
    "    _getComments(story)\n",
    "}\n",
    "\n",
    "val getItemText: (Comment) => String = (comment: Comment) => {\n",
    "    Jsoup.parse(comment.text).text()\n",
    "}\n",
    "val getItemLinks: (Comment) => Seq[String] = (comment: Comment) => {\n",
    "    val aTags: List[Element] = Jsoup.parse(comment.text).select(\"a\").toList\n",
    "    aTags.map((link: Element) => {\n",
    "        link.attr(\"href\")\n",
    "    })\n",
    "}\n",
    "\n",
    "def getStoryComments(storyId: Int) = {\n",
    "    val story = Seq(HackerNews.getItem(ItemId(storyId)).get)\n",
    "    sc.parallelize(story).flatMap((item: Item) => {\n",
    "        getComments(item)\n",
    "    })\n",
    "}   \n",
    "\n",
    "def getCommentLinks(comments: RDD[Comment]) = {\n",
    "    comments.flatMap((comment:Comment) => {\n",
    "        getItemLinks(comment)\n",
    "    })\n",
    "}\n",
    "\n",
    "def getCommentWordCounts(comments: RDD[Comment]) = {\n",
    "    val textDF = comments.map((comment:Comment) => {\n",
    "        getItemText(comment)\n",
    "    }).toDF\n",
    "    val tokenizedComments = tokenizer.transform(textDF)\n",
    "    val filteredWordCountsDF = remover.transform(tokenizedComments)\n",
    "    val terms = filteredWordCountsDF.flatMap((row: Row) =>{\n",
    "        row.getSeq[String](2)\n",
    "    })\n",
    "    val wordCounts = terms.map((word: String) => {\n",
    "        (word, 1)\n",
    "    }).reduceByKey(_+_)\n",
    "    wordCounts\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {}
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "case class Story(id: Long, title: String, url: String)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {}
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "def getTopNStories(n: Int=10) = {\n",
    "    val rdd = sc.parallelize(HackerNews.getTopStories(n))\n",
    "    rdd.cache()\n",
    "    val stories = rdd.map((item: Item) => {\n",
    "        Story(item.id.id, item.title.get, item.url.get)\n",
    "    })\n",
    "    stories\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 4,
        "hidden": true,
        "row": 0,
        "width": 12
       },
       "report_default": {}
      }
     }
    }
   },
   "source": [
    "## Adding Widgets\n",
    "Now we will begin integration the widget code into our demo. This begin by importing classes and initializing the library. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {}
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "import declarativewidgets._\n",
    "initWidgets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 4,
        "height": 4,
        "hidden": true,
        "row": 0,
        "width": 4
       },
       "report_default": {}
      }
     }
    }
   },
   "source": [
    "The [Jupyter Declarative Widget](https://github.com/jupyter-incubator/declarativewidgets) is built off of [Polymer webcomponents](https://www.polymer-project.org/1.0/) which allows us to incorporate any of the many components found online."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 8,
        "height": 4,
        "hidden": true,
        "row": 0,
        "width": 4
       },
       "report_default": {}
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "%%html\n",
    "<link rel=\"import\" href=\"urth_components/paper-card/paper-card.html\"\n",
    "    is=\"urth-core-import\" package=\"PolymerElements/paper-card\" >\n",
    "<link rel=\"import\" href=\"urth_components/paper-button/paper-button.html\"\n",
    "    is=\"urth-core-import\" package=\"PolymerElements/paper-button\" >\n",
    "\n",
    "<link rel=\"import\" href=\"urth_components/iron-list/iron-list.html\"\n",
    "    is=\"urth-core-import\" package=\"PolymerElements/iron-list\" >\n",
    "<link rel=\"import\" href=\"urth_components/paper-listbox/paper-listbox.html\"\n",
    "    is=\"urth-core-import\" package=\"PolymerElements/paper-listbox\" >\n",
    "<link rel=\"import\" href=\"urth_components/paper-item/paper-item.html\"\n",
    "    is=\"urth-core-import\" package=\"PolymerElements/paper-item\" >\n",
    "<link rel=\"import\" href=\"urth_components/juicy-html/juicy-html.html\"\n",
    "    is=\"urth-core-import\" package=\"Juicy/juicy-html\" >"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 4,
        "height": 4,
        "hidden": true,
        "row": 0,
        "width": 4
       },
       "report_default": {}
      }
     }
    }
   },
   "source": [
    "These are some wrapper functions for integrating with our widgets. They mainly handle serializing the values out to the widgets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {}
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "def storyComments(story: String): Seq[Map[String, String]] = {\n",
    "    getStoryComments(story.toInt).collect().map((comment: Comment) => {\n",
    "        Map[String, String](\"text\" -> comment.text)\n",
    "    })\n",
    "}\n",
    "\n",
    "def storyLinks(story: String) = {\n",
    "    getCommentLinks(getStoryComments(story.toInt)).collect()\n",
    "}\n",
    "\n",
    "def storyWords(story: String) = {\n",
    "    getCommentWordCounts(getStoryComments(story.toInt)).sortBy((wordCount: (String, Int)) => {\n",
    "        wordCount._2\n",
    "    }, ascending=false).take(50)\n",
    "}\n",
    "\n",
    "def getMoreInfo(story: String) = {\n",
    "    channel(\"default\").set(\"status\", \"Loading Comments\")\n",
    "    val comments = storyComments(story)\n",
    "    channel(\"default\").set(\"status\", \"Loading Links\")\n",
    "    val links = storyLinks(story)\n",
    "    channel(\"default\").set(\"status\", \"Loading Word Counts\")\n",
    "    val wordCounts = storyWords(story)\n",
    "    \n",
    "    channel(\"default\").set(\"comments\", comments)\n",
    "    channel(\"default\").set(\"links\", links)\n",
    "    channel(\"default\").set(\"words\", wordCounts)\n",
    "    channel(\"default\").set(\"status\", \"\")\n",
    "    true\n",
    "}\n",
    "\n",
    "def stories() = {\n",
    "    val stories = getTopNStories(10).collect().map((story: Story) => {\n",
    "        Map[String, String](\"id\"->story.id.toString, \"title\"->story.title, \"url\"->story.url)\n",
    "    })\n",
    "    channel(\"default\").set(\"stories\", stories)\n",
    "    stories\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 8,
        "height": 4,
        "hidden": true,
        "row": 0,
        "width": 4
       },
       "report_default": {}
      }
     }
    }
   },
   "source": [
    "This is the main \"UI\" for the application. A list of the Top 10 stories will be displayed. The user can then select the story to inspect. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 22,
        "hidden": false,
        "row": 0,
        "width": 12
       },
       "report_default": {}
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "%%html\n",
    "<style>\n",
    ".iron-selected {\n",
    "    background-color: #ddd;\n",
    "}\n",
    "</style>\n",
    "<template is=\"urth-core-bind\">\n",
    "    <urth-core-function auto ref=\"stories\"></urth-core-function>\n",
    "    <urth-core-function auto ref=\"getMoreInfo\" arg-story=\"{{selectedStoryId}}\"></urth-core-function>\n",
    "    <h1>Top 10 Hacker News Stories</h1>\n",
    "    <p>{{status}}</p>\n",
    "    <paper-listbox attr-for-selected=\"data-name\" selected=\"{{selectedStoryId}}\">\n",
    "        <template is=\"dom-repeat\" items=\"{{stories}}\">\n",
    "            <paper-item data-name$=\"{{item.id}}\">\n",
    "                <a href=\"{{item.url}}\">{{item.title}}</a>\n",
    "            </paper-item>\n",
    "        </template>\n",
    "    </paper-listbox>\n",
    "</template>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 11,
        "hidden": false,
        "row": 35,
        "width": 12
       },
       "report_default": {}
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "%%html\n",
    "<h2>Comments</h2>\n",
    "<template is=\"urth-core-bind\">\n",
    "    <iron-list style=\"height: 300px;\" target=\"document\" items=\"[[comments]]\">\n",
    "        <template>\n",
    "            <paper-card>\n",
    "                <div class=\"card-content\">\n",
    "                    <template is=\"juicy-html\" content$=\"{{item.text}}\"></template>\n",
    "                </div>\n",
    "            </paper-card>\n",
    "        </template>\n",
    "    </iron-list>\n",
    "</template>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 7,
        "height": 13,
        "hidden": false,
        "row": 22,
        "width": 5
       },
       "report_default": {}
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "%%html\n",
    "<h2>Word Counts</h2>\n",
    "<template is=\"urth-core-bind\">\n",
    "    <iron-list style=\"height: 300px;\" target=\"document\" items=\"[[words]]\">\n",
    "        <template>\n",
    "            <paper-card>\n",
    "                <div class=\"card-content\">\n",
    "                    <p>{{item}}</p>\n",
    "                </div>\n",
    "            </paper-card>\n",
    "        </template>\n",
    "    </iron-list>\n",
    "</template>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 13,
        "hidden": false,
        "row": 22,
        "width": 7
       },
       "report_default": {}
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "%%html\n",
    "<h2>Comment Links</h2>\n",
    "<template is=\"urth-core-bind\">\n",
    "    <iron-list style=\"height: 300px;\" target=\"document\" items=\"[[links]]\">\n",
    "        <template>\n",
    "            <paper-card>\n",
    "                <div class=\"card-content\">\n",
    "                    <a href=\"{{item}}\">{{item}}</a>\n",
    "                </div>\n",
    "            </paper-card>\n",
    "        </template>\n",
    "    </iron-list>\n",
    "</template>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {}
      }
     }
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "extensions": {
   "jupyter_dashboards": {
    "activeView": "grid_default",
    "version": 1,
    "views": {
     "grid_default": {
      "cellMargin": 10,
      "defaultCellHeight": 20,
      "maxColumns": 12,
      "name": "grid",
      "type": "grid"
     },
     "report_default": {
      "name": "report",
      "type": "report"
     }
    }
   }
  },
  "kernelspec": {
   "display_name": "Apache Toree - Scala",
   "language": "scala",
   "name": "apache_toree_scala"
  },
  "language_info": {
   "name": "scala"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}