explore_all_in_one.ipynb

{
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  },
  "name": ""
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "# \u4f7f\u7528\u8bf4\u660e\n",
      "\n",
      "\n",
      "## \u7b80\u4ecb\n",
      "\n",
      "\u5728\u4e4b\u524d\u7684\u300aAnki\u7cfb\u5217-\u7528Anki\u51c6\u5907GRE\u300b\u4e2d\uff0c\u6211\u627f\u8bfa\u63d0\u4f9b\u8f6c\u6362\u7528\u7684\u811a\u672c\u6587\u4ef6\u3002\u65e7\u7248\u672c\u7684\u811a\u672c\u4f7f\u7528\u8d77\u6765\u5f88\u9ebb\u70e6\uff0c\u6240\u4ee5\u91cd\u5199\u4e86\u4e00\u4efd\uff0c\u7cbe\u7b80\u4ee3\u7801\u7ed3\u6784\uff0c\u5e76\u4e14\u5c06\u5168\u6587\u4ee5markdown\u7684\u65b9\u5f0f\u5206\u4eab\u5230\u7b80\u4e66\uff0c\u65b9\u4fbf\u611f\u5174\u8da3\u7684\u4eba\u4e0e\u6211\u8ba8\u8bba\u3002\n",
      "\n",
      "\u8fd9\u4e2anotebook\u5c55\u793a\u4e86\u5904\u7406kindle\u7248\u300aGRE\u6838\u5fc3\u8bcd\u6c47\u8003\u6cd5\u7cbe\u6790\u300b\u3001\u300aGRE\u6838\u5fc3\u8bcd\u6c47\u52a9\u8bb0\u4e0e\u7cbe\u7ec3\u300b\u3001\u300aGRE\u9ad8\u5206\u5fc5\u5907\u77ed\u8bed\u642d\u914d\u300b\u7684\u8fc7\u7a0b\u3002\u76ee\u7684\u662f\u751f\u6210\u53ef\u4ee5\u5bfc\u5165Anki\u7684txt\u6587\u6863\u3002\u300aGRE\u6838\u5fc3\u8bcd\u6c47\u8003\u6cd5\u7cbe\u6790\u300b\u3001\u300aGRE\u6838\u5fc3\u8bcd\u6c47\u52a9\u8bb0\u4e0e\u7cbe\u7ec3\u300b\u751f\u6210\u7684\u5bfc\u5165\u6587\u4ef6\u5bf9\u5e94\u7684Note\u7ed3\u6784\u4e3aGreWord\uff0c\u300aGRE\u9ad8\u5206\u5fc5\u5907\u77ed\u8bed\u642d\u914d\u300b\u751f\u6210\u7684\u5219\u5bf9\u5e94PhraseGRE\u3002 \n",
      "\n",
      "\u9996\u5148\u4f60\u8981\u4eceAmazon.cn\u8d2d\u4e70\u8005\u4e09\u672c\u4e66\u7684\u7535\u5b50\u7248\uff08[1](http://www.amazon.cn/GRE/dp/B00GWD2L4W/)\u3001[2](http://www.amazon.cn/dp/B00NXCXBSK)\u3001[3](http://www.amazon.cn//dp/B00SMNMHDK/)\uff09\u3002\u4f60\u5e76\u4e0d\u9700\u8981\u62e5\u6709\u4e00\u53f0Kindle\u624d\u80fd\u8d2d\u4e70\u4e0a\u8ff0\u7535\u5b50\u4e66\u3002\u53ea\u9700\u8981\u5728\u7535\u8111\u4e0a\u4e0b\u8f7dKindle\u7684\u5ba2\u6237\u7aef\uff0c\u5b89\u88c5\u540e\u767b\u9646\u4f60\u7684Amazon\u8d26\u53f7\u5373\u53ef\u3002\u7136\u540e\uff0c\u4f60\u9700\u8981\u5229\u7528[Calibre](http://calibre-ebook.com/)\u5c06\u4e66\u7c4d\u8f6c\u6362\u4e3atxt\u683c\u5f0f\u4ee5\u4fbf\u8ba9python\u5904\u7406\u3002\u8f6c\u6362\u8fc7\u7a0b\u4e2d\uff0c\u9700\u8981\u5728\u201cTXT Output\u201d\u90a3\u91cc\u5c06Formatting\u8bbe\u7f6e\u4e3amarkdown\uff0c\u4ee5\u4fbf\u63d0\u53d6\u539f\u4e66\u4e2d\u7684\u4e00\u4e9b\u683c\u5f0f\u4fe1\u606f\uff1b\u201cLine ending style\u201d\u9009\u62e9unix\uff1b\u201cOutput Encoding\u201d\u9009\u62e9'utf-8'\u3002\n",
      "\n",
      "\u8f6c\u6362\u540e\uff0c\u9ed8\u8ba4\u6587\u6863\u540d\u5206\u522b\u4e3a\n",
      "\n",
      "    \"GREHe Xin Ci Hui Kao Fa Jing Xi  (Xin Dong Fang Da Yu Ying Yu Xue Xi Cong Shu ) - Chen Qi.txt\"\n",
      "    \"GREHe Xin Ci Hui Zhu Ji Yu Jing - Cao Tian Cheng.txt\"\n",
      "    \"GREGao Fen Bi Bei Duan Yu Da Pe - Yan Yu Zhen ,Gao Yu ,Chen Qi.txt\"\n",
      "    \n",
      "\u5047\u5b9a\u4f60\u6ca1\u6709\u4fee\u6539\u6587\u4ef6\u540d\uff0c\u5e76\u4e14\u6309\u7167\u76f8\u5bf9\u8def\u5f84\u5c06\u8fd93\u4e2atxt\u653e\u5230\u4e86\u4e0e\u8be5notebook\u76f8\u540c\u8def\u5f84\u7684\"base_data\"\u6587\u4ef6\u5939\u4e2d\u3002\u5982\u679c\u4f60\u5b89\u88c5\u4e86jupyter notebook\uff0c\u53ef\u4ee5\u6253\u5f00\u5e76\u8fd0\u884c\u8fd9\u4e2a.ipynb\u6587\u4ef6\u3002\u5b83\u4f1a\u81ea\u52a8\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e09\u4e2a`_base_d`\u6587\u4ef6\uff0c\u5bf9\u5e94\u4e09\u4e2atxt\u6e90\u6587\u4ef6\uff0c\u53ef\u4ee5\u88abAnkiImport\u811a\u672c\u8c03\u7528\u3002\u53e6\u5916\u8fd8\u4f1a\u751f\u6210\u4e09\u4e2apy\u811a\u672c\u6587\u4ef6\uff0c\u53ef\u4ee5\u72ec\u7acb\u4f7f\u7528\uff0c\u529f\u80fd\u90fd\u662f\u8bfb\u5165txt\u6e90\u6587\u4ef6\u5e76\u8f6c\u6362\uff0c\u751f\u6210\u5bf9\u5e94\u7684`_base_d`\u6587\u4ef6\u3002\n",
      "\n",
      "notebook\u5f00\u5934\u7684\u8f85\u52a9\u51fd\u6570\u90fd\u88ab\u5199\u5165\u4e86my_helpers.py\u6587\u4ef6\u3002\u81ea\u5b9a\u4e49\u7684magic command\u88ab\u5199\u5165sync_to_file_magic_command.py\u3002\u5176\u4ed6\u811a\u672c\u6216notebook\u53ef\u65b9\u4fbf\u7684\u8c03\u7528\u3002\n",
      "\n",
      "\u672cnotebook\u540e\u7eed\u4f1a\u7528\u4e09\u4e2a\u7ae0\u8282\u5206\u522b\u5904\u7406\u8fd9\u4e09\u4e2a\u6e90\u6587\u6863\u3002  \n",
      "\u7b2c\u4e00\u6b65\u5f53\u7136\u662f\u5c06\u6e90\u6587\u6863\u7684\u5185\u5bb9\u8bfb\u5165\u4e3a\u5b57\u7b26\u4e32\u3002  \n",
      "\u6bcf\u4e2a\u7ae0\u8282\uff0c\u4ee3\u7801\u7684\u6700\u7ec8\u76ee\u7684\u90fd\u662f\u5c06\u8bfb\u5165\u7684\u5b57\u7b26\u4e32\u4ee5\u5355\u8bcd\u91ca\u4e49\u4e3a\u5355\u4f4d\uff0c\u8f6c\u6362\u4e3apython\u4e2d\u7684\u5b57\u5178\u7ed3\u6784\u3002\u8fd9\u4e4b\u95f4\u9700\u8981\u5c06\u5b57\u7b26\u4e32\u4e00\u6b65\u6b65\u5207\u5206\uff08split\uff09\uff0c\u5254\u9664\u6389\u4e0d\u5fc5\u8981\u7684\u4fe1\u606f\u3002\u5207\u5206\u7684\u89c4\u5219\u901a\u8fc7\u89c2\u5bdftxt\u6587\u6863\u5e76\u5bfb\u627e\u89c4\u5f8b\u5f97\u6765\u3002 \n",
      "\n",
      "\n",
      "## \u52d8\u8bef\n",
      "\n",
      "\u5904\u7406\u8fc7\u7a0b\u4e2d\u53ef\u80fd\u4f1a\u53d1\u73b0\u6e90\u6587\u6863\u4e2d\u7684\u9519\u8bef\u3002\u53d1\u73b0\u65b9\u6cd5\u662f\u7a0b\u5e8f\u8f85\u52a9\u3001\u624b\u52a8\u5728txt\u6e90\u6587\u6863\u4e2d\u5b9a\u4f4d\u3002  \n",
      "\u81f3\u4e8e\u5904\u7406\u65b9\u6cd5\uff0c\u5c3d\u7ba1\u76f4\u63a5\u4fee\u6539txt\u6e90\u6587\u4ef6\u53ef\u80fd\u66f4\u65b9\u4fbf\uff0c\u672c\u6587\u4e3a\u4e86\u81ea\u52a8\u5316\u6240\u6709\u6d41\u7a0b\uff0c\u8fd8\u662f\u9009\u62e9\u4e86\u8d39\u4e8b\u4e9b\u7684\u65b9\u6cd5\uff0c\u5373\u624b\u5de5\u7f16\u5199\u89c4\u5219\uff0c\u8ba9\u7a0b\u5e8f\u81ea\u52a8\u5904\u7406\u5185\u5b58\u4e2d\u7684\u5bf9\u8c61\uff0c\u540c\u65f6\u4e5f\u53ef\u907f\u514d\u4fee\u6539txt\u6e90\u6587\u4ef6\u3002  \n",
      "\u6240\u6709\u6d89\u53ca\u5230\u52d8\u8bef\u7684\u5730\u65b9\uff0c\u90fd\u53ef\u901a\u8fc7\u641c\u7d22\u5173\u952e\u8bcd\u201c\u3010\u52d8\u8bef\u3011\u201d\u800c\u5b9a\u4f4d\u5230\u76f8\u5173\u7684\u6587\u5b57\u8bf4\u660e\uff0c\u641c\u7d22\u201crevise\u201d\u52a0\u4e0a\u6240\u4fee\u6539\u5355\u8bcd\u7684\u62fc\u5199\u53ef\u4ee5\u770b\u5230\u4ee3\u7801\u4e0a\u7684\u5b9e\u73b0\u3002\u6709\u65f6\u53d1\u73b0\u3010\u52d8\u8bef\u3011\u540e\u9700\u8981\u8fd4\u56de\u524d\u9762\u4fee\u6539\u4ee3\u7801\uff0c\u6240\u4ee5\u6587\u5b57\u8bf4\u660e\u548c\u4ee3\u7801\u53ef\u80fd\u6709\u9519\u4f4d\u3002\n",
      "\n",
      "## \u53d8\u91cf\u547d\u540d\u89c4\u5219\n",
      "\n",
      "\u4ee5\u6613\u4e8e\u7406\u89e3\u4e3a\u7b2c\u4e00\u76ee\u6807\u3002\u4e00\u4e2a\u53d8\u91cf\u901a\u5e38\u7531\u8868\u793a\u53d8\u91cf\u610f\u4e49\u7684\u4e3b\u4f53\u52a0\u4e0a\u8868\u793a\u53d8\u91cf\u7c7b\u522b\u7684\u540e\u7f00\u7ec4\u6210\u3002  \n",
      "\u540e\u7f00\u7528\u6765\u8868\u660e\u53d8\u91cf\u7684\u6570\u636e\u7c7b\u578b\uff0c\u5177\u4f53\u89c4\u5219\uff1a\n",
      "+ `_l\u3001_d` \u5206\u522b\u4ee3\u8868\u5217\u8868\u3001\u5b57\u5178\n",
      "+ `_str` \u4ee3\u8868\u5b57\u7b26\u4e32\u3002\u7531\u4e8e\u6d89\u53ca\u5230\u4e2d\u6587\u7684\u5904\u7406\uff0c\u4e00\u822c\u4f1a\u5c06\u5b57\u7b26\u4e32\u8f6c\u53d8\u4e3a\u5bf9\u5e94\u7684unicode\u5bf9\u8c61\uff0c\u800c\u540e\u8005\u5728\u64cd\u4f5c\u4e0a\u53c8\u4e0e\u4e00\u822c\u7684\u5b57\u7b26\u4e32\u6ca1\u6709\u533a\u522b\u3002\u6240\u4ee5\u4e5f\u7528\u8be5\u540e\u7f00\u4ee3\u8868unicode\u5bf9\u8c61\u3002\u53e6\u5916\uff0c\u4f7f\u7528`_str`\u53ef\u4ee5\u5f3a\u8c03\u8be5\u53d8\u91cf\u7684\u6587\u672c\u610f\u4e49\uff0c\u5c3d\u7ba1\u5728\u7a0b\u5e8f\u5185\u90e8\u53ef\u80fd\u662f\u4e00\u4e2aunicode\u3002\n",
      "+ `_uni` \u7279\u522b\u5f3a\u8c03\u8be5\u53d8\u91cf\u4ee3\u8868\u4e00\u4e2aunicode\u5bf9\u8c61\n",
      "+ `_re` \u8868\u793a\u7f16\u8bd1\u540e\u7684\u6b63\u5219\u8868\u8fbe\u5f0f\n",
      "+ `_fun` \u8868\u793a\u53d8\u91cf\u5185\u5b58\u653e\u7684\u662f\u4e00\u4e2a\u51fd\u6570\n",
      "+ `_boo` \u8868\u793a\u5e03\u5c14\u53d8\u91cf\n",
      "+ `_iter` \u6cdb\u6307\u4e00\u4e2a\u53ef\u4ee5\u987a\u5e8f\u904d\u5386\u7684\u5bf9\u8c61\n",
      "+ \u540e\u7f00\u53ef\u4ee5\u4e92\u76f8\u7ec4\u5408\uff0c\u8868\u793a\u5d4c\u5957\u7684\u6570\u636e\u7ed3\u6784\uff0c\u5de6\u4fa7\u540e\u7f00\u5d4c\u5957\u4e8e\u53f3\u4fa7\u540e\u7f00\u3002\u6bd4\u5982\u53cc\u91cd\u540e\u7f00`_str_l`\u8868\u793a\u4e00\u4e2a\u5b58\u653e\u5b57\u7b26\u4e32\u7684\u5217\u8868\uff0c`_l_l`\u8868\u793a\u4e00\u4e2a\u5d4c\u5957\u5217\u8868\uff0c\u5373\u4e00\u4e2a\u5217\u8868\u7684\u6bcf\u4e2a\u5143\u7d20\u672c\u8eab\u5373\u662f\u4e00\u4e2a\u5217\u8868\u3002\u50cf`_l_l`\u8fd9\u79cd\u540e\u7f00\uff0c\u610f\u4e49\u5728\u4e8e\u5f3a\u8c03\u8be5\u5982\u4f55\u4f7f\u7528\u53d8\u91cf\u3002\u5047\u5982\u4e0d\u5b58\u5728\u5f3a\u8c03\u7684\u76ee\u7684\uff0c\u5c31\u4f1a\u53ea\u4f7f\u7528`_l`\u8868\u793a\u4e00\u4e2a\u5d4c\u5957\u5217\u8868\uff0c\u4e5f\u53ef\u80fd\u4e0d\u4f7f\u7528\u540e\u7f00\u3002\n",
      "\n",
      "## \u51fd\u6570\u547d\u540d\u89c4\u5219\n",
      "\n",
      "\u66f4\u52a0\u7075\u6d3b\uff0c\u524d\u9762\u7684\u53d8\u91cf\u540e\u7f00\u4e5f\u53ef\u80fd\u7528\u4e8e\u51fd\u6570\u4e2d\uff0c\u4f46\u53ef\u80fd\u51fa\u73b0\u5728\u4efb\u610f\u4f4d\u7f6e\u3002\u4e3b\u8981\u76ee\u7684\u662f\u63d0\u793a\u51fd\u6570\u7684\u7528\u9014\u3001\u8fd4\u56de\u503c\u7684\u6570\u636e\u7c7b\u578b\u7b49\u3002  \n",
      "\u51fd\u6570\u7684\u53c2\u6570\u547d\u540d\u89c4\u5219\u4e0e\u53d8\u91cf\u547d\u540d\u89c4\u5219\u76f8\u540c\u3002\n",
      "\n",
      "## test, check, example\n",
      "\n",
      "+ test  \n",
      "\u4e00\u4e2a\u529f\u80fd\u5b9e\u73b0\u5b8c\u4e86\uff0c\u4e0d\u77e5\u9053\u80fd\u4e0d\u80fd\u7528\uff0c\u6240\u4ee5\u627e\u4e2a\u7b80\u5355\u7684\u4f8b\u5b50\u8bd5\u4e00\u4e0b\u3002test\u53ef\u4ee5\u5e2e\u52a9\u4fee\u590d\u4e00\u4e9b\u7b80\u5355\u7684bug\uff0c\u8fd1\u4f3c\u4e8e\u8349\u7a3f\u672c\u4e0a\u7684\u6f14\u7b97\u6b65\u9aa4\u3002\u56e0\u4e3a\u662f\u8349\u7a3f\uff0c\u6240\u4ee5\u786e\u8ba4\u65e0\u8bef\u540e\u4f1a\u6ce8\u91ca\u6389\uff0c\u8282\u7701\u7a7a\u95f4\u3002  \n",
      "\u8fd8\u6709\u4e00\u79cd\u60c5\u51b5\uff0c\u5199\u7a0b\u5e8f\u65f6\uff0c\u60f3\u77e5\u9053\u67d0\u4e2a\u53d8\u91cf\u7684\u5177\u4f53\u503c\u662f\u4ec0\u4e48\u3002\u4e00\u822c\u4f1a\u9009\u62e9`print`\u8fd9\u4e2a\u53d8\u91cf\u3002\u8fd9\u90e8\u5206\u4fe1\u606f\u53ef\u4ee5\u8f85\u52a9\u7a0b\u5e8f\u7f16\u5199\uff0c\u800c\u5199\u597d\u540e\u4e3a\u4e86\u8282\u7701\u7a7a\u95f4\uff0c\u4e00\u822c\u4f1a\u6ce8\u91ca\u6389`print`\u8bed\u53e5\u3002\u4e3a\u4e86\u65b9\u4fbf\u540e\u7eed\u7684\u68c0\u67e5\uff0c\u4ee5\u53ca\u672a\u6765\u53ef\u80fd\u7684\u4fee\u6539\uff0c\u53ea\u6ce8\u91ca\u5f15\u8d77\u7a0b\u5e8f\u8f93\u51fa\u7684\u8bed\u53e5\uff0c\u4fdd\u7559\u8d77\u6761\u4ef6\u5224\u65ad\u7684\u3002  \n",
      "\u603b\u4e4b\uff0c\u6ce8\u91catest\u8bed\u53e5\u7684\u539f\u5219\u662f\uff0c\u5c3d\u91cf\u51cf\u5c11\u672a\u6765\u9700\u8981\u518d\u6b21test\u65f6\u9700\u8981\u91cd\u5199\u7684\u4ee3\u7801\u6570\u91cf\u3002\n",
      "\n",
      "+ check  \n",
      "\u8c03\u7528\u4e86\u4e4b\u524d\u5b9e\u73b0\u7684\u67d0\u4e2a\u51fd\u6570\uff0c\u68c0\u67e5\u4e0b\u8fd4\u56de\u7ed3\u679c\u662f\u5426\u8fbe\u5230\u9884\u671f\u3002check\u53ef\u4ee5\u5e2e\u52a9\u53d1\u73b0\u4e00\u4e9b\u4e0d\u660e\u663e\u7684bug\uff0c\u540c\u65f6\u4fdd\u8bc1\u8fd4\u56de\u7ed3\u679c\u7684\u6b63\u786e\u6027\uff0c\u4ee5\u5907\u540e\u7eed\u4f7f\u7528\u3002  \n",
      "\u4e00\u822c\u5e0c\u671b\u5c55\u793acheck\u90e8\u5206\u7684\u8f93\u51fa\u7ed3\u679c\uff0c\u6240\u4ee5\u4e0d\u4f1a\u6ce8\u91ca\u6389\u76f8\u5173\u4ee3\u7801\u3002\n",
      "\n",
      "+ example  \n",
      "\u4e00\u4e2a\u529f\u80fd\u5b9e\u73b0\u5b8c\u4e86\uff0c\u786e\u4fe1\u6ca1\u95ee\u9898\uff0c\u4ee5\u793a\u4f8b\u5c55\u793a\u5982\u4f55\u4f7f\u7528\u6216\u8005\u67d0\u4e9b\u8f93\u5165\u503c\u7684\u8f93\u51fa\u3002"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "# Sync to file\n",
      "\n",
      "\u5199\u5b8c\u4e86\u624d\u53d1\u73b0\uff0c\u5df2\u7ecf\u6709\u4eba\u5199\u8fc7\u7c7b\u4f3c\u7684\u63d2\u4ef6\u4e86\u3002\u53c2\u89c1[writeandexecute.py](https://github.com/ipython-contrib/IPython-extensions/blob/master/ipyext/writeandexecute.py)\u3002\u4e0d\u8fc7\u529f\u80fd\u5404\u6709\u6240\u957f\u5427\uff0c\u529f\u592b\u4e5f\u4e0d\u7b97\u5b8c\u5168\u767d\u8d39\u3002\n",
      "\n",
      "## \u91cd\u65b0\u8bbe\u8ba1\u8f6e\u5b50\uff5e\n",
      "\n",
      "jupyter notebook\u5141\u8bb8\u4f60\u4ea4\u4e92\u5f0f\u7684\u7f16\u5199\u7a0b\u5e8f\uff0c\u5728\u4ee3\u7801\u4e4b\u95f4\u63d2\u5165markdown\uff0c\u7528\u66f4\u597d\u7684\u6392\u7248\u5448\u73b0\u5173\u4e8e\u4ee3\u7801\u7684\u89e3\u91ca\u3002\u4f46\u89e3\u91ca\u672c\u8eab\u4e0d\u662f\u76ee\u7684\uff0c\u91cd\u70b9\u5728\u4e8e\u90a3\u4e9b\u771f\u6b63\u505a\u4e8b\u7684\u4ee3\u7801\u3002\u6240\u4ee5\uff0c\u5f80\u5f80\u5e0c\u671b\u5728\u5c06\u6240\u6709cell\u987a\u5e8f\u6267\u884c\u4e00\u904d\u540e\uff0c\u67d0\u4e9b\u4ee3\u7801\u88ab\u7ed3\u5408\u5230\u4e00\u8d77\uff0c\u6784\u6210\u4e00\u4e2a\u5e72\u51c0\u7b80\u6d01\u7684\u811a\u672c\u3002\n",
      "\n",
      "ipython\u4e2d\uff0c%%writefile\u53ef\u4ee5\u5c06\u4ee3\u7801\u5199\u5165\u6587\u4ef6\uff0c\u4e0d\u8fc7\u6709\u4e2a\u7f3a\u9677\uff0c\u5373\u4e0d\u80fd\u5728\u5199\u5165\u6587\u4ef6\u7684\u540c\u65f6\uff0c\u6267\u884ccell\u4e2d\u7684\u4ee3\u7801\u3002\u901a\u8fc7\u9605\u8bfb[Defining custom magics](http://ipython.readthedocs.org/en/stable/config/custommagics.html)\uff0c\u5b66\u4e60\u5230\u5982\u4f55\u81ea\u5b9a\u4e49magic command\u6765\u5b9e\u73b0\u9700\u6c42\u3002\u4f46\u8fd9\u4e2a\u6587\u6863\u5e76\u6ca1\u6709\u8be6\u7ec6\u63cf\u8ff0ipython\u63d0\u4f9b\u7ed9magic command\u7684\u63a5\u53e3\u3002\u901a\u8fc7\u641c\u7d22\uff0c\u627e\u5230\u4e86ipython\u5728Github\u4e0a\u7684\u6e90\u7801\uff08\u5176\u5b9e\u672c\u5730\u4e5f\u6709\uff09\u3002\u5176\u4e2d\uff0c\u8ddf\u521a\u624d\u63d0\u5230\u7684\u9700\u6c42\u5bc6\u5207\u76f8\u5173\u7684\u76f8\u5173\u7684\u6709[magic.py](https://github.com/ipython/ipython/blob/master/IPython/core/magic.py)\u3001[magic_arguments.py](https://github.com/ipython/ipython/blob/master/IPython/core/magic_arguments.py)\u3002\u53e6\u5916\uff0c%%writefile\u7684\u5177\u4f53\u5b9e\u73b0\u5728[osm.py](https://github.com/ipython/ipython/blob/master/IPython/core/magics/osm.py)\u4e2d\u3002\n",
      "\n",
      "\u5199\u4ee3\u7801\u524d\uff0c\u5148\u89e3\u51b3\u4e24\u4e2a\u57fa\u672c\u95ee\u9898\u3002\n",
      "\n",
      "\u7b2c\u4e00\uff0c\u5982\u4f55\u5c06\u4ee3\u7801\u9012\u4ea4\u7ed9ipython\u6765\u6267\u884c\uff1f\n",
      "\u975e\u5e38\u7b80\u5355\u3002`self.shell.run_cell(cell)`\u3002\n",
      "\n",
      "\u7b2c\u4e8c\uff0c\u5e0c\u671b\u80fd\u540c\u6b65\u6587\u6863\uff0c\u800c\u4e0d\u662f\u7b80\u5355\u7684\u8986\u76d6\u3002\n",
      "\u4ee3\u7801\u4e0d\u5b58\u5728\u4e8e\u76ee\u6807\u6587\u6863\u4e2d\u65f6\uff0c\u5728\u6307\u5b9a\u4f4d\u7f6e\u5199\u5165\uff1b\u5b58\u5728\u4e8e\u76ee\u6807\u6587\u6863\u4e2d\u65f6\uff0c\u81ea\u52a8\u66f4\u65b0\u3002\u672c\u8d28\u4e0a\u5373mini\u7248\u7684git merge\u3002\u4e3a\u6b64\uff0c\u8003\u8651\u5982\u4e0b\u4e24\u70b9\u3002\n",
      "\n",
      "\u9996\u5148\uff0c\u5fc5\u987b\u5411magic command\u4f20\u9012\u4e00\u7cfb\u5217\u53c2\u6570\u3002python\u4e2d\uff0c\u5e93argparse\u53ef\u4ee5\u5f88\u597d\u7684\u89e3\u6790\u547d\u4ee4\u884c\u53c2\u6570\u3002\u6559\u7a0b[Argparse Tutorial](https://docs.python.org/2/howto/argparse.html)\u89e3\u91ca\u4e86\u7b80\u5355\u7684\u8bed\u6cd5\u3002ipython\u57fa\u4e8eargparse\uff0c\u5b9a\u4e49\u4e86\u81ea\u5df1\u5904\u7406magic command\u53c2\u6570\u7684\u6a21\u5757\uff0c\u5373[magic_arguments.py](https://github.com/ipython/ipython/blob/master/IPython/core/magic_arguments.py)\u3002\u8fd9\u4e2a\u811a\u672c\u540c\u65f6\u63d0\u4f9b\u4e86\u7b80\u5355\u7684\u4f7f\u7528\u6307\u5357\u3002\u57fa\u672c\u8bed\u6cd5\u5373\u901a\u8fc7[\u4fee\u9970\u5668](http://coolshell.cn/articles/11265.html)\u6765\u8ffd\u52a0arg\u3002\n",
      "\n",
      "\u63a5\u4e0b\u6765\uff0c\u8be5\u5982\u4f55\u5b9e\u73b0\u6240\u8c13\u7684\u540c\u6b65\u5462\uff1f\u5177\u4f53\u6765\u8bf4\uff0c\u5982\u4f55\u8bbe\u8ba1\u53c2\u6570\uff0c\u4ee5\u53ca\u5982\u4f55\u5b9e\u73b0\u53c2\u6570\u6240\u4ee3\u8868\u7684\u529f\u80fd\uff1f\u57fa\u672c\u60f3\u6cd5\u5982\u4e0b\u3002\n",
      "\n",
      "* \u5c06\u76ee\u6807\u6587\u6863\u8bfb\u5165\u4e3a\u5b57\u7b26\u4e32\uff0c\u540e\u79f0\u76ee\u6807\u5b57\u7b26\u4e32\u3002  \n",
      "\n",
      "* \u5148\u786e\u5b9a\u5728\u76ee\u6807\u5b57\u7b26\u4e32\u7684\u54ea\u4e2a\u8303\u56f4\u5185\u5339\u914dcell\uff0c\u5373\u4e24\u4e2a\u53d8\u91cf\uff0csearch_start_index\uff0csearch_end_index\u3002\n",
      "\u9ed8\u8ba4\u503c\u5206\u522b\u4e3a0\u548clen(target_str)\uff0c\u5373\u539f\u5b57\u7b26\u4e32\u7684\u5f00\u59cb\u548c\u7ed3\u675f\u4f4d\u7f6e\u3002\u4e24\u4e2a\u53d8\u91cf\u4e4b\u95f4\u7684\u8303\u56f4\u79f0\u4e4b\u4e3asearch scope\u3002\n",
      "\n",
      "* option \"-a --after\"\uff0c\u4ee5\u53ca\"-b --before\"\uff0c\u8fdb\u4e00\u6b65\u786e\u5b9a\u4e0a\u8ff0\u53d8\u91cf\u3002\u4e24\u4e2a\u9009\u9879\u5404\u63a5\u53d7\u4e00\u4e2a\u5b57\u7b26\u4e32\uff0c\u7528\u6765\u6784\u5efa\u4e24\u4e2a\u6b63\u5219\u8868\u8fbe\u5f0f\u3002\n",
      "\u540e\u6587\u4ee5\u53c2\u6570\u540d\u6765\u4ee3\u8868\u76f8\u5e94\u7684\u5339\u914d\u6a21\u5f0f\u3002\u5982\u679c-r\uff08\u89c1\u540e\u6587\uff09\u4e0d\u88ab\u6307\u5b9a\uff0c\u5b57\u7b26\u4e32\u4e2d\u7684\u5143\u5b57\u7b26\u4f1a\u88ab\u8f6c\u4e49\uff0c\u5373\u5f53\u4f5c\u666e\u901a\u5b57\u7b26\u5bf9\u5f85\u3002\n",
      "\u5f53-a -b\u540c\u65f6\u88ab\u6307\u5b9a\u65f6\uff0c\u5728\u76ee\u6807\u5b57\u7b26\u4e32\u4e2d\u5bfb\u627e\u4e24\u4e2a\u6a21\u5f0f\u4e4b\u95f4\u7684\u8303\u56f4\uff0c\u5373-a\u7684end\u4f4d\u7f6e\u548c-b\u7684start\u4f4d\u7f6e\uff0c\u4fdd\u5b58\u4e3a\u4e00\u7ec4index\uff0ctype\u4e3aint\u3002\u5982\u679c\u5b58\u5728\u591a\u7ec4index\uff0c\u53d6\u7b2c\u4e00\u7ec4\u3002\u5982\u679c\u4e0d\u80fd\u540c\u65f6\u5339\u914d-a\u548c-b\uff0c\u5219\u5148\u5339\u914d-b\uff0c\u4e0d\u6210\u529f\u518d\u5339\u914d-a\uff0c\u5982\u679c\u4ecd\u4e0d\u6210\u529f\uff0c\u4e24\u4e2a\u53d8\u91cf\u53d6\u9ed8\u8ba4\u503c\u3002\n",
      "\u5f53\u53ea\u6709-a\u6216\u76f8\u5f53\u4e8e\u53ea\u6709-a\u88ab\u6307\u5b9a\u65f6\uff0c\u5c06-a\u7684\u7b2c\u4e00\u4e2a\u6210\u529f\u5339\u914d\u7684end position\u8d4b\u7ed9`search_start_index`\u3002\u5982\u679c\u6ca1\u6709\u6210\u529f\u5339\u914d\uff0c\u6309\u7167\u9ed8\u8ba4\u5904\u7406\u3002\n",
      "\u5f53\u53ea\u6709-b\u6216\u76f8\u5f53\u4e8e\u53ea\u6709-b\u88ab\u6307\u5b9a\u65f6\uff0c\u5c06-b\u7684\u7b2c\u4e00\u4e2a\u6210\u529f\u5339\u914d\u7684start position\u8d4b\u7ed9`search_end_index`\u3002\u5982\u679c\u6ca1\u6709\u6210\u529f\u5339\u914d\uff0c\u6309\u7167\u9ed8\u8ba4\u5904\u7406\u3002\n",
      "\n",
      "\n",
      "* option \"-m --mode\"\uff0c\u6307\u660e\u5199\u5165\u6a21\u5f0f\uff0c\u53ef\u9009\u503c[i, a, di, da, o]\u3002  \n",
      "i: insert\uff0c\u628acell\u7684\u5185\u5bb9\u6dfb\u52a0\u5230search scope\u7684\u5f00\u5934\u3002  \n",
      "a: append\uff0c\u628acell\u7684\u5185\u5bb9\u6dfb\u52a0\u5230search scope\u7684\u7ed3\u5c3e\u3002  \n",
      "di: different and insert\uff0c\u5373cell\u4f5c\u4e3a\u6574\u4f53\u4e0d\u80fd\u5728\u76ee\u6807\u5b57\u7b26\u4e32\u4e2d\u5339\u914d\u540e\uff0c\u518dinsert\u3002\u5426\u5219\u7ef4\u6301\u539f\u6837\u3002  \n",
      "da\uff1adiffernt and append\u3002\u7c7b\u4f3cdi\u3002\n",
      "o: overwrite\u3002\u7528cell\u4e2d\u7684\u5185\u5bb9\u8986\u76d6\u6574\u4e2asearch scope\u3002\n",
      "\n",
      "* option \"-t --test\"\uff0c\u6d4b\u8bd5\u6a21\u5f0f\u3002  \n",
      "\u4e0d\u5199\u5165cell\u3002\u901a\u5e38\u4e0e-l\u4f7f\u7528\u3002\n",
      "\n",
      "* option \"-p --pass\"\uff0c\u51b3\u5b9a\u662f\u5426\u8fd0\u884ccell\u4e2d\u7684\u4ee3\u7801\u3002  \n",
      "\u9ed8\u8ba4\u5148\u8fd0\u884c\u4ee3\u7801\uff0c\u518d\u6267\u884c\u5199\u5165\u6587\u4ef6\u7684\u64cd\u4f5c\u3002\n",
      "\u5982\u679c\u6307\u5b9a-p\uff0c\u5219\u8df3\u8fc7\u8fd0\u884c\uff0c\u53ea\u5199\u5165\u3002\n",
      "\n",
      "* option \"-r --reg\"\uff0c\u6307\u660e-a -b\u7684\u5b57\u7b26\u4e32\u662f\u6b63\u5219\u8868\u8fbe\u5f0f\u3002\u63a5\u53d7\u4e00\u4e2a\u5b57\u7b26\u4e32\u53c2\u6570\uff0c\u4f5c\u4e3a\u6b63\u5219\u8868\u8fbe\u5f0f\u7684\u9009\u9879\u3002\u6bd4\u5982\uff0c`reU|re.I`\u3002  \n",
      "\u6ce8\u610f\uff0c\u7a0b\u5e8f\u5728\u5185\u90e8\u4e5f\u4f1a\u4f7f\u7528\u6b63\u5219\u8868\u8fbe\u5f0f\uff0c\u9ed8\u8ba4\u4f1a\u4f7f\u7528`re.S|re.M`\u3002\u4e0a\u8ff0\u9009\u9879\u4f1a\u88ab\u8ffd\u52a0\u5230\u9ed8\u8ba4\u9009\u9879\u540e\u3002  \n",
      "\u53e6\u5916\uff0c\u5982\u679c\u53ea\u60f3\u5f00\u542f\u6b63\u5219\u6a21\u5f0f\uff0c\u800c\u4e0d\u6307\u5b9aflag\uff0c\u9700\u952e\u5165`d`\n",
      "\n",
      "* option \"-i --indent\"\uff0c\u5f3a\u5236\u4ee3\u7801\u7f29\u8fdb\uff0c\u63a5\u53d7\u4e00\u4e2a\u6574\u578b\u53c2\u6570\u3002  \n",
      "\u9ed8\u8ba4\u662f0\u3002  \n",
      "\u5982\u679c\u6307\u5b9a-i\uff0c\u90a3\u4e48\u5728cell\u7684\u6bcf\u884c\u884c\u9996\u5f3a\u5236\u8ffd\u52a0\u6307\u5b9a\u6570\u76ee\u7684\u7a7a\u683c\u5b57\u7b26\u3002\n",
      "\n",
      "* option \"-l --log\"\uff0c\u662f\u5426\u8f93\u51fa\u7a0b\u5e8f\u8fd0\u884c\u65e5\u5fd7\u3002\u4e0d\u63a5\u53d7\u53c2\u6570\u3002  \n",
      "\u5f53\u7ed9\u5b9a\u7684\u6587\u672c\u6a21\u5f0f\uff0c\u5373-a-b\uff0ccell\u7684\u7b2c\u4e00\u884c\u548c\u6700\u540e\u4e00\u884c\uff0c\u51fa\u73b0\u4e0d\u80fd\u5339\u914d\u6216\u8005\u591a\u4e2a\u5339\u914d\u7684\u60c5\u51b5\u65f6\uff0c\u7a0b\u5e8f\u4f1a\u9759\u9ed8\u7684\u6267\u884c\u4e00\u7cfb\u5217\u5e94\u5bf9\u63aa\u65bd\u3002  \n",
      "\u8fd9\u4e9b\u884c\u4e3a\u88ab\u8bb0\u5f55\u5728\u53d8\u91cf`log_message_l`\u4e2d\uff08\u4e00\u4e2alist\uff09\u3002  \n",
      "\u5982\u679c\u6307\u5b9a-l\uff0c`log_message_l`\u4f1a\u88ab\u8f6c\u53d8\u4e3a\u5b57\u7b26\u4e32\u540e\u8f93\u51fa\u3002\n",
      "\n",
      "* arg \"file\"\uff0c\u6307\u660e\u9700\u540c\u6b65\u7684\u76ee\u6807\u6587\u4ef6\u3002\u8bfb\u5165\u81f3\u5c11\u4e00\u4e2a\u5b57\u7b26\u4e32\uff0c\u591a\u5b57\u7b26\u4e32\u4ee5\u7a7a\u683c\u95f4\u9694\uff0c\u5bf9\u5e94\u591a\u4e2a\u6587\u4ef6\u3002  \n",
      "file\u662f\u4e00\u4e2apositional argument\uff0c\u5373\u5728\u524d\u9762\u5217\u51fa\u7684option\u53ca\u5176argument\u90fd\u88ab\u5c1d\u8bd5\u5339\u914d\u540e\uff0c\u4ecd\u7136\u5269\u4e0b\u7684\u90e8\u5206\u3002  \n",
      "\u9700\u8981\u6ce8\u610f\uff0c\u867d\u7136\u662f\u5269\u4e0b\u7684\uff0c\u4f46\u5728parse\u4e4b\u524d\u5fc5\u987b\u8fde\u7eed\uff0c\u5373\u4e4b\u95f4\u4e0d\u80fd\u51fa\u73b0\u5176\u4ed6\u7684option\u3002"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## \u6784\u9020\n",
      "\n",
      "\u53c2\u8003\u811a\u672c `sync_to_file_magic_command.py`"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%run \"sync_to_file_magic_command.py\""
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## sync_to_file\u53c2\u6570\n",
      "\n",
      "\u4e4b\u524d\u5199\u4e86magic command `%%sync_to_file`\uff0c\u4e0b\u9762\u6307\u5b9a\u4e00\u4e9b\u8be5\u547d\u4ee4\u7684\u5e38\u7528\u53c2\u6570\uff0c\u901a\u8fc7`%%sync_to_file $var`\u6765\u8c03\u7528\u3002"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "configMyHelpers = 'my_helpers.py'"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "before_main_arg = ' -b \"def main(file_name=None):\"'\n",
      "after_main_arg = ' -a \"def main(file_name=None):\"'\n",
      "indent_arg = ' -i 4'"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "new3000_convert_script_name = 'convert_new3000.py'\n",
      "configNew3000 = new3000_convert_script_name\n",
      "configNew3000BeforeMain = new3000_convert_script_name + before_main_arg\n",
      "configNew3000AfterMain = new3000_convert_script_name + after_main_arg + indent_arg"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "zhuji_convert_script_name = 'convert_zhuji.py'\n",
      "configZhuji = zhuji_convert_script_name\n",
      "configZhujiBeforeMain = configZhuji + before_main_arg\n",
      "configZhujiAfterMain = configZhuji + after_main_arg + indent_arg"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "duanyu_convert_script_name = 'convert_duanyu.py'\n",
      "configDy = duanyu_convert_script_name\n",
      "configDyBeforeMain = configDy + before_main_arg\n",
      "configDyAfterMain = configDy + after_main_arg + indent_arg"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configNew3000 $configMyHelpers $configZhuji $configDy -m o\n",
      "\n",
      "# coding:utf-8\n",
      "import re\n",
      "import json\n",
      "import codecs\n",
      "import functools\n",
      "import os.path\n",
      "from random import random\n",
      "from random import randint\n",
      "from pprint import pprint\n",
      "from copy import deepcopy"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "with codecs.open(new3000_convert_script_name, 'a', encoding='utf-8') as f:\n",
      "    f.write('\\nfrom my_helpers import *')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "with codecs.open(zhuji_convert_script_name, 'a', encoding='utf-8') as f:\n",
      "    f.write('\\nfrom my_helpers import *')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "with codecs.open(duanyu_convert_script_name, 'a', encoding='utf-8') as f:\n",
      "    f.write('\\nfrom my_helpers import *')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## txt\u6e90\u6587\u4ef6\u6240\u5728\u76f8\u5bf9\u8def\u5f84\n",
      "\n",
      "\u6bcf\u4e2a\u81ea\u52a8\u751f\u6210\u7684\u8f6c\u6362\u811a\u672c\u90fd\u6709\u4e00\u4e2a\u4e3b\u51fd\u6570\uff0c\u8fd8\u6709\u4e00\u5806\u5199\u5728\u4e3b\u51fd\u6570\u4e0a\u9762\uff08\u5916\u9762\uff09\u7684\u8f85\u52a9\u51fd\u6570\u3002\u4e3b\u51fd\u6570\u7684\u4f5c\u7528\u662f\uff0c\u5148\u5224\u65ad\u6307\u5b9a\u6587\u4ef6\u662f\u5426\u5b58\u5728\uff0c\u5982\u679c\u5b58\u5728\uff0c\u5728\u987a\u5e8f\u6267\u884c\u540e\u7eed\u8bed\u53e5\u3002\u540e\u6587\u4e2d\uff0c\u4e00\u4e9b\u4ee3\u7801\u9700\u8981\u51fa\u73b0\u5728\u4e3b\u51fd\u6570\u4e4b\u524d\uff0c\u4e00\u4e9b\u9700\u8981\u51fa\u73b0\u5728\u4e4b\u540e\u3002\u4e0b\u9762\u7684\u4e09\u4e2acell\u5206\u522b\u5411\u4e09\u4e2a\u8f6c\u6362\u811a\u672c\u4e2d\u5199\u5165\u5404\u81ea\u7684\u4e3b\u51fd\u6570\u5f00\u5934\uff0c\u540c\u65f6\u4e3a\u540e\u7eed\u4ee3\u7801\u5199\u5165\u63d0\u4f9b\u4e3b\u51fd\u6570\u7684\u4f4d\u7f6e\u4f9d\u636e\u3002"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configNew3000\n",
      "file_new_3000 = \"base_data\\GREHe Xin Ci Hui Kao Fa Jing Xi  (Xin Dong Fang Da Yu Ying Yu Xue Xi Cong Shu ) - Chen Qi.txt\"\n",
      "def main(file_name=None):\n",
      "    if file_name is None:\n",
      "        file_name = file_new_3000\n",
      "    # for module call\n",
      "    if not os.path.isfile(file_name):\n",
      "        return"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configZhuji\n",
      "file_zhuji = \"base_data\\GREHe Xin Ci Hui Zhu Ji Yu Jing - Cao Tian Cheng.txt\"\n",
      "def main(file_name=None):\n",
      "    if file_name is None:\n",
      "        file_name = file_zhuji\n",
      "    # for module call\n",
      "    if not os.path.isfile(file_name):\n",
      "        return"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configDy\n",
      "file_duanyu = \"base_data\\GREGao Fen Bi Bei Duan Yu Da Pe - Yan Yu Zhen ,Gao Yu ,Chen Qi.txt\"\n",
      "def main(file_name=None):\n",
      "    if file_name is None:\n",
      "        file_name = file_duanyu\n",
      "    # for module call\n",
      "    if not os.path.isfile(file_name):\n",
      "        return"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "# \u8f85\u52a9\u51fd\u6570\n",
      "\n",
      "## \u5168\u89d2\u8f6c\u534a\u89d2"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configMyHelpers\n",
      "def strF2H(ustring):\n",
      "    '''\n",
      "    convert full width character to half width\n",
      "    input: a unicode object\n",
      "    return: a unicode object\n",
      "    '''\n",
      "    h_ustring = u\"\"\n",
      "    assert isinstance(ustring, unicode)\n",
      "    for uchar in ustring:\n",
      "        inside_code = ord(uchar)\n",
      "        if inside_code == 12288:\n",
      "            #  white space\n",
      "            inside_code = 32\n",
      "        elif 65281 <= inside_code <= 65374:\n",
      "            #  other characters\n",
      "            inside_code -= 65248\n",
      "\n",
      "        h_ustring += unichr(inside_code)\n",
      "    return h_ustring"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# exmple\n",
      "test_str = '\u5168\u89d2\u5b57\u7b26\uff1a\uff0b\uff0d\uff01\uff26\uff55\uff4c\uff4c\u3000\uff37\uff49\uff44\uff54\uff48\u3000\uff43\uff48\uff41\uff52\uff01'.decode('utf-8')\n",
      "print test_str, type(test_str)\n",
      "test_str = strF2H(test_str)\n",
      "print test_str, type(test_str)\n",
      "del test_str"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## print_if_main_call\n",
      "\n",
      "\u901a\u8fc7\u5c06print\u5c01\u88c5\u5728print_if_main_call\u4e2d\uff0c\u53ea\u5728\u88abnotebook\u81ea\u8eab\u8c03\u7528\u65f6\u624dprint\u8f93\u5165\u7684\u53c2\u6570"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def print_if_main_call(*_str):\n",
      "    if __name__ == \"__main__\":\n",
      "        for a_str in _str:\n",
      "            print a_str,"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## recursive print\n",
      "\n",
      "\u65e0\u8bba\u662f`pprint`\u8fd8\u662f`print`\uff0c\u90fd\u4e0d\u80fd\u5c06\u5d4c\u5957\u7ed3\u6784\u4e2d\u7684unicode\u89e3\u7801\u540e\u7684\u5b57\u7b26\u6253\u5370\u51fa\u6765\uff0c\u6240\u4ee5\u6709\u4e86\u4e0b\u9762\u8fd9\u4e2a`iter_print`\u51fd\u6570"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configMyHelpers\n",
      "# pretty print the embedded unicode of list or dict\n",
      "def iter_print(obj_iter, indent=0, increment=2, max_top_level_print=None, \n",
      "               top_level=True, top_level_extra_line_feed=False, print_list_index=True):\n",
      "    if not hasattr(obj_iter, '__iter__'):\n",
      "        if isinstance(obj_iter, basestring):\n",
      "            if obj_iter == u'':\n",
      "                pass\n",
      "            elif '\\n' in obj_iter:\n",
      "                for line in obj_iter.split('\\n'):\n",
      "                    if line:\n",
      "                        print ' '*indent, line\n",
      "            else:\n",
      "                print ' '*indent, obj_iter\n",
      "        else:\n",
      "            print ' '*indent, obj_iter\n",
      "        return\n",
      "    print_count = 0\n",
      "    if isinstance(obj_iter, dict):\n",
      "        for key, iter_sub_obj in obj_iter.iteritems():\n",
      "            print ' '*indent, key\n",
      "            iter_print(iter_sub_obj, indent+increment, increment, None, False, False, print_list_index)\n",
      "            if top_level:\n",
      "                print_count += 1\n",
      "                if max_top_level_print:\n",
      "                    if print_count >= max_top_level_print:\n",
      "                        break\n",
      "                if top_level_extra_line_feed:\n",
      "                    print '\\n'\n",
      "    else:\n",
      "        for list_index, sub_obj_iter in enumerate(obj_iter):\n",
      "            if print_list_index:\n",
      "                print ' '*indent, list_index\n",
      "            iter_print(sub_obj_iter, indent+increment, increment, None, False, False, print_list_index)\n",
      "            if top_level:\n",
      "                print_count += 1\n",
      "                if max_top_level_print:\n",
      "                    if print_count >= max_top_level_print:\n",
      "                        break\n",
      "                if top_level_extra_line_feed:\n",
      "                    print '\\n'"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "\u5206\u9694\u5b57\u7b26\u4e32"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "\u5728\u5207\u5272\u5b57\u7b26\u4e32\u7684\u8fc7\u7a0b\u4e2d\uff0c\u7ecf\u5e38\u9047\u5230\u8fd9\u6837\u4e00\u4e2a\u9700\u6c42\uff1a\u7ed9\u5b9a\u4e00\u4e2a\u6587\u672c\u6a21\u5f0f\uff0c\u5728\u4e00\u4e2a\u5b57\u7b26\u4e32\u4e2d\u4f1a\u9047\u5230\u8be5\u6a21\u5f0f\u82e5\u5e72\u6b21\uff0c\u4f9d\u6b21\u63d0\u53d6\u6bcf\u4e24\u6b21\u76f8\u9047\u4e4b\u95f4\u7684\u5185\u5bb9\uff0c\u4ee5\u53ca\u6700\u540e\u4e00\u6b21\u76f8\u9047\u76f4\u5230\u5b57\u7b26\u4e32\u7ed3\u5c3e\u7684\u5185\u5bb9\u3002  \n",
      "\u6bd4\u5982\uff0c\u4f9d\u6b21\u63d0\u53d6\"List 1\"\u5230\"List 2\"\u4e4b\u95f4\u3001\"List 2\"\u5230\"List 3\"\u4e4b\u95f4\u3001\"List 3\"\u5230\u5b57\u7b26\u4e32\u7ed3\u5c3e\u7684\u5185\u5bb9\u3002  \n",
      "\u8fd9\u4e00\u9700\u6c42\u88ab\u62bd\u8c61\u540e\u5b9e\u73b0\u4e3a\u5982\u4e0b\u51fd\u6570\u3002"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configMyHelpers\n",
      "def extract_content_between(obj_str, match_re, return_str_before_first_match=False):\n",
      "    '''\n",
      "    extract content between the start of two equal pattern found in a str,\n",
      "    also extract the content after the last match\n",
      "    input: obj_str, the string to extract content from, must be a unicode object\n",
      "           match_re, the pattern to be matched\n",
      "    return: a list of str\n",
      "    return_str_before_first_match: whether to return the str before the first match of the given patter\n",
      "    '''\n",
      "    assert isinstance(obj_str, unicode)\n",
      "    retype = type(re.compile(r'a str'))\n",
      "    assert isinstance(match_re, retype)\n",
      "    \n",
      "    match_results_iter = match_re.finditer(obj_str)\n",
      "    returned_str_l = []\n",
      "    start_index = None\n",
      "    end_index = None\n",
      "    first_start_index = None\n",
      "    for match_result in match_results_iter:\n",
      "        if first_start_index is None:\n",
      "            first_start_index = match_result.start()\n",
      "        if not (start_index is None):\n",
      "            end_index = match_result.start()\n",
      "            returned_str_l.append(obj_str[start_index:end_index])\n",
      "        start_index = match_result.start()\n",
      "    returned_str_l.append(obj_str[start_index:])\n",
      "    if return_str_before_first_match:\n",
      "        returned_str_l = [obj_str[:first_start_index]] + returned_str_l\n",
      "    return returned_str_l"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# exmple\n",
      "test_str = u'a/b/c'\n",
      "test_re = re.compile(u'/')\n",
      "print extract_content_between(test_str, test_re)\n",
      "del test_str, test_re"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "\u904d\u5386\u5d4c\u5957\u53d8\u91cf"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "\u5199\u4ee3\u7801\u65f6\uff0c\u7ecf\u5e38\u9700\u8981\u904d\u5386\u4e00\u4e9b\u7ed3\u6784\u975e\u5e38\u590d\u6742\u7684\u5d4c\u5957\u53d8\u91cf\u3002\u6bd4\u5982\uff0c\u5bf9\u4e8e\u4e00\u4e2a`_d_d`\u53d8\u91cf\uff0c\u904d\u5386\u9876\u5c42\u5b57\u5178\u7684\u6bcf\u4e00\u4e2a\u952e\u503c\u5bf9\uff0c\u4f46\u5bf9\u4e8e\u7b2c\u4e8c\u5c42\u7684\u5b57\u5178\uff0c\u53ea\u53d6\u67d0\u4e00\u4e2a\u952e\u503c\u5bf9\u3002\u4e0b\u9762\u7684\u4ee3\u7801\u904d\u5386\u4e00\u4e2a\u7ed3\u6784\u4e3a`_d_l_d_d`\u7684\u53d8\u91cf\u3002"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configMyHelpers\n",
      "def iter_value_of_key_through_d_l_d_d(obj_d_l_d_d, key_2nd_level, key_4th_level, \n",
      "                                      expected_draw=1.0, yield_top_key=False, yield_list_index=False):\n",
      "    '''\n",
      "    a function that return a generator\n",
      "    it will iter through all the values of the first level dict with every value being themself a dict\n",
      "    for every such value dict,\n",
      "        a key specified by key_2nd_level is used to access a list\n",
      "        for every elment of the list\n",
      "            a key specified by key_4th_level is used to access the corresponding value\n",
      "    so in total it is a two level nested loop\n",
      "    \n",
      "    key_2nd_level: what it points to must be a list \n",
      "    \n",
      "    expected_draw: roughly control the proportion of the innermost values to be sampled\n",
      "                   can be an integar, which will be converted to the corresponding probability\n",
      "    \n",
      "    yield_top_key: whether to include the top key\n",
      "    yield_list_index: whether to include the list index\n",
      "    note that (yield_top_key=False, yield_list_index=True) is a useless combination, so raise an ValueError\n",
      "    '''\n",
      "    if isinstance(expected_draw, int):\n",
      "        expected_draw = float(expected_draw)/len(obj_d_l_d_d)\n",
      "    assert isinstance(expected_draw, float)\n",
      "    for top_key, value_d_l_d in obj_d_l_d_d.iteritems():\n",
      "        assert isinstance(value_d_l_d[key_2nd_level], list)\n",
      "        for _list_index, value_d in enumerate(value_d_l_d[key_2nd_level]):\n",
      "            if random() <= expected_draw:\n",
      "                if (not yield_top_key) and (not yield_list_index):\n",
      "                    yield value_d[key_4th_level]\n",
      "                elif yield_top_key and (not yield_list_index):\n",
      "                    yield top_key, value_d[key_4th_level]\n",
      "                elif yield_top_key and yield_list_index:\n",
      "                    yield top_key, _list_index, value_d[key_4th_level]\n",
      "                else:\n",
      "                    raise ValueError('Invalid Combination of yield_top_key and yield_list_index')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "\u5bf9\u4e0a\u8ff0\u4ee3\u7801\u8fdb\u4e00\u6b65\u62bd\u8c61\u5f97\u5230"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configMyHelpers\n",
      "def iter_through_general(obj_iter, path, yield_flags=True, final_yield_object=None):\n",
      "    '''\n",
      "    iter through an object following the given path\n",
      "    yield_flags: control whether to yield the flags indicating the path at the global level\n",
      "    final_yield_object: internal parameter, don't modify\n",
      "    obj_iter: an iterable variable\n",
      "    path: a sequence, each element has the following structure\n",
      "        (how_to_iter, what_to_iter, yield_flag)\n",
      "        how_to_iter: a str, accept the following values\n",
      "            'all' or 'all_values': iter through key-value pair for dict, and all elements for other type\n",
      "                if yield_flag is True, attach key or index to the final yield object\n",
      "            'all_keys', only iter through the keys of a dict\n",
      "                obj_iter must be a dict\n",
      "            'key', iter through the value of a given key\n",
      "                what_to_iter must be a str representing a key in obj_iter\n",
      "                if yield_flag is True, attach key to the final yield object\n",
      "                ignored when obj_iter is not dict\n",
      "            'keys', iter through the values of a given set of keys\n",
      "                what_to_iter must be a tuple with elements reprenting keys in obj_iter\n",
      "                if yield_flag is True, attach key to the final yield object\n",
      "                ignored when obj_iter is not dict\n",
      "            'index', iter through a given element\n",
      "                what_to_iter must be an int within bound\n",
      "                if yield_flag is True, attach index to the final yield object\n",
      "                ignored when obj_iter is dict\n",
      "            'indexes', iter through the elements with given indexes\n",
      "                what_to_iter must be an list of int within bound\n",
      "                if yield_flag is True, attach key to the final yield object\n",
      "                ignored when obj_iter is dict\n",
      "        what_to_iter: content decided by how_to_iter\n",
      "            ignored for the following values of how_to_iter\n",
      "                all, all_values, all_keys\n",
      "        yield_flag: True or False\n",
      "            True: depending on how_to_iter, attch different flags to the final result\n",
      "            False: no flag wil be yield\n",
      "            ignored for the following values of how_to_iter\n",
      "                all_keys\n",
      "    '''\n",
      "    is_dict = isinstance(obj_iter, dict)\n",
      "    if final_yield_object is None:\n",
      "        final_yield_object = []\n",
      "    if len(path) == 0:\n",
      "        if yield_flags:\n",
      "            final_yield_object.append(obj_iter)\n",
      "            yield final_yield_object\n",
      "        else:\n",
      "            yield obj_iter\n",
      "    else:\n",
      "        how_to_iter, what_to_iter, yield_flag = path.pop(0)\n",
      "        assert isinstance(how_to_iter, basestring)\n",
      "        if how_to_iter in [u'all', u'all_values', u'keys', u'indexes']:\n",
      "            if how_to_iter in [u'keys', u'indexes']:\n",
      "                assert hasattr(what_to_iter, '__iter__')\n",
      "                for item in what_to_iter:\n",
      "                    if is_dict:\n",
      "                        assert how_to_iter == u'keys'\n",
      "                        assert isinstance(item, basestring)\n",
      "                        assert item in obj_iter\n",
      "                    else:\n",
      "                        assert how_to_iter == u'indexes'\n",
      "                        assert isinstance(item, int)\n",
      "                        assert item < len(obj_iter)\n",
      "                temp_iterator = ((item, obj_iter[item]) for item in what_to_iter)\n",
      "            else:\n",
      "                temp_iterator = obj_iter.iteritems() if is_dict else enumerate(obj_iter)\n",
      "            for flag, sub_obj_iter in temp_iterator:\n",
      "                final_yield_object_copy = deepcopy(final_yield_object)\n",
      "                if yield_flag:\n",
      "                    final_yield_object_copy.append(flag)\n",
      "                for value in iter_through_general(sub_obj_iter, deepcopy(path), yield_flags, final_yield_object_copy):\n",
      "                    yield value\n",
      "        elif how_to_iter == u'all_keys':\n",
      "            assert is_dict\n",
      "            for key in obj_iter.iterkeys():\n",
      "                if yield_flags:\n",
      "                    final_yield_object.append(key)\n",
      "                    yield final_yield_object\n",
      "                else:\n",
      "                    yield key\n",
      "        elif how_to_iter in [u'key', u'index']:\n",
      "            if is_dict:\n",
      "                assert how_to_iter == u'key'\n",
      "                assert isinstance(what_to_iter, basestring)\n",
      "                assert what_to_iter in obj_iter\n",
      "            else:\n",
      "                assert how_to_iter == u'index'\n",
      "                assert isinstance(what_to_iter, int)\n",
      "                assert what_to_iter < len(obj_iter)     \n",
      "            sub_obj_iter = obj_iter[what_to_iter]\n",
      "            if yield_flag:\n",
      "                final_yield_object.append(what_to_iter)\n",
      "            for value in iter_through_general(sub_obj_iter, deepcopy(path), yield_flags, final_yield_object):\n",
      "                yield value\n",
      "        else:\n",
      "            raise ValueError('Invalid path')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# example\n",
      "def unit_test():\n",
      "    test_dic = {'a':['a-a',2,3], 'b':['b-a',3,4], 'c':['c-a',2,3]}\n",
      "    print 'PATH ONE'\n",
      "    for value in iter_through_general(test_dic, [('all','',True),('indexes',[0,1],False)]):\n",
      "        print value,\n",
      "    print '\\nPATH TWO'\n",
      "    for value in iter_through_general(test_dic, [('keys', ['a', 'b'], False)]):\n",
      "        print value,\n",
      "unit_test()\n",
      "del unit_test"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## Reservoir_sampling\n",
      "\n",
      "\u6279\u5904\u7406\u5b8c\u540e\uff0c\u9700\u8981\u968f\u673a\u7684\u67e5\u770b\u4e00\u4e9b\u7ed3\u679c\uff0c\u68c0\u67e5\u6279\u5904\u7406\u4e2d\u662f\u5426\u6709bug\u3002\u4e0b\u9762\u8fd9\u4e2a\u51fd\u6570\u6309\u7167reservoir sampling\u7684\u65b9\u6cd5\uff0c\u4ece\u4e00\u4e2a\u672a\u77e5\u957f\u5ea6\u7684\u904d\u5386\u5bf9\u8c61\u4e2d\u968f\u673a\u9009\u53d6k\u4e2a\u5143\u7d20\u3002\n",
      "\n",
      "\u53c2\u89c1 https://en.wikipedia.org/wiki/Reservoir_sampling"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configMyHelpers\n",
      "def reservoir_sample_k(obj_iter, k):\n",
      "    assert isinstance(k, int)\n",
      "    assert hasattr(obj_iter, '__iter__')\n",
      "    # fit into k items\n",
      "    sampled_l = []\n",
      "    for _ in range(k):\n",
      "        sampled_l.append(obj_iter.next())\n",
      "    i = k\n",
      "    for item in obj_iter:\n",
      "        i += 1\n",
      "        j = randint(1, i)\n",
      "        if j <= k:\n",
      "            sampled_l[j-1] = item\n",
      "    return sampled_l"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "\u540c\u524d\u9762\u7684`iter_through_general`\u51fd\u6570\u7ed3\u5408\u4ea7\u751f"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configMyHelpers\n",
      "def iter_through_and_sample_k(obj_iter, k, path):\n",
      "    obj_iter_follow_path = iter_through_general(obj_iter, path)\n",
      "    return reservoir_sample_k(obj_iter_follow_path, k)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "\u5176\u4ed6"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configMyHelpers\n",
      "strip_white_space = lambda _str: _str.replace(' ', '')\n",
      "new_line_join = lambda str_l: '\\n'.join(str_l)\n",
      "def codecs_open_r_utf8(file_path):\n",
      "    with codecs.open(file_path, 'r', 'utf-8') as f:\n",
      "        returned_str = f.read()\n",
      "    return returned_str\n",
      "# merge blank lines\n",
      "def collapse_blank_line(base_str):\n",
      "    match_double_line_feed_re = re.compile(r'\\n\\n')\n",
      "    while match_double_line_feed_re.search(base_str):\n",
      "        base_str = match_double_line_feed_re.sub(r'\\n', base_str)\n",
      "    return base_str"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "# \u5904\u7406\u300aGRE\u6838\u5fc3\u8bcd\u6c47\u8003\u6cd5\u7cbe\u6790\u300b\n",
      "\n",
      "\u8bfb\u5165\"GREHe Xin Ci Hui Kao Fa Jing Xi - Chen Qi.txt\"\uff0c\u751f\u6210`new3000_base_d`"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configNew3000AfterMain\n",
      "new3000_base_str = codecs_open_r_utf8(file_new_3000)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print 'Note the type of new3000_base_str is', type(new3000_base_str)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## \u6309List\u63d0\u53d6\n",
      "\n",
      "\u6bcf\u4e2aList\u4ee5 \"`# List`\"+\u4e00\u4e2a\u7a7a\u683c+\u4e00\u4e2a\u6570\u5b57 \u5f00\u59cb\u3002\u4e24\u4e2a\u5f00\u59cb\u4f4d\u7f6e\u4e4b\u95f4\u7684\u5185\u5bb9\u5373\u4e3a\u4e00\u4e2aList\u3002\u67d0\u4e9bList\u7b2c\u4e00\u884c\u7684\u7ed3\u5c3e\u8fd8\u4f1a\u51fa\u73b0\"`\\*`\""
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configNew3000BeforeMain\n",
      "match_new3000_list_start_re = re.compile(ur'^# List \\d+', re.M)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configNew3000AfterMain\n",
      "new3000_base_list_data_l = extract_content_between(new3000_base_str, match_new3000_list_start_re)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# check\n",
      "print len(new3000_base_list_data_l), 'lists extracted out'\n",
      "print '\\nThe start of one list:\\n', new3000_base_list_data_l[1][0:200]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "\u6700\u540e\u4e00\u4e2aList\u76f4\u63a5\u5339\u914d\u5230\u5b57\u7b26\u4e32\u672b\u5c3e\uff0c\u901a\u8fc7\u89c2\u5bdftxt\u6587\u6863\uff0c\u53d1\u73b0List31\u5339\u914d\u5230\u5927\u91cf\u65e0\u5173\u5185\u5bb9\uff0c\u9700\u5355\u72ec\u5904\u7406\u3002"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configNew3000BeforeMain\n",
      "def strip_last_list(list_data):\n",
      "    strip_start_re = re.compile(ur'# Word List 1\u3000\u4e0e\u8bf4\u6709\u5173\u7684\u8bcd\u6839\u6784\u6210\u7684\u5355\u8bcd(.|\\n)*$')\n",
      "    return strip_start_re.sub('', list_data)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configNew3000AfterMain\n",
      "new3000_base_list_data_l[30] = strip_last_list(new3000_base_list_data_l[30])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## \u6309Unit\u63d0\u53d6\n",
      "\n",
      "\u6bcf\u4e2aUnit\u4ee5 \"`## Unit`\"+\u7a7a\u683c+\u4e00\u4e2a\u6570\u5b57+\"`\\n`\" \u5f00\u59cb\uff0c\u4e24\u4e2a\u5f00\u59cb\u4f4d\u7f6e\u4e4b\u95f4\u7684\u5185\u5bb9\u5373\u4e3a\u4e00\u4e2aUnit"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configNew3000BeforeMain\n",
      "match_unit_start_re = re.compile(ur'^## Unit \\d+', re.M)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configNew3000AfterMain\n",
      "new3000_base_unit_data_l_l = map(functools.partial(extract_content_between, \n",
      "                                                   match_re=match_unit_start_re), \n",
      "                                 new3000_base_list_data_l)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# check\n",
      "print 'the number of unit in a list\\n', map(len, new3000_base_unit_data_l_l)\n",
      "print '\\nthe start part of one unit\\n', new3000_base_unit_data_l_l[9][9][0:200]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## \u6309\u5355\u8bcd\u63d0\u53d6\n",
      "\n",
      "\u6bcf\u4e2a\u5355\u8bcd\u4ee5  \n",
      "\"`**`\"+\u82f1\u6587\u5355\u8bcd+\"`**`\"+\u4e00\u4e2a\u7a7a\u683c+\u5168\u89d2\u5b57\u7b26\"`\uff3b`\"+\u82e5\u5e72\u97f3\u6807\u5b57\u7b26+\u5168\u89d2\u5b57\u7b26\"`\uff3d`\"  \n",
      "\u5f00\u59cb\uff0c\u4e24\u4e2a\u5f00\u59cb\u4f4d\u7f6e\u4e4b\u95f4\u7684\u5185\u5bb9\u5373\u4e3a\u4e00\u4e2a\u5355\u8bcd  \n",
      "\n",
      "\u82f1\u6587\u5355\u8bcd\u90e8\u5206\u7684\u89c4\u5f8b\u5982\u4e0b\uff1a\n",
      "+ \u57fa\u672c\u90fd\u662f\u7531a\u81f3z\u7684\u5c0f\u5199\u5b57\u6bcd\u7ec4\u6210\n",
      "+ \u67d0\u4e9bGRE\u5355\u8bcd\u662f\u4ece\u6cd5\u8bed\u6f14\u53d8\u6765\u7684\u3002\u6bd4\u5982clich\u00e9\u8fd9\u79cd\u3002\u6309\u7406\u8bf4\u5e94\u8be5\u5339\u914d\u6240\u6709\u7684\u897f\u6b27\u5b57\u7b26\uff0c\u4f46\u7531\u4e8eGRE\u5355\u8bcd\u4e66\u4e2d\u8c8c\u4f3c\u53ea\u51fa\u73b0\u8fc7\u00e9\u00ef\uff0c\u6240\u4ee5\u5176\u4ed6\u7684\u5c31\u4e0d\u7528\u5339\u914d\u3002\n",
      "+ \u6709\u4e9b\u5355\u8bcd\u6d89\u53ca\u5230\u8fde\u5b57\u7b26\"`-`\"\n",
      "\n",
      "\u97f3\u6807\u90e8\u5206\uff0c\u5339\u914d\"`\uff3b`\"\u4e0e\"`\uff3d`\"\u4e4b\u95f4\u7684\u6240\u6709\u5b57\u7b26\u5373\u53ef\u3002\u6709\u7684\u591a\u97f3\u5355\u8bcd\uff0c\u6bd4\u5982addict\uff08List1Unit6\uff09\uff0c\u5355\u8bcd\u540e\u5e76\u6ca1\u6709\u97f3\u6807\uff0c\u4f46\u6bcf\u4e2a\u91ca\u4e49\u540e\u9762\u6709\u97f3\u6807"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configNew3000BeforeMain\n",
      "match_word_block_start = re.compile(ur'^\\*\\*(?P<word>[a-z\\-\u00e9\u00ef]+)\\*\\*(?P<phon>\uff3b.+\uff3d)?', re.U|re.M)\n",
      "# phon represent phonetic symbol"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# test\n",
      "def unit_test():\n",
      "    for result in match_word_block_start.finditer(new3000_base_unit_data_l_l[0][5]):\n",
      "        print result.group('word'),\n",
      "        print result.group('phon')\n",
      "# unit_test()\n",
      "del unit_test"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configNew3000BeforeMain\n",
      "def get_word_of_one_unit(unit_block_str, list_index, unit_index):\n",
      "    returned_words_d_d = {}\n",
      "    word_block_str_l = extract_content_between(unit_block_str, match_word_block_start)\n",
      "    for word_block_str in word_block_str_l:\n",
      "        first_line_match = match_word_block_start.match(word_block_str)\n",
      "        word = first_line_match.group('word')\n",
      "        phon = first_line_match.group('phon')\n",
      "        one_word_d = {'word_block_str': match_word_block_start.sub('', word_block_str), \n",
      "                      'phon': strF2H(phon) if phon else u'', \n",
      "                      'pos':(list_index, unit_index)}\n",
      "        returned_words_d_d[word] = one_word_d\n",
      "    return returned_words_d_d"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# example\n",
      "test_d = get_word_of_one_unit(new3000_base_unit_data_l_l[0][5], 1, 6)\n",
      "iter_print(test_d, max_top_level_print=2)\n",
      "del test_d"
     ],
     "language": "python",
     "metadata": {
      "scrolled": false
     },
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configNew3000BeforeMain\n",
      "def get_new3000_base_d(base_unit_data_l_l):\n",
      "    _new3000_base_d = {}\n",
      "    for list_index, unit_data_l in enumerate(base_unit_data_l_l):\n",
      "        for unit_index, unit_data in enumerate(unit_data_l):\n",
      "            _new3000_base_d.update(get_word_of_one_unit(unit_data, list_index+1, unit_index+1))\n",
      "    return _new3000_base_d"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configNew3000AfterMain\n",
      "new3000_base_d = get_new3000_base_d(new3000_base_unit_data_l_l)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# check\n",
      "print 'Total words: ', len(new3000_base_d)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## \u68c0\u89c6word_block\u5185\u90e8\u7ed3\u6784\n",
      "\n",
      "### \u793a\u4f8b"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print '\u5355\u4e49\u60c5\u5f62\uff1a'\n",
      "iter_print(new3000_base_d['dodge']['word_block_str'])\n",
      "print '\\n\u591a\u4e49\u60c5\u5f62\uff1a'\n",
      "iter_print(new3000_base_d['addict']['word_block_str'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "\u7edf\u8ba1\u6bcf\u884c\u7b2c\u4e00\u4e2a\u5b57\u7b26"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def count_line_start_char():\n",
      "    from collections import Counter\n",
      "    temp_start_types = Counter()\n",
      "    for value in new3000_base_d.itervalues():\n",
      "        word_block_str = value['word_block_str']\n",
      "        word_block_lines = word_block_str.split('\\n')\n",
      "        for line in word_block_lines:\n",
      "            if line == '' or line == '\\n':\n",
      "                continue\n",
      "            temp_start_types[line[0]] += 1\n",
      "            if line[0] == ' ' or line[0] == '*':\n",
      "                #print line\n",
      "                continue\n",
      "            if line[0] == 'a' or line[0] == 'H':\n",
      "                #print line\n",
      "                continue\n",
      "    for key, value in temp_start_types.iteritems():\n",
      "        print key, value, ',',\n",
      "count_line_start_char()\n",
      "del count_line_start_char"
     ],
     "language": "python",
     "metadata": {
      "scrolled": false
     },
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "+ \u6b63\u5e38\u60c5\u5f62\u4e0b\uff0c\u4e00\u884c\u5e94\u8be5\u4ee5\"\u3010\"\u3001\"\u4f8b\"\u3001\"\u8fd1\"\u3001\"\u53cd\"\u3001\"\u6d3e\"\u5f00\u59cb\u3002\u5f02\u5e38\u5b57\u7b26\u5f80\u5f80\u4ee3\u8868\u9700\u8981\u7279\u6b8a\u5904\u7406\u7684\u5730\u65b9\u3002\n",
      "+ \u901a\u8fc7\u68c0\u89c6\u539f\u6587\u6863\uff0c\u53d1\u73b0\uff1a\n",
      "  - \u59823\uff0c`\uff08`1\uff0c\uff0c\u62111\uff0c`\u2014`17\uff0c\u751f1\uff0c\u56f01\uff0c\u4eba2\uff0c\u6bcf2\uff0c\u65591\uff0c\u51e11\uff0c\u52e41\uff0c\u53ea1\uff0c\u67701\uff1a\u51fa\u73b0\u5728List\u7ed3\u5c3e\u5904\u7684\u540d\u4eba\u540d\u8a00\u4e2d\uff0c\u6bd4\u5982\u67d0\u4e9b\u540d\u4eba\u540d\u8a00\u7684\u4e2d\u6587\u7ffb\u8bd1\u4ee5\u201c\u5982\u201d\u5f00\u5934\u3002\u5bf9\u4e8e\u8fd9\u4e9b\u5b57\u7b26\uff0c\u5ffd\u7565\u5373\u53ef\u3002\n",
      "  - \u540c1\uff1a\u5355\u8bcdanarchist\u72ec\u6709\uff0c\u540e\u9762\u8ddf\u4e86\u4e24\u4e2a\u540c\u4e49\u8bcd\u3002\u800c\u4e14\u901a\u8fc7\u8fd9\u4e2a\u5355\u8bcd\uff0c\u53ef\u4ee5\u53d1\u73b0\u4e00\u4e2a\u5355\u8bcd\u7684\u6d3e\u751f\u8bcd\u4e5f\u6709\u81ea\u5df1\u5b8c\u6574\u7684\u7ed3\u6784\u3002\n",
      "  - \u4e00\u4e2a\u7a7a\u683c\u5b57\u7b261\uff1acompliment\u8fd9\u4e2a\u5355\u8bcd\uff0c\u5355\u8bcd\u8d77\u59cb\u884c\u4e3a\u201c`**compliment** *n.*\uff3b'k\u0251:mpl\u026am\u0259nt\uff3d *v.*\uff3b'k\u0251:mpl\u026ament\uff3d`\u201d\uff0c\u7279\u4f8b\u3002\u6839\u636e\u524d\u9762\u7684\u6b63\u5219\u89c4\u5219\u66ff\u6362\uff0c\u5c06\u201c`**compliment**`\u201d\u62ff\u8d70\u540e\uff0c\u7701\u4e0b\u4e86\u4e00\u4e2a\u4ee5\u7a7a\u767d\u5b57\u7b26\u8d77\u59cb\u7684\u884c\u3002\n",
      "  - `*`18\uff1a\u5927\u90e8\u5206\u5b58\u5728\u4e8e\u540d\u4eba\u540d\u8a00\u4e2d\u3002\u552f\u4e00\u7684\u4f8b\u5916\u662f\u7531compliment\u9020\u6210\u7684\uff0c\u524d\u9762\u63d0\u5230\u8fc7\u3002\n",
      "  - a1\uff1a\u5355\u8bcdantediluvian\uff0c\u8003\u6cd51\uff0c\u4e24\u4e2a\u4f8b\u53e5\u5206\u5360\u4e24\u884c\n",
      "  - H1\uff1a\u5355\u8bcdanecdote\uff0c\u4e24\u4e2a\u4f8b\u53e5\u5206\u5360\u4e24\u884c\n",
      "+ \u3010\u52d8\u8bef\u3011\u5904\u7406\u65b9\u6cd5\uff1a\n",
      "  - anarchist\uff0c\u5c06\u540c\u6539\u4e3a\u8fd1\n",
      "  - compliment\uff0c\u5355\u8bcd\u5757\u7b2c\u4e00\u884c\uff0c\u5c06\u97f3\u6807\u63d0\u53d6\u51fa\u6765\uff0c\u7ec4\u6210\u4e00\u4e2alist\uff0c\u7b2c\u4e00\u4e2a\u5143\u7d20\u662f\u540d\u8bcd\u97f3\u6807\uff0c\u7b2c\u4e8c\u4e2a\u5143\u7d20\u662f\u52a8\u8bcd\u97f3\u6807\u3002\u540e\u9762\u5904\u7406\u8003\u6cd5\u65f6\uff0c\u4e24\u4e2a\u97f3\u6807\u8fde\u6210\u4e00\u4e2a\u5b57\u7b26\u4e32\u7ed9\u8003\u6cd51\uff0c\u7b2c\u4e8c\u4e2a\u97f3\u6807\u7ed9\u8003\u6cd52\n",
      "  - antediluvian\u3001anecdote\uff0c\u5c06\u7b2c\u4e8c\u6b21\u9047\u5230\u7684\u4e24\u4e2a\u8fde\u7eed\u7684'\\n'\u66ff\u6362\u4e3a\"\u2016\""
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configNew3000BeforeMain\n",
      "# revise\n",
      "def revise_word_base_data(word_d):\n",
      "    # revise anarchist\n",
      "    word_block_str = 'word_block_str'\n",
      "    to_revise_word_d = word_d['anarchist']\n",
      "    to_revise_str = to_revise_word_d[word_block_str]\n",
      "    to_revise_word_d[word_block_str] = to_revise_str.replace(u'\u540c', u'\u8fd1')\n",
      "    # revise compliment\n",
      "    to_revise_word_d = word_d['compliment']\n",
      "    to_revise_str = to_revise_word_d[word_block_str]\n",
      "    to_revise_word_d['phon'] = [strF2H(phon) for phon in re.findall(ur'\uff3b.+?\uff3d', to_revise_str)]\n",
      "    to_revise_word_d[word_block_str] = '\\n'.join(to_revise_str.split('\\n')[1:])\n",
      "    # reviseantediluvian, revise anecdote\n",
      "    for to_revise_word in ['antediluvian', 'anecdote']:\n",
      "        to_revise_word_d = word_d[to_revise_word]\n",
      "        to_revise_str = to_revise_word_d[word_block_str]\n",
      "        temp_index = 0\n",
      "        for match_result in re.finditer(ur'\\n\\n', to_revise_str):\n",
      "            if temp_index == 2:\n",
      "                to_revise_str = to_revise_str[0:match_result.start()] + u'\u2016' + to_revise_str[match_result.end():]\n",
      "                break\n",
      "            temp_index += 1\n",
      "        to_revise_word_d[word_block_str] = to_revise_str\n",
      "    return word_d"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configNew3000AfterMain\n",
      "# revise\n",
      "subset_to_revise_d = {word:deepcopy(new3000_base_d[word]) for word in ['anarchist', 'compliment', 'antediluvian', 'anecdote']}\n",
      "subset_to_revise_d = revise_word_base_data(subset_to_revise_d)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# check\n",
      "print subset_to_revise_d['anarchist']['word_block_str'][0:150], '\\n'\n",
      "print subset_to_revise_d['compliment']['word_block_str'][0:50]\n",
      "print subset_to_revise_d['compliment']['phon'], '\\n'\n",
      "print subset_to_revise_d['antediluvian']['word_block_str'][0:210], '\\n'\n",
      "print subset_to_revise_d['anecdote']['word_block_str']"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configNew3000AfterMain\n",
      "new3000_base_d.update(subset_to_revise_d)\n",
      "del subset_to_revise_d, new3000_base_list_data_l, new3000_base_unit_data_l_l, new3000_base_str"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### \u5355\u8bcd\u9aa8\u67b6\u793a\u610f\n",
      "\n",
      "word|\u5355\u8bcd\n",
      "\n",
      "* usages: list #\u5355\u8bcd\u7684\u6240\u6709\u8003\u6cd5\n",
      " * usages[i]: dict {exp, examples, syn, ant, der, part of speech, ph_symbl} #\u5355\u8bcd\u7b2ci\u4e2a\u8003\u6cd5\uff0c\u5305\u62ec{\u89e3\u91ca\uff0c\u4f8b\u53e5\uff0c\u540c\u4e49\u8bcd\uff0c\u53cd\u4e49\u8bcd\uff0c\u6d3e\u751f\u8bcd\uff0c\u8bcd\u6027\uff0c\u7528\u6cd5\u4e13\u5c5e\u7684\u97f3\u6807}\n",
      "   + exp: dict {cn: str, en: str} #explanation expressed as cn and en | \u5355\u8bcd\u7528\u6cd5\u91ca\u4e49\uff0c\u5305\u62ec\u4e2d\u82f1\u6587\n",
      "   + examples: list\n",
      "      * examples[j]: dict {cn: str, en: str}\n",
      "   + syns: list\n",
      "      * syns[j]: str\n",
      "   + ants: list\n",
      "      * ants[j]: dict {cn: str, en: list}\n",
      "   + der: str #\u9700\u8981\u5355\u72ec\u5904\u7406\n",
      "   + pspeech: str #part of speech | \u8bcd\u6027\n",
      "   + ph_symbl : str #phonetic symbols\n",
      "* pos: (int, int) #(List, Unit) | existed\n",
      "* ph_symbl: str #phonetic symbols | \u97f3\u6807\uff0cexisted\n",
      "\n",
      "\n",
      "## \u521d\u6b65\u6784\u9020\n",
      "\n",
      "+ \u9700\u8981\u5c06\u6bcf\u4e2a\u5355\u8bcd\u7684'word_block_str'\u5b57\u6bb5\u8f6c\u6362\u4e3a'usages'\n",
      "+ \u5904\u7406\u8003\u6cd5\u65f6\uff0c\u6709\u4e9b\u5355\u8bcd\u7684\u97f3\u6807\u5728\u8003\u6cd5\u540e\u9762\uff1b\u53e6\u5916\uff0c\u5355\u8bcdcompliment\u9700\u8981\u7279\u6b8a\u5904\u7406\n",
      "+ \u5c06\u6d3e\u751f\u8bcd\u5f53\u4f5c\u4e00\u4e2a\u72ec\u7acb\u7684\u65b0\u8bcd\u5bf9\u5f85"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### \u5b58\u653e\u5404\u7c7b\u5224\u65ad\u51fd\u6570\u7684\u5b57\u5178\n",
      "\n",
      "* \u6bd4\u5982'examples'\u5b57\u6bb5\u5b58\u653e\u7740\u5224\u65ad\u8fd9\u884c\u662f\u4e0d\u662f\u4f8b\u53e5\u7684\u51fd\u6570\n",
      "* \u6ce8\u610f\u95ed\u5305\u9677\u9631"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configNew3000BeforeMain\n",
      "character_start = {'examples': '\u4f8b', \n",
      "                   'syns': '\u8fd1', \n",
      "                   'ants': '\u53cd', \n",
      "                   'der': '\u6d3e'}"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configNew3000BeforeMain\n",
      "is_str_start_with_character_fun_d = {}\n",
      "for key, value in character_start.iteritems():\n",
      "    def gen_match_fun_closure(_value):\n",
      "        return lambda s: s[0] == _value.decode('utf-8')\n",
      "    is_str_start_with_character_fun_d[key] = gen_match_fun_closure(value)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# an incorrect implementation\n",
      "incorrect_implementation = {key: lambda s: s[0] == value.decode('utf-8') for key, value in character_start.iteritems()}\n",
      "# pitfall: value is a global variable and ischanging dynamically, \n",
      "# so finally all lambda will refer to the last value, in this case it is syns!"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# example\n",
      "syn_str = u'\u8fd1 wild, bizarre'\n",
      "print 'Should be True, in fact is', incorrect_implementation['syns'](syn_str)\n",
      "print 'Should be True, in fact is', is_str_start_with_character_fun_d['syns'](syn_str)\n",
      "# anoter example\n",
      "example_str = u'\u4f8b abstract the 135-page'\n",
      "print 'Should be True, in fact is', incorrect_implementation['examples'](example_str)\n",
      "print 'Should be True, in fact is', is_str_start_with_character_fun_d['examples'](example_str)\n",
      "del syn_str, example_str"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### \u5c06`word_block_str`\u8f6c\u53d8\u4e3a`usages`\u7ed3\u6784\n",
      "\n",
      "* \u521d\u6b65\u8f6c\u6362\uff0c\u5373\u628a\u884c\u5bf9\u5e94\u5230\u5b57\u5178\u7684\u5b57\u6bb5\n",
      "* \u5c06`word_block_str`\u6309\u7167\u8003\u6cd5\u62c6\u5206\u4e3a\u91ca\u4e49\u5217\u8868\n",
      "* \u5bf9\u6bcf\u4e2a\u91ca\u4e49\uff0c\u6309`\\n`\u62c6\u5206\u4e3a\u82e5\u5e72\u884c\n",
      "* \u5bf9\u6bcf\u884c\u4f7f\u7528\u51fd\u6570`is_str_start_with_character_d`\u5224\u65ad\u8be5\u884c\u5c5e\u4e8e\u54ea\u4e2a\u5b57\u6bb5\n",
      "* \u67d0\u4e9b\u5355\u8bcd\u7684\u6d3e\u751f\u8bcd\u5f88\u590d\u6742\uff0c\u4ece\u4ee5\u201c\u6d3e\u201d\u5f00\u5934\u7684\u4e00\u884c\u5f00\u59cb\uff0c\u4e4b\u540e\u7684\u884c\uff0c\u90fd\u5c5e\u4e8e\u8be5\u6d3e\u751f\u8bcd\uff0c\u800c\u975e\u6e90\u5355\u8bcd\u3002\u5982compendium\u3001anarchist\n",
      "* \u5bf9\u4e8e\u590d\u6742\u6d3e\u751f\u8bcd\uff0c\u5c06\u76f8\u5173\u5b57\u6bb5\uff08\u51fa\u73b0\u5728\u6d3e\u751f\u8bcd\u6240\u5728\u884c\u4e4b\u540e\uff09\u7ec4\u5408\u6210\u4e00\u4e2a\u5b57\u7b26\u4e32\uff0c\u4ee5\u4fbf\u5229\u7528\u5df2\u6709\u51fd\u6570\u5904\u7406\n",
      "* \u590d\u6742\u6d3e\u751f\u8bcd\u72ec\u7acb\u4e3a\u65b0\u8bcd\u6761\u5904\u7406\n",
      "* \u901a\u8fc7\u89c2\u5bdf\u6240\u6709\u7684\u590d\u6742\u6d3e\u751f\u8bcd\uff0c\u53d1\u73b0\u5982\u4e0b\u7279\u70b9\uff1a\n",
      "  - \u6709\u8bcd\u6027\uff0c\u6ca1\u6709\u97f3\u6807\n",
      "  - \u6d3e\u751f\u8bcd\u7684\u62fc\u5199\uff1a\u4ee5\u6d3e\u5f00\u5934\u7684\u90a3\u884c\uff0c\u5339\u914d\u7b2c\u4e00\u4e2a\u7a7a\u683c\u540e\u7684\u8fde\u7eed\u51e0\u4e2a\u82f1\u6587\u5b57\u7b26\u5373\u53ef\uff0c\u6d89\u53ca\u5230\u7279\u6b8a\u5b57\u7b26\"/\"\n",
      "  - \u6d3e\u751f\u8bcd\u7684\u7b2c\u4e00\u884c\u662f\u89e3\u91ca\uff0c\u4e2d\u82f1\u6587\u4ee5\u201c:\u201d\u95f4\u9694\uff0c\u53ea\u6709\u4e2d\u6587\u65f6\u6ca1\u6709\u95f4\u9694\u7b26\n",
      "  - \u67d0\u4e9b\u5355\u8bcd\u4e00\u4e2a\u91ca\u4e49\u4e0b\u6709\u591a\u4e2a\u5e76\u5217\u7684\u590d\u6742\u6d3e\u751f\u8bcd\uff0c\u6bd4\u5982digress\u3002\u5bf9\u4e8e\u8fd9\u79cd\u60c5\u51b5\uff0c\u7528\u51fd\u6570`extract_content_between`\u5148\u62c6\u5206\u518d\u5904\u7406\n",
      "* \u3010\u52d8\u8bef\u3011\u5904\u7406\u8fc7\u7a0b\u4e2d\u53d1\u73b0\n",
      "  - \u5355\u8bcdplumb\u7684\u91ca\u4e493\u4e2d\uff0c\u4f8b\u6807\u6210\u4e86\u6d3e\n",
      "  - \u5355\u8bcddaunt\u7684\u4e24\u4e2a\u6d3e\u751f\u8bcd\u4ee5\u9017\u53f7\u52a0\u4e00\u4e2a\u7a7a\u683c\u76f8\u8fde"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configNew3000BeforeMain\n",
      "def revise_entry_name(words_d):\n",
      "    # revise random\n",
      "    words_d['random']['word_block_str'] = words_d['random']['word_block_str'].replace(u'\u4f8b\u3000aimless',\n",
      "                                                                                      u'\u8fd1\u3000aimless')\n",
      "    # revise sordid\n",
      "    words_d['sordid']['word_block_str'] = words_d['random']['word_block_str'].replace(u'\u8fd1\u3000Behind his generous',\n",
      "                                                                                      u'\u4f8b\u3000Behind his generous')\n",
      "    # revise clan\n",
      "    words_d['clan']['word_block_str'] = words_d['clan']['word_block_str'] .replace(u'\u53cd\u3000clannish', \n",
      "                                                                                   u'\u6d3e\u3000clannish')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configNew3000AfterMain\n",
      "revise_entry_name(new3000_base_d)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configNew3000BeforeMain\n",
      "match_usage_start_re = re.compile(ur'^\u3010\u8003(?:\u6cd5|\u70b9)\\d?\u3011(.*)$', re.M|re.U)\n",
      "match_der = re.compile(ur'^')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configNew3000BeforeMain\n",
      "def wb_str_2_usages_d_l(word_block_str):\n",
      "    '''\n",
      "    convert word block (string) to usages like structure\n",
      "    input: the 'word_block_str' attribute of a word dictionary\n",
      "    return: two lists, \n",
      "            the first with its 'i'th element indicating whether \n",
      "                the 'i'th usage has a complex der\n",
      "            the second is the list of usages\n",
      "    '''\n",
      "    usage_template = {'exp': '',\n",
      "                      'examples': '',\n",
      "                      'syns': '',\n",
      "                      'ants': '',\n",
      "                      'der': ''}\n",
      "    usages_str_l = extract_content_between(word_block_str, match_usage_start_re)\n",
      "    usages_d_l = []\n",
      "    is_complex_der_l = []\n",
      "    \n",
      "    for one_usage_str in usages_str_l:\n",
      "        one_usage_d = deepcopy(usage_template)\n",
      "        is_complex_der = False\n",
      "        has_der = False\n",
      "        one_usage_lines = one_usage_str.split('\\n')\n",
      "        one_usage_d['exp'] = match_usage_start_re.match(one_usage_lines[0]).group(1)\n",
      "        \n",
      "        for line in one_usage_lines[1:]:\n",
      "            has_been_matched = False\n",
      "            \n",
      "            if line == '' or line == '\\n':\n",
      "                continue\n",
      "            # match \"\u4f8b\" \"\u53cd\", etc.\n",
      "            for field_name, match_func in is_str_start_with_character_fun_d.iteritems():\n",
      "                if match_func(line):\n",
      "                    has_been_matched = True\n",
      "                    if has_der:\n",
      "                        one_usage_d['der'] += '\\n' + line.strip()\n",
      "                        is_complex_der = True\n",
      "                    else:\n",
      "                        # test\n",
      "                        if one_usage_d[field_name] != '':\n",
      "                            print '****Multi line field!****'\n",
      "                            print word_block_str\n",
      "                            pass\n",
      "                        one_usage_d[field_name] = line.strip()\n",
      "                    if field_name == 'der':\n",
      "                        # test\n",
      "                        if has_der:\n",
      "                            # print 'Warning! der in der!'\n",
      "                            # print one_usage_str\n",
      "                            pass\n",
      "                        has_der = True\n",
      "                    break\n",
      "            if not has_been_matched:\n",
      "                # after printed out, it can be seen that these lines are all aphorisms\n",
      "                # so, useless for our purpose\n",
      "                #print line\n",
      "                pass\n",
      "        usages_d_l.append(one_usage_d)\n",
      "        is_complex_der_l.append(is_complex_der)\n",
      "    return is_complex_der_l, usages_d_l"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# test\n",
      "def unit_test():\n",
      "    _, result = wb_str_2_usages_d_l(new3000_base_d['anarchist']['word_block_str'])\n",
      "    iter_print(result, 0, 4)\n",
      "#unit_test()\n",
      "del unit_test"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "\u3010\u52d8\u8bef\u3011  \n",
      "\u5df2\u7ecf\u8003\u8651\u8fc7\u590d\u6742\u6d3e\u751f\u8bcd\u7684\u524d\u63d0\u4e0b\uff0c\u5e94\u8be5\u4e00\u4e2afield\u5360\u636e\u4e00\u884c\u3002\u901a\u8fc7\u68c0\u67e5\u662f\u5426\u6709\u591a\u884cfield\uff0c\u53d1\u73b0\uff1a  \n",
      "* random\uff0c\u7b2c\u4e8c\u4e2a\u4f8b\u5e94\u8be5\u4e3a\u8fd1\uff0c\u8fd4\u56de\u5230\u5904\u7406word_block_str\u4e4b\u524d\u66f4\u6b63\n",
      "* sordid\uff0c\u8003\u6cd52\uff0c\u7b2c\u4e00\u4e2a\u8fd1\u5e94\u4e3a\u4f8b\uff0c\u8fd4\u56de\u5230\u5904\u7406word_block_str\u4e4b\u524d\u66f4\u6b63"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configNew3000BeforeMain\n",
      "def gen_usages_for_all_words(words_d):\n",
      "    match_der_word = re.compile(ur'^\u6d3e ([a-z,/\\-\u00e9\u00ef]+)', re.M)\n",
      "    complex_ders_d = {}\n",
      "    for word in words_d:\n",
      "        if words_d[word]['word_block_str'] == '':\n",
      "            print 'Empty word:', word\n",
      "            continue\n",
      "        is_complex_der_l, words_d[word]['usages'] = wb_str_2_usages_d_l(words_d[word]['word_block_str'])\n",
      "        if True in is_complex_der_l:\n",
      "            for i, one_usage in enumerate(words_d[word]['usages']):\n",
      "                # revise plumb\n",
      "                if i == 2 and word == u'plumb':\n",
      "                    one_usage['example'] = one_usage['der']\n",
      "                    one_usage['der'] = ''\n",
      "                    continue\n",
      "                if is_complex_der_l[i]:\n",
      "                    whole_der_block_str = strF2H(one_usage['der'])\n",
      "                    der_block_str_l = extract_content_between(whole_der_block_str, match_der_word)\n",
      "                    for der_block_str in der_block_str_l:\n",
      "                        # revise daunt\n",
      "                        if word == 'daunt':\n",
      "                            der_block_str = der_block_str.replace(', ',  '/')\n",
      "                        der_word = match_der_word.match(der_block_str).group(1)\n",
      "                        der_block_str = match_der_word.sub(ur'\u3010\u8003\u6cd5\u3011', der_block_str)\n",
      "                        complex_ders_d[der_word] = {}\n",
      "                        _, complex_ders_d[der_word]['usages'] = wb_str_2_usages_d_l(der_block_str)\n",
      "                        if len(complex_ders_d[der_word]['usages']) != 1:\n",
      "                            print 'Warning! Not unqiue explanation!'\n",
      "                            continue\n",
      "                        complex_ders_d[der_word]['usages'][0]['der'] = u'\u6e90 ' + word\n",
      "                        complex_ders_d[der_word]['phon'] = u''\n",
      "                        complex_ders_d[der_word]['pos'] = words_d[word]['pos']\n",
      "                        complex_ders_d[der_word]['word_block_str'] = u''\n",
      "                        # test\n",
      "                        #print der_word\n",
      "                        #iter_print(complex_ders_d[der_word]['usages'])\n",
      "        #del words_d[word]['word_block_str']\n",
      "    return complex_ders_d, words_d"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configNew3000AfterMain\n",
      "complex_ders_d, new3000_base_d = gen_usages_for_all_words(new3000_base_d)\n",
      "new3000_base_d.update(complex_ders_d)\n",
      "del complex_ders_d"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# test\n",
      "#iter_print(new3000_base_d['compliment'])\n",
      "# example\n",
      "print 'An example of the structure of a word dictionary'\n",
      "pprint(new3000_base_d['addict'])\n",
      "iter_print(new3000_base_d['addict'])"
     ],
     "language": "python",
     "metadata": {
      "scrolled": false
     },
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## \u5904\u7406\u91ca\u4e49\n",
      "\n",
      "* \u6bcf\u6b21\u90fd\u628a\u5b57\u7b26\u4e32\u4e2d\u5df2\u7ecf\u5339\u914d\u7684\u5185\u5bb9\u5220\u9664\n",
      "* \u5148\u628a\u8bcd\u6027\u5339\u914d\u51fa\u6765\n",
      "* \u518d\u770b\u91ca\u4e49\u662f\u5426\u5305\u62ec\u97f3\u6807\u3002\u5982\u679c\u5305\u62ec\uff0c\u5339\u914d\u51fa\u6765\u3002\u5982\u679c\u4e0d\u5305\u62ec\uff0c\u7559\u7a7a\uff0c\u672a\u6765\u8f93\u51fa\u65f6\uff0c\u5982\u679c\u53d1\u73b0\u7528\u6cd5\u4e0b\u9762\u7684\u97f3\u6807\u9879\u4e3a\u7a7a\uff0c\u5c31\u663e\u793a\u4e3b\u8bcd\u7684\u97f3\u6807\u3002\u6ce8\u610f\u5355\u72ec\u5904\u7406compliment\u7684\u97f3\u6807\u3002\n",
      "* \u4e2d\u82f1\u6587\u5212\u5206\u7684\u5927\u6982\u89c4\u5219\uff1a\u4ee5\u5206\u8bcd\u7b26\u201c:\u201d\u6765\u533a\u5206\u4e2d\u82f1\u6587\u91ca\u4e49\uff0c\u5de6\u4fa7\u4e3a\u4e2d\u6587\uff0c\u53f3\u4fa7\u4e3a\u82f1\u6587\uff1b\u5982\u679c\u6ca1\u6709\u5206\u8bcd\u7b26\uff0c\u90a3\u4e48\u53ea\u6709\u4e2d\u6587\u91ca\u4e49\n",
      "* \u6ce8\u610f\uff0c\u5bf9\u4e8e\u5192\u53f7\u5b57\u7b26\u6765\u8bf4\uff0c\u4e2d\u6587\u5168\u89d2\u3001\u4e2d\u6587\u534a\u89d2\u3001\u82f1\u6587\u5168\u89d2\u5728unicode\u4e2d\u662f\u7b49\u4ef7\u7684\u3002\u6240\u4ee5\u5148\u7528\u51fd\u6570`strF2H`\u5c06\u5192\u53f7\u7edf\u4e00\u4e3a\u82f1\u6587\u534a\u89d2\u5373\u53ef"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# check unicode\n",
      "print 'cn_f:', repr(u'\uff1a'), 'cn_h:', repr(u'\uff1a')\n",
      "print 'en_f:', repr(u'\uff1a'), 'en_h:', repr(u':')\n",
      "# example\n",
      "print 'Some examples of the raw string of the explanation field'\n",
      "_ = map(iter_print, iter_value_of_key_through_d_l_d_d(new3000_base_d, 'usages', 'exp', 5))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configNew3000BeforeMain\n",
      "match_phon_re = re.compile(ur'\uff3b.*\uff3d', re.U)\n",
      "match_pspeech_re = re.compile(ur'\\*([a-z\\/.]+\\.)\\*')\n",
      "has_cn_char_fun = lambda _str: re.compile(ur'[\\u4e00-\\u9fa5]').search(_str) is not None"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# test\n",
      "def unit_test():\n",
      "    test_str = u\"\uff3b\u0259'la\u026a\uff3d *v.* \u52a0\u5165\u8054\u76df\uff1ato **enter** into an alliance\"\n",
      "    print match_pspeech_re.search(test_str).group(1)\n",
      "    print match_phon_re.search(test_str).group()\n",
      "    print has_cn_char_fun(test_str)\n",
      "#unit_test()\n",
      "del unit_test"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configNew3000BeforeMain\n",
      "def process_exp(exp_field_str):\n",
      "    '''\n",
      "    input: a unicode object corresponding the explanation line of the word\n",
      "    return: dict {exp, pspeech, ph_symbl}\n",
      "    '''\n",
      "    if exp_field_str == '':\n",
      "        print 'Warning! No explanation!'\n",
      "        return\n",
      "    returned_d = {'exp': {'en': '', 'cn': '', 'en_cn': ''}, \n",
      "                  'pspeech': '',\n",
      "                  'ph_symbl': ''}\n",
      "    \n",
      "    result = match_pspeech_re.search(exp_field_str)\n",
      "    if result:\n",
      "        returned_d['pspeech'] = result.group(1)\n",
      "        exp_field_str = match_pspeech_re.sub('', exp_field_str, 1)\n",
      "    \n",
      "    result = match_phon_re.search(exp_field_str)\n",
      "    if result:\n",
      "        returned_d['ph_symbl'] = result.group()\n",
      "        exp_field_str = match_phon_re.sub('', exp_field_str, 1).strip()\n",
      "    \n",
      "    returned_d['exp']['en_cn'] = exp_field_str.strip()\n",
      "    \n",
      "    # seperate en and cn\n",
      "    spered_str_l = [_str.strip() for _str in strF2H(exp_field_str).split(u':')]\n",
      "    seperator_count = len(spered_str_l) - 1\n",
      "    if seperator_count == 0:\n",
      "        # test whether no seperator guarantees no chinese explanation\n",
      "        # print 'No sep', spered_str_l\n",
      "        returned_d['exp']['cn'] = spered_str_l[0]\n",
      "    elif seperator_count == 1:\n",
      "        returned_d['exp']['cn'], returned_d['exp']['en'] = spered_str_l\n",
      "    elif seperator_count == 2:\n",
      "        # test\n",
      "        # print 'Two sep: ', spered_str_l\n",
      "        has_char_cn_boo_l = map(has_cn_char_fun, spered_str_l)\n",
      "        returned_d['exp']['cn'] = u':'.join([spered_str_l[i] for i in range(seperator_count+1) if has_char_cn_boo_l[i]])\n",
      "        returned_d['exp']['en'] = u':'.join([spered_str_l[i] for i in range(seperator_count+1) if not has_char_cn_boo_l[i]])\n",
      "        # test\n",
      "        #iter_print(returned_d['exp'])\n",
      "    else:\n",
      "        # test \n",
      "        #print 'More than two sep: ', exp_field_str\n",
      "        pass\n",
      "    return returned_d"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "\u5206\u9694\u7b26\u6700\u591a\u67092\u4e2a\u3002\u5f53\u6709\u4e24\u4e2a\u5206\u9694\u7b26\u65f6\uff0c\u62c6\u5206\u540e\u7684\u5217\u8868\u4e2d\uff0c\u4e00\u822c\u662f\u540e\u4e24\u4e2astr\u5bf9\u5e94\u82f1\u6587\u91ca\u4e49\uff0c\u5c11\u6570\u60c5\u5f62\u4e0b\u524d\u4e24\u4e2astr\u5bf9\u5e94\u4e2d\u6587\u91ca\u4e49\u3002\u4f8b\u5916\uff1a\n",
      "* abuse\uff1a\u8003\u6cd52\u6709\u4e24\u5bf9\u4e2d\u82f1\u6587\u91ca\u4e49\n",
      "* \u3010\u52d8\u8bef\u3011disaffected\u6d3e\u751f\u8bcd\u7684\u91ca\u4e49\u884c\uff0c\u5305\u542b\u4e86\u4e24\u4e2a\u6d3e\u751f\u8bcd\uff0cdisaffect\u548cdisaffection\uff0c\u4ee5\u201c;\u201d\u95f4\u9694\u3002  \n",
      "\u5176\u4e2ddisaffect\u662f\u4e2a\u590d\u6742\u6d3e\u751f\u8bcd\uff0c\u5df2\u7ecf\u72ec\u7acb\u6210\u8bcd\u6761\uff0c\u4ee5disaffect\u7d22\u5f15\u3002\u800cdisaffection\u672c\u8eab\u662f\u4e00\u4e2a\u7b80\u5355\u6d3e\u751f\u8bcd\uff0c\u6240\u4ee5\u5ffd\u7565\u6389\u8fd9\u90e8\u5206\u89e3\u91ca\u5373\u53ef\u3002"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configNew3000BeforeMain\n",
      "def process_exp_field_for_all_words(words_d):\n",
      "    for word, usage_index, exp_str in iter_value_of_key_through_d_l_d_d(words_d, 'usages', 'exp', \n",
      "                                                                        yield_top_key=True, yield_list_index=True):\n",
      "        base_exp_d = None\n",
      "        # get base_exp_d\n",
      "        # revise abuse\n",
      "        if word == 'abuse' and usage_index == 1:\n",
      "            exp_str_l = exp_str.split(';')\n",
      "            base_exp_d, extra_exp_d = map(process_exp, exp_str_l)\n",
      "            base_exp_d['exp']['en'] = base_exp_d['exp']['en'] + ';' + extra_exp_d['exp']['en']\n",
      "            base_exp_d['exp']['cn'] = base_exp_d['exp']['cn'] + ';' + extra_exp_d['exp']['cn']\n",
      "            # test\n",
      "            #iter_print(base_exp_d)\n",
      "\n",
      "        # revise disaffected\n",
      "        if word == 'disaffect':\n",
      "            base_exp_d = process_exp(exp_str.split(';')[0])\n",
      "            # test\n",
      "            #iter_print(base_exp_d)\n",
      "\n",
      "        else:    \n",
      "            base_exp_d = process_exp(exp_str)\n",
      "        \n",
      "        # get phonic symbol from parent field\n",
      "        if base_exp_d['ph_symbl'] == u'':\n",
      "            # revise compliment\n",
      "            if word == 'compliment':\n",
      "                if usage_index == 0:\n",
      "                    base_exp_d['ph_symbl'] = 'n. ' + words_d[word]['phon'][0] + \\\n",
      "                                             ' v. ' + words_d[word]['phon'][1]\n",
      "                else:\n",
      "                    base_exp_d['ph_symbl'] = words_d[word]['phon'][0]\n",
      "            else:\n",
      "                # test\n",
      "                if usage_index > 2:\n",
      "                    #print word\n",
      "                    pass\n",
      "                base_exp_d['ph_symbl'] = words_d[word]['phon']\n",
      "        one_usage = words_d[word]['usages'][usage_index]\n",
      "        one_usage['ph_symbl'] = base_exp_d['ph_symbl']\n",
      "        del base_exp_d['ph_symbl']\n",
      "        one_usage['pspeech'] = base_exp_d['pspeech']\n",
      "        del base_exp_d['pspeech']\n",
      "        one_usage['exp_d'] = base_exp_d['exp']\n",
      "    return words_d"
     ],
     "language": "python",
     "metadata": {
      "scrolled": false
     },
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configNew3000AfterMain\n",
      "new3000_base_d = process_exp_field_for_all_words(new3000_base_d)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# test\n",
      "#iter_print(new3000_base_d['ensconce'], print_list_index=True)\n",
      "# check\n",
      "_ = map(pprint, iter_through_and_sample_k(new3000_base_d, 5, [('all','',True), ('key','usages',False),\n",
      "                                                              ('all','',False), ('key', 'exp_d', False),\n",
      "                                                              ('key', 'en', False)]))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## \u5904\u7406\u4f8b\u53e5\n",
      "\n",
      "### \u793a\u4f8b\n",
      "\n",
      "* \u4e00\u4e2a\u4f8b\u53e5  \n",
      "\u4f8b\u3000an audacious plan \u4e00\u4e2a\u5192\u8fdb\u7684\u8ba1\u5212\n",
      "\n",
      "* \u591a\u4e2a\u4f8b\u53e5\u4ee5\u201c\u2016\u201d(`\\u2016`)\u95f4\u9694  \n",
      "\u4f8b\u3000a bevy of ever-smiling aspirants for the Miss America title \u4e00\u7fa4\u60f3\u5f53\u7f8e\u56fd\u5c0f\u59d0\u7684\u5c11\u5973\u2016Envy can make oneself backward; self-confidence can tell oneself to be an aspirant. \u5992\u5fcc\u80fd\u4f7f\u81ea\u5df1\u843d\u540e\uff0c\u81ea\u4fe1\u80fd\u4f7f\u81ea\u5df1\u4e0a\u8fdb\u3002\n",
      "\n",
      "* \u4e2d\u6587\u7b26\u53f7\u51fa\u73b0\u5728\u82f1\u6587\u4e2d  \n",
      "\u201c\u2018Frankenstein\u2019...\u2018Dracula\u2019...\u2018Dr. Jekyll and Mr. Hyde\u2019... the archetypes that have influenced all subsequent horror stories\u201d\uff08New York Times\uff09\u201c\u2018\u5f17\u5170\u80af\u65af\u5766\u2019\u2026\u2018\u5fb7\u62c9\u5e93\u62c9\u2019\u2026\u2018\u6770\u57fa\u5c14\u535a\u58eb\u548c\u6d77\u5fb7\u5148\u751f\u2019\u662f\u5f71\u54cd\u4e86\u6240\u6709\u7ee7\u4e4b\u800c\u6765\u7684\u6050\u6016\u6545\u4e8b\u7684\u539f\u578b\u201d\uff08\u300a\u7ebd\u7ea6\u65f6\u62a5\u300b\uff09\n",
      "\n",
      "* \u7f3a\u5c11\u5206\u9694\u7b26  \n",
      "\u4f8b\u3000He became apoplectic about wasteful government spending. \u4ed6\u5bf9\u653f\u5e9c\u7684\u6d6a\u8d39\u5f00\u9500\u611f\u5230\u6012\u4e0d\u53ef\u904f\u3002 The coach was so apoplectic when the player missed the free throw that he threw his clipboard onto the court. \u6559\u7ec3\u5bf9\u7403\u5458\u7f5a\u7bee\u4e0d\u8fdb\u975e\u5e38\u607c\u6012\uff0c\u628a\u6218\u672f\u677f\u6254\u5230\u4e86\u7403\u573a\u4e0a\u3002"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# test\n",
      "path_to_example = [('all', '', True), ('key', 'usages', False), ('all','',False),('key','examples',False)]\n",
      "_ = map(functools.partial(iter_print, print_list_index=False), \n",
      "        iter_through_and_sample_k(new3000_base_d, 5, path_to_example))\n",
      "del path_to_example"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### \u5224\u65ad\u7f3a\u5c11\u5206\u9694\u7b26\u7684\u60c5\u5f62\n",
      " \n",
      "\u9996\u5148\uff0c[\u4e4b\u524d](#\u7edf\u8ba1\u6bcf\u884c\u7b2c\u4e00\u4e2a\u5b57\u7b26)\u5df2\u7ecf\u4fee\u6b63\u8fc7\u4e24\u4e2a\u4f8b\u53e5\u5360\u4e24\u884c\u7684\u60c5\u5f62\u3002  \n",
      "\u5047\u5b9a\u6240\u6709\u7684\u4f8b\u53e5\uff0c\u82f1\u6587\u90fd\u51fa\u73b0\u5728\u4e2d\u6587\u4e4b\u524d\u3002\u5982\u679c\u4e00\u884c\u5185\u5305\u542b\u591a\u4e2a\u4f8b\u53e5\u5374\u6ca1\u6709\u5206\u9694\u7b26\uff0c\u90a3\u4e48\u4e00\u5b9a\u6ee1\u8db3\n",
      "\n",
      "* \u5982\u679c\u4e00\u4e2a\u5b57\u7b26\u4e32\u4e2d\u5305\u542b\u591a\u7ec4\u4e2d\u6587\u5b57\u7b26\u4e32\uff0c\u90a3\u4e48\u9664\u6700\u540e\u4e00\u7ec4\u5916\uff0c\u6bcf\u7ec4\u4e2d\u6587\u5b57\u7b26\u4e32\u540e\u4e00\u5b9a\u4f1a\u8ddf\u4e00\u53e5\u82f1\u6587\n",
      "* \u82f1\u6587\u53e5\u5b50\u81f3\u5c11\u7531\u4e24\u4e2a\u5355\u8bcd\u6784\u6210\uff0c\u6240\u4ee5\u7528`(?=[a-z]+ [a-z]+)`\u6765\u5339\u914d\u4e00\u4e2a\u540e\u9762\u662f\u82f1\u6587\u53e5\u5b50\u7684\u4f4d\u7f6e\uff0c\u540c\u65f6\u7f16\u8bd1\u6a21\u5f0f\u9009\u62e9\u5ffd\u7565\u5927\u5c0f\u5199\n",
      "* \u6700\u540e\u4e00\u7ec4\u4e2d\u6587\u5b57\u7b26\u4e32\u4e00\u5b9a\u53ef\u4ee5\u5339\u914d\u4e00\u4e2a\u884c\u5c3e\u5b57\u7b26\n",
      "* \u5339\u914d\u6a21\u5f0f\u8981\u5141\u8bb8\u53e5\u9996\u51fa\u73b0\u4e2d\u6587\u6807\u70b9\uff08\u6bd4\u5982\u5f15\u53f7\uff09\uff0c\u82f1\u6587\u5355\u8bcd\uff08\u4e00\u822c\u662f\u4e13\u6709\u540d\u8bcd\uff09\u6216\u6570\u5b57\n",
      "* \u4e2d\u6587\u534a\u89d2\u9017\u53f7\u3001\u4e2d\u6587\u5168\u89d2\u9017\u53f7\u3001\u82f1\u6587\u5168\u89d2\u9017\u53f7\uff0cunicode\u4e2d\u662f\u4e00\u4e2a\u5b57\u7b26`\\uff0c`\n",
      "* `[\\u4e00-\\u9fa5]`\u5339\u914d\u5e38\u89c1\u7684\u4e2d\u6587\u5b57\u7b26\uff0c\u4e0d\u5305\u62ec\u6807\u70b9\u7b26\u53f7"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configNew3000BeforeMain\n",
      "match_all_cn_re = ur' ?[a-z0-9\uff1a\u3002\uff1b\uff0c\u201c\u201d\uff08\uff09\u3001\uff1f\u300a\u300b]*?[\\u4e00-\\u9fa5]+.*?(?=$|[a-z]+ [a-z]+)'\n",
      "match_all_cn_re = re.compile(match_all_cn_re, re.I)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# example\n",
      "test_str = u'He became apoplectic about wasteful government spending. \u4ed6\u5bf9\u653f\u5e9c\u7684\u6d6a\u8d39\u5f00\u9500\u611f\u5230\u6012\u4e0d\u53ef\u904f\u3002 \\\n",
      "            The coach was so apoplectic when the player missed the free throw that he threw his clipboard onto the court.\\\n",
      "            \u6559\u7ec3\u5bf9\u7403\u5458\u7f5a\u7bee\u4e0d\u8fdb\u975e\u5e38\u607c\u6012\uff0c\u628a\u6218\u672f\u677f\u6254\u5230\u4e86\u7403\u573a\u4e0a\u3002'\n",
      "results = match_all_cn_re.findall(test_str)\n",
      "for sen in results:\n",
      "    print sen\n",
      "del results, test_str"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "def detect_no_split_symbol(sentences_str):\n",
      "    if sentences_str == '':\n",
      "        return\n",
      "    if sentences_str.startswith(u'\u4f8b'):\n",
      "        sentences_str = sentences_str[1:]\n",
      "    else:\n",
      "        print 'Warning! Not start with \u4f8b:', sentences_str\n",
      "        return\n",
      "    if u'\\u2016' not in sentences_str:\n",
      "        results = match_all_cn_re.findall(sentences_str)\n",
      "        if len(results) > 1:\n",
      "            for sentence in results:\n",
      "                print sentence\n",
      "            print sentences_str\n",
      "            print '***********'\n",
      "            "
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def all_that_without_splt_symbol():\n",
      "    path_to_example = [('all', '', False), ('key', 'usages', False), ('all','',False),('key','examples',False)]\n",
      "    _ = map(detect_no_split_symbol, iter_through_general(new3000_base_d, path_to_example, False))\n",
      "all_that_without_splt_symbol()\n",
      "del all_that_without_splt_symbol"
     ],
     "language": "python",
     "metadata": {
      "scrolled": false
     },
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "\u3010\u52d8\u8bef\u3011\u5bf9\u4e8e\u6ca1\u6709\u5206\u9694\u7b26\u4f46\u5374\u51fa\u73b0\u591a\u6bb5\u4e2d\u6587\u5b57\u7b26\u7684\u53e5\u5b50  \n",
      "* \u5339\u914d\u4e00\u4e2a\u4e2d\u6587\u53e5\u53f7\u6216\u4e2d\u6587\u95ee\u53f7\uff0c\u518d\u52a0\u4e00\u4e2a\u82f1\u6587\u5b57\u7b26\uff0c\u5728\u4e2d\u6587\u7b26\u53f7\u540e\u52a0\u5206\u9694\u7b26\n",
      "* \u5bf9\u4e8e\u5355\u8bcdheckle\u3001carefree\uff0c\u5339\u914d\u4e00\u4e2a\u4e2d\u6587\u5b57\u7b26\u52a0\u82f1\u6587\u5b57\u7b26\uff0c\u5728\u4e2d\u6587\u5b57\u7b26\u540e\u52a0\u5206\u9694\u7b26\n",
      "* \u5bf9\u4e8e\u5355\u8bcdclarity\uff0c\u5c06\u201c;\u201d\u66ff\u6362\u4e3a\u5206\u9694\u7b26"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configNew3000BeforeMain\n",
      "match_cn_punc_with_en_char_fun = lambda _str: re.search(ur'[\u3002\uff1f]( )?(?=[a-z])', _str, re.I)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# example\n",
      "test_str = u\"I don't think that they could be compatible as roommates. \u6211\u4e0d\u89c9\u5f97\u4ed6\u4eec\u505a\u4e86\u5ba4\u53cb\u4e4b\u540e\u80fd\u548c\u8c10\u76f8\u5904\u3002a theory that is compatible with what we already know about early man \u4e00\u4e2a\u4e0e\u5bf9\u8fdc\u53e4\u4eba\u7c7b\u7684\u5df2\u6709\u77e5\u8bc6\u4e0d\u5b58\u5728\u77db\u76fe\u7684\u7406\u8bba\"\n",
      "cn_pun_index = match_cn_punc_with_en_char_fun(test_str).end()\n",
      "print test_str[:cn_pun_index] + u'\\u2016' + test_str[cn_pun_index:]\n",
      "del test_str, cn_pun_index"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configNew3000BeforeMain\n",
      "match_cn_char_with_en_char_fun = lambda _str: re.search(ur'[\\u4e00-\\u9fa5](?=[a-z])', _str, re.I)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# example\n",
      "test_str = u'passengers on a luxury cruise ship enjoying a carefree vacation \u5728\u8c6a\u534e\u6e38\u8f6e\u4e0a\u4eab\u53d7\u7740\u65e0\u5fe7\u65c5\u9014\u7684\u4e58\u5ba2carefree college students on spring break \u6625\u5047\u671f\u95f4\u65e0\u5fe7\u65e0\u8651\u7684\u5927\u5b66\u751f'\n",
      "cn_char_index = match_cn_char_with_en_char_fun(test_str).end()\n",
      "print test_str[:cn_char_index] + u'\\u2016' + test_str[cn_char_index:]\n",
      "del test_str, cn_char_index"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configNew3000BeforeMain\n",
      "# revise\n",
      "def revise_no_sep(words_d):\n",
      "    path_to_example = [('all', '', True), ('key', 'usages', False), ('all','',True),('key','examples',False)]\n",
      "    example_iter = iter_through_general(words_d, path_to_example)\n",
      "    for word, usage_index, example_str in example_iter:\n",
      "        if example_str == '':\n",
      "            continue\n",
      "        example_str = example_str[2:]\n",
      "        if u'\\u2016' not in example_str:\n",
      "            results = match_all_cn_re.findall(example_str)\n",
      "            if len(results) > 1:\n",
      "                index_to_add_sep = None\n",
      "                one_result = match_cn_punc_with_en_char_fun(example_str)\n",
      "                if one_result:\n",
      "                    index_to_add_sep = one_result.end()\n",
      "                elif word in [u'heckle', u'carefree']:\n",
      "                    one_result = match_cn_char_with_en_char_fun(example_str)\n",
      "                    index_to_add_sep = one_result.end()\n",
      "                elif word == 'clarify':\n",
      "                    example_str = example_str.replace(u';', u'\\u2016')\n",
      "                if index_to_add_sep:\n",
      "                    example_str = example_str[:index_to_add_sep] + u'\\u2016' + example_str[index_to_add_sep:]\n",
      "        words_d[word]['usages'][usage_index]['examples'] = u'\u4f8b ' + example_str       \n",
      "    return words_d"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configNew3000AfterMain\n",
      "new3000_base_d = revise_no_sep(new3000_base_d)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# check\n",
      "def unit_test():\n",
      "    path_to_example = [('all', '', False), ('key', 'usages', False), ('all','',False),('key','examples',False)]\n",
      "    _ = map(detect_no_split_symbol, iter_through_general(new3000_base_d, path_to_example, False))\n",
      "unit_test()\n",
      "del unit_test"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "\u81f3\u6b64\uff0c\u591a\u4e2a\u4f8b\u53e5\u5e94\u8be5\u90fd\u6709\u5206\u9694\u7b26\u95f4\u9694\u4e86\u3002\n",
      "\n",
      "### \u62c6\u5206\u4e2d\u82f1\u6587\n",
      "\n",
      "\u5c06\u6bcf\u4e2a\u4f8b\u53e5\u7684\u4e2d\u82f1\u6587\u5206\u9694\u5f00\u3002\u76f4\u63a5\u5339\u914d\u82f1\u6587\u90e8\u5206\u5373\u53ef\u3002  "
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configNew3000BeforeMain\n",
      "match_sentence_en_part_re = re.compile(ur'[a-z0-9\u00e9\u00ef\\'\";:,?!%()$\u2160\u2161.*/\\- \u2014\u3000\u2018\u2019\u201c\u201d\uff08\uff09]+(?=[\uff1c\u300a\u3008\\u4e00-\\u9fa5])', re.I)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# example\n",
      "test_str = u'     \u3000add spices to the stew with complete abandon \u8086\u65e0\u5fcc\u60ee\u5730\u5411\u7096\u83dc\u91cc\u9762\u52a0\u8c03\u6599'\n",
      "print match_sentence_en_part_re.match(test_str).group()\n",
      "del test_str"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "\u4e0a\u9762\u7684\u5339\u914d\u89c4\u5219\uff0c\u53ef\u80fd\u5bfc\u81f4\u4e2d\u6587\u90e8\u5206\u7684\u5185\u5bb9\u4e5f\u88ab\u5339\u914d\u3002\u6bd4\u5982  \n",
      "`A GPA of 1.0 flusters him. 1.0\u7684\u7ee9\u70b9\u8ba9\u4ed6\u5f88\u614c\u4e71\u3002`\u4f1a\u88ab\u5339\u914d\u4e3a`A GPA of 1.0 flusters him. 1.0`  \n",
      "\u6ce8\u610f\u5230\uff0c\u9519\u8bef\u5339\u914d\u7684\u90e8\u5206\uff0c\u4e00\u5b9a\u4f1a\u4e0e\u4e00\u4e2a\u5c5e\u4e8e\u4e2d\u6587\u90e8\u5206\u7684\u5b57\u7b26\u7d27\u5bc6\u76f8\u8fde\u3002  \n",
      "\n",
      "\u5982\u679c\u5339\u914d\u6b63\u786e\uff0c\u8fd9\u4e2a\u7d27\u5bc6\u76f8\u8fde\u7684\u90e8\u5206\u5e94\u8be5\u662f\u7a7a\u683c\u3001\u82f1\u6587\u53e5\u53f7\u3001\u4e2d\u6587\u7684\u53f3\u5f15\u53f7`\u201d`\u3001\u53f3\u62ec\u53f7`\uff09`  \n",
      "\u5982\u679c\u6ca1\u5339\u914d\u6b63\u786e\uff0c\u90a3\u4e48\u53ef\u80fd\u662f\u4e00\u4e2a\u4e2d\u6587\u5de6\u5f15\u53f7`\u201c`\u3001\u6570\u5b57\u3001\u82f1\u6587\u4eba\u540d\u3001\u4e13\u6709\u540d\u8bcd\u7b49\n",
      "* \u6570\u5b57\uff0c\u7ecf\u8fc7\u68c0\u67e5\uff0c\u786e\u5b9a\u65b9\u6848\u5982\u4e0b\u3002\u5339\u914d\u7684\u5b57\u7b26\u4e32\u4ece\u53f3\u81f3\u5de6\u904d\u5386\u81f3\u7b2c\u4e00\u4e2a\u7a7a\u683c\uff0c\u53d6\u7a7a\u683c\u5de6\u4fa7\u7684\u5185\u5bb9\n",
      "* `GRE`\u3001`IT`\u3001`DNA`\u3001`Jason`\uff0c\u5904\u7406\u65b9\u6cd5\u540c\u6570\u5b57\n",
      "* `\u201c`\u3001`\u201c\u2018`\uff0c\u53d6\u5b57\u7b26\u7ec4\u5408\u5de6\u4fa7\u7684\u5b57\u7b26\u4e32\u5373\u53ef"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configNew3000BeforeMain\n",
      "def sep_en_cn_sentence(sentences_str):\n",
      "    if sentences_str == '':\n",
      "        return '', '', '',\n",
      "    sentences_str = sentences_str[2:].replace(u'\\!', u'!')\n",
      "    is_number_fun = lambda _str: re.match('\\d', _str)\n",
      "    en_str_l = []\n",
      "    cn_str_l = []\n",
      "    en_cn_str_l= []\n",
      "    for sentence in sentences_str.split(u'\\u2016'):\n",
      "        sentence = sentence.strip(u' \u3000\\n')\n",
      "        en_cn_str_l.append(sentence)\n",
      "        result = match_sentence_en_part_re.match(sentence)\n",
      "        if result:\n",
      "            en_str = result.group()\n",
      "            # test\n",
      "            if not (en_str[-1] in [' ', '.', u'\uff09', u'\u201d']):\n",
      "                if en_str[-1] == u'\u201c':\n",
      "                    #print en_str\n",
      "                    en_str = en_str[:-1]\n",
      "                    #print en_str\n",
      "                elif is_number_fun(en_str[-1]) or (en_str[-2:] in ['RE', 'IT', 'on', 'NA']):\n",
      "                    #print en_str\n",
      "                    last_blank_space = len(en_str) - 1\n",
      "                    while en_str[last_blank_space] != ' ':\n",
      "                        last_blank_space -= 1\n",
      "                    en_str = en_str[:last_blank_space]\n",
      "                    #print en_str\n",
      "                elif en_str[-2:] == u'\u201c\u2018':\n",
      "                    #print en_str\n",
      "                    en_str = en_str[:-2]\n",
      "                    #print en_str\n",
      "                else:\n",
      "                    #print en_str\n",
      "                    #print sentence\n",
      "                    pass\n",
      "            en_str_l.append(strF2H(en_str).strip())\n",
      "            cn_str_l.append(sentence.replace(en_str, ''))\n",
      "        else:\n",
      "            print sentence\n",
      "            raise ValueError('Warning! No en part!')\n",
      "    return new_line_join(en_str_l), new_line_join(cn_str_l), new_line_join(en_cn_str_l)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# test\n",
      "#iter_print(sep_en_cn_sentence(new3000_base_d['abandon']['usages'][0]['examples']))"
     ],
     "language": "python",
     "metadata": {
      "scrolled": true
     },
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configNew3000BeforeMain\n",
      "def process_examples(words_d):\n",
      "    path_to_example = [('all', '', True), ('key', 'usages', False), ('all','',True),('key','examples',False)]\n",
      "    example_iter = iter_through_general(words_d, path_to_example)\n",
      "    for word, usage_index, example_str in example_iter:\n",
      "        examples_en, examples_cn, examples_encn = sep_en_cn_sentence(example_str)\n",
      "        words_d[word]['usages'][usage_index]['examples_d'] = {'en': examples_en, 'cn': examples_cn, 'en_cn': examples_encn}\n",
      "    return words_d"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configNew3000AfterMain\n",
      "new3000_base_d = process_examples(new3000_base_d)"
     ],
     "language": "python",
     "metadata": {
      "scrolled": false
     },
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# test\n",
      "path_to_example_d = [('all', '', False), ('key', 'usages', False), ('all','',False),('key','examples_d',False)]\n",
      "#_ = map(iter_print, iter_through_and_sample_k(new3000_base_d, 1, path_to_example_d))\n",
      "del path_to_example_d\n",
      "# check\n",
      "iter_print(new3000_base_d['abandon']['usages'][0])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## \u5904\u7406\u53cd\u4e49\u8bcd\n",
      "\n",
      "### \u793a\u4f8b\n",
      "\n",
      "* \u53cd\u3000timid, cowardice, cravenness, dastardliness, poltroonery \u80c6\u5c0f\uff0c\u61e6\u5f31"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# test\n",
      "path_to_ants = [('all','',False),('key','usages',False),('all','',False),('key','ants',False)]\n",
      "#_ = map(iter_print, iter_through_and_sample_k(new3000_base_d, 2, path_to_ants))\n",
      "del path_to_ants"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "\u62c6\u5206\u4e2d\u82f1\u6587"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configNew3000BeforeMain\n",
      "match_ants_en_part_re = re.compile(ur'[a-z\u00e9\u00ef][a-z\u00e9\u00ef ,-/]+(?=[\u3000\\u4e00-\\u9fa5\uff08]|$)', re.I)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# example\n",
      "test_str = u'unthreatening \u6ca1\u6709\u5a01\u80c1\u7684\uff1breassuring \u4ee4\u4eba\u5b89\u5fc3\u7684'\n",
      "print test_str\n",
      "iter_print(match_ants_en_part_re.findall(test_str))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configNew3000BeforeMain\n",
      "def sep_en_cn_ants(ants_str):\n",
      "    if ants_str == '':\n",
      "        return '', '', '', 0\n",
      "    ants_str = ants_str[2:]\n",
      "    num_ants_of_explanations = 0\n",
      "    en_str_l = match_ants_en_part_re.findall(ants_str)\n",
      "    num_ants_of_explanations = len(en_str_l)\n",
      "    # test\n",
      "    if num_ants_of_explanations == 0:\n",
      "        print 'Warning! No en part!', ants_str\n",
      "    cn_str = match_ants_en_part_re.sub('', ants_str).strip(' \\n')\n",
      "    search_en_fun = lambda _str: re.search(r'[a-z]', _str, re.I)\n",
      "    if search_en_fun(cn_str):\n",
      "        print 'Warning! en in cn part!', cn_str\n",
      "    en_cn = ants_str.strip(' \\n')\n",
      "    return '; '.join(en_str_l), cn_str, en_cn, num_ants_of_explanations"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configNew3000BeforeMain\n",
      "def process_all_ants(words_d):\n",
      "    path_to_ants = [('all','',True),('key','usages',False),('all','',True),('key','ants',False)]\n",
      "    ants_iter = iter_through_general(words_d, path_to_ants)\n",
      "    for word, usage_index, ant_str in ants_iter:\n",
      "        en_str, cn_str, en_cn_str, num_exps = sep_en_cn_ants(ant_str)\n",
      "        words_d[word]['usages'][usage_index]['ants_d'] = {'en': en_str, 'cn': cn_str, 'en_cn': en_cn_str}\n",
      "        # test\n",
      "        if num_exps > 1:\n",
      "            #print word\n",
      "            pass\n",
      "    return words_d"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "\u3010\u52d8\u8bef\u3011\n",
      "* enfranchise\u8003\u6cd52\uff0c`subjugate, subdue; enthrall`\u66ff\u6362\u4e3a`subjugate, subdue, enthrall`\n",
      "* clan\uff0c\u53cd\u4e49\u8bcd\u6761\u76ee\u5e94\u8be5\u662f\u6d3e\u751f\u8bcd\uff0c\u56e0\u4e3a\u5728\u8fd9\u4e4b\u524d\u5904\u7406\u7684\u6d3e\u751f\u8bcd\uff0c\u6240\u4ee5\u5fc5\u987b\u53bb\u524d\u9762\u66f4\u6b63"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configNew3000AfterMain\n",
      "new3000_base_d['enfranchise']['usages'][1]['ants'] = new3000_base_d['enfranchise']['usages'][1]['ants'].replace(u'subdue; enthrall', u'subdue, enthrall')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configNew3000AfterMain\n",
      "new3000_base_d = process_all_ants(new3000_base_d)"
     ],
     "language": "python",
     "metadata": {
      "scrolled": true
     },
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# test\n",
      "path_to_ants = [('all','',False),('key','usages',False),('all','',False),('key','ants_d',False)]\n",
      "#_ = map(iter_print, iter_through_and_sample_k(new3000_base_d, 10, path_to_ants))\n",
      "del path_to_ants\n",
      "#iter_print(new3000_base_d['polished'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## \u5904\u7406\u540c\u4e49\u8bcd\n",
      "\n",
      "\u57fa\u672c\u4e0d\u7528\u5904\u7406"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configNew3000BeforeMain\n",
      "strip_first_two_chars_fun = lambda _str: _str[2:]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configNew3000BeforeMain\n",
      "def process_all_syns(words_d):\n",
      "    path_to_syns = [('all','',True),('key','usages',False),('all','',True),('key','syns',False)]\n",
      "    for word, usage_index, syns_str in iter_through_general(words_d, path_to_syns):\n",
      "        usage_d = words_d[word]['usages'][usage_index]\n",
      "        usage_d['syns'] = strip_first_two_chars_fun(syns_str)\n",
      "    return words_d"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configNew3000AfterMain\n",
      "new3000_base_d = process_all_syns(new3000_base_d)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## \u8865\u5145\u53d1\u97f3\n",
      "\n",
      "\u539f\u4e66\u4e2d\uff0c\u5982\u679c\u4e00\u4e2a\u91ca\u4e49\u7684\u53d1\u97f3\u540c\u4e0a\u4e00\u4e2a\u4e00\u6837\uff0c\u5219\u7701\u7565\u6389\u3002\u8fd9\u91cc\u6211\u4eec\u5c06\u5176\u8865\u5145\u56de\u6765\u3002"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configNew3000BeforeMain\n",
      "def supplement_word_ph_symbl(words_d):\n",
      "    path_to_phsymb = [('all','',True),('key','usages',False),('all','',True),('key','ph_symbl',False)]\n",
      "    for word, usage_index, ph_symbl in iter_through_general(words_d, path_to_phsymb):\n",
      "        usage_d = words_d[word]['usages'][usage_index]\n",
      "        if usage_d['ph_symbl'] == '':\n",
      "            cur_pspeech = usage_d['pspeech']\n",
      "            if usage_index == 0:\n",
      "                # uncommend print if you want to check\n",
      "                #print 'Word %s has no phonetic symbol, maybe it is a derivative.'%word\n",
      "                continue\n",
      "            pre_usage_d = words_d[word]['usages'][usage_index-1]\n",
      "            pre_pspeech = pre_usage_d['pspeech']\n",
      "            pre_phsymbl = pre_usage_d['ph_symbl']\n",
      "            if pre_pspeech != cur_pspeech:\n",
      "                if not cur_pspeech.startswith('v'):\n",
      "                    # already check the v. vi. vt. case\n",
      "                    print 'Previous pspeech is different. Please check! Word %s'%word\n",
      "                    iter_print(usage_d)\n",
      "                    continue\n",
      "            usage_d['ph_symbl'] = pre_phsymbl\n",
      "    return words_d"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "\u3010\u52d8\u8bef\u3011 \u5355\u8bcdcompendium\u91ca\u4e492\u6ca1\u6709\u8bcd\u6027\uff0c\u662f\u540d\u5b57\uff0c\u8865\u5145\u4e0a\u3002"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configNew3000AfterMain\n",
      "\n",
      "# revise compendium\n",
      "new3000_base_d['compendium']['usages'][1]['pspeech'] = 'n.'"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configNew3000AfterMain\n",
      "new3000_base_d = supplement_word_ph_symbl(new3000_base_d)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "\u5199\u5165\u6587\u4ef6"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configNew3000AfterMain\n",
      "with codecs.open('new3000_base_d.txt', 'w', encoding='utf-8') as f:\n",
      "    json.dump(new3000_base_d, f)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%writefile -a $new3000_convert_script_name\n",
      "\n",
      "if __name__ == '__main__':\n",
      "    main()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "\u6700\u7ec8\u7ed3\u679c"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# example\n",
      "#iter_print(new3000_base_d['ad-lib'])\n",
      "iter_print(new3000_base_d['salutary'])\n",
      "#iter_print(new3000_base_d['fawn'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "\u5904\u7406\u300aGRE\u6838\u5fc3\u8bcd\u6c47\u52a9\u8bb0\u4e0e\u7cbe\u7ec3\u300b"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configZhujiAfterMain\n",
      "zhuji_base_str = codecs_open_r_utf8(file_zhuji)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "\u7531\u4e8e\u8f6c\u6362\u683c\u5f0f\u9009\u62e9\u4e86markdown\uff0c\u6240\u4ee5\u4f1a\u6709\u5927\u91cf\u7684\u8f6c\u4e49\u5b57\u7b26\u3002\u6bd4\u5982`\\[`\u3001`\\(`\u7b49\u3002\u7edf\u4e00\u5254\u9664\u524d\u9762\u7684\u8f6c\u4e49\u7b26\u3002"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configZhujiBeforeMain\n",
      "match_escape_char_re = re.compile(r'\\\\(?=[\\[\\]()*+])')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configZhujiAfterMain\n",
      "zhuji_base_str = match_escape_char_re.sub('', zhuji_base_str)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configZhujiAfterMain\n",
      "zhuji_base_str = collapse_blank_line(zhuji_base_str)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "\u5c06\u5408\u5e76\u7a7a\u884c\u540e\u7684\u6587\u672c\u8f93\u51fa\u5230\u4e34\u65f6\u6587\u4ef6\u3002\u4e4b\u540e\u4ee5\u8be5\u4e34\u65f6\u6587\u4ef6\u4e3a\u5bfb\u627e\u6587\u672c\u89c4\u5f8b\u7684\u4f9d\u636e\u3002"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configZhujiAfterMain\n",
      "with codecs.open('temp_zhuji_base_str.txt', 'w', encoding='utf-8') as f:\n",
      "    f.write(zhuji_base_str)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## \u63d0\u53d6\u601d\u8def\n",
      "\n",
      "+ \u524d\u8a00\u90e8\u5206\u8bf4\u660e\u4e86\u8bcd\u6761\u7684\u7ed3\u6784\n",
      "\n",
      "        \u4e09\u3001\u5355\u4e2a\u8bcd\u6761\u7684\u7f16\u6392\n",
      "\n",
      "        \u8bcd\u6761\u7ed3\u6784\n",
      "\n",
      "        \u6bcf\u4e00\u4e2a\u5355\u8bcd\u7684\u7b2c\u4e00\u884c\u662f\u62fc\u5199\u548c\u97f3\u6807\uff0c\u63a5\u4e0b\u6765\u662f\u5bf9\u8fd9\u4e2a\u5355\u8bcd\u7684\u52a9\u8bb0\u6cd5\u3002\u6709\u53ef\u80fd\u662f\u4ee5\u4e0b\u4e94\u79cd\u4e4b\u4e00\uff1a\n",
      "\n",
      "        \uff081\uff09[\u6839]\u8bcd\u6839\u8bcd\u7f00\u8bb0\u5fc6\u6cd5\uff1b\n",
      "\n",
      "        \uff082\uff09[\u8054]\u8054\u60f3\u52a9\u8bb0\u6cd5\uff1b\n",
      "\n",
      "        \uff083\uff09[\u6e90]\u901a\u8fc7\u8bcd\u6e90\u89e3\u91ca\u5355\u8bcd\uff1b\n",
      "\n",
      "        \uff084\uff09[\u8bb0]\u5176\u4ed6\u52a9\u8bb0\u6cd5\uff1b\n",
      "\n",
      "        \uff085\uff09[\u53c2]\u63d0\u4f9b\u76f8\u5173\u8bcd\u52a9\u8bb0\uff1b\n",
      "\n",
      "        \u6b64\u5916\uff0c\u6bcf\u6761\u52a9\u8bb0\u6cd5\u5728\u8bcd\u6761\u7684\u6700\u540e\u53ef\u80fd\u6709[\u6ce8]\uff0c\u7ed9\u51fa\u4e00\u4e9b\u5bf9\u7406\u89e3\u548c\u8bb0\u5fc6\u5355\u8bcd\u6709\u5e2e\u52a9\u7684\u5185\u5bb9\u3002\n",
      "    \n",
      " \u6240\u4ee5\uff0c\u6bcf\u4e2a\u5355\u8bcd\u5757\u4ee5\u82f1\u6587\u5355\u8bcd\u4e3a\u8d77\u59cb\uff0c\u4ee5\u4e0b\u4e00\u4e2a\u5355\u8bcd\u7684\u5f00\u59cb\u4e3a\u7ed3\u675f\n",
      "\n",
      "\n",
      "+ \u5355\u8bcd\u5757\u5305\u542b\u5728List\u91cc\uff0c\u5148\u8bc6\u522bList\u5757\uff0c\u518d\u8bc6\u522b\u5355\u8bcd\u5757\u3002  \n",
      "List\u4ee5\u201c### List\u201d\u52a0\u4e00\u4e2a\u7a7a\u683c\u52a0\u4e00\u4e2a\u6570\u5b57\u5f00\u59cb\n",
      "\n",
      "## Extract List Block\n",
      "\n",
      "\u53ea\u5904\u7406\u7b2c\u4e00\u7bc7\u7684\u5185\u5bb9"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configZhujiAfterMain\n",
      "zhuji_base_str = zhuji_base_str.split(u'# \u7b2c\u4e8c\u7bc7 \u6838\u5fc3\u8bcd\u6c47\u7ec3\u4e60')[0]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configZhujiBeforeMain\n",
      "match_zhuji_list_start_re = re.compile(ur'### List \\d+', re.M)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configZhujiAfterMain\n",
      "zhuji_base_list_l = extract_content_between(zhuji_base_str, match_zhuji_list_start_re)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# check\n",
      "print 'Should have 39 Lists. Extract', len(zhuji_base_list_l)\n",
      "# test\n",
      "#print zhuji_base_list_l[38]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## \u63d0\u53d6etyma_block\n",
      "\n",
      "+ \u5148\u628alist_block \u6309\u7167 \u201c\u5c0f\u7ed3&\u590d\u4e60\u201d \u62c6\u5206\u6210\u4e24\u90e8\u5206\uff0c\u7136\u540e\u53ea\u5904\u7406\u7b2c\u4e00\u90e8\u5206\n",
      "    \n",
      "+ \u5339\u914d\u4e00\u4e2a\u8bcd\u6839\u5757\uff08etyma_block\uff09\u7684\u8d77\u59cb    \n",
      "\u884c\u9996\u4e00\u4e2a\u6570\u5b57\u52a0\u7b26\u53f7\u201c.\u201d\uff0c\u7136\u540e\u5339\u914d\u4efb\u610f\u5b57\u7b26\u5230\u884c\u5c3e\u3002\u5bf9\u4e8eList37-List39\uff0c\u5339\u914d\u884c\u9996Unit\u3002\n",
      "\n",
      "+ \u6bcf\u4e2a\u8bcd\u6839\u5757\u5339\u7b2c\u4e00\u884c\u914d\u51fa\u8bcd\u6839\uff0c\u5e76\u4e14\u7b5b\u9009\u6389\u65e0\u610f\u4e49\u7684\u8bcd\u6839\n",
      "\n",
      "+ List 36\u7684\u8bcd\u6839\u5e94\u8be5\u52a0\u4e0a\u524d\u7f00\u201c\u4e0e\u52a8\u7269\u6709\u5173\u7684\u5355\u8bcd\u201d"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configZhujiBeforeMain\n",
      "def get_etyma_block_d_l_l(list_data_l):\n",
      "    match_etyma_block_start = re.compile(r'^\\d+\\.(.*)$\\n|^Unit \\d+$', re.M)\n",
      "    etyma_block_d_l_l = []\n",
      "    for list_index, base_list_str in enumerate(list_data_l):\n",
      "        if list_index > 38:\n",
      "            break\n",
      "        etyma_block_d_l = []\n",
      "        base_list_str = base_list_str.split(u'\u5c0f\u7ed3&\u590d\u4e60')[0]\n",
      "        etyma_block_str_l = extract_content_between(base_list_str, match_etyma_block_start)\n",
      "        for etyma_index, etyma_block_str in enumerate(etyma_block_str_l):\n",
      "            ety_str = match_etyma_block_start.search(etyma_block_str).group(1)\n",
      "            if ety_str is None:\n",
      "                ety_str = ''\n",
      "            ety_str = ety_str.strip()\n",
      "            if  ety_str == u'\u5176\u4ed6':\n",
      "                #print u'\u8bcd\u6839\u662f\u5176\u4ed6'\n",
      "                ety_str = ''\n",
      "            if list_index == 36-1:\n",
      "                ety_str = u'\u4e0e\u52a8\u7269\u6709\u5173\u7684\u5355\u8bcd\uff0c' + ety_str\n",
      "            ety_str = ety_str.strip()\n",
      "            etyma_block_str_and_summary_str = etyma_block_str.split(u'\u5c0f\u7ed3')\n",
      "            summary_str = etyma_block_str_and_summary_str[1] if len(etyma_block_str_and_summary_str) == 2 else ''\n",
      "            etyma_block_str = match_etyma_block_start.sub('', etyma_block_str_and_summary_str[0])\n",
      "            # revise surg, cit\n",
      "            if ety_str == 'surg, cit':\n",
      "                temp_str_l = etyma_block_str.split('\\n')\n",
      "                #iter_print(temp_str_l)\n",
      "                # insert line 5 after line 0\n",
      "                modified_str_l = [temp_str_l[0], temp_str_l[5]] + temp_str_l[1:5] + temp_str_l[6:]\n",
      "                etyma_block_str = '\\n'.join(modified_str_l)\n",
      "                #print etyma_block_str\n",
      "            # revise rejoice\n",
      "            if ety_str == u'\u6b22\u4e50\u4e0e\u559c\u60a6':\n",
      "                temp_str_l = etyma_block_str.split('\\n')\n",
      "                #iter_print(temp_str_l)\n",
      "                modified_str_l = [temp_str_l[0], temp_str_l[9]] + temp_str_l[1:9]\n",
      "                etyma_block_str = '\\n'.join(modified_str_l)\n",
      "                #print etyma_block_str\n",
      "            etyma_block_d = {'pos':(list_index+1, etyma_index+1), \n",
      "                             'ety': ety_str,\n",
      "                             'ety_block_str': etyma_block_str,\n",
      "                             'summary': summary_str}\n",
      "            etyma_block_d_l.append(etyma_block_d)\n",
      "        etyma_block_d_l_l.append(etyma_block_d_l)\n",
      "    return etyma_block_d_l_l"
     ],
     "language": "python",
     "metadata": {
      "scrolled": true
     },
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configZhujiAfterMain\n",
      "zhuji_base_d_l_l = get_etyma_block_d_l_l(zhuji_base_list_l)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# check\n",
      "print 'In total', len(zhuji_base_d_l_l), 'lists'"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configZhujiBeforeMain\n",
      "def revise_miss_etyma(base_d_l_l):\n",
      "    # revise list 25 etyma 3 revise tum\n",
      "    base_d_l_l[25-1][3-1]['ety'] = 'tum'\n",
      "    # revise list 5 etyma 4 revise post, pound\n",
      "    base_d_l_l[5-1][4-1]['ety'] = 'post, pound'\n",
      "    # revise list 6 etyma 7 revise vad, vag, ced\n",
      "    base_d_l_l[6-1][7-1]['ety'] = 'vad, vag, ced'"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configZhujiAfterMain\n",
      "revise_miss_etyma(zhuji_base_d_l_l)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# example\n",
      "iter_print(zhuji_base_d_l_l[24][2])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## \u5904\u7406ety_block_str\n",
      "\n",
      "* \u4e4b\u524d\u63d0\u53d6\u7684ety_block_str\u91cc\u5305\u542b\u4e86\u5bf9\u8bcd\u6839\u7ec4\u7684\u89e3\u91ca\uff08etyma_group_explanation\uff09\uff0c\u4ee5\u53ca\u6240\u6709\u7684\u540c\u6839\u8bcd\uff08cognate_block\uff09\u3002\n",
      "* cognate_block\u4ee5\u5355\u8bcd\u52a0\u4e0a\u97f3\u6807\u4e3a\u8d77\u59cb\uff0c\u7b2c\u4e00\u4e2acognate_block\u4e4b\u524d\u7684\u5c31\u662fetyma_group_explanation\uff0c\u4e24\u4e2a\u5f00\u59cb\u4f4d\u7f6e\u53ea\u95f4\u7684\u5c31\u662f\u4e00\u4e2acognate_block"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configZhujiBeforeMain\n",
      "match_cognate_block_start_re = re.compile(ur'^([a-z\u00e9\u00ef-]+)(.*?)(\\[.*\\])$', re.M|re.I)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# example\n",
      "iter_print(extract_content_between(zhuji_base_d_l_l[1-1][6-1]['ety_block_str'], match_cognate_block_start_re, True))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configZhujiBeforeMain\n",
      "def process_ety_block_str(base_d_l_l):\n",
      "    path_to_ety_block_str = [('all','',True),('all','',True),('key','ety_block_str',False)]\n",
      "    for list_index, ety_index, ety_block_str in iter_through_general(base_d_l_l, \n",
      "                                                                     path_to_ety_block_str):\n",
      "        etyma_block_d = base_d_l_l[list_index][ety_index]\n",
      "        returned_l = extract_content_between(ety_block_str, match_cognate_block_start_re, True)\n",
      "        ety_group_exp = returned_l.pop(0).strip()\n",
      "        etyma_block_d['etyma_group_explanation'] = ety_group_exp\n",
      "        etyma_block_d['cognate_block_str_l'] = returned_l"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configZhujiAfterMain\n",
      "process_ety_block_str(zhuji_base_d_l_l)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# test\n",
      "#iter_print(zhuji_base_d_l_l[26-1][3-1])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## \u5904\u7406cognate_block_str\n",
      "\n",
      "\u4e00\u4e2acognate_block\uff0c\u53ef\u4ee5\u4ece\u9996\u884c\u63d0\u53d6\u51faword\uff0c\u97f3\u6807\uff08phon\uff09  \n",
      "\u4e4b\u540e\uff0c\u6bcf\u884c\u5e94\u8be5\u4ee5\u201c[\u201d\u5f00\u59cb\u3002\u4ee5\u6b64\u6807\u51c6\uff0c\u67e5\u770b\u4f8b\u5916\u3002\u67e5\u770b\u540e\uff0c\u53d1\u73b0\u5982\u4e0b\u7279\u4f8b\u3002\n",
      "\n",
      "* \u3010\u52d8\u8bef\u3011\u201c\u6e90\u201d\u6ca1\u6709\u88ab\u5305\u62ec\u5728\u201c[]\u201d\u4e2d\n",
      "* \u5355\u8bcdfacilitate\u4e2d\u5305\u542b\u4e86\u5927\u91cf\u7684\u989d\u5916\u5185\u5bb9\uff0c\u6574\u5408\u5230\u4e00\u8d77\u5c31\u53ef\u4ee5\u4e86\n",
      "* \u3010\u52d8\u8bef\u3011 \u8bcd\u6839`surg, cit`\uff0c\u4ee5`(1)`\u3001`(2)`\u5206\u4e86\u4e24\u90e8\u5206\u3002\u53bb\u524d\u9762[\u63d0\u53d6etyma_block](#\u63d0\u53d6etyma_block)\u4e2d\u66f4\u6b63\n",
      "* List 26, \u7b2c2\u4e2a\u8bcd\u6839\uff0c\u6240\u6709\u5355\u8bcd\u53ea\u6709\u7b80\u5355\u7684\u91ca\u4e49\uff0c\u4e0d\u7528\u4fee\u6539\u3002  \n",
      "\u3010\u52d8\u8bef\u3011\u8be5\u8bcd\u6839\u5757\u7684\u6700\u540e\u4e00\u4e2a\u5355\u8bcd\uff0crejoice\u7684\u6700\u540e\u4e00\u884c\u5e94\u8be5\u52a0\u5230ety_group_explanation\u540e\u3002  \n",
      "\u53bb\u524d\u9762[\u63d0\u53d6etyma_block](#\u63d0\u53d6etyma_block)\u4e2d\u66f4\u6b63\n",
      "* List 2, \u7b2c6\u4e2a\u8bcd\u6839\uff0c\u884c\u201c`\u4ee5\u4e0b\u4e24\u4e2a\u5355\u8bcd\u4e2d\u7684ord\u548ccant\u4e0e\u201c\u8bf4\u201d\u6ca1\u6709\u5173\u7cfb`\u201d\u6ca1\u6709\u7528\u3002\u5ffd\u7565\u4e86\u3002\n",
      "* \u3010\u52d8\u8bef\u3011 List 13, \u7b2c3\u4e2a\u8bcd\u6839  \n",
      "`\u4ee5\u4e0b\u76844\u4e2a\u5355\u8bcd\u53ef\u4ee5\u5c06scru\u6309\u7167\u8bfb\u97f3\u8054\u60f3\u6210\u201c\u56db\u987e\u201d\uff0c\u8868\u793a\u201c (\u987e\u8651\u5730) \u770b\u201d\u3002`\u5e94\u8be5\u5c06scru\u63d0\u53d6\u51fa\u6765\uff0c\u4f5c\u4e3a\u540e4\u4e2a\u5355\u8bcd\u7684\u8bcd\u6839\u3002"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configZhujiBeforeMain\n",
      "# revise List 13, ety 3 revise scru\n",
      "def revise_scru(base_d_l_l):\n",
      "    '''\n",
      "    please only call it one time\n",
      "    or re-run the code cells starting from \n",
      "        \"zhuji_base_d_l_l = get_etyma_block_d_l_l(zhuji_base_list_l)\"\n",
      "    '''\n",
      "    to_revise_l = base_d_l_l[13-1][3-1]['cognate_block_str_l']\n",
      "    #iter_print(to_revise_l)\n",
      "    # remove element 3-5 and build new dict\n",
      "    new_l = to_revise_l[3:]\n",
      "    to_revise_l[2] = to_revise_l[2].replace(u'\u4ee5\u4e0b\u76844\u4e2a\u5355\u8bcd\u53ef\u4ee5\u5c06scru\u6309\u7167\u8bfb\u97f3\u8054\u60f3\u6210\u201c\u56db\u987e\u201d\uff0c\u8868\u793a\u201c (\u987e\u8651\u5730) \u770b\u201d\u3002', '')\n",
      "    to_revise_l = to_revise_l[:3]\n",
      "    new_ety = 'scru'\n",
      "    new_ety_group_exp = u'\u5c06scru\u6309\u7167\u8bfb\u97f3\u8054\u60f3\u6210\u201c\u56db\u987e\u201d\uff0c\u8868\u793a\u201c (\u987e\u8651\u5730) \u770b\u201d'\n",
      "    new_ety_d = {'cognate_block_str_l': new_l, 'pos': (13, 3),\n",
      "                 'ety': new_ety, \n",
      "                 'etyma_group_explanation': new_ety_group_exp,\n",
      "                 'summary':'', 'ety_block_str':''}\n",
      "    base_d_l_l[13-1].append(new_ety_d)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configZhujiAfterMain\n",
      "revise_scru(zhuji_base_d_l_l)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configZhujiBeforeMain\n",
      "def process_cognate_block(cognate_block_str):\n",
      "    cognate_dict = {}\n",
      "    cognate_lines_l = cognate_block_str.split('\\n')\n",
      "    first_line_match = match_cognate_block_start_re.match(cognate_lines_l.pop(0))\n",
      "    word = first_line_match.group(1)\n",
      "    if (word == '') or (word is None):\n",
      "        print 'Warning!'\n",
      "    cognate_dict['word'] = word\n",
      "    phon = first_line_match.group(3)\n",
      "    cognate_dict['phon'] = phon if not (phon is None) else ''\n",
      "    \n",
      "    modified_cognate_lines_l = []\n",
      "    for cognate_line in cognate_lines_l:\n",
      "        cognate_line = cognate_line.strip()\n",
      "        if cognate_line == '':\n",
      "            pass\n",
      "        elif cognate_line.startswith(u'\u6e90'):\n",
      "            # revise \u6e90\n",
      "            cognate_line = cognate_line.replace(u'\u6e90', u'[\u6e90]')\n",
      "            # print cognate_line\n",
      "        elif cognate_dict['word'] == u'facilitate':\n",
      "            pass\n",
      "        elif cognate_dict['word'] in ['jocular', 'jocund', 'jovial', 'rejoice']:\n",
      "            pass\n",
      "        elif cognate_line.startswith(u'\u4ee5\u4e0b\u4e24\u4e2a\u5355\u8bcd\u4e2d'):\n",
      "            pass\n",
      "        elif not cognate_line.startswith(u'['):\n",
      "            # test\n",
      "            print 'current line:', cognate_line, '\\ncurrent block\\n', cognate_block_str\n",
      "            break\n",
      "        else:\n",
      "            pass\n",
      "        modified_cognate_lines_l.append(cognate_line)\n",
      "    cognate_dict['content'] = '\\n'.join(modified_cognate_lines_l)\n",
      "    return cognate_dict"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configZhujiBeforeMain\n",
      "def process_all_cognate_block(base_data_d_l_l):\n",
      "    base_word_d = {}\n",
      "    path_to_cognate_block_str = [('all','',True),('all','',True),\n",
      "                                 ('key','cognate_block_str_l',False),('all','',True)]\n",
      "    for list_index, eytma_index, cognate_index, cognate_block_str in iter_through_general(base_data_d_l_l, \n",
      "                                                                                          path_to_cognate_block_str):\n",
      "        one_word_d = process_cognate_block(cognate_block_str)\n",
      "        word = one_word_d['word']\n",
      "        for _key in ['pos', 'ety', 'etyma_group_explanation', 'summary']:\n",
      "            one_word_d[_key] = base_data_d_l_l[list_index][eytma_index][_key]\n",
      "        one_word_d['pos'] = ', '.join([unicode(i) for i in one_word_d['pos']])\n",
      "        one_word_d['etyma_cognates_l'] = '' # waiting to be filled later\n",
      "        if word in base_word_d:\n",
      "            print 'Warning! word already exists!', word\n",
      "        base_word_d[word] = one_word_d\n",
      "    return base_word_d"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configZhujiAfterMain\n",
      "zhuji_base_word_d = process_all_cognate_block(zhuji_base_d_l_l)"
     ],
     "language": "python",
     "metadata": {
      "scrolled": false
     },
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## \u6dfb\u52a0\u540c\u6839\u8bcd\u5217\u8868\n",
      "\n",
      "\u4f9d\u636ecognate_block_str_l\u7ed9\u6bcf\u4e2a\u5355\u8bcd\u6dfb\u52a0\u540c\u6839\u8bcd\u5217\u8868\u3002\u53ea\u5bf9\u6709\u610f\u4e49\u7684\u8bcd\u6839\u6dfb\u52a0\u3002"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configZhujiBeforeMain\n",
      "def add_etyma_cognates_l(base_word_d, base_d_l_l):\n",
      "    path_to_etyma_d = [('all','',False),('all','',False)]\n",
      "    for etyma_d, in iter_through_general(base_d_l_l, path_to_etyma_d):\n",
      "        ety_str = etyma_d['ety']\n",
      "        ety_group_exp = etyma_d['etyma_group_explanation']\n",
      "        if ety_str != '' or ety_group_exp != '':\n",
      "            if ety_str == '':\n",
      "                # test\n",
      "                print ety_group_exp\n",
      "            etyma_cognates_l = []\n",
      "            for cognate_block_str in etyma_d['cognate_block_str_l']:\n",
      "                word = match_cognate_block_start_re.match(cognate_block_str).group(1)\n",
      "                etyma_cognates_l.append(word)\n",
      "            for word in etyma_cognates_l:\n",
      "                base_word_d[word]['etyma_cognates_l'] = ', '.join(etyma_cognates_l)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configZhujiAfterMain\n",
      "add_etyma_cognates_l(zhuji_base_word_d, zhuji_base_d_l_l)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "* \u3010\u52d8\u8bef\u3011List 25 \u7b2c3\u4e2a\u8bcd\u6839\uff0c\u5e94\u8be5\u662ftum\uff0c\u5728[\u5904\u7406ety_block_str](#\u5904\u7406ety_block_str)\u4e4b\u524d\u4fee\u8ba2\n",
      "* \u3010\u52d8\u8bef\u3011List 5 \u7b2c4\u4e2a\u8bcd\u6839\uff0c\u5e94\u8be5\u662f\u201cpost, pound\u201d\uff0c\u5728[\u5904\u7406ety_block_str](#\u5904\u7406ety_block_str)\u4e4b\u524d\u4fee\u8ba2\n",
      "* \u3010\u52d8\u8bef\u3011List 6 \u7b2c7\u4e2a\u8bcd\u6839\uff0c\u5e94\u8be5\u662f\u201cvad, vag, ced\u201d\uff0c\u5728[\u5904\u7406ety_block_str](#\u5904\u7406ety_block_str)\u4e4b\u524d\u4fee\u8ba2"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configZhujiAfterMain\n",
      "with codecs.open('zhuji_base_d.txt', 'w', encoding='utf-8') as f:\n",
      "    json.dump(zhuji_base_word_d, f)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%writefile -a $zhuji_convert_script_name\n",
      "\n",
      "if __name__ == '__main__':\n",
      "    main()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "\u6700\u7ec8\u6210\u679c"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# example\n",
      "pprint(zhuji_base_word_d['pervade'])\n",
      "iter_print(zhuji_base_word_d['pervade'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "\u5904\u7406\u300aGRE\u9ad8\u5206\u5fc5\u5907\u77ed\u8bed\u642d\u914d\u300b"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## \u6587\u672c\u7ed3\u6784\u5206\u6790\n",
      "\n",
      "+ \u6bcf\u4e00\u7ae0\u90fd\u4ee5\u201c# Unit\u201d\u52a0\u4e0a\u7a7a\u683c\u548c\u6570\u5b57\u5f00\u5934\n",
      "+ \u7b2cn\u6b21\u5339\u914d\u7684\u7ed3\u5c3e\u5230\u7b2cn+1\u6b21\u5339\u914d\u7684\u5f00\u5934\u4e4b\u95f4\uff0c\u5c31\u662f\u4e00\u4e2a\u5355\u5143\u7684\u5185\u5bb9\n",
      "+ \u6bcf\u4e2a\u5355\u5143\uff0c\u53ea\u62c6\u5206\u201c\u68c0\u6d4b\u7ec3\u4e60\u201d\u4e4b\u524d\u7684\u5185\u5bb9\uff0c\u5339\u914d\u201c## \u68c0\u6d4b\u7ec3\u4e60\u201d\n",
      "+ \u6bcf\u4e2a\u5355\u5143\u5185\u90e8\uff0c\u6bcf\u4e00\u4e2a\u8bcd\u7ec4\u7684\u7b2c\u4e00\u884c\u90fd\u4ee5\u201c`**`\u201d\u5f00\u59cb\u548c\u7ed3\u675f\u3002\n",
      "+ \u7b2c\u4e00\u884c\u5185\uff0c\u4e00\u822c\u5305\u542b\u8bcd\u7ec4\u82f1\u6587\u62fc\u5199\u548c\u4e2d\u6587\u89e3\u91ca\u3002\u6709\u7684\u8bcd\u7ec4\u591a\u4e2a\u91ca\u4e49\uff0c\u6240\u4ee5\u9996\u884c\u53ea\u6709\u62fc\u5199\u3002\n",
      "+ \u6bcf\u4e2a\u8bcd\u7ec4\u4e0b\uff0c\u6709\u4e09\u4e2a\u6761\u76ee\uff0c\u91ca\uff0c\u4f8b\uff0c\u9898\u3002\u91ca\u548c\u4f8b\u90fd\u5355\u72ec\u6210\u884c\uff0c\u9898\u5360\u4e86\u4e09\u884c\n",
      "\n",
      "## \u63d0\u53d6\u57fa\u672c\u5185\u5bb9\n",
      "\n",
      "+ \u53bb\u9664\u8f6c\u4e49\u5b57\u7b26\n",
      "+ \u6309\u5355\u5143\u63d0\u53d6\n",
      "+ \u5904\u7406\u7d22\u5f15\n",
      "+ \u6309\u8bcd\u7ec4\u63d0\u53d6\u3002"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configDyAfterMain\n",
      "dy_base_str = codecs_open_r_utf8(file_duanyu)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configDyAfterMain\n",
      "match_escape_char_re = re.compile(r'\\\\(?=[\\[\\]()*+])')\n",
      "dy_base_str = match_escape_char_re.sub('', dy_base_str)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configDyBeforeMain\n",
      "def extract_dy_unit(base_str):\n",
      "    base_str = base_str.split(u\"# \u7d22\u5f15\\n\")[0]\n",
      "    match_dy_unit_start_re = re.compile(ur'^# Unit \\d+', re.M)\n",
      "    base_unit_str_l = extract_content_between(base_str, match_dy_unit_start_re)\n",
      "    base_unit_str_l = [base_unit_str.split(u'## \u68c0\u6d4b\u7ec3\u4e60\\n')[0] for base_unit_str in base_unit_str_l]\n",
      "    return base_unit_str_l"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configDyAfterMain\n",
      "dy_base_unit_str_l = extract_dy_unit(dy_base_str)"
     ],
     "language": "python",
     "metadata": {
      "scrolled": true
     },
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# check\n",
      "print len(dy_base_unit_str_l), \"units extracted\"\n",
      "# print dy_base_unit_str_l[35]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configDyBeforeMain\n",
      "def extract_dy_index_content(base_str):\n",
      "    match_dy_index_cn_start_re = re.compile(u' (?=[\\u4e00-\\u9fa5\\u3010])')\n",
      "    index_str = base_str.split(u\"# \u7d22\u5f15\\n\")[1]\n",
      "    index_d = {}\n",
      "    for line_str in index_str.split('\\n'):\n",
      "        if line_str == '':\n",
      "            continue\n",
      "        line_str = strF2H(line_str)\n",
      "        en_cn = match_dy_index_cn_start_re.split(line_str)\n",
      "        if len(en_cn) == 2:\n",
      "            index_d[en_cn[0]] = en_cn[1]\n",
      "        else:\n",
      "            print 'Warning, no en or no cn:', en_cn\n",
      "    return index_d"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configDyAfterMain\n",
      "dy_index_d = extract_dy_index_content(dy_base_str)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# check\n",
      "print len(dy_index_d), 'phrases in total'\n",
      "print dy_index_d['a barrage of']"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configDyBeforeMain\n",
      "def extract_dy_phrase_d(base_unit_str_l):\n",
      "    base_phrase_d = {}\n",
      "    for unit_index, base_unit_str in enumerate(base_unit_str_l):\n",
      "        match_phrase_start_re = re.compile(ur'^\\*\\*([a-z].*?)([\\u3000\\u4e00-\\u9fa5].*)?\\*\\*$', \n",
      "                                        re.M|re.I)\n",
      "        phrase_block_str_l = extract_content_between(base_unit_str, match_phrase_start_re)\n",
      "        for phrase_block_str in phrase_block_str_l:\n",
      "            match_result = match_phrase_start_re.match(phrase_block_str)\n",
      "            if match_result is None:\n",
      "                print phrase_block_str\n",
      "            phrase_en = match_result.group(1)\n",
      "            phrase_exp_cn = match_result.group(2)\n",
      "            if phrase_exp_cn is None:\n",
      "                phrase_exp_cn = ''\n",
      "            else:\n",
      "                phrase_exp_cn = phrase_exp_cn.strip(u'\\u3000 ')\n",
      "            phrase_block_str = phrase_block_str[match_result.end():].strip('\\n ')\n",
      "            base_phrase_d[phrase_en] = {'exp_cn': phrase_exp_cn, \n",
      "                                        'phrase_block_str': phrase_block_str,\n",
      "                                        'pos': unit_index}\n",
      "    return base_phrase_d"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configDyAfterMain\n",
      "dy_phrase_d = extract_dy_phrase_d(dy_base_unit_str_l)"
     ],
     "language": "python",
     "metadata": {
      "scrolled": true
     },
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# example\n",
      "print len(dy_phrase_d)\n",
      "iter_print(dy_phrase_d['so far'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "\u4f9d\u636eindex\u6821\u8ba2"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "for word in dy_index_d:\n",
      "    if word not in dy_phrase_d:\n",
      "        print word\n",
      "print '****'\n",
      "for word in dy_phrase_d:\n",
      "    if word not in dy_index_d:\n",
      "        print word"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "\u3010\u52d8\u8bef\u3011\u4e0a\u9762\u4e24\u4e2a\u5355\u8bcd\uff0c\u5c06\u4e2d\u6587\u7684\u5355\u5f15\u53f7\u66ff\u6362\u4e3a\u82f1\u6587"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configDyAfterMain\n",
      "# revise \u2019'\n",
      "dy_phrase_d['under one\\'s control'] = dy_phrase_d[u'under one\u2019s control']\n",
      "dy_phrase_d['on one\\'s own'] = dy_phrase_d[u'on one\u2019s own']\n",
      "del dy_phrase_d[u'under one\u2019s control'], dy_phrase_d[u'on one\u2019s own']"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "# check\n",
      "for word in dy_index_d:\n",
      "    if word not in dy_phrase_d:\n",
      "        print word"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## \u5904\u7406phrase_block_str\n",
      "\n",
      "+ \u5148\u5224\u65ad\u8be5\u8bcd\u7ec4\u662f\u5426\u591a\u4e49\uff0c\u5373\u8bcd\u7ec4\u662f\u5426\u6709\u4e2d\u6587\u91ca\u4e49\u3002\n",
      "+ \u5982\u679c\u591a\u4e2a\u91ca\u4e49\uff0c\u6bcf\u4e2a\u91ca\u4e49\u5757\u7684\u7b2c\u4e00\u884c\u90fd\u4ee5\u6570\u5b57\u52a0\u82f1\u6587\u53e5\u53f7`.`\u5f00\u59cb\n",
      "+ \u5bf9\u4e8e\u6bcf\u4e2a\u91ca\u4e49\uff0c\u9664\u5374\u4e2d\u6587\u91ca\u4e49\u5916\uff0c\u91ca\u5bf9\u5e94\u82f1\u6587\u91ca\u4e49\uff0c\u53601\u884c\uff0c\u4f8b\u5bf9\u5e94\u4f8b\u53e5\uff0c\u53601\u884c\uff0c\u9898\u5bf9\u5e94GRE\u4f8b\u53e5\uff0c\u53603\u884c"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configDyBeforeMain\n",
      "def process_dy_phrase_block_str(base_d):\n",
      "    processed_phrase_d = {}\n",
      "    for phrase, base_phrase_d in base_d.iteritems():\n",
      "        phrase_block_str = base_phrase_d['phrase_block_str']\n",
      "        has_multiple_cn_exp = base_phrase_d['exp_cn'] == ''\n",
      "        match_dy_multi_cn_exp_start_re = re.compile(ur'^\\*\\*\\d+\\. (.*)\\*\\*$', re.M)\n",
      "        if has_multiple_cn_exp:\n",
      "            exp_cn_l = match_dy_multi_cn_exp_start_re.findall(phrase_block_str)\n",
      "            phrase_block_str_l = extract_content_between(phrase_block_str, \n",
      "                                                         match_dy_multi_cn_exp_start_re)\n",
      "        else:\n",
      "            exp_cn_l = [base_phrase_d['exp_cn']]\n",
      "            phrase_block_str_l = [phrase_block_str]\n",
      "        \n",
      "        match_en_exp_re = re.compile(ur'^\\*\\*\u91ca\\*\\* (.*)$', re.M)\n",
      "        match_example_re = re.compile(ur'^\\*\\*\u4f8b\\*\\* (.*)$', re.M)\n",
      "        match_gre_example = re.compile(ur'\\*\\*\u9898\\*\\* (.*)$', re.S)\n",
      "        \n",
      "        for usage_index, phrase_block_str in enumerate(phrase_block_str_l):\n",
      "\n",
      "            phrase_detailed_d = {}\n",
      "            exp_en = match_en_exp_re.search(phrase_block_str).group(1)\n",
      "            example = match_example_re.search(phrase_block_str).group(1)\n",
      "            gre_example_en_cn = match_gre_example.search(phrase_block_str).group(1).split('\\n')\n",
      "            gre_example_en = gre_example_en_cn[0]\n",
      "            gre_example_cn = gre_example_en_cn[2]\n",
      "            phrase_detailed_d = {'en_exp': exp_en, \n",
      "                                 'cn_exp': exp_cn_l[usage_index],\n",
      "                                 'example': example,\n",
      "                                 'gre_example_en': gre_example_en,\n",
      "                                 'gre_example_cn': gre_example_cn,\n",
      "                                 'pos': base_phrase_d['pos'],\n",
      "                                 'usage_index': unicode(usage_index + 1),\n",
      "                                 'phrase': phrase\n",
      "                                }\n",
      "            phrase_uid = phrase + unicode(usage_index+1)\n",
      "            processed_phrase_d[phrase_uid] = phrase_detailed_d\n",
      "    return processed_phrase_d"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configDyAfterMain\n",
      "dy_phrase_processed_d = process_dy_phrase_block_str(dy_phrase_d)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configDyAfterMain\n",
      "with codecs.open('duanyu_base_d.txt', 'w', encoding='utf-8') as f:\n",
      "    json.dump(dy_phrase_processed_d, f)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configDy -p\n",
      "\n",
      "if __name__ == '__main__':\n",
      "    main()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# example\n",
      "iter_print(dy_phrase_processed_d['so far2'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "! jupyter nbconvert explore_all_in_one.ipynb --to html"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "! jupyter nbconvert explore_all_in_one.ipynb --to markdown"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}