anki_import.ipynb

{
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  },
  "name": ""
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "# \u8bf4\u660e\n",
      "\n",
      "\u8fd9\u4e2anotebook\u5c55\u793a\u4e86\u5982\u4f55\u5c06\u4e00\u4e2ajson\u5bf9\u8c61\u8f6c\u6362\u4e3a\u53ef\u5bfc\u5165Anki\u7684\u6587\u4ef6\u3002\u91cd\u70b9\u5728\u4e8eAnki\u4e2dNoteType\u7684\u8bbe\u8ba1\u3002\u5185\u5bb9\u4e0a\u627f\u63a5explore_all_in_one.ipynb\u3002\n",
      "\n",
      "\u300aGRE\u6838\u5fc3\u8bcd\u6c47\u8003\u6cd5\u7cbe\u6790\u300b\u3001\u300aGRE\u6838\u5fc3\u8bcd\u6c47\u52a9\u8bb0\u4e0e\u7cbe\u7ec3\u300b\u4ee5\u53ca\u4ece\u7f51\u4e0a\u627e\u5230\u7684\u300a\u4e0d\u62e9\u624b\u6bb5\u80cc\u5355\u8bcd\u300b\u5bf9\u5e94NoteType\u4e3aGreWord\u3002\n",
      "\u300aGRE\u9ad8\u5206\u5fc5\u5907\u77ed\u8bed\u642d\u914d\u300b\u5bf9\u5e94NoteTpye\u4e3aGrePhrase\u3002\n",
      "\n",
      "notebook\u6267\u884c\u5b8c\u540e\uff0c\u4f1a\u81ea\u52a8\u751f\u6210\u4e24\u4e2a\u811a\u672c\uff0c\u540d\u5b57\u53c2\u89c1\u53d8\u91cffile_name_greword\uff0cfile_name_grephrase\u3002\u5355\u72ec\u8fd0\u884c\u4e24\u4e2a\u811a\u672c\u4e5f\u53ef\u5b8c\u6210\u8f6c\u6362\uff0c\u53ea\u8981\u6709python\u5c31\u53ef\u4f7f\u7528\u3002\n",
      "\n",
      "\u5b50\u7ae0\u8282\u300a\u5904\u7406\u53d1\u97f3\u6587\u4ef6\u300b\u548c\u300a\u6dfb\u52a0\u6587\u4ef6\u300b\uff0c\u9700\u8981\u8bb8\u591a\u5b9a\u5236\u6587\u4ef6\u3002\u6240\u4ee5\u6ca1\u6709\u5bfc\u51fa\u5230\u8f6c\u6362\u811a\u672c\u3002\u5982\u679c\u6ca1\u6709\u5bf9\u5e94\u6587\u4ef6\u7684\u8bdd\uff0c\u76f4\u63a5\u8fd0\u884c\u8fd9\u4e2anotebook\u800c\u4f1a\u62a5\u9519\u3002\u6240\u4ee5\u5982\u679c\u53ea\u60f3\u5f97\u5230\u65e0\u53d1\u97f3\u65e0\u7b14\u8bb0\u7248\u672c\u7684\u5bfc\u5165\u6587\u4ef6\uff0c\u8bf7\u8fd0\u884c\u90a3\u4e24\u4e2a\u8f6c\u6362\u811a\u672c\u3002\n",
      "\n",
      "\u8f6c\u6362\u51fa\u7684Anki\u5bfc\u5165\u6587\u4ef6\uff0c\u540d\u5b57\u53c2\u89c1\u53d8\u91cfoutput_file_GreWord\uff0coutput_file_GrePhrase\u3002"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%run sync_to_file_magic_command.py"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "file_name_greword = 'CreateAnkiImport_GreWord.py'\n",
      "file_name_grephrase = 'CreateAnkiImport_GrePhrase.py'\n",
      "configCreAnkiImpGreWord = file_name_greword\n",
      "configCreAnkiImpGrePhrase = file_name_grephrase\n",
      "configMyHelpers = 'my_helpers.py'"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "\u8865\u5145\u4e24\u4e2a\u8f85\u52a9\u51fd\u6570"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configMyHelpers\n",
      "def custom_html_element(_str):\n",
      "    \"\"\"\n",
      "    convert the markdown notations in a string to html tags\n",
      "    currently, only two kinds of markdown notation exist in all the strings\n",
      "    ** and *\n",
      "    \"\"\"\n",
      "    formatted_str = _str\n",
      "    # format double asterisk\n",
      "    match_double_asterisk_re = re.compile(u'\\*\\*(.*?)\\*\\*')\n",
      "    # replace **...** with <strong>...</strong>\n",
      "    #formatted_str = match_double_asterisk_re.sub(r'<strong>\\1</strong>', formatted_str)\n",
      "    # replace **...** with <ins>...</ins>\n",
      "    formatted_str = match_double_asterisk_re.sub(r'<ins>\\1</ins>', formatted_str)\n",
      "    # format single asterisk\n",
      "    # replace *...* with <i>...</i>\n",
      "    match_single_asterisk_re = re.compile(u'\\*(.*?)\\*')\n",
      "    formatted_str = match_single_asterisk_re.sub(r'<i>\\1</i>', formatted_str)\n",
      "    return formatted_str"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configMyHelpers\n",
      "def is_file_and_json_load(file_name_str):\n",
      "    if os.path.isfile(file_name_str):\n",
      "        with codecs.open(file_name_str, 'r', encoding='utf-8') as f:\n",
      "            json_d = json.load(f)\n",
      "        return json_d"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configCreAnkiImpGreWord $configCreAnkiImpGrePhrase -m o\n",
      "\n",
      "# coding:utf-8\n",
      "import json\n",
      "import codecs\n",
      "import os\n",
      "from my_helpers import *"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "GreWord"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# example\n",
      "test_str = 'to **put an end to**(something planned or previously agreed to)'\n",
      "print custom_html_element(test_str)\n",
      "del test_str"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configCreAnkiImpGreWord\n",
      "file_name_new3000 = 'new3000_base_d.txt'\n",
      "file_name_zhuji = 'zhuji_base_d.txt'\n",
      "file_name_bzsdbdc = 'base_data\\\\bzsdbdc_dic.txt'\n",
      "output_file_GreWord = 'AnkiImportData_GreWord.txt'\n",
      "new3000_base_d = None\n",
      "zhuji3000_base_d = None\n",
      "bzsdbdc_data = None\n",
      "new3000_base_d = is_file_and_json_load(file_name_new3000)\n",
      "zhuji3000_base_d = is_file_and_json_load(file_name_zhuji)\n",
      "bzsdbdc_data = is_file_and_json_load(file_name_bzsdbdc)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configCreAnkiImpGreWord\n",
      "no_data_new3000 = new3000_base_d is None\n",
      "no_data_zhuji = zhuji3000_base_d is None\n",
      "no_data_bzsdbdc = bzsdbdc_data is None"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "\u6838\u5fc3\u8f6c\u6362\u51fd\u6570"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configCreAnkiImpGreWord\n",
      "def add_extra_fields():\n",
      "    if no_data_new3000:\n",
      "        print 'New3000 data file does not exists! Nothing can be done...'\n",
      "        return\n",
      "    iter_path = [('all','',True), ('key','usages',False),('all','',False)]\n",
      "    for word, usage_d in iter_through_general(new3000_base_d, iter_path):\n",
      "        usage_d['audio'] = ''\n",
      "        usage_d['mynotes'] = ''\n",
      "    for word_d in new3000_base_d.itervalues():\n",
      "        word_d['similar_form'] = ''"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "add_extra_fields()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# test\n",
      "#pprint(new3000_base_d['abandon'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configCreAnkiImpGreWord\n",
      "def convert_to_GreWord():\n",
      "    if no_data_new3000:\n",
      "        print 'New3000 data file does not exists! Nothing can be done...'\n",
      "        return\n",
      "    if no_data_zhuji:\n",
      "        print 'No data of zhuji!'\n",
      "    if no_data_bzsdbdc:\n",
      "        print 'No data of bzsdbdc!'\n",
      "    output_list = []\n",
      "    None_repr = u''\n",
      "    join_by_line_break = u'<br>'.join\n",
      "    replace_with_br = lambda _str: _str.replace('\\n', '<br>')\n",
      "    tag_pos_prefix = ' in_'\n",
      "    for word in new3000_base_d:\n",
      "        # new 3000 part\n",
      "        \"\"\"\n",
      "        the structure of a word of new3000_base_d.txt\n",
      "\n",
      "        {'phon': u\"[\\u02cc\\xe6d'l\\u026ab]\",\n",
      "         'pos': (1, 6),\n",
      "         'usages': [{'ants': u'\\u53cd\\u3000considered, planned, premeditated, rehearsed \\u9884\\u5148\\u8ba1\\u5212\\u7684',\n",
      "                     'ants_d': {'cn': u'\\u9884\\u5148\\u8ba1\\u5212\\u7684',\n",
      "                                'en': u'considered, planned, premeditated, rehearsed ',\n",
      "                                'en_cn': u'considered, planned, premeditated, rehearsed \\u9884\\u5148\\u8ba1\\u5212\\u7684'},\n",
      "                     'der': '',\n",
      "                     'examples': u'content...',\n",
      "                                    'en': u'not bad for an ad-lib comedy routine',\n",
      "                                    'en_cn': u'content...'},\n",
      "                     'exp': u'*adj.* \\u5373\\u5174\\u7684\\uff1amade or done **without previous thought or preparation**',\n",
      "                     'exp_d': {'cn': u'\\u5373\\u5174\\u7684',\n",
      "                               'en': u'made or done **without previous thought or preparation**',\n",
      "                               'en_cn': u'\\u5373\\u5174\\u7684\\uff1amade or done **without previous thought or preparation**'},\n",
      "                     'ph_symbl': u\"[\\u02cc\\xe6d'l\\u026ab]\",\n",
      "                     'pspeech': u'adj.',\n",
      "                     'syns': u'content...'}\n",
      "        \"\"\"\n",
      "        one_new3000_word_d = new3000_base_d[word]\n",
      "        word_pos_L, word_pos_U = one_new3000_word_d['pos']\n",
      "        word_pos = u'L' + unicode(word_pos_L) + u' U' + unicode(word_pos_U)\n",
      "        num_usages = len(one_new3000_word_d['usages'])\n",
      "        usages_tag = unicode(num_usages) + u'_usage'\n",
      "\n",
      "        for usage_index, usage in enumerate(one_new3000_word_d['usages']):\n",
      "            word_phs = usage['ph_symbl'] \n",
      "            word_tags = usages_tag + tag_pos_prefix + 'zaiyaoniming3000'\n",
      "            if not no_data_zhuji:\n",
      "                if word in zhuji3000_base_d:\n",
      "                    word_tags += tag_pos_prefix + 'zhuji3000'\n",
      "            if not no_data_bzsdbdc:\n",
      "                if word in bzsdbdc_data:\n",
      "                    word_tags += tag_pos_prefix + 'bzsdbdc'\n",
      "            usage_index = unicode(usage_index+1)\n",
      "            word_uid = unicode(word) + usage_index\n",
      "            ph_symbl = usage['ph_symbl']\n",
      "            word_Audio = usage['audio']\n",
      "            pspeech = usage['pspeech']\n",
      "            exp_en = usage['exp_d']['en']\n",
      "            exp_cn = usage['exp_d']['cn']\n",
      "            exp_en_cn = usage['exp_d']['en_cn']\n",
      "            # combine other explanation\n",
      "            #usage_index_l = range(num_usages)\n",
      "            #usage_index_l.remove(usage_index)\n",
      "            #exp_other = ['**\u8003\u6cd5%d**:'%(i+1) + one_new3000_word_d['usages'][i]['exp_d']['en_cn'] +'\\n' for i in usage_index_l]\n",
      "            # use word_block_str as all explanation\n",
      "            exp_all = one_new3000_word_d['word_block_str']\n",
      "            examples_en = usage['examples_d']['en']\n",
      "            examples_cn = usage['examples_d']['cn']\n",
      "            examples_en_cn = usage['examples_d']['en_cn']\n",
      "            examples_others = ''\n",
      "            ants_en = usage['ants_d']['en']\n",
      "            ants_cn = usage['ants_d']['cn']\n",
      "            ants_en_cn = usage['ants_d']['en_cn']\n",
      "            syns = usage['syns']\n",
      "            # der from the book zaiyaoniming3000\n",
      "            der_new3000 = usage['der']\n",
      "            \n",
      "            # bzsdbdc part\n",
      "            how_to_mem_bzsdbdc = None_repr\n",
      "            if not no_data_bzsdbdc:\n",
      "                if word in bzsdbdc_data:\n",
      "                    how_to_mem_bzsdbdc = bzsdbdc_data[word]['combined']         \n",
      "\n",
      "            # zhuji3000 part\n",
      "            how_to_mem_zhuji3000, eytma_gr, eytma_gr_exp, eytma_cognates = None_repr, None_repr, None_repr, None_repr\n",
      "            '''\n",
      "            the structure of a word of zhuji3000_base_d\n",
      "            {'content': u'[\\u6839] per- [through] + vad [go] + -e [v.], go through, \\u904d\\u5e03 \\u2192 vt. \\u5f25\\u6f2b\\uff0c\\u5145\\u6ee1\\n',\n",
      "            'ety': 'vad, vag, ced',\n",
      "            'etyma_cognates_l': u'pervade, evasive, extravagant, vague, cessation, incessant',\n",
      "            'etyma_group_explanation': u'group explanation content',\n",
      "            'phon': u\"[p\\u0259r've\\u026ad]\",\n",
      "            'pos': u'6, 7',\n",
      "            'summary': u'summary content',\n",
      "            'word': u'pervade'}\n",
      "            '''\n",
      "            if not no_data_zhuji:\n",
      "                if word in zhuji3000_base_d:\n",
      "                    how_to_mem_zhuji3000 = zhuji3000_base_d[word]['content']\n",
      "                    eytma_gr = zhuji3000_base_d[word]['ety']\n",
      "                    eytma_gr_exp = zhuji3000_base_d[word]['etyma_group_explanation']\n",
      "                    eytma_cognates = zhuji3000_base_d[word]['etyma_cognates_l']\n",
      "            # extra fields\n",
      "            mynotes = usage['mynotes']\n",
      "            similar_form = one_new3000_word_d['similar_form']\n",
      "            \"\"\"\n",
      "            Anki GreWord Structure\n",
      "            word_uid  word  usage_index  ph_symbl  word_audio  pspeech  mynotes\n",
      "            exp_en exp_cn exp_en_cn exp_all\n",
      "            examples_en examples_cn examples_encn examples_others\n",
      "            ants_en ants_cn ants_encn\n",
      "            syns der_new3000 \n",
      "            how_to_mem_bzsdbdc how_to_mem_zhuji3000 \n",
      "            etyma_group etyma_group_exp etyma_cognates\n",
      "            position similar_form tags\n",
      "            \"\"\"\n",
      "            one_line = [word_uid, word, usage_index, ph_symbl, word_Audio, pspeech, mynotes, \n",
      "                        exp_en, exp_cn, exp_en_cn, exp_all, \n",
      "                        examples_en, examples_cn, examples_en_cn, examples_others,\n",
      "                        ants_en, ants_cn, ants_en_cn] +\\\n",
      "                       [syns, der_new3000, how_to_mem_bzsdbdc, how_to_mem_zhuji3000,\n",
      "                        eytma_gr, eytma_gr_exp, eytma_cognates, word_pos, similar_form, word_tags]\n",
      "            for index, _str in enumerate(one_line):\n",
      "                _str = replace_with_br(collapse_blank_line(_str).strip(' \\n'))\n",
      "                one_line[index] = custom_html_element(_str)\n",
      "            output_list.append(one_line)\n",
      "    output_list.sort(key=lambda x: x[0])\n",
      "    return output_list"
     ],
     "language": "python",
     "metadata": {
      "code_folding": []
     },
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "\u4e0a\u9762\u7684\u51fd\u6570\u6784\u5efa\u4e86\u57fa\u672c\u7684Anki\u5bfc\u5165\u6587\u4ef6\u3002\u73b0\u5728\u8fd8\u9700\u8981\u5c06\u53d1\u97f3\u6587\u4ef6\u7684\u6307\u9488\u6dfb\u52a0\u8fdb\u53bb\u3002  \n",
      "\u5982\u679c\u662f\u66f4\u65b0\u539f\u6709\u7684note\uff0c\u90a3\u4e48\u8fd8\u9700\u8981\u5c06\u539f\u6709note\u7684mynotes\u5b57\u6bb5\u53d6\u51fa\u6765\uff0c\u653e\u5230output_list\u7684\u5bf9\u5e94\u4f4d\u7f6e\u3002  \n",
      "\u6240\u4ee5\u5148\u4e0d\u6267\u884c\u4e0b\u9762\u7684\u51fd\u6570\u3002\u7b49\u5230\u6570\u636e\u8865\u5145\u9f50\u5168\u540e\u518d\u8fd0\u884c\u3002"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configCreAnkiImpGreWord\n",
      "def main():\n",
      "    add_field_audio_and_mynotes()\n",
      "    output_list = convert_to_GreWord()\n",
      "    if output_list is None:\n",
      "        return\n",
      "    with codecs.open(output_file_GreWord, 'w', encoding='utf-8') as f:\n",
      "        for one_line in output_list:\n",
      "            one_string = u'\\t'.join(one_line) + '\\n'\n",
      "            f.write(one_string) \n",
      "    del output_list"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configCreAnkiImpGreWord -p\n",
      "if __name__ == '__main__':\n",
      "    main()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## \u5904\u7406\u53d1\u97f3\u6587\u4ef6\u7684\u601d\u8def\n",
      "\n",
      "Anki\u4e2d\uff0c\u6dfb\u52a0\u53d1\u97f3\u6587\u4ef6\u7684\u8bed\u6cd5\u662f`[sound:\u53d1\u97f3\u6587\u4ef6\u6307\u9488]`\u3002\u53d1\u97f3\u6587\u4ef6\u6307\u9488\u5373\u53d1\u97f3\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u3002\u6240\u6709\u76f8\u5173\u6587\u4ef6\u5fc5\u987b\u653e\u5728Anki\u81ea\u5df1\u7684`collection.media`\u6587\u4ef6\u5939\u91cc\u3002\u6240\u4ee5\u8def\u5f84\u5e94\u8be5\u4f7f\u7528\u76f8\u5bf9\u5f15\u7528\u3002\n",
      "\n",
      "\u63a5\u4e0b\u6765\uff0c\u4ece\u5404\u4e2a\u53d1\u97f3\u5e93\u62bd\u53d6\u6587\u4ef6\u6307\u9488\uff0c\u5e76\u4e14\u5c06\u76f8\u5e94\u6587\u4ef6\u62f7\u8d1d\u5230Anki\u7684`collection.media`\u6587\u4ef6\u5939\u4e0b\uff0c\u540c\u65f6\u5c06\u6307\u9488\u6dfb\u52a0\u5230new3000_base_d\u4e2d\u3002"
     ]
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "\u518d\u8981\u4f60\u547d3000\u4e2d\u7684\u591a\u97f3\u8bcd"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print new3000_base_d['addict']['usages'][0].keys()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "path_to_pron = [('all','',True), ('key','usages',False), ('all','',True),('key','ph_symbl',False)]\n",
      "pre_word_pron = None\n",
      "multi_pron_word_set = set()\n",
      "for word, usage_index, word_pron in iter_through_general(new3000_base_d, deepcopy(path_to_pron)):\n",
      "    if usage_index > 0:\n",
      "        if word_pron != pre_word_pron:\n",
      "            multi_pron_word_set.add(word)\n",
      "    else:\n",
      "        pre_word_pron = word_pron"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print multi_pron_word_set"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## \u6765\u6e90\uff1adsl\u683c\u5f0f\u5b57\u5178\n",
      "\n",
      "dsl\u683c\u5f0f\u7684Longman Pronunciation Dictionary 3rd Ed.\n",
      "\n",
      "\u5173\u4e8e\u5904\u7406dsl\u7684\u57fa\u672c\u77e5\u8bc6\uff0c\u53c2\u8003[Full Text Search in GoldenDict](https://lisok3ajr.wordpress.com/2012/09/18/full-text-search-in-goldendict/)"
     ]
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "\u8bfb\u53d6\u6570\u636e"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "import gzip"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "file_pronunciation = 'D:\\Eudict\\dsl\\En-En_Longman_Pronunciation3\\En-En-Longman_Pronunciation.dsl.dz'\n",
      "dsl_str = gzip.open(file_pronunciation, mode='r').read().decode('utf-16')\n",
      "print dsl_str[100:400]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "match_word_fun = lambda word: re.search('^(%s)[ \\t]*$(.*?)(?=^[^ \\t])'%word, dsl_str, re.M|re.S)\n",
      "findall_word_fun = lambda word: re.findall('^(%s)[ \\t]*$(.*?)(?=^[^ \\t])'%word, dsl_str, re.M|re.S)\n",
      "match_us_pron_re = re.compile('\\[s\\](us.*?)\\[/s\\]')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "\u6709\u7684\u5355\u8bcd\uff0c\u5176\u4e0b\u5c5e\u6d3e\u751f\u8bcd\uff0c\u4ee5\u25b7\u6807\u8bc6\uff0c\u4e5f\u6709\u81ea\u5df1\u7684\u97f3\u6807\u3002\u8fd9\u90e8\u5206\u4e2d\u53ef\u80fd\u51fa\u73b0\u659c\u4f53\u5b57\uff0c\u4ee5[i]..[/i]\u6807\u8bc6\u3002\u53ea\u6709\u4e3b\u91ca\u4e49\u5355\u8bcd\u540e\u9762\u7684\u659c\u4f53\u624d\u662f\u97f3\u6807\u3002"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "match_pspeech_re = re.compile('^[ \\t]*?\\[m1\\]\\[b\\].*?\\[/b\\] \\[i\\] ([a-z, ]+).*?\\[/i\\]', re.M)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# test\n",
      "def unit_test():\n",
      "    result = match_word_fun('content')\n",
      "    result_str = result.group()\n",
      "    print result_str\n",
      "    print 'All pronunciation files: ', match_us_pron_re.findall(result_str)\n",
      "    print 'All part of speech: ', match_pspeech_re.findall(result_str)\n",
      "#unit_test()    \n",
      "del unit_test"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "\u5c06dsl_str\u8f6c\u6362\u4e3adict"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "extract_word_block_re = re.compile(ur'^([a-z-]+)[ \\t]*$(.*?)(?=^[^ \\t])', re.M|re.S|re.I)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "# test\n",
      "#extract_word_block_re.findall(dsl_str[0:5000])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dsl_pron_d = {}\n",
      "for one_match_obj in extract_word_block_re.finditer(dsl_str):\n",
      "    word = one_match_obj.group(1)\n",
      "    if word in dsl_pron_d:\n",
      "        print '%s already exists!'%word\n",
      "    one_word_d = {}\n",
      "    word_block = one_match_obj.group(2)\n",
      "    one_word_d['word_block'] = word_block\n",
      "    one_word_d['pspeech_l'] = match_pspeech_re.findall(word_block)\n",
      "    one_word_d['ph_symbol_l'] = match_us_pron_re.findall(word_block)  \n",
      "    if word in multi_pron_word_set:\n",
      "        #print 'check pspeech'\n",
      "        #print word, one_word_d['pspeech_l']\n",
      "        pass\n",
      "    dsl_pron_d[word] = one_word_d"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# example\n",
      "iter_print(dsl_pron_d['content'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "\u7edf\u8ba1\u8bcd\u6027\u5bf9\u5e94\u5173\u7cfb"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "def summary_pspeech():\n",
      "    #dsl\n",
      "    dsl_pspeech_set = set()\n",
      "    for word, word_d in dsl_pron_d.iteritems():\n",
      "        dsl_pspeech_l = word_d['pspeech_l']\n",
      "        for pspeech in dsl_pspeech_l:\n",
      "            dsl_pspeech_set.add(pspeech)\n",
      "    # new3000\n",
      "    new3000_pspeech_set = set()\n",
      "    path_to_pspeech = path_to_pron = [('all','',True), ('key','usages',False), ('all','',False),('key','pspeech',False)]\n",
      "    for word, pspeech in iter_through_general(new3000_base_d, path_to_pspeech):\n",
      "        for sub_pspeech in pspeech.split('/'):\n",
      "            new3000_pspeech_set.add(sub_pspeech)\n",
      "        stripped_pspeech = pspeech.strip('.')\n",
      "        if word in dsl_pron_d:\n",
      "            for dsl_pspeech in dsl_pron_d[word]['pspeech_l']:\n",
      "                if dsl_pspeech.startswith(stripped_pspeech):\n",
      "                    break\n",
      "            else:\n",
      "                if len(dsl_pron_d[word]['ph_symbol_l']) > 1:\n",
      "                    #print 'pspeech of %s in new3000 not match with dsl'%word\n",
      "                    # a lot!\n",
      "                    pass\n",
      "    print dsl_pspeech_set\n",
      "    print new3000_pspeech_set"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# summary_pspeech()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "dsl_pron_n\u4e2d\u7684\u6709\u6548\u8bcd\u6027\u7c7b\u522b\uff1aadjective verb pronoun preposition adverb  \n",
      "\u6240\u4ee5\uff0c\u53ea\u8981\u770b\u770bdsl_pron_n\u4e2d\u7684\u8bcd\u6027\u662f\u4e0d\u662f\u4ee5new3000_base_d\u4e2d\u7684\u5f00\u5934\u5c31\u53ef\u4ee5\u3002"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## \u518d\u8981\u4f60\u547d3000\u540cdsl_d\u6bd4\u8f83\n",
      "\n",
      "\u5c06\u897f\u6b27\u5b57\u7b26\u8f6c\u6362\u4e3a\u666e\u901a\u5b57\u7b26\uff0c\u5373\u00e9\u00ef\u8f6c\u4e3aei"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "def check_pron_in_new3000_and_dsl(word, print_only_bad_result = True):\n",
      "    word_converted = word.replace(u'\u00e9', 'e').replace(u'\u00ef', 'i').split('/')[0]\n",
      "    return_message_l = []\n",
      "    not_found = False\n",
      "    if not (word_converted in dsl_pron_d):\n",
      "        return_message_l.append('**%s** not found in dsl'%word)\n",
      "        not_found = True\n",
      "    else:\n",
      "        pron_in_dsl_l = dsl_pron_d[word_converted]['ph_symbol_l']\n",
      "        pspeech_in_dsl_l = dsl_pron_d[word_converted]['pspeech_l']\n",
      "        pron_new3000_l = []\n",
      "        pspeech_new3000_l = []\n",
      "        for usage_d in new3000_base_d[word]['usages']:\n",
      "            pron_new3000_l.append(usage_d['ph_symbl'])\n",
      "            pspeech_new3000_l.append(usage_d['pspeech'])\n",
      "        diff_pron_new3000_set = set(pron_new3000_l)\n",
      "        if len(pron_in_dsl_l) < len(diff_pron_new3000_set):\n",
      "            message = '**%s** in dsl has less pron'%word\n",
      "            message += '\\n' + str(len(pron_in_dsl_l)) + ', ' + str(len(diff_pron_new3000_set))\n",
      "            message += '\\n' + ','.join(pron_in_dsl_l)\n",
      "            message += '\\n' + ','.join(pron_new3000_l)\n",
      "            return_message_l.append(message)\n",
      "        else:\n",
      "            if not print_only_bad_result:\n",
      "                 return_message_l.append('**%s** in dsl has enough pron'%word)\n",
      "    return '\\n'.join(return_message_l), not_found"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "result_l = []\n",
      "not_found_word_l = []\n",
      "for word in new3000_base_d.iterkeys():\n",
      "    message_str, not_found = check_pron_in_new3000_and_dsl(word)\n",
      "    if message_str != '':\n",
      "        result_l.append(message_str)\n",
      "    if not_found:\n",
      "        not_found_word_l.append(word)\n",
      "        if word in multi_pron_word_set:\n",
      "            print 'Warning! **%s** in multi_pron_word_set'%word"
     ],
     "language": "python",
     "metadata": {
      "scrolled": false
     },
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "with codecs.open('temp_check_pron_log.txt', 'w', encoding='utf-8') as f:\n",
      "    json.dump(result_l, f, indent=5)\n",
      "    json.dump(not_found_word_l, f, indent=2)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print '%d words not found'%len(not_found_word_l)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "\u867d\u7136\u8fd8\u6709153\u4e2a\u6ca1\u627e\u5230\uff0c\u4f46\u6ce8\u610f\u5230\uff0c\u591a\u97f3\u8bcd\u90fd\u5728\u5176\u4e2d\u3002"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## \u7528\u97e6\u6c0f\u53d1\u97f3\u5e93\u8865\u5145\n",
      "\n",
      "\u4ece\u7f51\u4e0a\u627e\u7684\u97e6\u6c0f\u53d1\u97f3\u5e93\uff0c\u7f51\u5740\uff1ahttp://blog.emagic.org.cn/content/i1931.html\n",
      "\n",
      "ed2k\u94fe\u63a5\n",
      "\n",
      "        ed2k://|file|%E9%9F%A6%E6%B0%8F%E5%B8%B8%E7%94%A8%E5%8D%95%E8%AF%8D%E8%AF%AD%E9%9F%B3%E5%BA%93.rar|315458082|88b70fe90a6658cec689352f66a7af6c|h=4rblspftuskt5gfvmpbnfkdvhi2ey3fn|/"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "path_of_media_source = 'D:\\\\mvoice\\\\'\n",
      "word_list_file = 'word_list.txt'"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "media_path_dict = {}\n",
      "match_word = r'([a-z1-9 ~]+)\\.mp3'\n",
      "match_word_re = re.compile(match_word, re.I|re.M)\n",
      "with codecs.open(path_of_media_source + word_list_file, encoding='utf-8') as f:\n",
      "    for line in f:\n",
      "        result = match_word_re.search(line)\n",
      "        if not (result is None):\n",
      "            media_path_dict[result.group(1)] = line.strip()\n",
      "        else:\n",
      "            #print line\n",
      "            pass"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print media_path_dict['habit']"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "count = 0\n",
      "still_not_found_word_l = []\n",
      "for word in not_found_word_l:\n",
      "    word_converted = word.replace(u'\u00e9', 'e').replace(u'\u00ef', 'i').split('/')[0]\n",
      "    if word_converted in media_path_dict:\n",
      "        count += 1\n",
      "        #print 'found', word\n",
      "    else:\n",
      "        still_not_found_word_l.append(word)\n",
      "print 'found %d of %d'%(count, len(not_found_word_l))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## \u7528mdict\u8865\u5145\n",
      "\n",
      "\u4f7f\u7528\u6717\u6587\u5f53\u4ee3\u7b2c5\u7248\u7684mdx\u548cmdd\u6587\u4ef6\n",
      "\n",
      "\u4f7f\u7528\u63d2\u4ef6 https://bitbucket.org/xwang/mdict-analysis\n"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from readmdict import MDX, MDD\n",
      "from bs4 import BeautifulSoup\n",
      "file_to_longman_mdx = \"D:\\Eudict\\Frequent\\Longman Dictionary of Contemporary English.mdx\"\n",
      "mdx = MDX(file_to_longman_mdx)\n",
      "longman_mdx_iter = mdx.items()\n",
      "longman_in_new3000_d = {}\n",
      "for word, word_block in longman_mdx_iter:\n",
      "    if word in new3000_base_d:\n",
      "        longman_in_new3000_d[word] = word_block\n",
      "print 'In longman, found %d words of new3000 (%d in total)'%(len(longman_in_new3000_d), len(new3000_base_d))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "\u62bd\u53d6\u97f3\u9891\u5730\u5740"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# this is the pattern we gonna use\n",
      "soup = BeautifulSoup(longman_in_new3000_d['abandon'],\"lxml\")\n",
      "print soup.find_all(href=re.compile('sound.*?US'))[0]['href'][8:]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "count = 0\n",
      "still_still_not_found_word_l = []\n",
      "longman_found_word_d = {}\n",
      "for word in still_not_found_word_l:\n",
      "    founded = False\n",
      "    word_converted = word.replace(u'\u00e9', 'e').replace(u'\u00ef', 'i').split('/')[0]\n",
      "    if word_converted in longman_in_new3000_d:\n",
      "        soup = BeautifulSoup(longman_in_new3000_d[word_converted],\"lxml\")\n",
      "        find_result = soup.find_all(href=re.compile('sound.*?US'))\n",
      "        if len(find_result) != 0:\n",
      "            count += 1\n",
      "            #print word\n",
      "            founded = True\n",
      "            longman_found_word_d[word] = find_result[0]['href'][8:]\n",
      "    if not founded:\n",
      "        still_still_not_found_word_l.append(word)\n",
      "print 'found %d of %d'%(count, len(still_not_found_word_l))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# example\n",
      "longman_found_word_d['ingratiating']"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# unzip the mdd mdx file. \n",
      "# Warning! This take a lot of time. I have already unpacked it, so commend the next line\n",
      "#! python readmdict.py -x \"D:\\Eudict\\Frequent\\Longman Dictionary of Contemporary English.mdx\""
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "\u6dfb\u52a0\u97f3\u9891\u6307\u9488"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import shutil"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "anki_media_collection = os.path.expanduser('~\\\\Documents\\\\Anki\\\\xiaohang\\\\collection.media')\n",
      "dsl_source_media_path = 'D:\\Eudict\\dsl\\En-En_Longman_Pronunciation3\\En-En-Longman_Pronunciation.dsl.dz.files'\n",
      "longman_source_media_path = 'D:\\Eudict\\Frequent\\data'"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def add_audio_pointer(word):\n",
      "    word_converted = word.replace(u'\u00e9', 'e').replace(u'\u00ef', 'i').split('/')[0]\n",
      "    word_d = new3000_base_d[word]\n",
      "    for usage_d in word_d['usages']:\n",
      "        usage_d['audio'] = ''\n",
      "        source_audio_file_name = None\n",
      "        first_pspeech_match_obj = re.search('^([a-z]+)\\.', usage_d['pspeech'])\n",
      "        if first_pspeech_match_obj is None:\n",
      "            print '%s has no pspeech'%word\n",
      "            new3000_pspeech = ''\n",
      "        else:\n",
      "            new3000_pspeech = first_pspeech_match_obj.group(1)\n",
      "        if new3000_pspeech in ['vt', 'vi']:\n",
      "            new3000_pspeech = 'v'\n",
      "        new_audio_pointer_without_ext = word_converted + '_' + new3000_pspeech\n",
      "        new_audio_file_name_without_ext = anki_media_collection + '\\\\' + new_audio_pointer_without_ext\n",
      "        new_audio_pointer_without_ext = '[sound:' + new_audio_pointer_without_ext\n",
      "        existed = False\n",
      "        for file_ext in ['.wav', '.mp3', '.spx']:\n",
      "            if os.path.isfile(new_audio_file_name_without_ext + file_ext):\n",
      "                # print 'existed!'\n",
      "                existed = True\n",
      "                usage_d['audio'] = new_audio_pointer_without_ext + file_ext + ']'\n",
      "                break\n",
      "        if existed:\n",
      "            continue\n",
      "        if word_converted in dsl_pron_d:\n",
      "            dsl_word_d = dsl_pron_d[word_converted]\n",
      "            if word in multi_pron_word_set:\n",
      "                # check pspeech\n",
      "                for index, dsl_pspeech in enumerate(dsl_word_d['pspeech_l']):\n",
      "                    for dsl_sub_pspeech in dsl_pspeech.split(','):\n",
      "                        if dsl_sub_pspeech.strip().startswith(new3000_pspeech):\n",
      "                            source_audio_file_name = dsl_source_media_path + '\\\\' + dsl_word_d['ph_symbol_l'][index]\n",
      "                            break\n",
      "                else:\n",
      "                    print 'no match of pspeech, word %s'%word\n",
      "                    print dsl_word_d['pspeech_l'], new3000_pspeech\n",
      "                    pass\n",
      "            else:\n",
      "                # use the first audio pointer\n",
      "                source_audio_file_name = dsl_source_media_path + '\\\\' + dsl_word_d['ph_symbol_l'][0] \n",
      "            if not (source_audio_file_name is None):\n",
      "                new_audio_pointer = new_audio_pointer_without_ext + '.wav]'\n",
      "                new_audio_file_name = new_audio_file_name_without_ext + '.wav'\n",
      "        else:\n",
      "            # the not found word\n",
      "            if word_converted in media_path_dict:\n",
      "                # try webster\n",
      "                source_audio_file_name = media_path_dict[word_converted]\n",
      "                new_audio_pointer = new_audio_pointer_without_ext + '.mp3]'\n",
      "                new_audio_file_name = new_audio_file_name_without_ext + '.mp3'\n",
      "            elif word in longman_found_word_d:\n",
      "                # try longman\n",
      "                source_audio_file_name = longman_source_media_path + '\\\\' + longman_found_word_d[word]\n",
      "                new_audio_pointer = new_audio_pointer_without_ext + '.spx]'\n",
      "                new_audio_file_name = new_audio_file_name_without_ext + '.spx'\n",
      "        if not (source_audio_file_name is None):\n",
      "            usage_d['audio'] = new_audio_pointer\n",
      "            shutil.copy(source_audio_file_name, new_audio_file_name)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "for word in new3000_base_d:\n",
      "    add_audio_pointer(word)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# example\n",
      "word = 'compendium'\n",
      "for index, usage_d in enumerate(new3000_base_d[word]['usages']):\n",
      "    print usage_d['audio']"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## \u8f6c\u6362\u4e3amp3\n",
      "\n",
      "\u5230\u8fd9\u91cc\uff0c\u7535\u8111\u4e0a\u5df2\u7ecf\u53ef\u4ee5\u53d1\u97f3\u4e86\u3002\u4f46\u624b\u673a\u53ea\u652f\u6301mp3\u683c\u5f0f\uff0c\u6240\u4ee5\u8981\u5c06collection.media\u4e2d\u7684wav\u548cspx\u8f6c\u6362\u4e3amp3\u3002\n",
      "\n",
      "\u4f7f\u7528pydub+ffmpeg\n",
      "\n",
      "\u53c2\u8003[Pydub ](https://github.com/jiaaro/pydub/)"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "from pydub import AudioSegment\n",
      "import glob"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def convert_to_mp3():\n",
      "    owd = os.getcwd()\n",
      "    os.chdir(anki_media_collection)\n",
      "    extension_list = ('*.wav', '*.spx')\n",
      "    for extension in extension_list:\n",
      "        for audio in glob.glob(extension):\n",
      "            mp3_filename = os.path.splitext(os.path.basename(audio))[0] + '.mp3'\n",
      "            if not os.path.isfile(mp3_filename):\n",
      "                AudioSegment.from_file(audio).export(mp3_filename, format='mp3')\n",
      "    os.chdir(owd)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "convert_to_mp3()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "def modify_audio_pointer():\n",
      "    path_to_usage_d = path_to_pron = [('all','',False), ('key','usages',False), ('all','',False)]\n",
      "    for usage_d, in iter_through_general(new3000_base_d, path_to_usage_d):\n",
      "        old_audio_name = usage_d['audio']\n",
      "        if old_audio_name != '':\n",
      "            new_audio_name = os.path.splitext(os.path.basename(old_audio_name))[0] + '.mp3]'\n",
      "            usage_d['audio'] = new_audio_name"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "modify_audio_pointer()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# test\n",
      "#iter_print(new3000_base_d['chaperone'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "\u6dfb\u52a0\u7b14\u8bb0"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "old_anki_GreWord_file_name = 'old_anki_greword.txt'"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "def add_mynotes():\n",
      "    if not os.path.isfile(old_anki_GreWord_file_name):\n",
      "        return\n",
      "    old_data_line_l = codecs_open_r_utf8(old_anki_GreWord_file_name).split('\\n')\n",
      "    for line in old_data_line_l:\n",
      "        field_l = line.split('\\t')\n",
      "        word = field_l[1]\n",
      "        usage_index = int(field_l[2])\n",
      "        my_note = field_l[6]\n",
      "        if my_note != '':\n",
      "            new3000_base_d[word]['usages'][usage_index-1]['mynotes'] = my_note"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "add_mynotes()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## \u6dfb\u52a0\u5f62\u8fd1\u8bcd\n",
      "\n",
      "\u7b2c\u4e00\u904d\u80cc\u8bf5\u65f6\uff0c\u7531\u4e8e\u4e0d\u4f1a\u7cbe\u786e\u7684\u8bb0\u5fc6\u8bcd\u5f62\uff0c\u7ecf\u5e38\u51fa\u73b0\u5404\u79cd\u8bef\u8bc6\u522b\u3002\u6240\u4ee5\u52a0\u5165\u4e00\u4e2a\u73af\u8282\uff0c\u6839\u636e[Wagner\u2013Fischer\u7b97\u6cd5](https://en.wikipedia.org/wiki/Wagner%E2%80%93Fischer_algorithm)\u81ea\u52a8\u751f\u6210\u5f62\u8fd1\u8bcd\uff0c\u5e76\u5c06\u5f62\u8fd1\u8bcd\u7684\u7b80\u5355\u4e2d\u6587\u91ca\u4e49\u6dfb\u52a0\u5230\u5176\u540e\u3002\u8ba1\u7b97\u5355\u8bcd\u7684Levenshtein distance\u65f6\uff0c\u4f7f\u7528\u7684\u522b\u4eba\u5199\u597d\u7684[\u7b97\u6cd5](https://gist.github.com/kylebgorman/8034009)\uff0c\u5e76\u6ca1\u505a\u4f18\u5316\uff0c\u6240\u4ee5\u53ef\u80fd\u6162\u4e00\u4e9b\u3002\u53e6\u5916\uff0c\u4e5f\u6ca1\u6709\u7f13\u5b58\u8ddd\u79bb\uff0c\u5373\u8ba1\u7b97\u5b8cA\u3001B\u95f4\u7684\u8ddd\u79bb\u540e\uff0cB\u3001A\u95f4\u8fd8\u4f1a\u518d\u8ba1\u7b97\u4e00\u6b21\u3002\n",
      "\n",
      "\u7531\u4e8eipython\u6267\u884cmultiprocess\u6709bug\uff0c[multiprocessing](https://docs.python.org/2/library/multiprocessing.html)\u63d0\u5230\uff1a\n",
      "    \n",
      "    This means that some examples, such as the Pool examples will not work in the interactive interpreter. \n",
      "        \n",
      "\u6240\u4ee5\uff0c\u5177\u4f53\u7a0b\u5e8f\u53c2\u89c1\u811a\u672c`add_similar_word.py`\u3002\u8be5\u811a\u672c\u751f\u6210`similar_word.txt`\uff0c\u91cc\u9762\u6709\u9700\u8981\u7684\u6570\u636e\u3002"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "with open('similar_word.txt') as f:\n",
      "    similar_word_l = json.load(f)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "similar_word_d = {pair[0]:pair[1] for pair in similar_word_l}"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "def add_similar_word_to_new3000_base_d():\n",
      "    for word in similar_word_d:\n",
      "        new3000_base_d[word]['similar_form'] = similar_word_d[word]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "add_similar_word_to_new3000_base_d()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "\u751f\u6210\u6587\u4ef6"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "greword_import_data_l = convert_to_GreWord()\n",
      "with codecs.open(output_file_GreWord, 'w', encoding='utf-8') as f:\n",
      "    for one_line in greword_import_data_l:\n",
      "        one_string = u'\\t'.join(one_line) + '\\n'\n",
      "        f.write(one_string)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# test\n",
      "#iter_print(new3000_base_d['hike'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {
      "collapsed": true
     },
     "source": [
      "GrePhrase"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "%%sync_to_file $configCreAnkiImpGrePhrase\n",
      "file_name_duanyu = 'duanyu_base_d.txt'\n",
      "duanyu_base_d = is_file_and_json_load(file_name_duanyu)\n",
      "output_file_GrePhrase = 'AnkiImportData_GrePhrase.txt'"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print 'The structure of duanyu_base_d'\n",
      "pprint(duanyu_base_d['under one\\'s control1'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $configCreAnkiImpGrePhrase\n",
      "def convert_to_GrePhrase():\n",
      "    with codecs.open(output_file_GrePhrase, 'w', encoding='utf-8') as f:\n",
      "        my_notes = ''\n",
      "        for phrase_uid, phrase_dict in duanyu_base_d.iteritems():\n",
      "            one_line = [phrase_uid, phrase_dict['phrase'], phrase_dict['usage_index'], my_notes,\n",
      "                        phrase_dict['en_exp'], phrase_dict['cn_exp'], \n",
      "                        phrase_dict['example'], phrase_dict['gre_example_cn'],\n",
      "                        phrase_dict['gre_example_en']]\n",
      "            one_line = '\\t'.join(one_line) + '\\n'\n",
      "            f.write(one_line)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "convert_to_GrePhrase()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%sync_to_file $file_name_grephrase -p\n",
      "\n",
      "if __name__ == '__main__':\n",
      "    if not (duanyu_base_d is None):\n",
      "        convert_to_GrePhrase()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "! jupyter nbconvert anki_import.ipynb --to markdown\n",
      "! jupyter nbconvert anki_import.ipynb -- to html"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}